From 2403c0e1981d6cd3ca60a88c3fc38be7e3e84810 Mon Sep 17 00:00:00 2001 From: Igor Perepilitsyn Date: Sat, 18 Jun 2022 15:00:47 +0300 Subject: [PATCH 01/42] toggle json logging via config --- cmd/headscale/cli/root.go | 4 ++++ config-example.yaml | 1 + config.go | 4 ++++ 3 files changed, 9 insertions(+) diff --git a/cmd/headscale/cli/root.go b/cmd/headscale/cli/root.go index 459a99f..659358f 100644 --- a/cmd/headscale/cli/root.go +++ b/cmd/headscale/cli/root.go @@ -55,6 +55,10 @@ func initConfig() { zerolog.SetGlobalLevel(zerolog.Disabled) } + if cfg.JSONLogs { + log.Logger = log.Output(os.Stdout) + } + if !cfg.DisableUpdateCheck && !machineOutput { if (runtime.GOOS == "linux" || runtime.GOOS == "darwin") && Version != "dev" { diff --git a/config-example.yaml b/config-example.yaml index 2019a13..883d81f 100644 --- a/config-example.yaml +++ b/config-example.yaml @@ -173,6 +173,7 @@ tls_cert_path: "" tls_key_path: "" log_level: info +json_logs: false # Path to a file containg ACL policies. # ACLs can be defined as YAML or HUJSON. diff --git a/config.go b/config.go index 0024731..83cfc62 100644 --- a/config.go +++ b/config.go @@ -37,6 +37,7 @@ type Config struct { NoisePrivateKeyPath string BaseDomain string LogLevel zerolog.Level + JSONLogs bool DisableUpdateCheck bool DERP DERPConfig @@ -147,6 +148,7 @@ func LoadConfig(path string, isFile bool) error { viper.SetDefault("tls_client_auth_mode", "relaxed") viper.SetDefault("log_level", "info") + viper.SetDefault("json_logs", false) viper.SetDefault("dns_config", nil) @@ -434,6 +436,7 @@ func GetHeadscaleConfig() (*Config, error) { if err != nil { logLevel = zerolog.DebugLevel } + jsonLogs := viper.GetBool("json_logs") legacyPrefixField := viper.GetString("ip_prefix") if len(legacyPrefixField) > 0 { @@ -488,6 +491,7 @@ func GetHeadscaleConfig() (*Config, error) { GRPCAllowInsecure: viper.GetBool("grpc_allow_insecure"), DisableUpdateCheck: viper.GetBool("disable_check_updates"), LogLevel: logLevel, + JSONLogs: jsonLogs, IPPrefixes: prefixes, PrivateKeyPath: AbsolutePathFromConfigPath( From bb6b07dedcb15b9f84138ece6dac503b1065bcd4 Mon Sep 17 00:00:00 2001 From: Igor Perepilitsyn Date: Fri, 26 Aug 2022 13:43:25 +0200 Subject: [PATCH 02/42] FIXES #768 add new config entry to the old itegration tests --- integration_test/etc/alt-config.dump.gold.yaml | 1 + integration_test/etc/alt-env-config.dump.gold.yaml | 1 + integration_test/etc/config.dump.gold.yaml | 1 + 3 files changed, 3 insertions(+) diff --git a/integration_test/etc/alt-config.dump.gold.yaml b/integration_test/etc/alt-config.dump.gold.yaml index 3d38b12..07544d4 100644 --- a/integration_test/etc/alt-config.dump.gold.yaml +++ b/integration_test/etc/alt-config.dump.gold.yaml @@ -29,6 +29,7 @@ ip_prefixes: - 100.64.0.0/10 listen_addr: 0.0.0.0:18080 log_level: disabled +json_logs: false logtail: enabled: false metrics_listen_addr: 127.0.0.1:19090 diff --git a/integration_test/etc/alt-env-config.dump.gold.yaml b/integration_test/etc/alt-env-config.dump.gold.yaml index f3ebd08..805c25a 100644 --- a/integration_test/etc/alt-env-config.dump.gold.yaml +++ b/integration_test/etc/alt-env-config.dump.gold.yaml @@ -28,6 +28,7 @@ ip_prefixes: - 100.64.0.0/10 listen_addr: 0.0.0.0:18080 log_level: disabled +json_logs: false logtail: enabled: false metrics_listen_addr: 127.0.0.1:19090 diff --git a/integration_test/etc/config.dump.gold.yaml b/integration_test/etc/config.dump.gold.yaml index 91ca5b9..ff61c6f 100644 --- a/integration_test/etc/config.dump.gold.yaml +++ b/integration_test/etc/config.dump.gold.yaml @@ -29,6 +29,7 @@ ip_prefixes: - 100.64.0.0/10 listen_addr: 0.0.0.0:8080 log_level: disabled +json_logs: false logtail: enabled: false metrics_listen_addr: 127.0.0.1:9090 From 90e840c3c93cae2c0ac133eb1cf67d637946fa02 Mon Sep 17 00:00:00 2001 From: Mike Lloyd Date: Sun, 4 Sep 2022 09:42:23 -0700 Subject: [PATCH 03/42] Add reverse proxy documentation --- docs/README.md | 1 + docs/reverse-proxy.md | 43 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+) create mode 100644 docs/reverse-proxy.md diff --git a/docs/README.md b/docs/README.md index 9f8e681..dfad611 100644 --- a/docs/README.md +++ b/docs/README.md @@ -28,6 +28,7 @@ written by community members. It is _not_ verified by `headscale` developers. - [Running headscale in a container](running-headscale-container.md) - [Running headscale on OpenBSD](running-headscale-openbsd.md) +- [Running headscale behind a reverse proxy](reverse-proxy.md) ## Misc diff --git a/docs/reverse-proxy.md b/docs/reverse-proxy.md new file mode 100644 index 0000000..c9a1d1b --- /dev/null +++ b/docs/reverse-proxy.md @@ -0,0 +1,43 @@ +# Running behind a reverse proxy + +Running Headscale behind a reverse proxy is suitable for container-based deployments. This is especially useful on a server were port 443 is already being used for other web services. + +Headscale can be configured not to use TLS, leaving it to the reverse proxy to handle. Add the following configuration values to your headscale config file. + +```yaml +server_url: https:// # This should be the FQDN at which headscale will be served +listen_addr: 0.0.0.0:8080 +metrics_listen_addr: 0.0.0.0:9090 +tls_cert_path: "" +tls_key_path: "" +``` + +## nginx +The following example configuration can be used in your nginx setup, substituting values as necessary. `` should be the IP address and port where headscale is running. In most cases, this will be `http://localhost:8080`. + +```Nginx +server { + listen 80; + listen [::]:80; + + listen 443 ssl http2; + listen [::]:443 ssl http2; + + server_name ; + + ssl_certificate ; + ssl_certificate_key ; + ssl_protocols TLSv1.2 TLSv1.3; + + location / { + proxy_pass http://; + proxy_set_header Host $server_name; + proxy_redirect http:// https://; + proxy_buffering off; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $http_x_forwarded_proto; + add_header Strict-Transport-Security "max-age=15552000; includeSubDomains" always; + } +} +``` From 02ab3a2cb66c6f8b711fc0fe5f29390474a491c6 Mon Sep 17 00:00:00 2001 From: Mike Lloyd Date: Sun, 4 Sep 2022 09:46:11 -0700 Subject: [PATCH 04/42] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cf0afb7..e194a7a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ - Target Go 1.19 for Headscale [#778](https://github.com/juanfont/headscale/pull/778) - Target Tailscale v1.30.0 to build Headscale [#780](https://github.com/juanfont/headscale/pull/780) - Give a warning when running Headscale with reverse proxy improperly configured for WebSockets [#788](https://github.com/juanfont/headscale/pull/788) +- Add documentation for running behind a reverse proxy. ## 0.16.4 (2022-08-21) From f6e83413e5871cf501047c165cd318ccb655d380 Mon Sep 17 00:00:00 2001 From: Mike Lloyd Date: Sun, 4 Sep 2022 09:49:34 -0700 Subject: [PATCH 05/42] Add PR link to changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e194a7a..3504797 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ - Target Go 1.19 for Headscale [#778](https://github.com/juanfont/headscale/pull/778) - Target Tailscale v1.30.0 to build Headscale [#780](https://github.com/juanfont/headscale/pull/780) - Give a warning when running Headscale with reverse proxy improperly configured for WebSockets [#788](https://github.com/juanfont/headscale/pull/788) -- Add documentation for running behind a reverse proxy. +- Add documentation for running behind a reverse proxy. [#790](https://github.com/juanfont/headscale/pull/790) ## 0.16.4 (2022-08-21) From dc18d64286534f8f9bfb013a3e4e6773b46d2afe Mon Sep 17 00:00:00 2001 From: Mike Lloyd Date: Sun, 4 Sep 2022 17:26:33 -0700 Subject: [PATCH 06/42] Add websockets config --- docs/reverse-proxy.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/reverse-proxy.md b/docs/reverse-proxy.md index c9a1d1b..0dbd75f 100644 --- a/docs/reverse-proxy.md +++ b/docs/reverse-proxy.md @@ -29,8 +29,17 @@ server { ssl_certificate_key ; ssl_protocols TLSv1.2 TLSv1.3; + map $http_upgrade $connection_upgrade { + default keep-alive; + 'websocket' upgrade; + '' close; + } + location / { proxy_pass http://; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; proxy_set_header Host $server_name; proxy_redirect http:// https://; proxy_buffering off; From 3a042471b798af6facf6e5f36dd3366168fbbd5b Mon Sep 17 00:00:00 2001 From: Mike Lloyd Date: Sun, 4 Sep 2022 17:39:51 -0700 Subject: [PATCH 07/42] Add web sockets section --- docs/reverse-proxy.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/reverse-proxy.md b/docs/reverse-proxy.md index 0dbd75f..db698d8 100644 --- a/docs/reverse-proxy.md +++ b/docs/reverse-proxy.md @@ -2,6 +2,10 @@ Running Headscale behind a reverse proxy is suitable for container-based deployments. This is especially useful on a server were port 443 is already being used for other web services. +### Web Sockets +The reverse proxy _must_ be configured to support websockets if you are running headscale 0.17.x+ and tailscale v1.30+. + +### TLS Headscale can be configured not to use TLS, leaving it to the reverse proxy to handle. Add the following configuration values to your headscale config file. ```yaml From 45df6e77ff39d9d68104d3b06a55508aaf5086e9 Mon Sep 17 00:00:00 2001 From: Mike Lloyd <49411532+mike-lloyd03@users.noreply.github.com> Date: Tue, 6 Sep 2022 15:37:39 -0700 Subject: [PATCH 08/42] Apply suggestions from code review Thanks for the pointers! Co-authored-by: Juan Font --- docs/reverse-proxy.md | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/docs/reverse-proxy.md b/docs/reverse-proxy.md index db698d8..1f66d3c 100644 --- a/docs/reverse-proxy.md +++ b/docs/reverse-proxy.md @@ -1,9 +1,13 @@ -# Running behind a reverse proxy +# Running headscale behind a reverse proxy + +Running headscale behind a reverse proxy is useful when running multiple applications on the same server, and you want to reuse the same external IP and port - usually tcp/443 for HTTPS. + +### WebSockets +The reverse proxy MUST be configured to support WebSockets, as it is needed for clients running Tailscale v1.30+. + +WebSockets support is required when using the headscale embedded DERP server. In this case, you will also need to expose the UDP port used for STUN (by default, udp/3478). Please check our [config-example.yaml](https://github.com/juanfont/headscale/blob/main/config-example.yaml). -Running Headscale behind a reverse proxy is suitable for container-based deployments. This is especially useful on a server were port 443 is already being used for other web services. -### Web Sockets -The reverse proxy _must_ be configured to support websockets if you are running headscale 0.17.x+ and tailscale v1.30+. ### TLS Headscale can be configured not to use TLS, leaving it to the reverse proxy to handle. Add the following configuration values to your headscale config file. From 0fe3c21223d22ac46e433507f405cbd6f77b8f26 Mon Sep 17 00:00:00 2001 From: Mike Lloyd Date: Tue, 6 Sep 2022 16:12:20 -0700 Subject: [PATCH 09/42] Move map block out of server block --- docs/reverse-proxy.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/reverse-proxy.md b/docs/reverse-proxy.md index 1f66d3c..0809d04 100644 --- a/docs/reverse-proxy.md +++ b/docs/reverse-proxy.md @@ -24,6 +24,12 @@ tls_key_path: "" The following example configuration can be used in your nginx setup, substituting values as necessary. `` should be the IP address and port where headscale is running. In most cases, this will be `http://localhost:8080`. ```Nginx +map $http_upgrade $connection_upgrade { + default keep-alive; + 'websocket' upgrade; + '' close; +} + server { listen 80; listen [::]:80; @@ -37,12 +43,6 @@ server { ssl_certificate_key ; ssl_protocols TLSv1.2 TLSv1.3; - map $http_upgrade $connection_upgrade { - default keep-alive; - 'websocket' upgrade; - '' close; - } - location / { proxy_pass http://; proxy_http_version 1.1; From b27b789e286c3043a6e988a4bc3e15fcf20b7ecb Mon Sep 17 00:00:00 2001 From: Juan Font Alonso Date: Wed, 7 Sep 2022 18:40:02 +0200 Subject: [PATCH 10/42] Added base config file template --- integration_test/etc_oidc/base_config.yaml | 17 +++++++++++++++++ integration_test/etc_oidc/tls/server.crt | 22 ++++++++++++++++++++++ 2 files changed, 39 insertions(+) create mode 100644 integration_test/etc_oidc/base_config.yaml create mode 100644 integration_test/etc_oidc/tls/server.crt diff --git a/integration_test/etc_oidc/base_config.yaml b/integration_test/etc_oidc/base_config.yaml new file mode 100644 index 0000000..3aa68a4 --- /dev/null +++ b/integration_test/etc_oidc/base_config.yaml @@ -0,0 +1,17 @@ +log_level: trace +acl_policy_path: "" +db_type: sqlite3 +ephemeral_node_inactivity_timeout: 30m +node_update_check_interval: 10s +ip_prefixes: + - fd7a:115c:a1e0::/48 + - 100.64.0.0/10 +db_path: /tmp/integration_test_db.sqlite3 +private_key_path: private.key +noise: + private_key_path: noise_private.key +listen_addr: 0.0.0.0:8443 +server_url: https://headscale:8443 +tls_cert_path: "/etc/headscale/tls/server.crt" +tls_key_path: "/etc/headscale/tls/server.key" +tls_client_auth_mode: disabled diff --git a/integration_test/etc_oidc/tls/server.crt b/integration_test/etc_oidc/tls/server.crt new file mode 100644 index 0000000..9555649 --- /dev/null +++ b/integration_test/etc_oidc/tls/server.crt @@ -0,0 +1,22 @@ + +-----BEGIN CERTIFICATE----- +MIIC8jCCAdqgAwIBAgIULbu+UbSTMG/LtxooLLh7BgSEyqEwDQYJKoZIhvcNAQEL +BQAwFDESMBAGA1UEAwwJaGVhZHNjYWxlMCAXDTIyMDMwNTE2NDgwM1oYDzI1MjEx +MTA0MTY0ODAzWjAUMRIwEAYDVQQDDAloZWFkc2NhbGUwggEiMA0GCSqGSIb3DQEB +AQUAA4IBDwAwggEKAoIBAQDqcfpToLZUF0rlNwXkkt3lbyw4Cl4TJdx36o2PKaOK +U+tze/IjRsCWeMwrcR1o9TNZcxsD+c2J48D1WATuQJlMeg+2UJXGaTGRKkkbPMy3 +5m7AFf/Q16UEOgm2NYjZaQ8faRGIMYURG/6sXmNeETJvBixpBev9yKJuVXgqHNS4 +NpEkNwdOCuAZXrmw0HCbiusawJOay4tFvhH14rav8Uimonl8UTNVXufMzyUOuoaQ +TGflmzYX3hIoswRnTPlIWFoqObvx2Q8H+of3uQJXy0m8I6OrIoXLNxnqYMfFls79 +9SYgVc2jPsCbh5fwyRbx2Hof7sIZ1K/mNgxJRG1E3ZiLAgMBAAGjOjA4MBQGA1Ud +EQQNMAuCCWhlYWRzY2FsZTALBgNVHQ8EBAMCB4AwEwYDVR0lBAwwCgYIKwYBBQUH +AwEwDQYJKoZIhvcNAQELBQADggEBANGlVN7NCsJaKz0k0nhlRGK+tcxn2p1PXN/i +Iy+JX8ahixPC4ocRwOhrXgb390ZXLLwq08HrWYRB/Wi1VUzCp5d8dVxvrR43dJ+v +L2EOBiIKgcu2C3pWW1qRR46/EoXUU9kSH2VNBvIhNufi32kEOidoDzxtQf6qVCoF +guUt1JkAqrynv1UvR/2ZRM/WzM/oJ8qfECwrwDxyYhkqU5Z5jCWg0C6kPIBvNdzt +B0eheWS+ZxVwkePTR4e17kIafwknth3lo+orxVrq/xC+OVM1bGrt2ZyD64ZvEqQl +w6kgbzBdLScAQptWOFThwhnJsg0UbYKimZsnYmjVEuN59TJv92M= +-----END CERTIFICATE----- + +(Expires on Nov 4 16:48:03 2521 GMT) + From cb70d7c705a4a0a170445f8672253e861fed8a58 Mon Sep 17 00:00:00 2001 From: Juan Font Alonso Date: Wed, 7 Sep 2022 23:53:31 +0200 Subject: [PATCH 11/42] Return the results on error --- integration_common_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_common_test.go b/integration_common_test.go index fb5abb1..71db0e5 100644 --- a/integration_common_test.go +++ b/integration_common_test.go @@ -121,7 +121,7 @@ func ExecuteCommand( return stdout.String(), nil case <-time.After(execConfig.timeout): - return "", fmt.Errorf("command timed out after %s", execConfig.timeout) + return stderr.String(), fmt.Errorf("command timed out after %s", execConfig.timeout) } } From fca380587a9342a4ae22189ba6935693aca2d7b5 Mon Sep 17 00:00:00 2001 From: Juan Font Alonso Date: Wed, 7 Sep 2022 23:53:46 +0200 Subject: [PATCH 12/42] Initial work on OIDC tests --- integration_oidc_test.go | 461 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 461 insertions(+) create mode 100644 integration_oidc_test.go diff --git a/integration_oidc_test.go b/integration_oidc_test.go new file mode 100644 index 0000000..0fc3f61 --- /dev/null +++ b/integration_oidc_test.go @@ -0,0 +1,461 @@ +//go:build integration_oidc + +package headscale + +import ( + "bytes" + "context" + "crypto/tls" + "fmt" + "log" + "net" + "net/http" + "os" + "path" + "strings" + "sync" + "testing" + "time" + + "github.com/oauth2-proxy/mockoidc" + "github.com/ory/dockertest/v3" + "github.com/ory/dockertest/v3/docker" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +const ( + oidcNamespaceName = "oidcnamespace" + totalOidcContainers = 3 +) + +type IntegrationOIDCTestSuite struct { + suite.Suite + stats *suite.SuiteInformation + + oidc *mockoidc.MockOIDC + pool dockertest.Pool + network dockertest.Network + headscale dockertest.Resource + saveLogs bool + + tailscales map[string]dockertest.Resource + joinWaitGroup sync.WaitGroup +} + +func TestOIDCIntegrationTestSuite(t *testing.T) { + saveLogs, err := GetEnvBool("HEADSCALE_INTEGRATION_SAVE_LOG") + if err != nil { + saveLogs = false + } + + s := new(IntegrationOIDCTestSuite) + + s.tailscales = make(map[string]dockertest.Resource) + s.saveLogs = saveLogs + + suite.Run(t, s) + + // HandleStats, which allows us to check if we passed and save logs + // is called after TearDown, so we cannot tear down containers before + // we have potentially saved the logs. + if s.saveLogs { + for _, tailscale := range s.tailscales { + if err := s.pool.Purge(&tailscale); err != nil { + log.Printf("Could not purge resource: %s\n", err) + } + } + + if !s.stats.Passed() { + err := s.saveLog(&s.headscale, "test_output") + if err != nil { + log.Printf("Could not save log: %s\n", err) + } + } + if err := s.pool.Purge(&s.headscale); err != nil { + t.Logf("Could not purge resource: %s\n", err) + } + + if err := s.network.Close(); err != nil { + log.Printf("Could not close network: %s\n", err) + } + } +} + +func (s *IntegrationOIDCTestSuite) SetupSuite() { + if ppool, err := dockertest.NewPool(""); err == nil { + s.pool = *ppool + } else { + s.FailNow(fmt.Sprintf("Could not connect to docker: %s", err), "") + } + + if pnetwork, err := s.pool.CreateNetwork("headscale-test"); err == nil { + s.network = *pnetwork + } else { + s.FailNow(fmt.Sprintf("Could not create network: %s", err), "") + } + + // Create does not give us an updated version of the resource, so we need to + // get it again. + networks, err := s.pool.NetworksByName("headscale-test") + if err != nil { + s.FailNow(fmt.Sprintf("Could not get network: %s", err), "") + } + s.network = networks[0] + + s.Suite.T().Log("Setting up mock OIDC") + oidc, _ := mockoidc.NewServer(nil) + ln, _ := net.Listen("tcp", fmt.Sprintf("%s:0", s.network.Network.IPAM.Config[0].Gateway)) + oidc.Start(ln, nil) + s.oidc = oidc + + // we now parse the Issuer URL and replace the host with the docker internal hostname + // urlIssuer, _ := url.Parse(s.oidc.Issuer()) + // urlIssuer.Host = fmt.Sprintf("host-gateway:%s", urlIssuer.Port()) + // issuer := urlIssuer.String() + + oidcCfg := fmt.Sprintf(` +oidc: + issuer: %s + client_id: %s + client_secret: %s + strip_email_domain: true`, + s.oidc.Issuer(), + s.oidc.Config().ClientID, + s.oidc.Config().ClientSecret) + + fmt.Println(oidcCfg) + + headscaleBuildOptions := &dockertest.BuildOptions{ + Dockerfile: "Dockerfile.debug", + ContextDir: ".", + } + + currentPath, err := os.Getwd() + if err != nil { + s.FailNow(fmt.Sprintf("Could not determine current path: %s", err), "") + } + + baseConfig, _ := os.ReadFile("integration_test/etc_oidc/base_config.yaml") + config := string(baseConfig) + oidcCfg + + configPath := path.Join(currentPath, "integration_test/etc_oidc/config.yaml") + err = os.WriteFile(configPath, []byte(config), 0644) + + headscaleOptions := &dockertest.RunOptions{ + Name: "headscale", + Mounts: []string{ + fmt.Sprintf( + "%s/integration_test/etc_oidc:/etc/headscale", + currentPath, + ), + }, + Cmd: []string{"headscale", "serve"}, + ExposedPorts: []string{"8443/tcp", "3478/udp"}, + PortBindings: map[docker.Port][]docker.PortBinding{ + "8443/tcp": {{HostPort: "8443"}}, + "3478/udp": {{HostPort: "3478"}}, + }, + } + + err = s.pool.RemoveContainerByName("headscale") + if err != nil { + s.FailNow( + fmt.Sprintf( + "Could not remove existing container before building test: %s", + err, + ), + "", + ) + } + + s.Suite.T().Logf("Creating headscale container for OIDC integration tests") + if pheadscale, err := s.pool.BuildAndRunWithBuildOptions(headscaleBuildOptions, headscaleOptions, DockerRestartPolicy); err == nil { + s.headscale = *pheadscale + } else { + s.FailNow(fmt.Sprintf("Could not start headscale container: %s", err), "") + } + s.Suite.T().Logf("Created headscale container for embedded OIDC tests") + + s.Suite.T().Logf("Creating tailscale containers for embedded OIDC tests") + + for i := 0; i < totalOidcContainers; i++ { + version := tailscaleVersions[i%len(tailscaleVersions)] + hostname, container := s.tailscaleContainer( + fmt.Sprint(i), + version, + ) + s.tailscales[hostname] = *container + } + + s.Suite.T().Logf("Waiting for headscale to be ready for embedded OIDC tests") + hostEndpoint := fmt.Sprintf("localhost:%s", s.headscale.GetPort("8443/tcp")) + + if err := s.pool.Retry(func() error { + url := fmt.Sprintf("https://%s/health", hostEndpoint) + insecureTransport := http.DefaultTransport.(*http.Transport).Clone() + insecureTransport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} + client := &http.Client{Transport: insecureTransport} + resp, err := client.Get(url) + if err != nil { + fmt.Printf("headscale for embedded OIDC tests is not ready: %s\n", err) + return err + } + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("status code not OK") + } + + return nil + }); err != nil { + // TODO(kradalby): If we cannot access headscale, or any other fatal error during + // test setup, we need to abort and tear down. However, testify does not seem to + // support that at the moment: + // https://github.com/stretchr/testify/issues/849 + return // fmt.Errorf("Could not connect to headscale: %s", err) + } + s.Suite.T().Log("headscale container is ready for embedded OIDC tests") + + s.Suite.T().Logf("Creating headscale namespace: %s\n", oidcNamespaceName) + result, err := ExecuteCommand( + &s.headscale, + []string{"headscale", "namespaces", "create", oidcNamespaceName}, + []string{}, + ) + log.Println("headscale create namespace result: ", result) + assert.Nil(s.T(), err) + + // log.Printf("Creating pre auth key for %s\n", oidcNamespaceName) + // preAuthResult, err := ExecuteCommand( + // &s.headscale, + // []string{ + // "headscale", + // "--namespace", + // oidcNamespaceName, + // "preauthkeys", + // "create", + // "--reusable", + // "--expiration", + // "24h", + // "--output", + // "json", + // }, + // []string{"LOG_LEVEL=error"}, + // ) + // assert.Nil(s.T(), err) + + // var preAuthKey v1.PreAuthKey + // err = json.Unmarshal([]byte(preAuthResult), &preAuthKey) + // assert.Nil(s.T(), err) + // assert.True(s.T(), preAuthKey.Reusable) + + headscaleEndpoint := fmt.Sprintf( + "https://headscale:%s", + s.headscale.GetPort("8443/tcp"), + ) + + log.Printf( + "Joining tailscale containers to headscale at %s\n", + headscaleEndpoint, + ) + for hostname, tailscale := range s.tailscales { + s.joinWaitGroup.Add(1) + go s.Join(headscaleEndpoint, hostname, tailscale) + } + + s.joinWaitGroup.Wait() + + // The nodes need a bit of time to get their updated maps from headscale + // TODO: See if we can have a more deterministic wait here. + time.Sleep(60 * time.Second) +} + +func (s *IntegrationOIDCTestSuite) Join( + endpoint, hostname string, + tailscale dockertest.Resource, +) { + defer s.joinWaitGroup.Done() + + command := []string{ + "tailscale", + "up", + "-login-server", + endpoint, + "--hostname", + hostname, + } + + log.Println("Join command:", command) + log.Printf("Running join command for %s\n", hostname) + result, err := ExecuteCommand( + &tailscale, + command, + []string{}, + ) + + // https://github.com/tailscale/tailscale/blob/main/cmd/tailscale/cli/up.go#L584 + url := strings.ReplaceAll(result, "\nTo authenticate, visit:\n\n\t", "") + url = strings.TrimSpace(url) + + log.Println(url) + assert.Nil(s.T(), err) + log.Printf("%s joined\n", hostname) +} + +func (s *IntegrationOIDCTestSuite) tailscaleContainer( + identifier, version string, +) (string, *dockertest.Resource) { + tailscaleBuildOptions := getDockerBuildOptions(version) + + hostname := fmt.Sprintf( + "tailscale-%s-%s", + strings.Replace(version, ".", "-", -1), + identifier, + ) + tailscaleOptions := &dockertest.RunOptions{ + Name: hostname, + Networks: []*dockertest.Network{&s.network}, + Cmd: []string{ + "tailscaled", "--tun=tsdev", + }, + + // expose the host IP address, so we can access it from inside the container + ExtraHosts: []string{ + "host.docker.internal:host-gateway", + "headscale:host-gateway", + }, + } + + pts, err := s.pool.BuildAndRunWithBuildOptions( + tailscaleBuildOptions, + tailscaleOptions, + DockerRestartPolicy, + DockerAllowLocalIPv6, + DockerAllowNetworkAdministration, + ) + if err != nil { + log.Fatalf("Could not start tailscale container version %s: %s", version, err) + } + log.Printf("Created %s container\n", hostname) + + return hostname, pts +} + +func (s *IntegrationOIDCTestSuite) TearDownSuite() { + s.oidc.Shutdown() + + if !s.saveLogs { + for _, tailscale := range s.tailscales { + if err := s.pool.Purge(&tailscale); err != nil { + log.Printf("Could not purge resource: %s\n", err) + } + } + + if err := s.pool.Purge(&s.headscale); err != nil { + log.Printf("Could not purge resource: %s\n", err) + } + + if err := s.network.Close(); err != nil { + log.Printf("Could not close network: %s\n", err) + } + } +} + +func (s *IntegrationOIDCTestSuite) HandleStats( + suiteName string, + stats *suite.SuiteInformation, +) { + s.stats = stats +} + +func (s *IntegrationOIDCTestSuite) saveLog( + resource *dockertest.Resource, + basePath string, +) error { + err := os.MkdirAll(basePath, os.ModePerm) + if err != nil { + return err + } + + var stdout bytes.Buffer + var stderr bytes.Buffer + + err = s.pool.Client.Logs( + docker.LogsOptions{ + Context: context.TODO(), + Container: resource.Container.ID, + OutputStream: &stdout, + ErrorStream: &stderr, + Tail: "all", + RawTerminal: false, + Stdout: true, + Stderr: true, + Follow: false, + Timestamps: false, + }, + ) + if err != nil { + return err + } + + log.Printf("Saving logs for %s to %s\n", resource.Container.Name, basePath) + + err = os.WriteFile( + path.Join(basePath, resource.Container.Name+".stdout.log"), + []byte(stdout.String()), + 0o644, + ) + if err != nil { + return err + } + + err = os.WriteFile( + path.Join(basePath, resource.Container.Name+".stderr.log"), + []byte(stdout.String()), + 0o644, + ) + if err != nil { + return err + } + + return nil +} + +func (s *IntegrationOIDCTestSuite) TestPingAllPeersByHostname() { + hostnames, err := getDNSNames(&s.headscale) + assert.Nil(s.T(), err) + + log.Printf("Hostnames: %#v\n", hostnames) + + for hostname, tailscale := range s.tailscales { + for _, peername := range hostnames { + if strings.Contains(peername, hostname) { + continue + } + s.T().Run(fmt.Sprintf("%s-%s", hostname, peername), func(t *testing.T) { + command := []string{ + "tailscale", "ping", + "--timeout=10s", + "--c=5", + "--until-direct=false", + peername, + } + + log.Printf( + "Pinging using hostname from %s to %s\n", + hostname, + peername, + ) + log.Println(command) + result, err := ExecuteCommand( + &tailscale, + command, + []string{}, + ) + assert.Nil(t, err) + log.Printf("Result for %s: %s\n", hostname, result) + assert.Contains(t, result, "via DERP(headscale)") + }) + } + } +} From 5f384c63239d5fe2ce0313e9b16580ed5122369c Mon Sep 17 00:00:00 2001 From: Juan Font Alonso Date: Thu, 8 Sep 2022 18:11:41 +0200 Subject: [PATCH 13/42] Removed old code and minor changes --- integration_oidc_test.go | 33 +++++---------------------------- 1 file changed, 5 insertions(+), 28 deletions(-) diff --git a/integration_oidc_test.go b/integration_oidc_test.go index 0fc3f61..9b4766d 100644 --- a/integration_oidc_test.go +++ b/integration_oidc_test.go @@ -25,8 +25,9 @@ import ( ) const ( - oidcNamespaceName = "oidcnamespace" - totalOidcContainers = 3 + oidcHeadscaleHostname = "headscale" + oidcNamespaceName = "oidcnamespace" + totalOidcContainers = 3 ) type IntegrationOIDCTestSuite struct { @@ -143,7 +144,7 @@ oidc: err = os.WriteFile(configPath, []byte(config), 0644) headscaleOptions := &dockertest.RunOptions{ - Name: "headscale", + Name: oidcHeadscaleHostname, Mounts: []string{ fmt.Sprintf( "%s/integration_test/etc_oidc:/etc/headscale", @@ -158,7 +159,7 @@ oidc: }, } - err = s.pool.RemoveContainerByName("headscale") + err = s.pool.RemoveContainerByName(oidcHeadscaleHostname) if err != nil { s.FailNow( fmt.Sprintf( @@ -225,30 +226,6 @@ oidc: log.Println("headscale create namespace result: ", result) assert.Nil(s.T(), err) - // log.Printf("Creating pre auth key for %s\n", oidcNamespaceName) - // preAuthResult, err := ExecuteCommand( - // &s.headscale, - // []string{ - // "headscale", - // "--namespace", - // oidcNamespaceName, - // "preauthkeys", - // "create", - // "--reusable", - // "--expiration", - // "24h", - // "--output", - // "json", - // }, - // []string{"LOG_LEVEL=error"}, - // ) - // assert.Nil(s.T(), err) - - // var preAuthKey v1.PreAuthKey - // err = json.Unmarshal([]byte(preAuthResult), &preAuthKey) - // assert.Nil(s.T(), err) - // assert.True(s.T(), preAuthKey.Reusable) - headscaleEndpoint := fmt.Sprintf( "https://headscale:%s", s.headscale.GetPort("8443/tcp"), From f33e3e3b818c75fdf2e15c7910480dc7a8c1b561 Mon Sep 17 00:00:00 2001 From: Juan Font Alonso Date: Thu, 8 Sep 2022 19:32:11 +0200 Subject: [PATCH 14/42] Parse the OIDC login URL --- integration_oidc_test.go | 55 +++++++++++++++++++++++++++++++++------- 1 file changed, 46 insertions(+), 9 deletions(-) diff --git a/integration_oidc_test.go b/integration_oidc_test.go index 9b4766d..5469837 100644 --- a/integration_oidc_test.go +++ b/integration_oidc_test.go @@ -7,9 +7,11 @@ import ( "context" "crypto/tls" "fmt" + "io" "log" "net" "net/http" + "net/url" "os" "path" "strings" @@ -237,7 +239,7 @@ oidc: ) for hostname, tailscale := range s.tailscales { s.joinWaitGroup.Add(1) - go s.Join(headscaleEndpoint, hostname, tailscale) + go s.AuthenticateOIDC(headscaleEndpoint, hostname, tailscale) } s.joinWaitGroup.Wait() @@ -247,12 +249,40 @@ oidc: time.Sleep(60 * time.Second) } -func (s *IntegrationOIDCTestSuite) Join( +func (s *IntegrationOIDCTestSuite) AuthenticateOIDC( endpoint, hostname string, tailscale dockertest.Resource, ) { defer s.joinWaitGroup.Done() + loginURL, err := s.joinOIDC(endpoint, hostname, tailscale) + if err != nil { + s.FailNow(fmt.Sprintf("Could not join OIDC node: %s", err), "") + } + + insecureTransport := &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } + client := &http.Client{Transport: insecureTransport} + resp, err := client.Get(loginURL.String()) + if err != nil { + s.FailNow(fmt.Sprintf("Could not get login page: %s", err), "") + } + // read the body + body, err := io.ReadAll(resp.Body) + if err != nil { + s.FailNow(fmt.Sprintf("Could not read login page: %s", err), "") + } + + panic(string(body)) + +} + +func (s *IntegrationOIDCTestSuite) joinOIDC( + endpoint, hostname string, + tailscale dockertest.Resource, +) (*url.URL, error) { + command := []string{ "tailscale", "up", @@ -264,19 +294,26 @@ func (s *IntegrationOIDCTestSuite) Join( log.Println("Join command:", command) log.Printf("Running join command for %s\n", hostname) - result, err := ExecuteCommand( + result, _ := ExecuteCommand( &tailscale, command, []string{}, ) - // https://github.com/tailscale/tailscale/blob/main/cmd/tailscale/cli/up.go#L584 - url := strings.ReplaceAll(result, "\nTo authenticate, visit:\n\n\t", "") - url = strings.TrimSpace(url) + // This piece of code just gets the login URL out of the output of the tailscale client. + // See https://github.com/tailscale/tailscale/blob/main/cmd/tailscale/cli/up.go#L584. + urlStr := strings.ReplaceAll(result, "\nTo authenticate, visit:\n\n\t", "") + urlStr = strings.TrimSpace(urlStr) - log.Println(url) - assert.Nil(s.T(), err) - log.Printf("%s joined\n", hostname) + // parse URL + loginUrl, err := url.Parse(urlStr) + if err != nil { + log.Printf("Could not parse login URL: %s", err) + log.Printf("Original join command result: %s", result) + return nil, err + } + + return loginUrl, nil } func (s *IntegrationOIDCTestSuite) tailscaleContainer( From 71b712356fa0711ef3f8cd756288a7cf565c9410 Mon Sep 17 00:00:00 2001 From: Juan Font Alonso Date: Thu, 8 Sep 2022 19:47:29 +0200 Subject: [PATCH 15/42] Minor change on the base config for OIDC --- integration_test/etc_oidc/base_config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_test/etc_oidc/base_config.yaml b/integration_test/etc_oidc/base_config.yaml index 3aa68a4..4a321e8 100644 --- a/integration_test/etc_oidc/base_config.yaml +++ b/integration_test/etc_oidc/base_config.yaml @@ -11,7 +11,7 @@ private_key_path: private.key noise: private_key_path: noise_private.key listen_addr: 0.0.0.0:8443 -server_url: https://headscale:8443 +server_url: https://localhost:8443 tls_cert_path: "/etc/headscale/tls/server.crt" tls_key_path: "/etc/headscale/tls/server.key" tls_client_auth_mode: disabled From 9c0cf4595a46582b29f876622edaab06f6e7328d Mon Sep 17 00:00:00 2001 From: Juan Font Alonso Date: Thu, 8 Sep 2022 19:47:47 +0200 Subject: [PATCH 16/42] OIDC integration tests working --- integration_oidc_test.go | 78 ++++++++++++++++++++-------------------- 1 file changed, 40 insertions(+), 38 deletions(-) diff --git a/integration_oidc_test.go b/integration_oidc_test.go index 5469837..238ceaf 100644 --- a/integration_oidc_test.go +++ b/integration_oidc_test.go @@ -265,17 +265,16 @@ func (s *IntegrationOIDCTestSuite) AuthenticateOIDC( } client := &http.Client{Transport: insecureTransport} resp, err := client.Get(loginURL.String()) - if err != nil { - s.FailNow(fmt.Sprintf("Could not get login page: %s", err), "") - } - // read the body + assert.Nil(s.T(), err) + body, err := io.ReadAll(resp.Body) + assert.Nil(s.T(), err) + if err != nil { s.FailNow(fmt.Sprintf("Could not read login page: %s", err), "") } - panic(string(body)) - + log.Printf("Login page for %s: %s", hostname, string(body)) } func (s *IntegrationOIDCTestSuite) joinOIDC( @@ -435,41 +434,44 @@ func (s *IntegrationOIDCTestSuite) saveLog( return nil } -func (s *IntegrationOIDCTestSuite) TestPingAllPeersByHostname() { - hostnames, err := getDNSNames(&s.headscale) - assert.Nil(s.T(), err) - - log.Printf("Hostnames: %#v\n", hostnames) - +func (s *IntegrationOIDCTestSuite) TestPingAllPeersByAddress() { for hostname, tailscale := range s.tailscales { - for _, peername := range hostnames { - if strings.Contains(peername, hostname) { - continue - } - s.T().Run(fmt.Sprintf("%s-%s", hostname, peername), func(t *testing.T) { - command := []string{ - "tailscale", "ping", - "--timeout=10s", - "--c=5", - "--until-direct=false", - peername, + ips, err := getIPs(s.tailscales) + assert.Nil(s.T(), err) + for peername, peerIPs := range ips { + for i, ip := range peerIPs { + // We currently cant ping ourselves, so skip that. + if peername == hostname { + continue } + s.T(). + Run(fmt.Sprintf("%s-%s-%d", hostname, peername, i), func(t *testing.T) { + // We are only interested in "direct ping" which means what we + // might need a couple of more attempts before reaching the node. + command := []string{ + "tailscale", "ping", + "--timeout=1s", + "--c=10", + "--until-direct=true", + ip.String(), + } - log.Printf( - "Pinging using hostname from %s to %s\n", - hostname, - peername, - ) - log.Println(command) - result, err := ExecuteCommand( - &tailscale, - command, - []string{}, - ) - assert.Nil(t, err) - log.Printf("Result for %s: %s\n", hostname, result) - assert.Contains(t, result, "via DERP(headscale)") - }) + log.Printf( + "Pinging from %s to %s (%s)\n", + hostname, + peername, + ip, + ) + result, err := ExecuteCommand( + &tailscale, + command, + []string{}, + ) + assert.Nil(t, err) + log.Printf("Result for %s: %s\n", hostname, result) + assert.Contains(t, result, "pong") + }) + } } } } From 41353a57c87185bea493aece55bac592d4b10a9f Mon Sep 17 00:00:00 2001 From: Juan Font Alonso Date: Thu, 8 Sep 2022 19:48:27 +0200 Subject: [PATCH 17/42] Added integration tests for OIDC on Makefile --- Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Makefile b/Makefile index 651ff5c..9132d2a 100644 --- a/Makefile +++ b/Makefile @@ -35,6 +35,9 @@ test_integration_derp: test_integration_general: go test -failfast -tags integration_general,integration -timeout 30m -count=1 ./... +test_integration_oidc: + go test -failfast -tags integration_oidc,integration -timeout 30m -count=1 ./... + coverprofile_func: go tool cover -func=coverage.out From 5774b32e552255d513e3e37486cd70f1f9eb1b07 Mon Sep 17 00:00:00 2001 From: Juan Font Alonso Date: Thu, 8 Sep 2022 19:48:51 +0200 Subject: [PATCH 18/42] Include OIDC in the full execution --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 9132d2a..84cb63c 100644 --- a/Makefile +++ b/Makefile @@ -24,7 +24,7 @@ dev: lint test build test: @go test -coverprofile=coverage.out ./... -test_integration: test_integration_cli test_integration_derp test_integration_general +test_integration: test_integration_cli test_integration_derp test_integration_oidc test_integration_general test_integration_cli: go test -failfast -tags integration_cli,integration -timeout 30m -count=1 ./... From b2f3ffbc5ac50c74a12798de173973d5b0055233 Mon Sep 17 00:00:00 2001 From: Juan Font Alonso Date: Thu, 8 Sep 2022 19:49:37 +0200 Subject: [PATCH 19/42] Run integration tests in Actions --- .github/workflows/test-integration.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/test-integration.yml b/.github/workflows/test-integration.yml index de896cb..f2adfa1 100644 --- a/.github/workflows/test-integration.yml +++ b/.github/workflows/test-integration.yml @@ -48,6 +48,15 @@ jobs: retry_on: error command: nix develop --command -- make test_integration_derp + - name: Run OIDC integration tests + if: steps.changed-files.outputs.any_changed == 'true' + uses: nick-fields/retry@v2 + with: + timeout_minutes: 240 + max_attempts: 5 + retry_on: error + command: nix develop --command -- make test_integration_oidc + - name: Run general integration tests if: steps.changed-files.outputs.any_changed == 'true' uses: nick-fields/retry@v2 From 99307d1576a391863cd7c195a5e8d0e6c9a51301 Mon Sep 17 00:00:00 2001 From: Juan Font Alonso Date: Thu, 8 Sep 2022 20:36:44 +0200 Subject: [PATCH 20/42] Update nix sum --- flake.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index d7eb203..7927221 100644 --- a/flake.nix +++ b/flake.nix @@ -24,7 +24,7 @@ # When updating go.mod or go.sum, a new sha will need to be calculated, # update this if you have a mismatch after doing a change to thos files. - vendorSha256 = "sha256-kc8EU+TkwRlsKM2+ljm/88aWe5h2QMgd/ZGPSgdd9QQ="; + vendorSha256 = "sha256-DosFCSiQ5FURbIrt4NcPGkExc84t2MGMqe9XLxNHdIM="; ldflags = [ "-s" "-w" "-X github.com/juanfont/headscale/cmd/headscale/cli.Version=v${version}" ]; }; From dd155dca97809d72a2ac6e031608b23f4d37a1fd Mon Sep 17 00:00:00 2001 From: Igor Perepilitsyn Date: Sun, 11 Sep 2022 21:37:23 +0200 Subject: [PATCH 21/42] Create a distinct log section in config --- cmd/headscale/cli/root.go | 4 +- config-example.yaml | 6 ++- config.go | 54 ++++++++++++++----- .../etc/alt-config.dump.gold.yaml | 5 +- integration_test/etc/alt-config.yaml | 3 +- .../etc/alt-env-config.dump.gold.yaml | 5 +- integration_test/etc/alt-env-config.yaml | 3 +- integration_test/etc/config.dump.gold.yaml | 5 +- integration_test/etc/config.yaml | 3 +- 9 files changed, 62 insertions(+), 26 deletions(-) diff --git a/cmd/headscale/cli/root.go b/cmd/headscale/cli/root.go index 659358f..60186b5 100644 --- a/cmd/headscale/cli/root.go +++ b/cmd/headscale/cli/root.go @@ -47,7 +47,7 @@ func initConfig() { machineOutput := HasMachineOutputFlag() - zerolog.SetGlobalLevel(cfg.LogLevel) + zerolog.SetGlobalLevel(cfg.Log.Level) // If the user has requested a "machine" readable format, // then disable login so the output remains valid. @@ -55,7 +55,7 @@ func initConfig() { zerolog.SetGlobalLevel(zerolog.Disabled) } - if cfg.JSONLogs { + if cfg.Log.Format == headscale.JSONLogFormat { log.Logger = log.Output(os.Stdout) } diff --git a/config-example.yaml b/config-example.yaml index 883d81f..69672b2 100644 --- a/config-example.yaml +++ b/config-example.yaml @@ -172,8 +172,10 @@ tls_letsencrypt_listen: ":http" tls_cert_path: "" tls_key_path: "" -log_level: info -json_logs: false +log: + # Output formatting for logs: text or json + format: text + level: info # Path to a file containg ACL policies. # ACLs can be defined as YAML or HUJSON. diff --git a/config.go b/config.go index ed007c1..b000c56 100644 --- a/config.go +++ b/config.go @@ -22,6 +22,9 @@ import ( const ( tlsALPN01ChallengeType = "TLS-ALPN-01" http01ChallengeType = "HTTP-01" + + JSONLogFormat = "json" + TextLogFormat = "text" ) // Config contains the initial Headscale configuration. @@ -37,8 +40,7 @@ type Config struct { PrivateKeyPath string NoisePrivateKeyPath string BaseDomain string - LogLevel zerolog.Level - JSONLogs bool + Log LogConfig DisableUpdateCheck bool DERP DERPConfig @@ -125,6 +127,11 @@ type ACLConfig struct { PolicyPath string } +type LogConfig struct { + Format string + Level zerolog.Level +} + func LoadConfig(path string, isFile bool) error { if isFile { viper.SetConfigFile(path) @@ -148,8 +155,8 @@ func LoadConfig(path string, isFile bool) error { viper.SetDefault("tls_letsencrypt_challenge_type", http01ChallengeType) viper.SetDefault("tls_client_auth_mode", "relaxed") - viper.SetDefault("log_level", "info") - viper.SetDefault("json_logs", false) + viper.SetDefault("log.level", "info") + viper.SetDefault("log.format", TextLogFormat) viper.SetDefault("dns_config", nil) @@ -336,6 +343,34 @@ func GetACLConfig() ACLConfig { } } +func GetLogConfig() LogConfig { + logLevelStr := viper.GetString("log.level") + logLevel, err := zerolog.ParseLevel(logLevelStr) + if err != nil { + logLevel = zerolog.DebugLevel + } + + logFormatOpt := viper.GetString("log.format") + var logFormat string + switch logFormatOpt { + case "json": + logFormat = JSONLogFormat + case "text": + logFormat = TextLogFormat + case "": + logFormat = TextLogFormat + default: + log.Error(). + Str("func", "GetLogConfig"). + Msgf("Could not parse log format: %s. Valid choices are 'json' or 'text'", logFormatOpt) + } + + return LogConfig{ + Format: logFormat, + Level: logLevel, + } +} + func GetDNSConfig() (*tailcfg.DNSConfig, string) { if viper.IsSet("dns_config") { dnsConfig := &tailcfg.DNSConfig{} @@ -432,13 +467,6 @@ func GetHeadscaleConfig() (*Config, error) { configuredPrefixes := viper.GetStringSlice("ip_prefixes") parsedPrefixes := make([]netip.Prefix, 0, len(configuredPrefixes)+1) - logLevelStr := viper.GetString("log_level") - logLevel, err := zerolog.ParseLevel(logLevelStr) - if err != nil { - logLevel = zerolog.DebugLevel - } - jsonLogs := viper.GetBool("json_logs") - legacyPrefixField := viper.GetString("ip_prefix") if len(legacyPrefixField) > 0 { log. @@ -491,8 +519,6 @@ func GetHeadscaleConfig() (*Config, error) { GRPCAddr: viper.GetString("grpc_listen_addr"), GRPCAllowInsecure: viper.GetBool("grpc_allow_insecure"), DisableUpdateCheck: viper.GetBool("disable_check_updates"), - LogLevel: logLevel, - JSONLogs: jsonLogs, IPPrefixes: prefixes, PrivateKeyPath: AbsolutePathFromConfigPath( @@ -554,5 +580,7 @@ func GetHeadscaleConfig() (*Config, error) { }, ACL: GetACLConfig(), + + Log: GetLogConfig(), }, nil } diff --git a/integration_test/etc/alt-config.dump.gold.yaml b/integration_test/etc/alt-config.dump.gold.yaml index 07544d4..c9bd39b 100644 --- a/integration_test/etc/alt-config.dump.gold.yaml +++ b/integration_test/etc/alt-config.dump.gold.yaml @@ -28,8 +28,9 @@ ip_prefixes: - fd7a:115c:a1e0::/48 - 100.64.0.0/10 listen_addr: 0.0.0.0:18080 -log_level: disabled -json_logs: false +log: + level: disabled + format: text logtail: enabled: false metrics_listen_addr: 127.0.0.1:19090 diff --git a/integration_test/etc/alt-config.yaml b/integration_test/etc/alt-config.yaml index 179fdcd..837ba6c 100644 --- a/integration_test/etc/alt-config.yaml +++ b/integration_test/etc/alt-config.yaml @@ -1,4 +1,5 @@ -log_level: trace +log: + level: trace acl_policy_path: "" db_type: sqlite3 ephemeral_node_inactivity_timeout: 30m diff --git a/integration_test/etc/alt-env-config.dump.gold.yaml b/integration_test/etc/alt-env-config.dump.gold.yaml index 805c25a..4df4bf4 100644 --- a/integration_test/etc/alt-env-config.dump.gold.yaml +++ b/integration_test/etc/alt-env-config.dump.gold.yaml @@ -27,8 +27,9 @@ ip_prefixes: - fd7a:115c:a1e0::/48 - 100.64.0.0/10 listen_addr: 0.0.0.0:18080 -log_level: disabled -json_logs: false +log: + level: disabled + format: text logtail: enabled: false metrics_listen_addr: 127.0.0.1:19090 diff --git a/integration_test/etc/alt-env-config.yaml b/integration_test/etc/alt-env-config.yaml index 4f19526..3856048 100644 --- a/integration_test/etc/alt-env-config.yaml +++ b/integration_test/etc/alt-env-config.yaml @@ -1,4 +1,5 @@ -log_level: trace +log: + level: trace acl_policy_path: "" db_type: sqlite3 ephemeral_node_inactivity_timeout: 30m diff --git a/integration_test/etc/config.dump.gold.yaml b/integration_test/etc/config.dump.gold.yaml index ff61c6f..158a195 100644 --- a/integration_test/etc/config.dump.gold.yaml +++ b/integration_test/etc/config.dump.gold.yaml @@ -28,8 +28,9 @@ ip_prefixes: - fd7a:115c:a1e0::/48 - 100.64.0.0/10 listen_addr: 0.0.0.0:8080 -log_level: disabled -json_logs: false +log: + format: text + level: disabled logtail: enabled: false metrics_listen_addr: 127.0.0.1:9090 diff --git a/integration_test/etc/config.yaml b/integration_test/etc/config.yaml index da842cc..8b4d7db 100644 --- a/integration_test/etc/config.yaml +++ b/integration_test/etc/config.yaml @@ -1,4 +1,5 @@ -log_level: trace +log: + level: trace acl_policy_path: "" db_type: sqlite3 ephemeral_node_inactivity_timeout: 30m From ae4f2cc4b520a560752c6a458527ee6fe3600f44 Mon Sep 17 00:00:00 2001 From: Igor Perepilitsyn Date: Sun, 11 Sep 2022 21:37:38 +0200 Subject: [PATCH 22/42] Update changelog --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cf0afb7..a829f12 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,11 +2,17 @@ ## 0.17.0 (2022-XX-XX) +### BREAKING + +- Log level option `log_level` was moved to a distinct `log` config section and renamed to `level` [#768](https://github.com/juanfont/headscale/pull/768) +### Changes + - Added support for Tailscale TS2021 protocol [#738](https://github.com/juanfont/headscale/pull/738) - Add ability to specify config location via env var `HEADSCALE_CONFIG` [#674](https://github.com/juanfont/headscale/issues/674) - Target Go 1.19 for Headscale [#778](https://github.com/juanfont/headscale/pull/778) - Target Tailscale v1.30.0 to build Headscale [#780](https://github.com/juanfont/headscale/pull/780) - Give a warning when running Headscale with reverse proxy improperly configured for WebSockets [#788](https://github.com/juanfont/headscale/pull/788) +- Added support for JSON logs [#653](https://github.com/juanfont/headscale/issues/653) ## 0.16.4 (2022-08-21) From 874d6aaf6b9ce8563e09d3d4da8a3b100c4f04b5 Mon Sep 17 00:00:00 2001 From: Igor Perepilitsyn Date: Sun, 11 Sep 2022 21:44:28 +0200 Subject: [PATCH 23/42] Make styling fixes --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a829f12..0d251a5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### BREAKING - Log level option `log_level` was moved to a distinct `log` config section and renamed to `level` [#768](https://github.com/juanfont/headscale/pull/768) + ### Changes - Added support for Tailscale TS2021 protocol [#738](https://github.com/juanfont/headscale/pull/738) From 3abca99b0c31e6c63607db3cd71f651994a3b728 Mon Sep 17 00:00:00 2001 From: Juan Font Alonso Date: Wed, 14 Sep 2022 23:32:19 +0200 Subject: [PATCH 24/42] Add logs for issues in Actions --- integration_oidc_test.go | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/integration_oidc_test.go b/integration_oidc_test.go index 238ceaf..0a028ea 100644 --- a/integration_oidc_test.go +++ b/integration_oidc_test.go @@ -139,18 +139,26 @@ oidc: s.FailNow(fmt.Sprintf("Could not determine current path: %s", err), "") } - baseConfig, _ := os.ReadFile("integration_test/etc_oidc/base_config.yaml") + baseConfig, err := os.ReadFile( + path.Join(currentPath, "integration_test/etc_oidc/base_config.yaml")) + if err != nil { + s.FailNow(fmt.Sprintf("Could not read base config: %s", err), "") + } config := string(baseConfig) + oidcCfg + log.Println(config) + configPath := path.Join(currentPath, "integration_test/etc_oidc/config.yaml") err = os.WriteFile(configPath, []byte(config), 0644) + if err != nil { + s.FailNow(fmt.Sprintf("Could not write config: %s", err), "") + } headscaleOptions := &dockertest.RunOptions{ Name: oidcHeadscaleHostname, Mounts: []string{ - fmt.Sprintf( - "%s/integration_test/etc_oidc:/etc/headscale", - currentPath, + path.Join(currentPath, + "integration_test/etc_oidc:/etc/headscale", ), }, Cmd: []string{"headscale", "serve"}, From c21479cb9c1420fe7554bd7c1c16f802f5beaa01 Mon Sep 17 00:00:00 2001 From: Juan Font Alonso Date: Thu, 15 Sep 2022 00:06:17 +0200 Subject: [PATCH 25/42] Print docker network config --- integration_oidc_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/integration_oidc_test.go b/integration_oidc_test.go index 0a028ea..31c1714 100644 --- a/integration_oidc_test.go +++ b/integration_oidc_test.go @@ -106,6 +106,8 @@ func (s *IntegrationOIDCTestSuite) SetupSuite() { } s.network = networks[0] + log.Printf("Network config: %v", s.network.Network.IPAM.Config[0]) + s.Suite.T().Log("Setting up mock OIDC") oidc, _ := mockoidc.NewServer(nil) ln, _ := net.Listen("tcp", fmt.Sprintf("%s:0", s.network.Network.IPAM.Config[0].Gateway)) From 33ae56acfaf96d6e4d3e8d15c91868b47ba5ed6c Mon Sep 17 00:00:00 2001 From: Kristoffer Dalby Date: Sun, 18 Sep 2022 11:36:35 +0200 Subject: [PATCH 26/42] Add primary routes to node Signed-off-by: Kristoffer Dalby --- machine.go | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/machine.go b/machine.go index 9fe450a..b43c457 100644 --- a/machine.go +++ b/machine.go @@ -35,6 +35,11 @@ const ( maxHostnameLength = 255 ) +var ( + ExitRouteV4 = netip.MustParsePrefix("0.0.0.0/0") + ExitRouteV6 = netip.MustParsePrefix("::/0") +) + // Machine is a Headscale client. type Machine struct { ID uint64 `gorm:"primary_key"` @@ -633,10 +638,17 @@ func (machine Machine) toNode( []netip.Prefix{}, addrs...) // we append the node own IP, as it is required by the clients - // TODO(kradalby): Needs investigation, We probably dont need this condition - // now that we dont have shared nodes - if includeRoutes { - allowedIPs = append(allowedIPs, machine.EnabledRoutes...) + allowedIPs = append(allowedIPs, machine.EnabledRoutes...) + + primaryRoutes := []netip.Prefix{} + if len(machine.EnabledRoutes) > 0 { + for _, route := range machine.EnabledRoutes { + if route == ExitRouteV4 || route == ExitRouteV6 { + continue + } + + primaryRoutes = append(primaryRoutes, route) + } } var derp string @@ -691,6 +703,7 @@ func (machine Machine) toNode( DiscoKey: discoKey, Addresses: addrs, AllowedIPs: allowedIPs, + PrimaryRoutes: primaryRoutes, Endpoints: machine.Endpoints, DERP: derp, From 356b76fc566a7610d903080a78b9097aac947aea Mon Sep 17 00:00:00 2001 From: Kristoffer Dalby Date: Sun, 18 Sep 2022 11:37:38 +0200 Subject: [PATCH 27/42] Format Signed-off-by: Kristoffer Dalby --- machine.go | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/machine.go b/machine.go index b43c457..ad31c86 100644 --- a/machine.go +++ b/machine.go @@ -26,9 +26,11 @@ const ( ) ErrCouldNotConvertMachineInterface = Error("failed to convert machine interface") ErrHostnameTooLong = Error("Hostname too long") - ErrDifferentRegisteredNamespace = Error("machine was previously registered with a different namespace") - MachineGivenNameHashLength = 8 - MachineGivenNameTrimSize = 2 + ErrDifferentRegisteredNamespace = Error( + "machine was previously registered with a different namespace", + ) + MachineGivenNameHashLength = 8 + MachineGivenNameTrimSize = 2 ) const ( @@ -695,17 +697,17 @@ func (machine Machine) toNode( StableID: tailcfg.StableNodeID( strconv.FormatUint(machine.ID, Base10), ), // in headscale, unlike tailcontrol server, IDs are permanent - Name: hostname, - User: tailcfg.UserID(machine.NamespaceID), - Key: nodeKey, - KeyExpiry: keyExpiry, - Machine: machineKey, - DiscoKey: discoKey, - Addresses: addrs, - AllowedIPs: allowedIPs, + Name: hostname, + User: tailcfg.UserID(machine.NamespaceID), + Key: nodeKey, + KeyExpiry: keyExpiry, + Machine: machineKey, + DiscoKey: discoKey, + Addresses: addrs, + AllowedIPs: allowedIPs, PrimaryRoutes: primaryRoutes, - Endpoints: machine.Endpoints, - DERP: derp, + Endpoints: machine.Endpoints, + DERP: derp, Online: &online, Hostinfo: hostInfo.View(), @@ -820,7 +822,8 @@ func (h *Headscale) RegisterMachineFromAuthCallback( } // Registration of expired machine with different namespace - if registrationMachine.ID != 0 && registrationMachine.NamespaceID != namespace.ID { + if registrationMachine.ID != 0 && + registrationMachine.NamespaceID != namespace.ID { return nil, ErrDifferentRegisteredNamespace } From f2da1a1665cc53fe4c426af0b11d6fc24b2ec591 Mon Sep 17 00:00:00 2001 From: Kristoffer Dalby Date: Sun, 18 Sep 2022 12:14:49 +0200 Subject: [PATCH 28/42] Add comment and update changelog Signed-off-by: Kristoffer Dalby --- CHANGELOG.md | 1 + machine.go | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cf0afb7..ac1e383 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ - Target Go 1.19 for Headscale [#778](https://github.com/juanfont/headscale/pull/778) - Target Tailscale v1.30.0 to build Headscale [#780](https://github.com/juanfont/headscale/pull/780) - Give a warning when running Headscale with reverse proxy improperly configured for WebSockets [#788](https://github.com/juanfont/headscale/pull/788) +- Fix subnet routers with Primary Routes [#811](https://github.com/juanfont/headscale/pull/811) ## 0.16.4 (2022-08-21) diff --git a/machine.go b/machine.go index ad31c86..92d714e 100644 --- a/machine.go +++ b/machine.go @@ -642,6 +642,11 @@ func (machine Machine) toNode( allowedIPs = append(allowedIPs, machine.EnabledRoutes...) + // TODO(kradalby): This is kind of a hack where we say that + // all the announced routes (except exit), is presented as primary + // routes. This might be problematic if two nodes expose the same route. + // This was added to address an issue where subnet routers stopped working + // when we only populated AllowedIPs. primaryRoutes := []netip.Prefix{} if len(machine.EnabledRoutes) > 0 { for _, route := range machine.EnabledRoutes { From b117ca77206177507f3a3978761909bea10495d9 Mon Sep 17 00:00:00 2001 From: Juan Font Date: Sun, 18 Sep 2022 21:26:47 +0000 Subject: [PATCH 29/42] Added missing TLS key for testing --- integration_test/etc_oidc/tls/server.key | 28 ++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 integration_test/etc_oidc/tls/server.key diff --git a/integration_test/etc_oidc/tls/server.key b/integration_test/etc_oidc/tls/server.key new file mode 100644 index 0000000..8a2df34 --- /dev/null +++ b/integration_test/etc_oidc/tls/server.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEwAIBADANBgkqhkiG9w0BAQEFAASCBKowggSmAgEAAoIBAQDqcfpToLZUF0rl +NwXkkt3lbyw4Cl4TJdx36o2PKaOKU+tze/IjRsCWeMwrcR1o9TNZcxsD+c2J48D1 +WATuQJlMeg+2UJXGaTGRKkkbPMy35m7AFf/Q16UEOgm2NYjZaQ8faRGIMYURG/6s +XmNeETJvBixpBev9yKJuVXgqHNS4NpEkNwdOCuAZXrmw0HCbiusawJOay4tFvhH1 +4rav8Uimonl8UTNVXufMzyUOuoaQTGflmzYX3hIoswRnTPlIWFoqObvx2Q8H+of3 +uQJXy0m8I6OrIoXLNxnqYMfFls799SYgVc2jPsCbh5fwyRbx2Hof7sIZ1K/mNgxJ +RG1E3ZiLAgMBAAECggEBALu1Ni/u5Qy++YA8ZcN0s6UXNdhItLmv/q0kZuLQ+9et +CT8VZfFInLndTdsaXenDKLHdryunviFA8SV+q7P2lMbek+Xs735EiyMnMBFWxLIZ +FWNGOeQERGL19QCmLEOmEi2b+iWJQHlKaMWpbPXL3w11a+lKjIBNO4ALfoJ5QveZ +cGMKsJdm/mpqBvLeNeh2eAFk3Gp6sT1g80Ge8NkgyzFBNIqnut0eerM15kPTc6Qz +12JLaOXUuV3PrcB4PN4nOwrTDg88GDNOQtc1Pc9r4nOHyLfr8X7QEtj1wXSwmOuK +d6ynMnAmoxVA9wEnupLbil1bzohRzpsTpkmDruYaBEECgYEA/Z09I8D6mt2NVqIE +KyvLjBK39ijSV9r3/lvB2Ple2OOL5YQEd+yTrIFy+3zdUnDgD1zmNnXjmjvHZ9Lc +IFf2o06AF84QLNB5gLPdDQkGNFdDqUxljBrfAfE3oANmPS/B0SijMGOOOiDO2FtO +xl1nfRr78mswuRs9awoUWCdNRKUCgYEA7KaTYKIQW/FEjw9lshp74q5vbn6zoXF5 +7N8VkwI+bBVNvRbM9XZ8qhfgRdu9eXs5oL/N4mSYY54I8fA//pJ0Z2vpmureMm1V +mL5WBUmSD9DIbAchoK+sRiQhVmNMBQC6cHMABA7RfXvBeGvWrm9pKCS6ZLgLjkjp +PsmAcaXQcW8CgYEA2inAxljjOwUK6FNGsrxhxIT1qtNC3kCGxE+6WSNq67gSR8Vg +8qiX//T7LEslOB3RIGYRwxd2St7RkgZZRZllmOWWWuPwFhzf6E7RAL2akLvggGov +kG4tGEagSw2hjVDfsUT73ExHtMk0Jfmlsg33UC8+PDLpHtLH6qQpDAwC8+ECgYEA +o+AqOIWhvHmT11l7O915Ip1WzvZwYADbxLsrDnVEUsZh4epTHjvh0kvcY6PqTqCV +ZIrOANNWb811Nkz/k8NJVoD08PFp0xPBbZeIq/qpachTsfMyRzq/mobUiyUR9Hjv +ooUQYr78NOApNsG+lWbTNBhS9wI4BlzZIECbcJe5g4MCgYEAndRoy8S+S0Hx/S8a +O3hzXeDmivmgWqn8NVD4AKOovpkz4PaIVVQbAQkiNfAx8/DavPvjEKAbDezJ4ECV +j7IsOWtDVI7pd6eF9fTcECwisrda8aUoiOap8AQb48153Vx+g2N4Vy3uH0xJs4cz +TDALZPOBg8VlV+HEFDP43sp9Bf0= +-----END PRIVATE KEY----- From 9c58395bb3d10dfc3d42857467ad8655766fc769 Mon Sep 17 00:00:00 2001 From: Juan Font Date: Sun, 18 Sep 2022 21:40:52 +0000 Subject: [PATCH 30/42] Removed unused param after routes fix --- api_common.go | 4 ++-- machine.go | 4 +--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/api_common.go b/api_common.go index 5ffbed0..b4983cc 100644 --- a/api_common.go +++ b/api_common.go @@ -13,7 +13,7 @@ func (h *Headscale) generateMapResponse( Str("func", "generateMapResponse"). Str("machine", mapRequest.Hostinfo.Hostname). Msg("Creating Map response") - node, err := machine.toNode(h.cfg.BaseDomain, h.cfg.DNSConfig, true) + node, err := machine.toNode(h.cfg.BaseDomain, h.cfg.DNSConfig) if err != nil { log.Error(). Caller(). @@ -37,7 +37,7 @@ func (h *Headscale) generateMapResponse( profiles := getMapResponseUserProfiles(*machine, peers) - nodePeers, err := peers.toNodes(h.cfg.BaseDomain, h.cfg.DNSConfig, true) + nodePeers, err := peers.toNodes(h.cfg.BaseDomain, h.cfg.DNSConfig) if err != nil { log.Error(). Caller(). diff --git a/machine.go b/machine.go index 92d714e..da98053 100644 --- a/machine.go +++ b/machine.go @@ -573,12 +573,11 @@ func (machines MachinesP) String() string { func (machines Machines) toNodes( baseDomain string, dnsConfig *tailcfg.DNSConfig, - includeRoutes bool, ) ([]*tailcfg.Node, error) { nodes := make([]*tailcfg.Node, len(machines)) for index, machine := range machines { - node, err := machine.toNode(baseDomain, dnsConfig, includeRoutes) + node, err := machine.toNode(baseDomain, dnsConfig) if err != nil { return nil, err } @@ -594,7 +593,6 @@ func (machines Machines) toNodes( func (machine Machine) toNode( baseDomain string, dnsConfig *tailcfg.DNSConfig, - includeRoutes bool, ) (*tailcfg.Node, error) { var nodeKey key.NodePublic err := nodeKey.UnmarshalText([]byte(NodePublicKeyEnsurePrefix(machine.NodeKey))) From 1c267f72e0a7f2756c59fd4949c0d268f10921ea Mon Sep 17 00:00:00 2001 From: Juan Font Date: Mon, 19 Sep 2022 23:07:47 +0000 Subject: [PATCH 31/42] Capture listen error on mockoidc --- integration_oidc_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/integration_oidc_test.go b/integration_oidc_test.go index 31c1714..d6e1d62 100644 --- a/integration_oidc_test.go +++ b/integration_oidc_test.go @@ -110,7 +110,10 @@ func (s *IntegrationOIDCTestSuite) SetupSuite() { s.Suite.T().Log("Setting up mock OIDC") oidc, _ := mockoidc.NewServer(nil) - ln, _ := net.Listen("tcp", fmt.Sprintf("%s:0", s.network.Network.IPAM.Config[0].Gateway)) + ln, err := net.Listen("tcp", fmt.Sprintf("%s:0", s.network.Network.IPAM.Config[0].Gateway)) + if err != nil { + s.FailNow(fmt.Sprintf("Could not listen on port: %s", err), "") + } oidc.Start(ln, nil) s.oidc = oidc From a3f18f248c766cfc1a96e715aff760a6c8c74951 Mon Sep 17 00:00:00 2001 From: Juan Font Date: Tue, 20 Sep 2022 19:58:36 +0000 Subject: [PATCH 32/42] Add internal mockoidc command --- cmd/headscale/cli/mockoidc.go | 89 +++++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 cmd/headscale/cli/mockoidc.go diff --git a/cmd/headscale/cli/mockoidc.go b/cmd/headscale/cli/mockoidc.go new file mode 100644 index 0000000..07248d4 --- /dev/null +++ b/cmd/headscale/cli/mockoidc.go @@ -0,0 +1,89 @@ +package cli + +import ( + "fmt" + "net" + "os" + "strconv" + "time" + + "github.com/oauth2-proxy/mockoidc" + "github.com/rs/zerolog/log" + "github.com/spf13/cobra" +) + +func init() { + rootCmd.AddCommand(mockOidcCmd) +} + +var mockOidcCmd = &cobra.Command{ + Use: "mockoidc", + Short: "Runs a mock OIDC server for testing", + Long: "This internal command runs a OpenID Connect for testing purposes", + Run: func(cmd *cobra.Command, args []string) { + err := mockOIDC() + if err != nil { + fmt.Println(err) + os.Exit(1) + } + }, +} + +func mockOIDC() error { + clientID := os.Getenv("MOCKOIDC_CLIENT_ID") + if clientID == "" { + return fmt.Errorf("MOCKOIDC_CLIENT_ID not set") + } + clientSecret := os.Getenv("MOCKOIDC_CLIENT_SECRET") + if clientSecret == "" { + return fmt.Errorf("MOCKOIDC_CLIENT_SECRET not set") + } + portStr := os.Getenv("MOCKOIDC_PORT") + if portStr == "" { + return fmt.Errorf("MOCKOIDC_PORT not set") + } + + port, err := strconv.Atoi(portStr) + if err != nil { + return err + } + + mock, err := getMockOIDC(clientID, clientSecret) + if err != nil { + return err + } + + ln, err := net.Listen("tcp", fmt.Sprintf("mockoidc:%d", port)) + if err != nil { + return err + } + + mock.Start(ln, nil) + log.Info().Msgf("Mock OIDC server listening on %s", ln.Addr().String()) + log.Info().Msgf("Issuer: %s", mock.Issuer()) + c := make(chan struct{}) + <-c + + return nil +} + +func getMockOIDC(clientID string, clientSecret string) (*mockoidc.MockOIDC, error) { + keypair, err := mockoidc.NewKeypair(nil) + if err != nil { + return nil, err + } + + mock := mockoidc.MockOIDC{ + ClientID: clientID, + ClientSecret: clientSecret, + AccessTTL: time.Duration(10) * time.Minute, + RefreshTTL: time.Duration(60) * time.Minute, + CodeChallengeMethodsSupported: []string{"plain", "S256"}, + Keypair: keypair, + SessionStore: mockoidc.NewSessionStore(), + UserQueue: &mockoidc.UserQueue{}, + ErrorQueue: &mockoidc.ErrorQueue{}, + } + + return &mock, nil +} From b3a53bf6422ccd220c29e8942881099535556f4a Mon Sep 17 00:00:00 2001 From: Juan Font Date: Tue, 20 Sep 2022 19:59:22 +0000 Subject: [PATCH 33/42] Do not load the config for CLI mockoidc (and version) --- cmd/headscale/cli/root.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cmd/headscale/cli/root.go b/cmd/headscale/cli/root.go index 459a99f..c60614d 100644 --- a/cmd/headscale/cli/root.go +++ b/cmd/headscale/cli/root.go @@ -15,6 +15,10 @@ import ( var cfgFile string = "" func init() { + if len(os.Args) > 1 && os.Args[1] == "version" || os.Args[1] == "mockoidc" { + return + } + cobra.OnInitialize(initConfig) rootCmd.PersistentFlags(). StringVarP(&cfgFile, "config", "c", "", "config file (default is /etc/headscale/config.yaml)") From 2e97119db838e5d1197bd169b0e04337f01911b2 Mon Sep 17 00:00:00 2001 From: Juan Font Date: Tue, 20 Sep 2022 20:42:12 +0000 Subject: [PATCH 34/42] Added derp config to OIDC etc --- integration_test/etc_oidc/base_config.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/integration_test/etc_oidc/base_config.yaml b/integration_test/etc_oidc/base_config.yaml index 4a321e8..10fa775 100644 --- a/integration_test/etc_oidc/base_config.yaml +++ b/integration_test/etc_oidc/base_config.yaml @@ -15,3 +15,8 @@ server_url: https://localhost:8443 tls_cert_path: "/etc/headscale/tls/server.crt" tls_key_path: "/etc/headscale/tls/server.key" tls_client_auth_mode: disabled +derp: + urls: + - https://controlplane.tailscale.com/derpmap/default + auto_update_enabled: true + update_frequency: 1m From 1563d7555f21a82086cbb9006ebf36f24ef5acfc Mon Sep 17 00:00:00 2001 From: Juan Font Date: Tue, 20 Sep 2022 20:42:50 +0000 Subject: [PATCH 35/42] Use Headscale container to run mockoidc --- integration_oidc_test.go | 71 ++++++++++++++++++++++++---------------- 1 file changed, 42 insertions(+), 29 deletions(-) diff --git a/integration_oidc_test.go b/integration_oidc_test.go index d6e1d62..27f3f8d 100644 --- a/integration_oidc_test.go +++ b/integration_oidc_test.go @@ -9,7 +9,6 @@ import ( "fmt" "io" "log" - "net" "net/http" "net/url" "os" @@ -19,7 +18,6 @@ import ( "testing" "time" - "github.com/oauth2-proxy/mockoidc" "github.com/ory/dockertest/v3" "github.com/ory/dockertest/v3/docker" "github.com/stretchr/testify/assert" @@ -36,10 +34,10 @@ type IntegrationOIDCTestSuite struct { suite.Suite stats *suite.SuiteInformation - oidc *mockoidc.MockOIDC pool dockertest.Pool network dockertest.Network headscale dockertest.Resource + mockOidc dockertest.Resource saveLogs bool tailscales map[string]dockertest.Resource @@ -75,6 +73,11 @@ func TestOIDCIntegrationTestSuite(t *testing.T) { log.Printf("Could not save log: %s\n", err) } } + + if err := s.pool.Purge(&s.mockOidc); err != nil { + log.Printf("Could not purge resource: %s\n", err) + } + if err := s.pool.Purge(&s.headscale); err != nil { t.Logf("Could not purge resource: %s\n", err) } @@ -109,36 +112,43 @@ func (s *IntegrationOIDCTestSuite) SetupSuite() { log.Printf("Network config: %v", s.network.Network.IPAM.Config[0]) s.Suite.T().Log("Setting up mock OIDC") - oidc, _ := mockoidc.NewServer(nil) - ln, err := net.Listen("tcp", fmt.Sprintf("%s:0", s.network.Network.IPAM.Config[0].Gateway)) - if err != nil { - s.FailNow(fmt.Sprintf("Could not listen on port: %s", err), "") + mockOidcOptions := &dockertest.RunOptions{ + Name: "mockoidc", + Hostname: "mockoidc", + Cmd: []string{"headscale", "mockoidc"}, + ExposedPorts: []string{"10000/tcp"}, + Networks: []*dockertest.Network{&s.network}, + PortBindings: map[docker.Port][]docker.PortBinding{ + "10000/tcp": {{HostPort: "10000"}}, + }, + Env: []string{ + "MOCKOIDC_PORT=10000", + "MOCKOIDC_CLIENT_ID=superclient", + "MOCKOIDC_CLIENT_SECRET=supersecret", + }, } - oidc.Start(ln, nil) - s.oidc = oidc - - // we now parse the Issuer URL and replace the host with the docker internal hostname - // urlIssuer, _ := url.Parse(s.oidc.Issuer()) - // urlIssuer.Host = fmt.Sprintf("host-gateway:%s", urlIssuer.Port()) - // issuer := urlIssuer.String() - - oidcCfg := fmt.Sprintf(` -oidc: - issuer: %s - client_id: %s - client_secret: %s - strip_email_domain: true`, - s.oidc.Issuer(), - s.oidc.Config().ClientID, - s.oidc.Config().ClientSecret) - - fmt.Println(oidcCfg) headscaleBuildOptions := &dockertest.BuildOptions{ Dockerfile: "Dockerfile.debug", ContextDir: ".", } + if pmockoidc, err := s.pool.BuildAndRunWithBuildOptions( + headscaleBuildOptions, + mockOidcOptions, + DockerRestartPolicy); err == nil { + s.mockOidc = *pmockoidc + } else { + s.FailNow(fmt.Sprintf("Could not start mockOIDC container: %s", err), "") + } + + oidcCfg := fmt.Sprintf(` +oidc: + issuer: http://%s:10000/oidc + client_id: superclient + client_secret: supersecret + strip_email_domain: true`, s.mockOidc.GetIPInNetwork(&s.network)) + currentPath, err := os.Getwd() if err != nil { s.FailNow(fmt.Sprintf("Could not determine current path: %s", err), "") @@ -160,7 +170,8 @@ oidc: } headscaleOptions := &dockertest.RunOptions{ - Name: oidcHeadscaleHostname, + Name: oidcHeadscaleHostname, + Networks: []*dockertest.Network{&s.network}, Mounts: []string{ path.Join(currentPath, "integration_test/etc_oidc:/etc/headscale", @@ -368,8 +379,6 @@ func (s *IntegrationOIDCTestSuite) tailscaleContainer( } func (s *IntegrationOIDCTestSuite) TearDownSuite() { - s.oidc.Shutdown() - if !s.saveLogs { for _, tailscale := range s.tailscales { if err := s.pool.Purge(&tailscale); err != nil { @@ -381,6 +390,10 @@ func (s *IntegrationOIDCTestSuite) TearDownSuite() { log.Printf("Could not purge resource: %s\n", err) } + if err := s.pool.Purge(&s.mockOidc); err != nil { + log.Printf("Could not purge resource: %s\n", err) + } + if err := s.network.Close(); err != nil { log.Printf("Could not close network: %s\n", err) } From 7a171cf5eaa0d2c9bd15be0c8caec6203813140e Mon Sep 17 00:00:00 2001 From: Juan Font Date: Tue, 20 Sep 2022 20:54:58 +0000 Subject: [PATCH 36/42] Added sleep to workaround #814 --- integration_oidc_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/integration_oidc_test.go b/integration_oidc_test.go index 27f3f8d..a7baad7 100644 --- a/integration_oidc_test.go +++ b/integration_oidc_test.go @@ -264,6 +264,7 @@ oidc: for hostname, tailscale := range s.tailscales { s.joinWaitGroup.Add(1) go s.AuthenticateOIDC(headscaleEndpoint, hostname, tailscale) + time.Sleep(1 * time.Second) } s.joinWaitGroup.Wait() From 083d2a871c1efb0ae81fb1803c8a43d626600fc1 Mon Sep 17 00:00:00 2001 From: Juan Font Date: Tue, 20 Sep 2022 21:02:44 +0000 Subject: [PATCH 37/42] Linting fixes --- cmd/headscale/cli/mockoidc.go | 27 +++++++++++++++++++-------- integration_oidc_test.go | 2 +- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/cmd/headscale/cli/mockoidc.go b/cmd/headscale/cli/mockoidc.go index 07248d4..179a7d4 100644 --- a/cmd/headscale/cli/mockoidc.go +++ b/cmd/headscale/cli/mockoidc.go @@ -12,6 +12,14 @@ import ( "github.com/spf13/cobra" ) +const ( + errMockOidcClientIDNotDefined = Error("MOCKOIDC_CLIENT_ID not defined") + errMockOidcClientSecretNotDefined = Error("MOCKOIDC_CLIENT_SECRET not defined") + errMockOidcPortNotDefined = Error("MOCKOIDC_PORT not defined") + accessTTL = 10 * time.Minute + refreshTTL = 60 * time.Minute +) + func init() { rootCmd.AddCommand(mockOidcCmd) } @@ -32,15 +40,15 @@ var mockOidcCmd = &cobra.Command{ func mockOIDC() error { clientID := os.Getenv("MOCKOIDC_CLIENT_ID") if clientID == "" { - return fmt.Errorf("MOCKOIDC_CLIENT_ID not set") + return errMockOidcClientIDNotDefined } clientSecret := os.Getenv("MOCKOIDC_CLIENT_SECRET") if clientSecret == "" { - return fmt.Errorf("MOCKOIDC_CLIENT_SECRET not set") + return errMockOidcClientSecretNotDefined } portStr := os.Getenv("MOCKOIDC_PORT") if portStr == "" { - return fmt.Errorf("MOCKOIDC_PORT not set") + return errMockOidcPortNotDefined } port, err := strconv.Atoi(portStr) @@ -53,13 +61,16 @@ func mockOIDC() error { return err } - ln, err := net.Listen("tcp", fmt.Sprintf("mockoidc:%d", port)) + listener, err := net.Listen("tcp", fmt.Sprintf("mockoidc:%d", port)) if err != nil { return err } - mock.Start(ln, nil) - log.Info().Msgf("Mock OIDC server listening on %s", ln.Addr().String()) + err = mock.Start(listener, nil) + if err != nil { + return err + } + log.Info().Msgf("Mock OIDC server listening on %s", listener.Addr().String()) log.Info().Msgf("Issuer: %s", mock.Issuer()) c := make(chan struct{}) <-c @@ -76,8 +87,8 @@ func getMockOIDC(clientID string, clientSecret string) (*mockoidc.MockOIDC, erro mock := mockoidc.MockOIDC{ ClientID: clientID, ClientSecret: clientSecret, - AccessTTL: time.Duration(10) * time.Minute, - RefreshTTL: time.Duration(60) * time.Minute, + AccessTTL: accessTTL, + RefreshTTL: refreshTTL, CodeChallengeMethodsSupported: []string{"plain", "S256"}, Keypair: keypair, SessionStore: mockoidc.NewSessionStore(), diff --git a/integration_oidc_test.go b/integration_oidc_test.go index a7baad7..fc1667b 100644 --- a/integration_oidc_test.go +++ b/integration_oidc_test.go @@ -225,7 +225,7 @@ oidc: client := &http.Client{Transport: insecureTransport} resp, err := client.Get(url) if err != nil { - fmt.Printf("headscale for embedded OIDC tests is not ready: %s\n", err) + log.Printf("headscale for embedded OIDC tests is not ready: %s\n", err) return err } From e87b4709967d4635ec626ee6035199750886fcb6 Mon Sep 17 00:00:00 2001 From: Juan Font Date: Tue, 20 Sep 2022 21:06:43 +0000 Subject: [PATCH 38/42] Removed fmt.Println for linting --- cmd/headscale/cli/mockoidc.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/headscale/cli/mockoidc.go b/cmd/headscale/cli/mockoidc.go index 179a7d4..4313bbf 100644 --- a/cmd/headscale/cli/mockoidc.go +++ b/cmd/headscale/cli/mockoidc.go @@ -31,7 +31,7 @@ var mockOidcCmd = &cobra.Command{ Run: func(cmd *cobra.Command, args []string) { err := mockOIDC() if err != nil { - fmt.Println(err) + log.Error().Err(err).Msgf("Error running mock OIDC server") os.Exit(1) } }, From f2928d7dcb0df9c91bc90a0737759d1737b105ed Mon Sep 17 00:00:00 2001 From: Juan Font Date: Tue, 20 Sep 2022 21:26:11 +0000 Subject: [PATCH 39/42] Removed gin from go.sum (Github security notice) --- go.sum | 2 -- 1 file changed, 2 deletions(-) diff --git a/go.sum b/go.sum index 9d51d32..0567a05 100644 --- a/go.sum +++ b/go.sum @@ -273,8 +273,6 @@ github.com/fzipp/gocyclo v0.3.1/go.mod h1:DJHO6AUmbdqj2ET4Z9iArSuwWgYDRryYt2wASx github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= -github.com/gin-gonic/gin v1.6.3 h1:ahKqKTFpO5KTPHxWZjEdPScmYaGtLo8Y4DMHoEsnp14= -github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M= github.com/glebarez/go-sqlite v1.17.3 h1:Rji9ROVSTTfjuWD6j5B+8DtkNvPILoUC3xRhkQzGxvk= github.com/glebarez/go-sqlite v1.17.3/go.mod h1:Hg+PQuhUy98XCxWEJEaWob8x7lhJzhNYF1nZbUiRGIY= github.com/glebarez/go-sqlite v1.18.1 h1:w0xtxKWktqYsUsXg//SQK+l1IcpKb3rGOQHmMptvL2U= From e286ba817b52fbeb2fd9cc904ecfc12b1ea83a08 Mon Sep 17 00:00:00 2001 From: Mike Lloyd Date: Tue, 20 Sep 2022 20:12:45 -0700 Subject: [PATCH 40/42] Format reverse-proxy.md --- docs/reverse-proxy.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/reverse-proxy.md b/docs/reverse-proxy.md index 0809d04..74bbff7 100644 --- a/docs/reverse-proxy.md +++ b/docs/reverse-proxy.md @@ -3,13 +3,13 @@ Running headscale behind a reverse proxy is useful when running multiple applications on the same server, and you want to reuse the same external IP and port - usually tcp/443 for HTTPS. ### WebSockets -The reverse proxy MUST be configured to support WebSockets, as it is needed for clients running Tailscale v1.30+. + +The reverse proxy MUST be configured to support WebSockets, as it is needed for clients running Tailscale v1.30+. WebSockets support is required when using the headscale embedded DERP server. In this case, you will also need to expose the UDP port used for STUN (by default, udp/3478). Please check our [config-example.yaml](https://github.com/juanfont/headscale/blob/main/config-example.yaml). - - ### TLS + Headscale can be configured not to use TLS, leaving it to the reverse proxy to handle. Add the following configuration values to your headscale config file. ```yaml @@ -21,12 +21,13 @@ tls_key_path: "" ``` ## nginx + The following example configuration can be used in your nginx setup, substituting values as necessary. `` should be the IP address and port where headscale is running. In most cases, this will be `http://localhost:8080`. ```Nginx -map $http_upgrade $connection_upgrade { +map $http_upgrade $connection_upgrade { default keep-alive; - 'websocket' upgrade; + 'websocket' upgrade; '' close; } From 95948e03c999fc6021a21e11da3ef3a2d5bfd527 Mon Sep 17 00:00:00 2001 From: Juan Font Date: Wed, 21 Sep 2022 14:47:48 +0000 Subject: [PATCH 41/42] Added indication of workaround for #814 --- integration_oidc_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/integration_oidc_test.go b/integration_oidc_test.go index fc1667b..b7e032c 100644 --- a/integration_oidc_test.go +++ b/integration_oidc_test.go @@ -264,6 +264,8 @@ oidc: for hostname, tailscale := range s.tailscales { s.joinWaitGroup.Add(1) go s.AuthenticateOIDC(headscaleEndpoint, hostname, tailscale) + + // TODO(juan): Workaround for https://github.com/juanfont/headscale/issues/814 time.Sleep(1 * time.Second) } From 695359862e4d147a649148a03111598153209818 Mon Sep 17 00:00:00 2001 From: Juan Font Date: Wed, 21 Sep 2022 15:01:26 +0000 Subject: [PATCH 42/42] Return stderr too in ExecuteCommand --- integration_cli_test.go | 122 +++++++++++++++--------------- integration_common_test.go | 18 ++--- integration_embedded_derp_test.go | 8 +- integration_general_test.go | 24 +++--- integration_oidc_test.go | 16 ++-- 5 files changed, 94 insertions(+), 94 deletions(-) diff --git a/integration_cli_test.go b/integration_cli_test.go index d2e28be..4dc7455 100644 --- a/integration_cli_test.go +++ b/integration_cli_test.go @@ -129,7 +129,7 @@ func (s *IntegrationCLITestSuite) HandleStats( } func (s *IntegrationCLITestSuite) createNamespace(name string) (*v1.Namespace, error) { - result, err := ExecuteCommand( + result, _, err := ExecuteCommand( &s.headscale, []string{ "headscale", @@ -172,7 +172,7 @@ func (s *IntegrationCLITestSuite) TestNamespaceCommand() { assert.Equal(s.T(), names[2], namespaces[2].Name) // Test list namespaces - listResult, err := ExecuteCommand( + listResult, _, err := ExecuteCommand( &s.headscale, []string{ "headscale", @@ -194,7 +194,7 @@ func (s *IntegrationCLITestSuite) TestNamespaceCommand() { assert.Equal(s.T(), names[2], listedNamespaces[2].Name) // Test rename namespace - renameResult, err := ExecuteCommand( + renameResult, _, err := ExecuteCommand( &s.headscale, []string{ "headscale", @@ -216,7 +216,7 @@ func (s *IntegrationCLITestSuite) TestNamespaceCommand() { assert.Equal(s.T(), renamedNamespace.Name, "newname") // Test list after rename namespaces - listAfterRenameResult, err := ExecuteCommand( + listAfterRenameResult, _, err := ExecuteCommand( &s.headscale, []string{ "headscale", @@ -247,7 +247,7 @@ func (s *IntegrationCLITestSuite) TestPreAuthKeyCommand() { assert.Nil(s.T(), err) for i := 0; i < count; i++ { - preAuthResult, err := ExecuteCommand( + preAuthResult, _, err := ExecuteCommand( &s.headscale, []string{ "headscale", @@ -275,7 +275,7 @@ func (s *IntegrationCLITestSuite) TestPreAuthKeyCommand() { assert.Len(s.T(), keys, 5) // Test list of keys - listResult, err := ExecuteCommand( + listResult, _, err := ExecuteCommand( &s.headscale, []string{ "headscale", @@ -335,7 +335,7 @@ func (s *IntegrationCLITestSuite) TestPreAuthKeyCommand() { // Expire three keys for i := 0; i < 3; i++ { - _, err := ExecuteCommand( + _, _, err := ExecuteCommand( &s.headscale, []string{ "headscale", @@ -351,7 +351,7 @@ func (s *IntegrationCLITestSuite) TestPreAuthKeyCommand() { } // Test list pre auth keys after expire - listAfterExpireResult, err := ExecuteCommand( + listAfterExpireResult, _, err := ExecuteCommand( &s.headscale, []string{ "headscale", @@ -396,7 +396,7 @@ func (s *IntegrationCLITestSuite) TestPreAuthKeyCommandWithoutExpiry() { namespace, err := s.createNamespace("pre-auth-key-without-exp-namespace") assert.Nil(s.T(), err) - preAuthResult, err := ExecuteCommand( + preAuthResult, _, err := ExecuteCommand( &s.headscale, []string{ "headscale", @@ -417,7 +417,7 @@ func (s *IntegrationCLITestSuite) TestPreAuthKeyCommandWithoutExpiry() { assert.Nil(s.T(), err) // Test list of keys - listResult, err := ExecuteCommand( + listResult, _, err := ExecuteCommand( &s.headscale, []string{ "headscale", @@ -449,7 +449,7 @@ func (s *IntegrationCLITestSuite) TestPreAuthKeyCommandReusableEphemeral() { namespace, err := s.createNamespace("pre-auth-key-reus-ephm-namespace") assert.Nil(s.T(), err) - preAuthReusableResult, err := ExecuteCommand( + preAuthReusableResult, _, err := ExecuteCommand( &s.headscale, []string{ "headscale", @@ -472,7 +472,7 @@ func (s *IntegrationCLITestSuite) TestPreAuthKeyCommandReusableEphemeral() { assert.True(s.T(), preAuthReusableKey.GetReusable()) assert.False(s.T(), preAuthReusableKey.GetEphemeral()) - preAuthEphemeralResult, err := ExecuteCommand( + preAuthEphemeralResult, _, err := ExecuteCommand( &s.headscale, []string{ "headscale", @@ -514,7 +514,7 @@ func (s *IntegrationCLITestSuite) TestPreAuthKeyCommandReusableEphemeral() { // assert.NotNil(s.T(), err) // Test list of keys - listResult, err := ExecuteCommand( + listResult, _, err := ExecuteCommand( &s.headscale, []string{ "headscale", @@ -548,7 +548,7 @@ func (s *IntegrationCLITestSuite) TestNodeTagCommand() { assert.Nil(s.T(), err) for index, machineKey := range machineKeys { - _, err := ExecuteCommand( + _, _, err := ExecuteCommand( &s.headscale, []string{ "headscale", @@ -567,7 +567,7 @@ func (s *IntegrationCLITestSuite) TestNodeTagCommand() { ) assert.Nil(s.T(), err) - machineResult, err := ExecuteCommand( + machineResult, _, err := ExecuteCommand( &s.headscale, []string{ "headscale", @@ -592,7 +592,7 @@ func (s *IntegrationCLITestSuite) TestNodeTagCommand() { } assert.Len(s.T(), machines, len(machineKeys)) - addTagResult, err := ExecuteCommand( + addTagResult, _, err := ExecuteCommand( &s.headscale, []string{ "headscale", @@ -612,7 +612,7 @@ func (s *IntegrationCLITestSuite) TestNodeTagCommand() { assert.Equal(s.T(), []string{"tag:test"}, machine.ForcedTags) // try to set a wrong tag and retrieve the error - wrongTagResult, err := ExecuteCommand( + wrongTagResult, _, err := ExecuteCommand( &s.headscale, []string{ "headscale", @@ -634,7 +634,7 @@ func (s *IntegrationCLITestSuite) TestNodeTagCommand() { assert.Contains(s.T(), errorOutput.Error, "tag must start with the string 'tag:'") // Test list all nodes after added seconds - listAllResult, err := ExecuteCommand( + listAllResult, _, err := ExecuteCommand( &s.headscale, []string{ "headscale", @@ -684,7 +684,7 @@ func (s *IntegrationCLITestSuite) TestNodeCommand() { assert.Nil(s.T(), err) for index, machineKey := range machineKeys { - _, err := ExecuteCommand( + _, _, err := ExecuteCommand( &s.headscale, []string{ "headscale", @@ -703,7 +703,7 @@ func (s *IntegrationCLITestSuite) TestNodeCommand() { ) assert.Nil(s.T(), err) - machineResult, err := ExecuteCommand( + machineResult, _, err := ExecuteCommand( &s.headscale, []string{ "headscale", @@ -730,7 +730,7 @@ func (s *IntegrationCLITestSuite) TestNodeCommand() { assert.Len(s.T(), machines, len(machineKeys)) // Test list all nodes after added seconds - listAllResult, err := ExecuteCommand( + listAllResult, _, err := ExecuteCommand( &s.headscale, []string{ "headscale", @@ -769,7 +769,7 @@ func (s *IntegrationCLITestSuite) TestNodeCommand() { assert.Nil(s.T(), err) for index, machineKey := range otherNamespaceMachineKeys { - _, err := ExecuteCommand( + _, _, err := ExecuteCommand( &s.headscale, []string{ "headscale", @@ -788,7 +788,7 @@ func (s *IntegrationCLITestSuite) TestNodeCommand() { ) assert.Nil(s.T(), err) - machineResult, err := ExecuteCommand( + machineResult, _, err := ExecuteCommand( &s.headscale, []string{ "headscale", @@ -815,7 +815,7 @@ func (s *IntegrationCLITestSuite) TestNodeCommand() { assert.Len(s.T(), otherNamespaceMachines, len(otherNamespaceMachineKeys)) // Test list all nodes after added otherNamespace - listAllWithotherNamespaceResult, err := ExecuteCommand( + listAllWithotherNamespaceResult, _, err := ExecuteCommand( &s.headscale, []string{ "headscale", @@ -845,7 +845,7 @@ func (s *IntegrationCLITestSuite) TestNodeCommand() { assert.Equal(s.T(), "otherNamespace-machine-2", listAllWithotherNamespace[6].Name) // Test list all nodes after added otherNamespace - listOnlyotherNamespaceMachineNamespaceResult, err := ExecuteCommand( + listOnlyotherNamespaceMachineNamespaceResult, _, err := ExecuteCommand( &s.headscale, []string{ "headscale", @@ -884,7 +884,7 @@ func (s *IntegrationCLITestSuite) TestNodeCommand() { ) // Delete a machines - _, err = ExecuteCommand( + _, _, err = ExecuteCommand( &s.headscale, []string{ "headscale", @@ -902,7 +902,7 @@ func (s *IntegrationCLITestSuite) TestNodeCommand() { assert.Nil(s.T(), err) // Test: list main namespace after machine is deleted - listOnlyMachineNamespaceAfterDeleteResult, err := ExecuteCommand( + listOnlyMachineNamespaceAfterDeleteResult, _, err := ExecuteCommand( &s.headscale, []string{ "headscale", @@ -943,7 +943,7 @@ func (s *IntegrationCLITestSuite) TestNodeExpireCommand() { assert.Nil(s.T(), err) for index, machineKey := range machineKeys { - _, err := ExecuteCommand( + _, _, err := ExecuteCommand( &s.headscale, []string{ "headscale", @@ -962,7 +962,7 @@ func (s *IntegrationCLITestSuite) TestNodeExpireCommand() { ) assert.Nil(s.T(), err) - machineResult, err := ExecuteCommand( + machineResult, _, err := ExecuteCommand( &s.headscale, []string{ "headscale", @@ -988,7 +988,7 @@ func (s *IntegrationCLITestSuite) TestNodeExpireCommand() { assert.Len(s.T(), machines, len(machineKeys)) - listAllResult, err := ExecuteCommand( + listAllResult, _, err := ExecuteCommand( &s.headscale, []string{ "headscale", @@ -1014,7 +1014,7 @@ func (s *IntegrationCLITestSuite) TestNodeExpireCommand() { assert.True(s.T(), listAll[4].Expiry.AsTime().IsZero()) for i := 0; i < 3; i++ { - _, err := ExecuteCommand( + _, _, err := ExecuteCommand( &s.headscale, []string{ "headscale", @@ -1028,7 +1028,7 @@ func (s *IntegrationCLITestSuite) TestNodeExpireCommand() { assert.Nil(s.T(), err) } - listAllAfterExpiryResult, err := ExecuteCommand( + listAllAfterExpiryResult, _, err := ExecuteCommand( &s.headscale, []string{ "headscale", @@ -1070,7 +1070,7 @@ func (s *IntegrationCLITestSuite) TestNodeRenameCommand() { assert.Nil(s.T(), err) for index, machineKey := range machineKeys { - _, err := ExecuteCommand( + _, _, err := ExecuteCommand( &s.headscale, []string{ "headscale", @@ -1089,7 +1089,7 @@ func (s *IntegrationCLITestSuite) TestNodeRenameCommand() { ) assert.Nil(s.T(), err) - machineResult, err := ExecuteCommand( + machineResult, _, err := ExecuteCommand( &s.headscale, []string{ "headscale", @@ -1115,7 +1115,7 @@ func (s *IntegrationCLITestSuite) TestNodeRenameCommand() { assert.Len(s.T(), machines, len(machineKeys)) - listAllResult, err := ExecuteCommand( + listAllResult, _, err := ExecuteCommand( &s.headscale, []string{ "headscale", @@ -1141,7 +1141,7 @@ func (s *IntegrationCLITestSuite) TestNodeRenameCommand() { assert.Contains(s.T(), listAll[4].GetGivenName(), "machine-5") for i := 0; i < 3; i++ { - _, err := ExecuteCommand( + _, _, err := ExecuteCommand( &s.headscale, []string{ "headscale", @@ -1156,7 +1156,7 @@ func (s *IntegrationCLITestSuite) TestNodeRenameCommand() { assert.Nil(s.T(), err) } - listAllAfterRenameResult, err := ExecuteCommand( + listAllAfterRenameResult, _, err := ExecuteCommand( &s.headscale, []string{ "headscale", @@ -1182,7 +1182,7 @@ func (s *IntegrationCLITestSuite) TestNodeRenameCommand() { assert.Contains(s.T(), listAllAfterRename[4].GetGivenName(), "machine-5") // Test failure for too long names - result, err := ExecuteCommand( + result, _, err := ExecuteCommand( &s.headscale, []string{ "headscale", @@ -1197,7 +1197,7 @@ func (s *IntegrationCLITestSuite) TestNodeRenameCommand() { assert.Nil(s.T(), err) assert.Contains(s.T(), result, "not be over 63 chars") - listAllAfterRenameAttemptResult, err := ExecuteCommand( + listAllAfterRenameAttemptResult, _, err := ExecuteCommand( &s.headscale, []string{ "headscale", @@ -1233,7 +1233,7 @@ func (s *IntegrationCLITestSuite) TestRouteCommand() { // Randomly generated machine keys machineKey := "9b2ffa7e08cc421a3d2cca9012280f6a236fd0de0b4ce005b30a98ad930306fe" - _, err = ExecuteCommand( + _, _, err = ExecuteCommand( &s.headscale, []string{ "headscale", @@ -1256,7 +1256,7 @@ func (s *IntegrationCLITestSuite) TestRouteCommand() { ) assert.Nil(s.T(), err) - machineResult, err := ExecuteCommand( + machineResult, _, err := ExecuteCommand( &s.headscale, []string{ "headscale", @@ -1280,7 +1280,7 @@ func (s *IntegrationCLITestSuite) TestRouteCommand() { assert.Equal(s.T(), uint64(1), machine.Id) assert.Equal(s.T(), "route-machine", machine.Name) - listAllResult, err := ExecuteCommand( + listAllResult, _, err := ExecuteCommand( &s.headscale, []string{ "headscale", @@ -1305,7 +1305,7 @@ func (s *IntegrationCLITestSuite) TestRouteCommand() { assert.Empty(s.T(), listAll.EnabledRoutes) - enableTwoRoutesResult, err := ExecuteCommand( + enableTwoRoutesResult, _, err := ExecuteCommand( &s.headscale, []string{ "headscale", @@ -1337,7 +1337,7 @@ func (s *IntegrationCLITestSuite) TestRouteCommand() { assert.Contains(s.T(), enableTwoRoutes.EnabledRoutes, "192.168.1.0/24") // Enable only one route, effectively disabling one of the routes - enableOneRouteResult, err := ExecuteCommand( + enableOneRouteResult, _, err := ExecuteCommand( &s.headscale, []string{ "headscale", @@ -1366,7 +1366,7 @@ func (s *IntegrationCLITestSuite) TestRouteCommand() { assert.Contains(s.T(), enableOneRoute.EnabledRoutes, "10.0.0.0/8") // Enable only one route, effectively disabling one of the routes - failEnableNonAdvertisedRoute, err := ExecuteCommand( + failEnableNonAdvertisedRoute, _, err := ExecuteCommand( &s.headscale, []string{ "headscale", @@ -1390,7 +1390,7 @@ func (s *IntegrationCLITestSuite) TestRouteCommand() { ) // Enable all routes on host - enableAllRouteResult, err := ExecuteCommand( + enableAllRouteResult, _, err := ExecuteCommand( &s.headscale, []string{ "headscale", @@ -1425,7 +1425,7 @@ func (s *IntegrationCLITestSuite) TestApiKeyCommand() { keys := make([]string, count) for i := 0; i < count; i++ { - apiResult, err := ExecuteCommand( + apiResult, _, err := ExecuteCommand( &s.headscale, []string{ "headscale", @@ -1451,7 +1451,7 @@ func (s *IntegrationCLITestSuite) TestApiKeyCommand() { assert.Len(s.T(), keys, 5) // Test list of keys - listResult, err := ExecuteCommand( + listResult, _, err := ExecuteCommand( &s.headscale, []string{ "headscale", @@ -1513,7 +1513,7 @@ func (s *IntegrationCLITestSuite) TestApiKeyCommand() { // Expire three keys for i := 0; i < 3; i++ { - _, err := ExecuteCommand( + _, _, err := ExecuteCommand( &s.headscale, []string{ "headscale", @@ -1530,7 +1530,7 @@ func (s *IntegrationCLITestSuite) TestApiKeyCommand() { } // Test list pre auth keys after expire - listAfterExpireResult, err := ExecuteCommand( + listAfterExpireResult, _, err := ExecuteCommand( &s.headscale, []string{ "headscale", @@ -1573,7 +1573,7 @@ func (s *IntegrationCLITestSuite) TestNodeMoveCommand() { // Randomly generated machine key machineKey := "688411b767663479632d44140f08a9fde87383adc7cdeb518f62ce28a17ef0aa" - _, err = ExecuteCommand( + _, _, err = ExecuteCommand( &s.headscale, []string{ "headscale", @@ -1592,7 +1592,7 @@ func (s *IntegrationCLITestSuite) TestNodeMoveCommand() { ) assert.Nil(s.T(), err) - machineResult, err := ExecuteCommand( + machineResult, _, err := ExecuteCommand( &s.headscale, []string{ "headscale", @@ -1619,7 +1619,7 @@ func (s *IntegrationCLITestSuite) TestNodeMoveCommand() { machineId := fmt.Sprintf("%d", machine.Id) - moveToNewNSResult, err := ExecuteCommand( + moveToNewNSResult, _, err := ExecuteCommand( &s.headscale, []string{ "headscale", @@ -1641,7 +1641,7 @@ func (s *IntegrationCLITestSuite) TestNodeMoveCommand() { assert.Equal(s.T(), machine.Namespace, newNamespace) - listAllNodesResult, err := ExecuteCommand( + listAllNodesResult, _, err := ExecuteCommand( &s.headscale, []string{ "headscale", @@ -1664,7 +1664,7 @@ func (s *IntegrationCLITestSuite) TestNodeMoveCommand() { assert.Equal(s.T(), allNodes[0].Namespace, machine.Namespace) assert.Equal(s.T(), allNodes[0].Namespace, newNamespace) - moveToNonExistingNSResult, err := ExecuteCommand( + moveToNonExistingNSResult, _, err := ExecuteCommand( &s.headscale, []string{ "headscale", @@ -1688,7 +1688,7 @@ func (s *IntegrationCLITestSuite) TestNodeMoveCommand() { ) assert.Equal(s.T(), machine.Namespace, newNamespace) - moveToOldNSResult, err := ExecuteCommand( + moveToOldNSResult, _, err := ExecuteCommand( &s.headscale, []string{ "headscale", @@ -1710,7 +1710,7 @@ func (s *IntegrationCLITestSuite) TestNodeMoveCommand() { assert.Equal(s.T(), machine.Namespace, oldNamespace) - moveToSameNSResult, err := ExecuteCommand( + moveToSameNSResult, _, err := ExecuteCommand( &s.headscale, []string{ "headscale", @@ -1742,7 +1742,7 @@ func (s *IntegrationCLITestSuite) TestLoadConfigFromCommand() { altEnvConfig, err := os.ReadFile("integration_test/etc/alt-env-config.dump.gold.yaml") assert.Nil(s.T(), err) - _, err = ExecuteCommand( + _, _, err = ExecuteCommand( &s.headscale, []string{ "headscale", @@ -1757,7 +1757,7 @@ func (s *IntegrationCLITestSuite) TestLoadConfigFromCommand() { assert.YAMLEq(s.T(), string(defaultConfig), string(defaultDumpConfig)) - _, err = ExecuteCommand( + _, _, err = ExecuteCommand( &s.headscale, []string{ "headscale", @@ -1774,7 +1774,7 @@ func (s *IntegrationCLITestSuite) TestLoadConfigFromCommand() { assert.YAMLEq(s.T(), string(altConfig), string(altDumpConfig)) - _, err = ExecuteCommand( + _, _, err = ExecuteCommand( &s.headscale, []string{ "headscale", @@ -1791,7 +1791,7 @@ func (s *IntegrationCLITestSuite) TestLoadConfigFromCommand() { assert.YAMLEq(s.T(), string(altEnvConfig), string(altEnvDumpConfig)) - _, err = ExecuteCommand( + _, _, err = ExecuteCommand( &s.headscale, []string{ "headscale", diff --git a/integration_common_test.go b/integration_common_test.go index 71db0e5..9cce12f 100644 --- a/integration_common_test.go +++ b/integration_common_test.go @@ -68,7 +68,7 @@ func ExecuteCommand( cmd []string, env []string, options ...ExecuteCommandOption, -) (string, error) { +) (string, string, error) { var stdout bytes.Buffer var stderr bytes.Buffer @@ -78,7 +78,7 @@ func ExecuteCommand( for _, opt := range options { if err := opt(&execConfig); err != nil { - return "", fmt.Errorf("execute-command/options: %w", err) + return "", "", fmt.Errorf("execute-command/options: %w", err) } } @@ -107,7 +107,7 @@ func ExecuteCommand( select { case res := <-resultChan: if res.err != nil { - return "", res.err + return stdout.String(), stderr.String(), res.err } if res.exitCode != 0 { @@ -115,13 +115,13 @@ func ExecuteCommand( fmt.Println("stdout: ", stdout.String()) fmt.Println("stderr: ", stderr.String()) - return "", fmt.Errorf("command failed with: %s", stderr.String()) + return stdout.String(), stderr.String(), fmt.Errorf("command failed with: %s", stderr.String()) } - return stdout.String(), nil + return stdout.String(), stderr.String(), nil case <-time.After(execConfig.timeout): - return stderr.String(), fmt.Errorf("command timed out after %s", execConfig.timeout) + return stdout.String(), stderr.String(), fmt.Errorf("command timed out after %s", execConfig.timeout) } } @@ -200,7 +200,7 @@ func getIPs( for hostname, tailscale := range tailscales { command := []string{"tailscale", "ip"} - result, err := ExecuteCommand( + result, _, err := ExecuteCommand( &tailscale, command, []string{}, @@ -228,7 +228,7 @@ func getIPs( func getDNSNames( headscale *dockertest.Resource, ) ([]string, error) { - listAllResult, err := ExecuteCommand( + listAllResult, _, err := ExecuteCommand( headscale, []string{ "headscale", @@ -261,7 +261,7 @@ func getDNSNames( func getMagicFQDN( headscale *dockertest.Resource, ) ([]string, error) { - listAllResult, err := ExecuteCommand( + listAllResult, _, err := ExecuteCommand( headscale, []string{ "headscale", diff --git a/integration_embedded_derp_test.go b/integration_embedded_derp_test.go index 37ce82c..a31006e 100644 --- a/integration_embedded_derp_test.go +++ b/integration_embedded_derp_test.go @@ -187,7 +187,7 @@ func (s *IntegrationDERPTestSuite) SetupSuite() { log.Println("headscale container is ready for embedded DERP tests") log.Printf("Creating headscale namespace: %s\n", namespaceName) - result, err := ExecuteCommand( + result, _, err := ExecuteCommand( &s.headscale, []string{"headscale", "namespaces", "create", namespaceName}, []string{}, @@ -196,7 +196,7 @@ func (s *IntegrationDERPTestSuite) SetupSuite() { assert.Nil(s.T(), err) log.Printf("Creating pre auth key for %s\n", namespaceName) - preAuthResult, err := ExecuteCommand( + preAuthResult, _, err := ExecuteCommand( &s.headscale, []string{ "headscale", @@ -259,7 +259,7 @@ func (s *IntegrationDERPTestSuite) Join( log.Println("Join command:", command) log.Printf("Running join command for %s\n", hostname) - _, err := ExecuteCommand( + _, _, err := ExecuteCommand( &tailscale, command, []string{}, @@ -414,7 +414,7 @@ func (s *IntegrationDERPTestSuite) TestPingAllPeersByHostname() { peername, ) log.Println(command) - result, err := ExecuteCommand( + result, _, err := ExecuteCommand( &tailscale, command, []string{}, diff --git a/integration_general_test.go b/integration_general_test.go index 66652d7..5abdccb 100644 --- a/integration_general_test.go +++ b/integration_general_test.go @@ -163,7 +163,7 @@ func (s *IntegrationTestSuite) Join( log.Println("Join command:", command) log.Printf("Running join command for %s\n", hostname) - _, err := ExecuteCommand( + _, _, err := ExecuteCommand( &tailscale, command, []string{}, @@ -305,7 +305,7 @@ func (s *IntegrationTestSuite) SetupSuite() { for namespace, scales := range s.namespaces { log.Printf("Creating headscale namespace: %s\n", namespace) - result, err := ExecuteCommand( + result, _, err := ExecuteCommand( &s.headscale, []string{"headscale", "namespaces", "create", namespace}, []string{}, @@ -314,7 +314,7 @@ func (s *IntegrationTestSuite) SetupSuite() { assert.Nil(s.T(), err) log.Printf("Creating pre auth key for %s\n", namespace) - preAuthResult, err := ExecuteCommand( + preAuthResult, _, err := ExecuteCommand( &s.headscale, []string{ "headscale", @@ -386,7 +386,7 @@ func (s *IntegrationTestSuite) HandleStats( func (s *IntegrationTestSuite) TestListNodes() { for namespace, scales := range s.namespaces { log.Println("Listing nodes") - result, err := ExecuteCommand( + result, _, err := ExecuteCommand( &s.headscale, []string{"headscale", "--namespace", namespace, "nodes", "list"}, []string{}, @@ -518,7 +518,7 @@ func (s *IntegrationTestSuite) TestPingAllPeersByAddress() { peername, ip, ) - result, err := ExecuteCommand( + result, _, err := ExecuteCommand( &tailscale, command, []string{}, @@ -552,7 +552,7 @@ func (s *IntegrationTestSuite) TestTailDrop() { for hostname, tailscale := range scales.tailscales { command := []string{"touch", fmt.Sprintf("/tmp/file_from_%s", hostname)} - _, err := ExecuteCommand( + _, _, err := ExecuteCommand( &tailscale, command, []string{}, @@ -586,7 +586,7 @@ func (s *IntegrationTestSuite) TestTailDrop() { hostname, peername, ) - _, err := ExecuteCommand( + _, _, err := ExecuteCommand( &tailscale, command, []string{}, @@ -606,7 +606,7 @@ func (s *IntegrationTestSuite) TestTailDrop() { "get", "/tmp/", } - _, err := ExecuteCommand( + _, _, err := ExecuteCommand( &tailscale, command, []string{}, @@ -628,7 +628,7 @@ func (s *IntegrationTestSuite) TestTailDrop() { peername, ip, ) - result, err := ExecuteCommand( + result, _, err := ExecuteCommand( &tailscale, command, []string{}, @@ -672,7 +672,7 @@ func (s *IntegrationTestSuite) TestPingAllPeersByHostname() { hostname, peername, ) - result, err := ExecuteCommand( + result, _, err := ExecuteCommand( &tailscale, command, []string{}, @@ -724,7 +724,7 @@ func (s *IntegrationTestSuite) TestMagicDNS() { peername, hostname, ) - result, err := ExecuteCommand( + result, _, err := ExecuteCommand( &tailscale, command, []string{}, @@ -757,7 +757,7 @@ func getAPIURLs( "/run/tailscale/tailscaled.sock", "http://localhost/localapi/v0/file-targets", } - result, err := ExecuteCommand( + result, _, err := ExecuteCommand( &tailscale, command, []string{}, diff --git a/integration_oidc_test.go b/integration_oidc_test.go index b7e032c..70f793b 100644 --- a/integration_oidc_test.go +++ b/integration_oidc_test.go @@ -244,7 +244,7 @@ oidc: s.Suite.T().Log("headscale container is ready for embedded OIDC tests") s.Suite.T().Logf("Creating headscale namespace: %s\n", oidcNamespaceName) - result, err := ExecuteCommand( + result, _, err := ExecuteCommand( &s.headscale, []string{"headscale", "namespaces", "create", oidcNamespaceName}, []string{}, @@ -320,22 +320,22 @@ func (s *IntegrationOIDCTestSuite) joinOIDC( log.Println("Join command:", command) log.Printf("Running join command for %s\n", hostname) - result, _ := ExecuteCommand( + _, stderr, _ := ExecuteCommand( &tailscale, command, []string{}, ) - // This piece of code just gets the login URL out of the output of the tailscale client. + // This piece of code just gets the login URL out of the stderr of the tailscale client. // See https://github.com/tailscale/tailscale/blob/main/cmd/tailscale/cli/up.go#L584. - urlStr := strings.ReplaceAll(result, "\nTo authenticate, visit:\n\n\t", "") + urlStr := strings.ReplaceAll(stderr, "\nTo authenticate, visit:\n\n\t", "") urlStr = strings.TrimSpace(urlStr) // parse URL loginUrl, err := url.Parse(urlStr) if err != nil { log.Printf("Could not parse login URL: %s", err) - log.Printf("Original join command result: %s", result) + log.Printf("Original join command result: %s", stderr) return nil, err } @@ -491,14 +491,14 @@ func (s *IntegrationOIDCTestSuite) TestPingAllPeersByAddress() { peername, ip, ) - result, err := ExecuteCommand( + stdout, stderr, err := ExecuteCommand( &tailscale, command, []string{}, ) assert.Nil(t, err) - log.Printf("Result for %s: %s\n", hostname, result) - assert.Contains(t, result, "pong") + log.Printf("result for %s: stdout: %s, stderr: %s\n", hostname, stdout, stderr) + assert.Contains(t, stdout, "pong") }) } }