Merge pull request #844 from kradalby/container-exist-fix

Run integration tests inside docker, dont depend on local platform

Makefile

@@ -27,16 +27,40 @@ test:
 test_integration: test_integration_cli test_integration_derp test_integration_oidc test_integration_general
 
 test_integration_cli:
-	go test -failfast -tags integration_cli,integration -timeout 30m -count=1 ./...
+	docker network rm $$(docker network ls --filter name=headscale --quiet) || true
+	docker network create headscale-test || true
+	docker run -t --rm \
+		--network headscale-test \
+		-v $$PWD:$$PWD -w $$PWD \
+		-v /var/run/docker.sock:/var/run/docker.sock golang:1 \
+		go test -failfast -tags integration_cli,integration -timeout 30m -count=1 ./...
 
 test_integration_derp:
-	go test -failfast -tags integration_derp,integration -timeout 30m -count=1 ./...
+	docker network rm $$(docker network ls --filter name=headscale --quiet) || true
+	docker network create headscale-test || true
+	docker run -t --rm \
+		--network headscale-test \
+		-v $$PWD:$$PWD -w $$PWD \
+		-v /var/run/docker.sock:/var/run/docker.sock golang:1 \
+		go test -failfast -tags integration_derp,integration -timeout 30m -count=1 ./...
 
 test_integration_general:
-	go test -failfast -tags integration_general,integration -timeout 30m -count=1 ./...
+	docker network rm $$(docker network ls --filter name=headscale --quiet) || true
+	docker network create headscale-test || true
+	docker run -t --rm \
+		--network headscale-test \
+		-v $$PWD:$$PWD -w $$PWD \
+		-v /var/run/docker.sock:/var/run/docker.sock golang:1 \
+		go test -failfast -tags integration_general,integration -timeout 30m -count=1 ./...
 
 test_integration_oidc:
-	go test -failfast -tags integration_oidc,integration -timeout 30m -count=1 ./...
+	docker network rm $$(docker network ls --filter name=headscale --quiet) || true
+	docker network create headscale-test || true
+	docker run -t --rm \
+		--network headscale-test \
+		-v $$PWD:$$PWD -w $$PWD \
+		-v /var/run/docker.sock:/var/run/docker.sock golang:1 \
+		go test -failfast -tags integration_oidc,integration -timeout 30m -count=1 ./...
 
 coverprofile_func:
 	go tool cover -func=coverage.out

cmd/headscale/cli/mockoidc.go

@@ -46,6 +46,10 @@ func mockOIDC() error {
 	if clientSecret == "" {
 		return errMockOidcClientSecretNotDefined
 	}
+	addrStr := os.Getenv("MOCKOIDC_ADDR")
+	if addrStr == "" {
+		return errMockOidcPortNotDefined
+	}
 	portStr := os.Getenv("MOCKOIDC_PORT")
 	if portStr == "" {
 		return errMockOidcPortNotDefined
@@ -61,7 +65,7 @@ func mockOIDC() error {
 		return err
 	}
 
-	listener, err := net.Listen("tcp", fmt.Sprintf("mockoidc:%d", port))
+	listener, err := net.Listen("tcp", fmt.Sprintf("%s:%d", addrStr, port))
 	if err != nil {
 		return err
 	}

flake.nix

@@ -128,7 +128,13 @@
       };
     in rec {
       # `nix develop`
-      devShell = pkgs.mkShell {buildInputs = devDeps;};
+      devShell = pkgs.mkShell {
+        buildInputs = devDeps;
+
+        shellHook = ''
+          export GOFLAGS=-tags="integration,integration_general,integration_oidc,integration_cli,integration_derp"
+        '';
+      };
 
       # `nix build`
       packages = with pkgs; {

integration_cli_test.go

@@ -13,6 +13,7 @@ import (
 
 	v1 "github.com/juanfont/headscale/gen/go/headscale/v1"
 	"github.com/ory/dockertest/v3"
+	"github.com/ory/dockertest/v3/docker"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/suite"
 )
@@ -42,11 +43,11 @@ func (s *IntegrationCLITestSuite) SetupTest() {
 		s.FailNow(fmt.Sprintf("Could not connect to docker: %s", err), "")
 	}
 
-	if pnetwork, err := s.pool.CreateNetwork("headscale-test"); err == nil {
+	network, err := GetFirstOrCreateNetwork(&s.pool, headscaleNetwork)
-		s.network = *pnetwork
+	if err != nil {
-	} else {
+		s.FailNow(fmt.Sprintf("Failed to create or get network: %s", err), "")
-		s.FailNow(fmt.Sprintf("Could not create network: %s", err), "")
 	}
+	s.network = network
 
 	headscaleBuildOptions := &dockertest.BuildOptions{
 		Dockerfile: "Dockerfile",
@@ -63,8 +64,12 @@ func (s *IntegrationCLITestSuite) SetupTest() {
 		Mounts: []string{
 			fmt.Sprintf("%s/integration_test/etc:/etc/headscale", currentPath),
 		},
-		Networks: []*dockertest.Network{&s.network},
+		Cmd:          []string{"headscale", "serve"},
-		Cmd:      []string{"headscale", "serve"},
+		Networks:     []*dockertest.Network{&s.network},
+		ExposedPorts: []string{"8080/tcp"},
+		PortBindings: map[docker.Port][]docker.PortBinding{
+			"8080/tcp": {{HostPort: "8080"}},
+		},
 	}
 
 	err = s.pool.RemoveContainerByName(headscaleHostname)
@@ -87,7 +92,9 @@ func (s *IntegrationCLITestSuite) SetupTest() {
 	fmt.Println("Created headscale container for CLI tests")
 
 	fmt.Println("Waiting for headscale to be ready for CLI tests")
-	hostEndpoint := fmt.Sprintf("localhost:%s", s.headscale.GetPort("8080/tcp"))
+	hostEndpoint := fmt.Sprintf("%s:%s",
+		s.headscale.GetIPInNetwork(&s.network),
+		s.headscale.GetPort("8080/tcp"))
 
 	if err := s.pool.Retry(func() error {
 		url := fmt.Sprintf("http://%s/health", hostEndpoint)

integration_common_test.go

@@ -19,7 +19,8 @@ import (
 )
 
 const (
-	headscaleHostname      = "headscale-derp"
+	headscaleNetwork       = "headscale-test"
+	headscaleHostname      = "headscale"
 	DOCKER_EXECUTE_TIMEOUT = 10 * time.Second
 )
 
@@ -32,7 +33,7 @@ var (
 	tailscaleVersions = []string{
 		// "head",
 		// "unstable",
-		"1.30.0",
+		"1.30.2",
 		"1.28.0",
 		"1.26.2",
 		"1.24.2",
@@ -115,13 +116,19 @@ func ExecuteCommand(
 			fmt.Println("stdout: ", stdout.String())
 			fmt.Println("stderr: ", stderr.String())
 
-			return stdout.String(), stderr.String(), fmt.Errorf("command failed with: %s", stderr.String())
+			return stdout.String(), stderr.String(), fmt.Errorf(
+				"command failed with: %s",
+				stderr.String(),
+			)
 		}
 
 		return stdout.String(), stderr.String(), nil
 	case <-time.After(execConfig.timeout):
 
-		return stdout.String(), stderr.String(), fmt.Errorf("command timed out after %s", execConfig.timeout)
+		return stdout.String(), stderr.String(), fmt.Errorf(
+			"command timed out after %s",
+			execConfig.timeout,
+		)
 	}
 }
 
@@ -316,3 +323,22 @@ func GetEnvBool(key string) (bool, error) {
 
 	return v, nil
 }
+
+func GetFirstOrCreateNetwork(pool *dockertest.Pool, name string) (dockertest.Network, error) {
+	networks, err := pool.NetworksByName(name)
+	if err != nil || len(networks) == 0 {
+
+		if _, err := pool.CreateNetwork(name); err == nil {
+			// Create does not give us an updated version of the resource, so we need to
+			// get it again.
+			networks, err := pool.NetworksByName(name)
+			if err != nil {
+				return dockertest.Network{}, err
+			}
+
+			return networks[0], nil
+		}
+	}
+
+	return networks[0], nil
+}

integration_embedded_derp_test.go

@@ -27,18 +27,20 @@ import (
 )
 
 const (
-	namespaceName   = "derpnamespace"
+	headscaleDerpHostname = "headscale-derp"
-	totalContainers = 3
+	namespaceName         = "derpnamespace"
+	totalContainers       = 3
 )
 
 type IntegrationDERPTestSuite struct {
 	suite.Suite
 	stats *suite.SuiteInformation
 
-	pool      dockertest.Pool
+	pool              dockertest.Pool
-	networks  map[int]dockertest.Network // so we keep the containers isolated
+	network           dockertest.Network
-	headscale dockertest.Resource
+	containerNetworks map[int]dockertest.Network // so we keep the containers isolated
-	saveLogs  bool
+	headscale         dockertest.Resource
+	saveLogs          bool
 
 	tailscales    map[string]dockertest.Resource
 	joinWaitGroup sync.WaitGroup
@@ -53,7 +55,7 @@ func TestDERPIntegrationTestSuite(t *testing.T) {
 	s := new(IntegrationDERPTestSuite)
 
 	s.tailscales = make(map[string]dockertest.Resource)
-	s.networks = make(map[int]dockertest.Network)
+	s.containerNetworks = make(map[int]dockertest.Network)
 	s.saveLogs = saveLogs
 
 	suite.Run(t, s)
@@ -78,7 +80,7 @@ func TestDERPIntegrationTestSuite(t *testing.T) {
 			log.Printf("Could not purge resource: %s\n", err)
 		}
 
-		for _, network := range s.networks {
+		for _, network := range s.containerNetworks {
 			if err := network.Close(); err != nil {
 				log.Printf("Could not close network: %s\n", err)
 			}
@@ -93,9 +95,15 @@ func (s *IntegrationDERPTestSuite) SetupSuite() {
 		s.FailNow(fmt.Sprintf("Could not connect to docker: %s", err), "")
 	}
 
+	network, err := GetFirstOrCreateNetwork(&s.pool, headscaleNetwork)
+	if err != nil {
+		s.FailNow(fmt.Sprintf("Failed to create or get network: %s", err), "")
+	}
+	s.network = network
+
 	for i := 0; i < totalContainers; i++ {
 		if pnetwork, err := s.pool.CreateNetwork(fmt.Sprintf("headscale-derp-%d", i)); err == nil {
-			s.networks[i] = *pnetwork
+			s.containerNetworks[i] = *pnetwork
 		} else {
 			s.FailNow(fmt.Sprintf("Could not create network: %s", err), "")
 		}
@@ -112,7 +120,8 @@ func (s *IntegrationDERPTestSuite) SetupSuite() {
 	}
 
 	headscaleOptions := &dockertest.RunOptions{
-		Name: headscaleHostname,
+
+		Name: headscaleDerpHostname,
 		Mounts: []string{
 			fmt.Sprintf(
 				"%s/integration_test/etc_embedded_derp:/etc/headscale",
@@ -120,6 +129,7 @@ func (s *IntegrationDERPTestSuite) SetupSuite() {
 			),
 		},
 		Cmd:          []string{"headscale", "serve"},
+		Networks:     []*dockertest.Network{&s.network},
 		ExposedPorts: []string{"8443/tcp", "3478/udp"},
 		PortBindings: map[docker.Port][]docker.PortBinding{
 			"8443/tcp": {{HostPort: "8443"}},
@@ -127,7 +137,7 @@ func (s *IntegrationDERPTestSuite) SetupSuite() {
 		},
 	}
 
-	err = s.pool.RemoveContainerByName(headscaleHostname)
+	err = s.pool.RemoveContainerByName(headscaleDerpHostname)
 	if err != nil {
 		s.FailNow(
 			fmt.Sprintf(
@@ -153,13 +163,15 @@ func (s *IntegrationDERPTestSuite) SetupSuite() {
 		hostname, container := s.tailscaleContainer(
 			fmt.Sprint(i),
 			version,
-			s.networks[i],
+			s.containerNetworks[i],
 		)
 		s.tailscales[hostname] = *container
 	}
 
 	log.Println("Waiting for headscale to be ready for embedded DERP tests")
-	hostEndpoint := fmt.Sprintf("localhost:%s", s.headscale.GetPort("8443/tcp"))
+	hostEndpoint := fmt.Sprintf("%s:%s",
+		s.headscale.GetIPInNetwork(&s.network),
+		s.headscale.GetPort("8443/tcp"))
 
 	if err := s.pool.Retry(func() error {
 		url := fmt.Sprintf("https://%s/health", hostEndpoint)
@@ -320,7 +332,7 @@ func (s *IntegrationDERPTestSuite) TearDownSuite() {
 			log.Printf("Could not purge resource: %s\n", err)
 		}
 
-		for _, network := range s.networks {
+		for _, network := range s.containerNetworks {
 			if err := network.Close(); err != nil {
 				log.Printf("Could not close network: %s\n", err)
 			}
@@ -428,7 +440,9 @@ func (s *IntegrationDERPTestSuite) TestPingAllPeersByHostname() {
 }
 
 func (s *IntegrationDERPTestSuite) TestDERPSTUN() {
-	headscaleSTUNAddr := fmt.Sprintf("localhost:%s", s.headscale.GetPort("3478/udp"))
+	headscaleSTUNAddr := fmt.Sprintf("%s:%s",
+		s.headscale.GetIPInNetwork(&s.network),
+		s.headscale.GetPort("3478/udp"))
 	client := stun.NewClient()
 	client.SetVerbose(true)
 	client.SetVVerbose(true)

integration_general_test.go

@@ -191,6 +191,17 @@ func (s *IntegrationTestSuite) tailscaleContainer(
 		},
 	}
 
+	err := s.pool.RemoveContainerByName(hostname)
+	if err != nil {
+		s.FailNow(
+			fmt.Sprintf(
+				"Could not remove existing container before building test: %s",
+				err,
+			),
+			"",
+		)
+	}
+
 	pts, err := s.pool.BuildAndRunWithBuildOptions(
 		tailscaleBuildOptions,
 		tailscaleOptions,
@@ -219,11 +230,11 @@ func (s *IntegrationTestSuite) SetupSuite() {
 		s.FailNow(fmt.Sprintf("Could not connect to docker: %s", err), "")
 	}
 
-	if pnetwork, err := s.pool.CreateNetwork("headscale-test"); err == nil {
+	network, err := GetFirstOrCreateNetwork(&s.pool, headscaleNetwork)
-		s.network = *pnetwork
+	if err != nil {
-	} else {
+		s.FailNow(fmt.Sprintf("Failed to create or get network: %s", err), "")
-		s.FailNow(fmt.Sprintf("Could not create network: %s", err), "")
 	}
+	s.network = network
 
 	headscaleBuildOptions := &dockertest.BuildOptions{
 		Dockerfile: "Dockerfile",
@@ -236,10 +247,14 @@ func (s *IntegrationTestSuite) SetupSuite() {
 	}
 
 	headscaleOptions := &dockertest.RunOptions{
-		Name: "headscale",
+		Name: headscaleHostname,
 		Mounts: []string{
 			fmt.Sprintf("%s/integration_test/etc:/etc/headscale", currentPath),
 		},
+		ExposedPorts: []string{"8080/tcp"},
+		PortBindings: map[docker.Port][]docker.PortBinding{
+			"8080/tcp": {{HostPort: "8080"}},
+		},
 		Networks: []*dockertest.Network{&s.network},
 		Cmd:      []string{"headscale", "serve"},
 	}
@@ -278,7 +293,9 @@ func (s *IntegrationTestSuite) SetupSuite() {
 	}
 
 	log.Println("Waiting for headscale to be ready for core integration tests")
-	hostEndpoint := fmt.Sprintf("localhost:%s", s.headscale.GetPort("8080/tcp"))
+	hostEndpoint := fmt.Sprintf("%s:%s",
+		s.headscale.GetIPInNetwork(&s.network),
+		s.headscale.GetPort("8080/tcp"))
 
 	if err := s.pool.Retry(func() error {
 		url := fmt.Sprintf("http://%s/health", hostEndpoint)

integration_oidc_test.go

@@ -25,7 +25,8 @@ import (
 )
 
 const (
-	oidcHeadscaleHostname = "headscale"
+	oidcHeadscaleHostname = "headscale-oidc"
+	oidcMockHostname      = "headscale-mock-oidc"
 	oidcNamespaceName     = "oidcnamespace"
 	totalOidcContainers   = 3
 )
@@ -95,33 +96,25 @@ func (s *IntegrationOIDCTestSuite) SetupSuite() {
 		s.FailNow(fmt.Sprintf("Could not connect to docker: %s", err), "")
 	}
 
-	if pnetwork, err := s.pool.CreateNetwork("headscale-test"); err == nil {
+	network, err := GetFirstOrCreateNetwork(&s.pool, headscaleNetwork)
-		s.network = *pnetwork
-	} else {
-		s.FailNow(fmt.Sprintf("Could not create network: %s", err), "")
-	}
-
-	// Create does not give us an updated version of the resource, so we need to
-	// get it again.
-	networks, err := s.pool.NetworksByName("headscale-test")
 	if err != nil {
-		s.FailNow(fmt.Sprintf("Could not get network: %s", err), "")
+		s.FailNow(fmt.Sprintf("Failed to create or get network: %s", err), "")
 	}
-	s.network = networks[0]
+	s.network = network
 
 	log.Printf("Network config: %v", s.network.Network.IPAM.Config[0])
 
 	s.Suite.T().Log("Setting up mock OIDC")
 	mockOidcOptions := &dockertest.RunOptions{
-		Name:         "mockoidc",
+		Name:         oidcMockHostname,
-		Hostname:     "mockoidc",
 		Cmd:          []string{"headscale", "mockoidc"},
 		ExposedPorts: []string{"10000/tcp"},
-		Networks:     []*dockertest.Network{&s.network},
 		PortBindings: map[docker.Port][]docker.PortBinding{
 			"10000/tcp": {{HostPort: "10000"}},
 		},
+		Networks: []*dockertest.Network{&s.network},
 		Env: []string{
+			fmt.Sprintf("MOCKOIDC_ADDR=%s", oidcMockHostname),
 			"MOCKOIDC_PORT=10000",
 			"MOCKOIDC_CLIENT_ID=superclient",
 			"MOCKOIDC_CLIENT_SECRET=supersecret",
@@ -133,6 +126,17 @@ func (s *IntegrationOIDCTestSuite) SetupSuite() {
 		ContextDir: ".",
 	}
 
+	err = s.pool.RemoveContainerByName(oidcMockHostname)
+	if err != nil {
+		s.FailNow(
+			fmt.Sprintf(
+				"Could not remove existing container before building test: %s",
+				err,
+			),
+			"",
+		)
+	}
+
 	if pmockoidc, err := s.pool.BuildAndRunWithBuildOptions(
 		headscaleBuildOptions,
 		mockOidcOptions,
@@ -142,6 +146,35 @@ func (s *IntegrationOIDCTestSuite) SetupSuite() {
 		s.FailNow(fmt.Sprintf("Could not start mockOIDC container: %s", err), "")
 	}
 
+	s.Suite.T().Logf("Waiting for headscale mock oidc to be ready for tests")
+	hostEndpoint := fmt.Sprintf(
+		"%s:%s",
+		s.mockOidc.GetIPInNetwork(&s.network),
+		s.mockOidc.GetPort("10000/tcp"),
+	)
+
+	if err := s.pool.Retry(func() error {
+		url := fmt.Sprintf("http://%s/oidc/.well-known/openid-configuration", hostEndpoint)
+		resp, err := http.Get(url)
+		if err != nil {
+			log.Printf("headscale mock OIDC tests is not ready: %s\n", err)
+			return err
+		}
+
+		if resp.StatusCode != http.StatusOK {
+			return fmt.Errorf("status code not OK")
+		}
+
+		return nil
+	}); err != nil {
+		// TODO(kradalby): If we cannot access headscale, or any other fatal error during
+		// test setup, we need to abort and tear down. However, testify does not seem to
+		// support that at the moment:
+		// https://github.com/stretchr/testify/issues/849
+		return // fmt.Errorf("Could not connect to headscale: %s", err)
+	}
+	s.Suite.T().Log("headscale-mock-oidc container is ready for embedded OIDC tests")
+
 	oidcCfg := fmt.Sprintf(`
 oidc:
   issuer: http://%s:10000/oidc
@@ -216,10 +249,14 @@ oidc:
 	}
 
 	s.Suite.T().Logf("Waiting for headscale to be ready for embedded OIDC tests")
-	hostEndpoint := fmt.Sprintf("localhost:%s", s.headscale.GetPort("8443/tcp"))
+	hostMockEndpoint := fmt.Sprintf(
+		"%s:%s",
+		s.headscale.GetIPInNetwork(&s.network),
+		s.headscale.GetPort("8443/tcp"),
+	)
 
 	if err := s.pool.Retry(func() error {
-		url := fmt.Sprintf("https://%s/health", hostEndpoint)
+		url := fmt.Sprintf("https://%s/health", hostMockEndpoint)
 		insecureTransport := http.DefaultTransport.(*http.Transport).Clone()
 		insecureTransport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
 		client := &http.Client{Transport: insecureTransport}
@@ -294,6 +331,8 @@ func (s *IntegrationOIDCTestSuite) AuthenticateOIDC(
 	resp, err := client.Get(loginURL.String())
 	assert.Nil(s.T(), err)
 
+	log.Printf("auth body, err: %#v, %s", resp, err)
+
 	body, err := io.ReadAll(resp.Body)
 	assert.Nil(s.T(), err)
 
@@ -308,7 +347,6 @@ func (s *IntegrationOIDCTestSuite) joinOIDC(
 	endpoint, hostname string,
 	tailscale dockertest.Resource,
 ) (*url.URL, error) {
-
 	command := []string{
 		"tailscale",
 		"up",
@@ -374,7 +412,13 @@ func (s *IntegrationOIDCTestSuite) tailscaleContainer(
 		DockerAllowNetworkAdministration,
 	)
 	if err != nil {
-		log.Fatalf("Could not start tailscale container version %s: %s", version, err)
+		s.FailNow(
+			fmt.Sprintf(
+				"Could not start tailscale container version %s: %s",
+				version,
+				err,
+			),
+		)
 	}
 	log.Printf("Created %s container\n", hostname)
 
@@ -497,7 +541,12 @@ func (s *IntegrationOIDCTestSuite) TestPingAllPeersByAddress() {
 							[]string{},
 						)
 						assert.Nil(t, err)
-						log.Printf("result for %s: stdout: %s, stderr: %s\n", hostname, stdout, stderr)
+						log.Printf(
+							"result for %s: stdout: %s, stderr: %s\n",
+							hostname,
+							stdout,
+							stderr,
+						)
 						assert.Contains(t, stdout, "pong")
 					})
 			}

integration_test/etc_oidc/base_config.yaml

@@ -11,7 +11,7 @@ private_key_path: private.key
 noise:
   private_key_path: noise_private.key
 listen_addr: 0.0.0.0:8443
-server_url: https://localhost:8443
+server_url: https://headscale-oidc:8443
 tls_cert_path: "/etc/headscale/tls/server.crt"
 tls_key_path: "/etc/headscale/tls/server.key"
 tls_client_auth_mode: disabled