myriation/headscale
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity

Merge pull request #667 from kradalby/rerun-docker

Browse source 
Make integration tests retry on failure.
This commit is contained in:
Juan Font 2022-06-27 17:04:39 +02:00 committed by GitHub
commit e0b15c18ce
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 146 additions and 45 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
.github/workflows/test-integration.yml vendored
View file

@ -27,4 +27,9 @@ jobs:
- name: Run Integration tests
if: steps.changed-files.outputs.any_changed == 'true'
run: nix develop --command -- make test_integration
uses: nick-fields/retry@v2
with:
timeout_minutes: 240
max_attempts: 5
retry_on: error
command: nix develop --command -- make test_integration

13
integration_cli_test.go
View file

@ -40,13 +40,13 @@ func (s *IntegrationCLITestSuite) SetupTest() {
if ppool, err := dockertest.NewPool(""); err == nil {
s.pool = *ppool
} else {
log.Fatalf("Could not connect to docker: %s", err)
s.FailNow(fmt.Sprintf("Could not connect to docker: %s", err), "")
}
if pnetwork, err := s.pool.CreateNetwork("headscale-test"); err == nil {
s.network = *pnetwork
} else {
log.Fatalf("Could not create network: %s", err)
s.FailNow(fmt.Sprintf("Could not create network: %s", err), "")
}
headscaleBuildOptions := &dockertest.BuildOptions{
@ -56,7 +56,7 @@ func (s *IntegrationCLITestSuite) SetupTest() {
currentPath, err := os.Getwd()
if err != nil {
log.Fatalf("Could not determine current path: %s", err)
s.FailNow(fmt.Sprintf("Could not determine current path: %s", err), "")
}
headscaleOptions := &dockertest.RunOptions{
@ -68,11 +68,16 @@ func (s *IntegrationCLITestSuite) SetupTest() {
Cmd: []string{"headscale", "serve"},
}
err = s.pool.RemoveContainerByName(headscaleHostname)
if err != nil {
s.FailNow(fmt.Sprintf("Could not remove existing container before building test: %s", err), "")
}
fmt.Println("Creating headscale container")
if pheadscale, err := s.pool.BuildAndRunWithBuildOptions(headscaleBuildOptions, headscaleOptions, DockerRestartPolicy); err == nil {
s.headscale = *pheadscale
} else {
log.Fatalf("Could not start headscale container: %s", err)
s.FailNow(fmt.Sprintf("Could not start headscale container: %s", err), "")
}
fmt.Println("Created headscale container")

31
integration_common_test.go
View file

@ -6,7 +6,10 @@ package headscale
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"os"
"strconv"
"strings"
"time"
@ -16,9 +19,13 @@ import (
"inet.af/netaddr"
)
const DOCKER_EXECUTE_TIMEOUT = 10 * time.Second
const (
DOCKER_EXECUTE_TIMEOUT = 10 * time.Second
)
var (
errEnvVarEmpty = errors.New("getenv: environment variable empty")
IpPrefix4 = netaddr.MustParseIPPrefix("100.64.0.0/10")
IpPrefix6 = netaddr.MustParseIPPrefix("fd7a:115c:a1e0::/48")
@ -283,3 +290,25 @@ func getMagicFQDN(
return hostnames, nil
}
func GetEnvStr(key string) (string, error) {
v := os.Getenv(key)
if v == "" {
return v, errEnvVarEmpty
}
return v, nil
}
func GetEnvBool(key string) (bool, error) {
s, err := GetEnvStr(key)
if err != nil {
return false, err
}
v, err := strconv.ParseBool(s)
if err != nil {
return false, err
}
return v, nil
}

71
integration_embedded_derp_test.go
View file

@ -40,41 +40,50 @@ type IntegrationDERPTestSuite struct {
pool dockertest.Pool
networks map[int]dockertest.Network // so we keep the containers isolated
headscale dockertest.Resource
saveLogs bool
tailscales map[string]dockertest.Resource
joinWaitGroup sync.WaitGroup
}
func TestDERPIntegrationTestSuite(t *testing.T) {
saveLogs, err := GetEnvBool("HEADSCALE_INTEGRATION_SAVE_LOG")
if err != nil {
saveLogs = false
}
s := new(IntegrationDERPTestSuite)
s.tailscales = make(map[string]dockertest.Resource)
s.networks = make(map[int]dockertest.Network)
s.saveLogs = saveLogs
suite.Run(t, s)
// HandleStats, which allows us to check if we passed and save logs
// is called after TearDown, so we cannot tear down containers before
// we have potentially saved the logs.
for _, tailscale := range s.tailscales {
if err := s.pool.Purge(&tailscale); err != nil {
if s.saveLogs {
for _, tailscale := range s.tailscales {
if err := s.pool.Purge(&tailscale); err != nil {
log.Printf("Could not purge resource: %s\n", err)
}
}
if !s.stats.Passed() {
err := s.saveLog(&s.headscale, "test_output")
if err != nil {
log.Printf("Could not save log: %s\n", err)
}
}
if err := s.pool.Purge(&s.headscale); err != nil {
log.Printf("Could not purge resource: %s\n", err)
}
}
if !s.stats.Passed() {
err := s.saveLog(&s.headscale, "test_output")
if err != nil {
log.Printf("Could not save log: %s\n", err)
}
}
if err := s.pool.Purge(&s.headscale); err != nil {
log.Printf("Could not purge resource: %s\n", err)
}
for _, network := range s.networks {
if err := network.Close(); err != nil {
log.Printf("Could not close network: %s\n", err)
for _, network := range s.networks {
if err := network.Close(); err != nil {
log.Printf("Could not close network: %s\n", err)
}
}
}
}
@ -83,14 +92,14 @@ func (s *IntegrationDERPTestSuite) SetupSuite() {
if ppool, err := dockertest.NewPool(""); err == nil {
s.pool = *ppool
} else {
log.Fatalf("Could not connect to docker: %s", err)
s.FailNow(fmt.Sprintf("Could not connect to docker: %s", err), "")
}
for i := 0; i < totalContainers; i++ {
if pnetwork, err := s.pool.CreateNetwork(fmt.Sprintf("headscale-derp-%d", i)); err == nil {
s.networks[i] = *pnetwork
} else {
log.Fatalf("Could not create network: %s", err)
s.FailNow(fmt.Sprintf("Could not create network: %s", err), "")
}
}
@ -101,7 +110,7 @@ func (s *IntegrationDERPTestSuite) SetupSuite() {
currentPath, err := os.Getwd()
if err != nil {
log.Fatalf("Could not determine current path: %s", err)
s.FailNow(fmt.Sprintf("Could not determine current path: %s", err), "")
}
headscaleOptions := &dockertest.RunOptions{
@ -120,11 +129,16 @@ func (s *IntegrationDERPTestSuite) SetupSuite() {
},
}
err = s.pool.RemoveContainerByName(headscaleHostname)
if err != nil {
s.FailNow(fmt.Sprintf("Could not remove existing container before building test: %s", err), "")
}
log.Println("Creating headscale container")
if pheadscale, err := s.pool.BuildAndRunWithBuildOptions(headscaleBuildOptions, headscaleOptions, DockerRestartPolicy); err == nil {
s.headscale = *pheadscale
} else {
log.Fatalf("Could not start headscale container: %s", err)
s.FailNow(fmt.Sprintf("Could not start headscale container: %s", err), "")
}
log.Println("Created headscale container to test DERP")
@ -290,6 +304,23 @@ func (s *IntegrationDERPTestSuite) tailscaleContainer(
}
func (s *IntegrationDERPTestSuite) TearDownSuite() {
if !s.saveLogs {
for _, tailscale := range s.tailscales {
if err := s.pool.Purge(&tailscale); err != nil {
log.Printf("Could not purge resource: %s\n", err)
}
}
if err := s.pool.Purge(&s.headscale); err != nil {
log.Printf("Could not purge resource: %s\n", err)
}
for _, network := range s.networks {
if err := network.Close(); err != nil {
log.Printf("Could not close network: %s\n", err)
}
}
}
}
func (s *IntegrationDERPTestSuite) HandleStats(

69
integration_test.go
View file

@ -36,6 +36,7 @@ type IntegrationTestSuite struct {
pool dockertest.Pool
network dockertest.Network
headscale dockertest.Resource
saveLogs bool
namespaces map[string]TestNamespace
@ -43,6 +44,11 @@ type IntegrationTestSuite struct {
}
func TestIntegrationTestSuite(t *testing.T) {
saveLogs, err := GetEnvBool("HEADSCALE_INTEGRATION_SAVE_LOG")
if err != nil {
saveLogs = false
}
s := new(IntegrationTestSuite)
s.namespaces = map[string]TestNamespace{
@ -55,32 +61,35 @@ func TestIntegrationTestSuite(t *testing.T) {
tailscales: make(map[string]dockertest.Resource),
},
}
s.saveLogs = saveLogs
suite.Run(t, s)
// HandleStats, which allows us to check if we passed and save logs
// is called after TearDown, so we cannot tear down containers before
// we have potentially saved the logs.
for _, scales := range s.namespaces {
for _, tailscale := range scales.tailscales {
if err := s.pool.Purge(&tailscale); err != nil {
log.Printf("Could not purge resource: %s\n", err)
if s.saveLogs {
for _, scales := range s.namespaces {
for _, tailscale := range scales.tailscales {
if err := s.pool.Purge(&tailscale); err != nil {
log.Printf("Could not purge resource: %s\n", err)
}
}
}
}
if !s.stats.Passed() {
err := s.saveLog(&s.headscale, "test_output")
if err != nil {
log.Printf("Could not save log: %s\n", err)
if !s.stats.Passed() {
err := s.saveLog(&s.headscale, "test_output")
if err != nil {
log.Printf("Could not save log: %s\n", err)
}
}
if err := s.pool.Purge(&s.headscale); err != nil {
log.Printf("Could not purge resource: %s\n", err)
}
}
if err := s.pool.Purge(&s.headscale); err != nil {
log.Printf("Could not purge resource: %s\n", err)
}
if err := s.network.Close(); err != nil {
log.Printf("Could not close network: %s\n", err)
if err := s.network.Close(); err != nil {
log.Printf("Could not close network: %s\n", err)
}
}
}
@ -209,13 +218,13 @@ func (s *IntegrationTestSuite) SetupSuite() {
if ppool, err := dockertest.NewPool(""); err == nil {
s.pool = *ppool
} else {
log.Fatalf("Could not connect to docker: %s", err)
s.FailNow(fmt.Sprintf("Could not connect to docker: %s", err), "")
}
if pnetwork, err := s.pool.CreateNetwork("headscale-test"); err == nil {
s.network = *pnetwork
} else {
log.Fatalf("Could not create network: %s", err)
s.FailNow(fmt.Sprintf("Could not create network: %s", err), "")
}
headscaleBuildOptions := &dockertest.BuildOptions{
@ -225,7 +234,7 @@ func (s *IntegrationTestSuite) SetupSuite() {
currentPath, err := os.Getwd()
if err != nil {
log.Fatalf("Could not determine current path: %s", err)
s.FailNow(fmt.Sprintf("Could not determine current path: %s", err), "")
}
headscaleOptions := &dockertest.RunOptions{
@ -237,11 +246,16 @@ func (s *IntegrationTestSuite) SetupSuite() {
Cmd: []string{"headscale", "serve"},
}
err = s.pool.RemoveContainerByName(headscaleHostname)
if err != nil {
s.FailNow(fmt.Sprintf("Could not remove existing container before building test: %s", err), "")
}
log.Println("Creating headscale container")
if pheadscale, err := s.pool.BuildAndRunWithBuildOptions(headscaleBuildOptions, headscaleOptions, DockerRestartPolicy); err == nil {
s.headscale = *pheadscale
} else {
log.Fatalf("Could not start headscale container: %s", err)
s.FailNow(fmt.Sprintf("Could not start headscale container: %s", err), "")
}
log.Println("Created headscale container")
@ -338,6 +352,23 @@ func (s *IntegrationTestSuite) SetupSuite() {
}
func (s *IntegrationTestSuite) TearDownSuite() {
if !s.saveLogs {
for _, scales := range s.namespaces {
for _, tailscale := range scales.tailscales {
if err := s.pool.Purge(&tailscale); err != nil {
log.Printf("Could not purge resource: %s\n", err)
}
}
}
if err := s.pool.Purge(&s.headscale); err != nil {
log.Printf("Could not purge resource: %s\n", err)
}
if err := s.network.Close(); err != nil {
log.Printf("Could not close network: %s\n", err)
}
}
}
func (s *IntegrationTestSuite) HandleStats(