024d6ee7c3
This commit adds initial integration tests for shared nodes, it adds them and verifies that they are shared. It does not yet manage to ping the shared node because it does not seem to establish the connection.
553 lines
14 KiB
Go
553 lines
14 KiB
Go
//go:build integration
|
|
// +build integration
|
|
|
|
package headscale
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io/ioutil"
|
|
"log"
|
|
"net/http"
|
|
"os"
|
|
"path"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/ory/dockertest/v3"
|
|
"github.com/ory/dockertest/v3/docker"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/suite"
|
|
"tailscale.com/ipn/ipnstate"
|
|
|
|
"inet.af/netaddr"
|
|
)
|
|
|
|
var (
|
|
integrationTmpDir string
|
|
ih Headscale
|
|
)
|
|
|
|
var (
|
|
pool dockertest.Pool
|
|
network dockertest.Network
|
|
headscale dockertest.Resource
|
|
)
|
|
|
|
var tailscaleVersions = []string{"1.14.3", "1.12.3"}
|
|
|
|
type TestNamespace struct {
|
|
count int
|
|
tailscales map[string]dockertest.Resource
|
|
}
|
|
|
|
type IntegrationTestSuite struct {
|
|
suite.Suite
|
|
stats *suite.SuiteInformation
|
|
|
|
namespaces map[string]TestNamespace
|
|
}
|
|
|
|
func TestIntegrationTestSuite(t *testing.T) {
|
|
s := new(IntegrationTestSuite)
|
|
|
|
s.namespaces = map[string]TestNamespace{
|
|
"main": {
|
|
count: 20,
|
|
tailscales: make(map[string]dockertest.Resource),
|
|
},
|
|
"shared": {
|
|
count: 5,
|
|
tailscales: make(map[string]dockertest.Resource),
|
|
},
|
|
}
|
|
|
|
suite.Run(t, s)
|
|
|
|
// HandleStats, which allows us to check if we passed and save logs
|
|
// is called after TearDown, so we cannot tear down containers before
|
|
// we have potentially saved the logs.
|
|
for _, scales := range s.namespaces {
|
|
for _, tailscale := range scales.tailscales {
|
|
if err := pool.Purge(&tailscale); err != nil {
|
|
log.Printf("Could not purge resource: %s\n", err)
|
|
}
|
|
}
|
|
}
|
|
|
|
if !s.stats.Passed() {
|
|
err := saveLog(&headscale, "test_output")
|
|
if err != nil {
|
|
log.Printf("Could not save log: %s\n", err)
|
|
}
|
|
}
|
|
if err := pool.Purge(&headscale); err != nil {
|
|
log.Printf("Could not purge resource: %s\n", err)
|
|
}
|
|
|
|
if err := network.Close(); err != nil {
|
|
log.Printf("Could not close network: %s\n", err)
|
|
}
|
|
}
|
|
|
|
func executeCommand(resource *dockertest.Resource, cmd []string) (string, error) {
|
|
var stdout bytes.Buffer
|
|
var stderr bytes.Buffer
|
|
|
|
exitCode, err := resource.Exec(
|
|
cmd,
|
|
dockertest.ExecOptions{
|
|
StdOut: &stdout,
|
|
StdErr: &stderr,
|
|
},
|
|
)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
|
|
if exitCode != 0 {
|
|
fmt.Println("Command: ", cmd)
|
|
fmt.Println("stdout: ", stdout.String())
|
|
fmt.Println("stderr: ", stderr.String())
|
|
return "", fmt.Errorf("command failed with: %s", stderr.String())
|
|
}
|
|
|
|
return stdout.String(), nil
|
|
}
|
|
|
|
func saveLog(resource *dockertest.Resource, basePath string) error {
|
|
err := os.MkdirAll(basePath, os.ModePerm)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
var stdout bytes.Buffer
|
|
var stderr bytes.Buffer
|
|
|
|
err = pool.Client.Logs(
|
|
docker.LogsOptions{
|
|
Context: context.TODO(),
|
|
Container: resource.Container.ID,
|
|
OutputStream: &stdout,
|
|
ErrorStream: &stderr,
|
|
Tail: "all",
|
|
RawTerminal: false,
|
|
Stdout: true,
|
|
Stderr: true,
|
|
Follow: false,
|
|
Timestamps: false,
|
|
},
|
|
)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
fmt.Printf("Saving logs for %s to %s\n", resource.Container.Name, basePath)
|
|
|
|
err = ioutil.WriteFile(path.Join(basePath, resource.Container.Name+".stdout.log"), []byte(stdout.String()), 0o644)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
err = ioutil.WriteFile(path.Join(basePath, resource.Container.Name+".stderr.log"), []byte(stdout.String()), 0o644)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func dockerRestartPolicy(config *docker.HostConfig) {
|
|
// set AutoRemove to true so that stopped container goes away by itself
|
|
config.AutoRemove = true
|
|
config.RestartPolicy = docker.RestartPolicy{
|
|
Name: "no",
|
|
}
|
|
}
|
|
|
|
func tailscaleContainer(namespace, identifier, version string) (string, *dockertest.Resource) {
|
|
tailscaleBuildOptions := &dockertest.BuildOptions{
|
|
Dockerfile: "Dockerfile.tailscale",
|
|
ContextDir: ".",
|
|
BuildArgs: []docker.BuildArg{
|
|
{
|
|
Name: "TAILSCALE_VERSION",
|
|
Value: version,
|
|
},
|
|
},
|
|
}
|
|
hostname := fmt.Sprintf("%s-tailscale-%s-%s", namespace, strings.Replace(version, ".", "-", -1), identifier)
|
|
tailscaleOptions := &dockertest.RunOptions{
|
|
Name: hostname,
|
|
Networks: []*dockertest.Network{&network},
|
|
Cmd: []string{"tailscaled", "--tun=userspace-networking", "--socks5-server=localhost:1055"},
|
|
}
|
|
|
|
pts, err := pool.BuildAndRunWithBuildOptions(tailscaleBuildOptions, tailscaleOptions, dockerRestartPolicy)
|
|
if err != nil {
|
|
log.Fatalf("Could not start resource: %s", err)
|
|
}
|
|
fmt.Printf("Created %s container\n", hostname)
|
|
return hostname, pts
|
|
}
|
|
|
|
func (s *IntegrationTestSuite) SetupSuite() {
|
|
var err error
|
|
h = Headscale{
|
|
dbType: "sqlite3",
|
|
dbString: "integration_test_db.sqlite3",
|
|
}
|
|
|
|
if ppool, err := dockertest.NewPool(""); err == nil {
|
|
pool = *ppool
|
|
} else {
|
|
log.Fatalf("Could not connect to docker: %s", err)
|
|
}
|
|
|
|
if pnetwork, err := pool.CreateNetwork("headscale-test"); err == nil {
|
|
network = *pnetwork
|
|
} else {
|
|
log.Fatalf("Could not create network: %s", err)
|
|
}
|
|
|
|
headscaleBuildOptions := &dockertest.BuildOptions{
|
|
Dockerfile: "Dockerfile",
|
|
ContextDir: ".",
|
|
}
|
|
|
|
currentPath, err := os.Getwd()
|
|
if err != nil {
|
|
log.Fatalf("Could not determine current path: %s", err)
|
|
}
|
|
|
|
headscaleOptions := &dockertest.RunOptions{
|
|
Name: "headscale",
|
|
Mounts: []string{
|
|
fmt.Sprintf("%s/integration_test/etc:/etc/headscale", currentPath),
|
|
fmt.Sprintf("%s/derp.yaml:/etc/headscale/derp.yaml", currentPath),
|
|
},
|
|
Networks: []*dockertest.Network{&network},
|
|
Cmd: []string{"headscale", "serve"},
|
|
PortBindings: map[docker.Port][]docker.PortBinding{
|
|
"8080/tcp": {{HostPort: "8080"}},
|
|
},
|
|
}
|
|
|
|
fmt.Println("Creating headscale container")
|
|
if pheadscale, err := pool.BuildAndRunWithBuildOptions(headscaleBuildOptions, headscaleOptions, dockerRestartPolicy); err == nil {
|
|
headscale = *pheadscale
|
|
} else {
|
|
log.Fatalf("Could not start resource: %s", err)
|
|
}
|
|
fmt.Println("Created headscale container")
|
|
|
|
fmt.Println("Creating tailscale containers")
|
|
for namespace, scales := range s.namespaces {
|
|
for i := 0; i < scales.count; i++ {
|
|
version := tailscaleVersions[i%len(tailscaleVersions)]
|
|
|
|
hostname, container := tailscaleContainer(namespace, fmt.Sprint(i), version)
|
|
scales.tailscales[hostname] = *container
|
|
}
|
|
}
|
|
|
|
fmt.Println("Waiting for headscale to be ready")
|
|
hostEndpoint := fmt.Sprintf("localhost:%s", headscale.GetPort("8080/tcp"))
|
|
|
|
if err := pool.Retry(func() error {
|
|
url := fmt.Sprintf("http://%s/health", hostEndpoint)
|
|
resp, err := http.Get(url)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if resp.StatusCode != http.StatusOK {
|
|
return fmt.Errorf("status code not OK")
|
|
}
|
|
return nil
|
|
}); err != nil {
|
|
log.Fatalf("Could not connect to docker: %s", err)
|
|
}
|
|
fmt.Println("headscale container is ready")
|
|
|
|
for namespace, scales := range s.namespaces {
|
|
fmt.Printf("Creating headscale namespace: %s\n", namespace)
|
|
result, err := executeCommand(
|
|
&headscale,
|
|
[]string{"headscale", "namespaces", "create", namespace},
|
|
)
|
|
assert.Nil(s.T(), err)
|
|
fmt.Println("headscale create namespace result: ", result)
|
|
|
|
fmt.Printf("Creating pre auth key for %s\n", namespace)
|
|
authKey, err := executeCommand(
|
|
&headscale,
|
|
[]string{"headscale", "--namespace", namespace, "preauthkeys", "create", "--reusable", "--expiration", "24h"},
|
|
)
|
|
assert.Nil(s.T(), err)
|
|
|
|
headscaleEndpoint := fmt.Sprintf("http://headscale:%s", headscale.GetPort("8080/tcp"))
|
|
|
|
fmt.Printf("Joining tailscale containers to headscale at %s\n", headscaleEndpoint)
|
|
for hostname, tailscale := range scales.tailscales {
|
|
command := []string{"tailscale", "up", "-login-server", headscaleEndpoint, "--authkey", strings.TrimSuffix(authKey, "\n"), "--hostname", hostname}
|
|
|
|
fmt.Println("Join command:", command)
|
|
fmt.Printf("Running join command for %s\n", hostname)
|
|
result, err := executeCommand(
|
|
&tailscale,
|
|
command,
|
|
)
|
|
fmt.Println("tailscale result: ", result)
|
|
assert.Nil(s.T(), err)
|
|
fmt.Printf("%s joined\n", hostname)
|
|
}
|
|
}
|
|
|
|
// The nodes need a bit of time to get their updated maps from headscale
|
|
// TODO: See if we can have a more deterministic wait here.
|
|
time.Sleep(60 * time.Second)
|
|
}
|
|
|
|
func (s *IntegrationTestSuite) TearDownSuite() {
|
|
}
|
|
|
|
func (s *IntegrationTestSuite) HandleStats(suiteName string, stats *suite.SuiteInformation) {
|
|
s.stats = stats
|
|
}
|
|
|
|
func (s *IntegrationTestSuite) TestListNodes() {
|
|
for namespace, scales := range s.namespaces {
|
|
fmt.Println("Listing nodes")
|
|
result, err := executeCommand(
|
|
&headscale,
|
|
[]string{"headscale", "--namespace", namespace, "nodes", "list"},
|
|
)
|
|
assert.Nil(s.T(), err)
|
|
|
|
fmt.Printf("List nodes: \n%s\n", result)
|
|
|
|
// Chck that the correct count of host is present in node list
|
|
lines := strings.Split(result, "\n")
|
|
assert.Equal(s.T(), len(scales.tailscales), len(lines)-2)
|
|
|
|
for hostname := range scales.tailscales {
|
|
assert.Contains(s.T(), result, hostname)
|
|
}
|
|
}
|
|
}
|
|
|
|
func (s *IntegrationTestSuite) TestGetIpAddresses() {
|
|
for _, scales := range s.namespaces {
|
|
ipPrefix := netaddr.MustParseIPPrefix("100.64.0.0/10")
|
|
ips, err := getIPs(scales.tailscales)
|
|
assert.Nil(s.T(), err)
|
|
|
|
for hostname := range scales.tailscales {
|
|
s.T().Run(hostname, func(t *testing.T) {
|
|
ip := ips[hostname]
|
|
|
|
fmt.Printf("IP for %s: %s\n", hostname, ip)
|
|
|
|
// c.Assert(ip.Valid(), check.IsTrue)
|
|
assert.True(t, ip.Is4())
|
|
assert.True(t, ipPrefix.Contains(ip))
|
|
|
|
ips[hostname] = ip
|
|
})
|
|
}
|
|
}
|
|
}
|
|
|
|
func (s *IntegrationTestSuite) TestStatus() {
|
|
for _, scales := range s.namespaces {
|
|
ips, err := getIPs(scales.tailscales)
|
|
assert.Nil(s.T(), err)
|
|
|
|
for hostname, tailscale := range scales.tailscales {
|
|
s.T().Run(hostname, func(t *testing.T) {
|
|
command := []string{"tailscale", "status", "--json"}
|
|
|
|
fmt.Printf("Getting status for %s\n", hostname)
|
|
result, err := executeCommand(
|
|
&tailscale,
|
|
command,
|
|
)
|
|
assert.Nil(t, err)
|
|
|
|
var status ipnstate.Status
|
|
err = json.Unmarshal([]byte(result), &status)
|
|
assert.Nil(s.T(), err)
|
|
|
|
// TODO(kradalby): Replace this check with peer length of SAME namespace
|
|
// Check if we have as many nodes in status
|
|
// as we have IPs/tailscales
|
|
// lines := strings.Split(result, "\n")
|
|
// assert.Equal(t, len(ips), len(lines)-1)
|
|
// assert.Equal(t, len(scales.tailscales), len(lines)-1)
|
|
|
|
peerIps := getIPsfromIPNstate(status)
|
|
|
|
// Check that all hosts is present in all hosts status
|
|
for ipHostname, ip := range ips {
|
|
if hostname != ipHostname {
|
|
assert.Contains(t, peerIps, ip)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
}
|
|
|
|
func getIPsfromIPNstate(status ipnstate.Status) []netaddr.IP {
|
|
ips := make([]netaddr.IP, 0)
|
|
|
|
for _, peer := range status.Peer {
|
|
ips = append(ips, peer.TailscaleIPs...)
|
|
}
|
|
|
|
return ips
|
|
}
|
|
|
|
func (s *IntegrationTestSuite) TestPingAllPeers() {
|
|
for _, scales := range s.namespaces {
|
|
ips, err := getIPs(scales.tailscales)
|
|
assert.Nil(s.T(), err)
|
|
|
|
for hostname, tailscale := range scales.tailscales {
|
|
for peername, ip := range ips {
|
|
s.T().Run(fmt.Sprintf("%s-%s", hostname, peername), func(t *testing.T) {
|
|
// We currently cant ping ourselves, so skip that.
|
|
if peername != hostname {
|
|
// We are only interested in "direct ping" which means what we
|
|
// might need a couple of more attempts before reaching the node.
|
|
command := []string{
|
|
"tailscale", "ping",
|
|
"--timeout=1s",
|
|
"--c=20",
|
|
"--until-direct=true",
|
|
ip.String(),
|
|
}
|
|
|
|
fmt.Printf("Pinging from %s (%s) to %s (%s)\n", hostname, ips[hostname], peername, ip)
|
|
result, err := executeCommand(
|
|
&tailscale,
|
|
command,
|
|
)
|
|
assert.Nil(t, err)
|
|
fmt.Printf("Result for %s: %s\n", hostname, result)
|
|
assert.Contains(t, result, "pong")
|
|
}
|
|
})
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func (s *IntegrationTestSuite) TestSharedNodes() {
|
|
main := s.namespaces["main"]
|
|
shared := s.namespaces["shared"]
|
|
|
|
result, err := executeCommand(
|
|
&headscale,
|
|
[]string{"headscale", "nodes", "list", "-o", "json", "--namespace", "shared"},
|
|
)
|
|
assert.Nil(s.T(), err)
|
|
|
|
var machineList []Machine
|
|
err = json.Unmarshal([]byte(result), &machineList)
|
|
assert.Nil(s.T(), err)
|
|
|
|
for _, machine := range machineList {
|
|
|
|
result, err := executeCommand(
|
|
&headscale,
|
|
[]string{"headscale", "nodes", "share", "--namespace", "shared", fmt.Sprint(machine.ID), "main"},
|
|
)
|
|
assert.Nil(s.T(), err)
|
|
|
|
fmt.Println("Shared node with result: ", result)
|
|
}
|
|
|
|
result, err = executeCommand(
|
|
&headscale,
|
|
[]string{"headscale", "nodes", "list", "--namespace", "main"},
|
|
)
|
|
assert.Nil(s.T(), err)
|
|
fmt.Println("Nodelist after sharing", result)
|
|
|
|
// Chck that the correct count of host is present in node list
|
|
lines := strings.Split(result, "\n")
|
|
assert.Equal(s.T(), len(main.tailscales)+len(shared.tailscales), len(lines)-2)
|
|
|
|
for hostname := range main.tailscales {
|
|
assert.Contains(s.T(), result, hostname)
|
|
}
|
|
|
|
for hostname := range shared.tailscales {
|
|
assert.Contains(s.T(), result, hostname)
|
|
}
|
|
|
|
// TODO(kradalby): Figure out why these connections are not set up
|
|
// // TODO: See if we can have a more deterministic wait here.
|
|
// time.Sleep(100 * time.Second)
|
|
|
|
// mainIps, err := getIPs(main.tailscales)
|
|
// assert.Nil(s.T(), err)
|
|
|
|
// sharedIps, err := getIPs(shared.tailscales)
|
|
// assert.Nil(s.T(), err)
|
|
|
|
// for hostname, tailscale := range main.tailscales {
|
|
// for peername, ip := range sharedIps {
|
|
// s.T().Run(fmt.Sprintf("%s-%s", hostname, peername), func(t *testing.T) {
|
|
// // We currently cant ping ourselves, so skip that.
|
|
// if peername != hostname {
|
|
// // We are only interested in "direct ping" which means what we
|
|
// // might need a couple of more attempts before reaching the node.
|
|
// command := []string{
|
|
// "tailscale", "ping",
|
|
// "--timeout=1s",
|
|
// "--c=20",
|
|
// "--until-direct=true",
|
|
// ip.String(),
|
|
// }
|
|
|
|
// fmt.Printf("Pinging from %s (%s) to %s (%s)\n", hostname, mainIps[hostname], peername, ip)
|
|
// result, err := executeCommand(
|
|
// &tailscale,
|
|
// command,
|
|
// )
|
|
// assert.Nil(t, err)
|
|
// fmt.Printf("Result for %s: %s\n", hostname, result)
|
|
// assert.Contains(t, result, "pong")
|
|
// }
|
|
// })
|
|
// }
|
|
// }
|
|
}
|
|
|
|
func getIPs(tailscales map[string]dockertest.Resource) (map[string]netaddr.IP, error) {
|
|
ips := make(map[string]netaddr.IP)
|
|
for hostname, tailscale := range tailscales {
|
|
command := []string{"tailscale", "ip"}
|
|
|
|
result, err := executeCommand(
|
|
&tailscale,
|
|
command,
|
|
)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
ip, err := netaddr.ParseIP(strings.TrimSuffix(result, "\n"))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
ips[hostname] = ip
|
|
}
|
|
return ips, nil
|
|
}
|