2023-02-02 08:05:52 -07:00
package integration
import (
2023-09-10 02:00:12 -06:00
"os"
2023-08-31 10:37:18 -06:00
"strings"
2024-02-18 15:22:18 -07:00
"sync"
2023-02-02 08:05:52 -07:00
"testing"
2023-04-13 15:09:09 -06:00
"time"
2024-02-23 02:59:24 -07:00
"github.com/juanfont/headscale/hscontrol/util"
2023-04-13 15:09:09 -06:00
"github.com/juanfont/headscale/integration/tsic"
2024-02-08 23:26:41 -07:00
"github.com/stretchr/testify/assert"
2023-02-02 08:05:52 -07:00
)
2023-04-13 15:10:47 -06:00
const (
derpPingTimeout = 2 * time . Second
derpPingCount = 10
)
2023-08-29 00:33:33 -06:00
func assertNoErr ( t * testing . T , err error ) {
t . Helper ( )
assertNoErrf ( t , "unexpected error: %s" , err )
}
func assertNoErrf ( t * testing . T , msg string , err error ) {
t . Helper ( )
if err != nil {
t . Fatalf ( msg , err )
}
}
2023-12-09 10:09:24 -07:00
func assertNotNil ( t * testing . T , thing interface { } ) {
t . Helper ( )
if thing == nil {
t . Fatal ( "got unexpected nil" )
}
}
2023-08-29 00:33:33 -06:00
func assertNoErrHeadscaleEnv ( t * testing . T , err error ) {
t . Helper ( )
assertNoErrf ( t , "failed to create headscale environment: %s" , err )
}
func assertNoErrGetHeadscale ( t * testing . T , err error ) {
t . Helper ( )
assertNoErrf ( t , "failed to get headscale: %s" , err )
}
func assertNoErrListClients ( t * testing . T , err error ) {
t . Helper ( )
assertNoErrf ( t , "failed to list clients: %s" , err )
}
func assertNoErrListClientIPs ( t * testing . T , err error ) {
t . Helper ( )
assertNoErrf ( t , "failed to get client IPs: %s" , err )
}
func assertNoErrSync ( t * testing . T , err error ) {
t . Helper ( )
assertNoErrf ( t , "failed to have all clients sync up: %s" , err )
}
func assertNoErrListFQDN ( t * testing . T , err error ) {
t . Helper ( )
assertNoErrf ( t , "failed to list FQDNs: %s" , err )
}
func assertNoErrLogout ( t * testing . T , err error ) {
t . Helper ( )
assertNoErrf ( t , "failed to log out tailscale nodes: %s" , err )
}
2023-08-31 10:37:18 -06:00
func assertContains ( t * testing . T , str , subStr string ) {
t . Helper ( )
if ! strings . Contains ( str , subStr ) {
t . Fatalf ( "%#v does not contain %#v" , str , subStr )
}
}
2023-12-09 10:09:24 -07:00
func pingAllHelper ( t * testing . T , clients [ ] TailscaleClient , addrs [ ] string , opts ... tsic . PingOption ) int {
2023-02-02 08:05:52 -07:00
t . Helper ( )
success := 0
for _ , client := range clients {
for _ , addr := range addrs {
2023-12-09 10:09:24 -07:00
err := client . Ping ( addr , opts ... )
2023-02-02 08:05:52 -07:00
if err != nil {
2024-02-08 23:26:41 -07:00
t . Errorf ( "failed to ping %s from %s: %s" , addr , client . Hostname ( ) , err )
2023-02-02 08:05:52 -07:00
} else {
success ++
}
}
}
return success
}
2023-04-13 15:09:09 -06:00
func pingDerpAllHelper ( t * testing . T , clients [ ] TailscaleClient , addrs [ ] string ) int {
t . Helper ( )
success := 0
for _ , client := range clients {
for _ , addr := range addrs {
if isSelfClient ( client , addr ) {
continue
}
2023-04-23 05:41:23 -06:00
err := client . Ping (
2023-04-13 15:09:09 -06:00
addr ,
2023-04-13 15:10:47 -06:00
tsic . WithPingTimeout ( derpPingTimeout ) ,
tsic . WithPingCount ( derpPingCount ) ,
2023-04-23 05:41:23 -06:00
tsic . WithPingUntilDirect ( false ) ,
2023-04-13 15:09:09 -06:00
)
if err != nil {
2023-08-29 00:33:33 -06:00
t . Fatalf ( "failed to ping %s from %s: %s" , addr , client . Hostname ( ) , err )
2023-04-13 15:09:09 -06:00
} else {
success ++
}
}
}
return success
}
2024-02-08 23:26:41 -07:00
// assertClientsState validates the status and netmap of a list of
// clients for the general case of all to all connectivity.
func assertClientsState ( t * testing . T , clients [ ] TailscaleClient ) {
t . Helper ( )
2024-02-18 15:22:18 -07:00
var wg sync . WaitGroup
2024-02-08 23:26:41 -07:00
for _ , client := range clients {
2024-02-18 15:22:18 -07:00
wg . Add ( 1 )
c := client // Avoid loop pointer
go func ( ) {
defer wg . Done ( )
assertValidStatus ( t , c )
assertValidNetcheck ( t , c )
assertValidNetmap ( t , c )
} ( )
2024-02-08 23:26:41 -07:00
}
2024-02-18 15:22:18 -07:00
t . Logf ( "waiting for client state checks to finish" )
wg . Wait ( )
2024-02-08 23:26:41 -07:00
}
// assertValidNetmap asserts that the netmap of a client has all
// the minimum required fields set to a known working config for
// the general case. Fields are checked on self, then all peers.
// This test is not suitable for ACL/partial connection tests.
// This test can only be run on clients from 1.56.1. It will
// automatically pass all clients below that and is safe to call
// for all versions.
func assertValidNetmap ( t * testing . T , client TailscaleClient ) {
t . Helper ( )
2024-02-23 02:59:24 -07:00
if ! util . TailscaleVersionNewerOrEqual ( "1.56" , client . Version ( ) ) {
t . Logf ( "%q has version %q, skipping netmap check..." , client . Hostname ( ) , client . Version ( ) )
2024-02-18 15:22:18 -07:00
2024-02-23 02:59:24 -07:00
return
}
2024-02-18 15:22:18 -07:00
t . Logf ( "Checking netmap of %q" , client . Hostname ( ) )
2024-02-08 23:26:41 -07:00
netmap , err := client . Netmap ( )
if err != nil {
t . Fatalf ( "getting netmap for %q: %s" , client . Hostname ( ) , err )
}
assert . Truef ( t , netmap . SelfNode . Hostinfo ( ) . Valid ( ) , "%q does not have Hostinfo" , client . Hostname ( ) )
if hi := netmap . SelfNode . Hostinfo ( ) ; hi . Valid ( ) {
assert . LessOrEqual ( t , 1 , netmap . SelfNode . Hostinfo ( ) . Services ( ) . Len ( ) , "%q does not have enough services, got: %v" , client . Hostname ( ) , netmap . SelfNode . Hostinfo ( ) . Services ( ) )
}
assert . NotEmptyf ( t , netmap . SelfNode . AllowedIPs ( ) , "%q does not have any allowed IPs" , client . Hostname ( ) )
assert . NotEmptyf ( t , netmap . SelfNode . Addresses ( ) , "%q does not have any addresses" , client . Hostname ( ) )
2024-02-23 02:59:24 -07:00
if netmap . SelfNode . Online ( ) != nil {
assert . Truef ( t , * netmap . SelfNode . Online ( ) , "%q is not online" , client . Hostname ( ) )
} else {
t . Errorf ( "Online should not be nil for %s" , client . Hostname ( ) )
}
2024-02-08 23:26:41 -07:00
assert . Falsef ( t , netmap . SelfNode . Key ( ) . IsZero ( ) , "%q does not have a valid NodeKey" , client . Hostname ( ) )
assert . Falsef ( t , netmap . SelfNode . Machine ( ) . IsZero ( ) , "%q does not have a valid MachineKey" , client . Hostname ( ) )
assert . Falsef ( t , netmap . SelfNode . DiscoKey ( ) . IsZero ( ) , "%q does not have a valid DiscoKey" , client . Hostname ( ) )
for _ , peer := range netmap . Peers {
assert . NotEqualf ( t , "127.3.3.40:0" , peer . DERP ( ) , "peer (%s) has no home DERP in %q's netmap, got: %s" , peer . ComputedName ( ) , client . Hostname ( ) , peer . DERP ( ) )
assert . Truef ( t , peer . Hostinfo ( ) . Valid ( ) , "peer (%s) of %q does not have Hostinfo" , peer . ComputedName ( ) , client . Hostname ( ) )
if hi := peer . Hostinfo ( ) ; hi . Valid ( ) {
assert . LessOrEqualf ( t , 3 , peer . Hostinfo ( ) . Services ( ) . Len ( ) , "peer (%s) of %q does not have enough services, got: %v" , peer . ComputedName ( ) , client . Hostname ( ) , peer . Hostinfo ( ) . Services ( ) )
// Netinfo is not always set
2024-02-18 15:22:18 -07:00
// assert.Truef(t, hi.NetInfo().Valid(), "peer (%s) of %q does not have NetInfo", peer.ComputedName(), client.Hostname())
2024-02-08 23:26:41 -07:00
if ni := hi . NetInfo ( ) ; ni . Valid ( ) {
assert . NotEqualf ( t , 0 , ni . PreferredDERP ( ) , "peer (%s) has no home DERP in %q's netmap, got: %s" , peer . ComputedName ( ) , client . Hostname ( ) , peer . Hostinfo ( ) . NetInfo ( ) . PreferredDERP ( ) )
}
}
assert . NotEmptyf ( t , peer . Endpoints ( ) , "peer (%s) of %q does not have any endpoints" , peer . ComputedName ( ) , client . Hostname ( ) )
assert . NotEmptyf ( t , peer . AllowedIPs ( ) , "peer (%s) of %q does not have any allowed IPs" , peer . ComputedName ( ) , client . Hostname ( ) )
assert . NotEmptyf ( t , peer . Addresses ( ) , "peer (%s) of %q does not have any addresses" , peer . ComputedName ( ) , client . Hostname ( ) )
assert . Truef ( t , * peer . Online ( ) , "peer (%s) of %q is not online" , peer . ComputedName ( ) , client . Hostname ( ) )
assert . Falsef ( t , peer . Key ( ) . IsZero ( ) , "peer (%s) of %q does not have a valid NodeKey" , peer . ComputedName ( ) , client . Hostname ( ) )
assert . Falsef ( t , peer . Machine ( ) . IsZero ( ) , "peer (%s) of %q does not have a valid MachineKey" , peer . ComputedName ( ) , client . Hostname ( ) )
assert . Falsef ( t , peer . DiscoKey ( ) . IsZero ( ) , "peer (%s) of %q does not have a valid DiscoKey" , peer . ComputedName ( ) , client . Hostname ( ) )
}
}
// assertValidStatus asserts that the status of a client has all
// the minimum required fields set to a known working config for
// the general case. Fields are checked on self, then all peers.
// This test is not suitable for ACL/partial connection tests.
func assertValidStatus ( t * testing . T , client TailscaleClient ) {
t . Helper ( )
2024-02-23 02:59:24 -07:00
status , err := client . Status ( true )
2024-02-08 23:26:41 -07:00
if err != nil {
t . Fatalf ( "getting status for %q: %s" , client . Hostname ( ) , err )
}
assert . NotEmptyf ( t , status . Self . HostName , "%q does not have HostName set, likely missing Hostinfo" , client . Hostname ( ) )
assert . NotEmptyf ( t , status . Self . OS , "%q does not have OS set, likely missing Hostinfo" , client . Hostname ( ) )
assert . NotEmptyf ( t , status . Self . Relay , "%q does not have a relay, likely missing Hostinfo/Netinfo" , client . Hostname ( ) )
assert . NotEmptyf ( t , status . Self . TailscaleIPs , "%q does not have Tailscale IPs" , client . Hostname ( ) )
// This seem to not appear until version 1.56
if status . Self . AllowedIPs != nil {
assert . NotEmptyf ( t , status . Self . AllowedIPs , "%q does not have any allowed IPs" , client . Hostname ( ) )
}
assert . NotEmptyf ( t , status . Self . Addrs , "%q does not have any endpoints" , client . Hostname ( ) )
assert . Truef ( t , status . Self . Online , "%q is not online" , client . Hostname ( ) )
assert . Truef ( t , status . Self . InNetworkMap , "%q is not in network map" , client . Hostname ( ) )
// This isnt really relevant for Self as it wont be in its own socket/wireguard.
// assert.Truef(t, status.Self.InMagicSock, "%q is not tracked by magicsock", client.Hostname())
// assert.Truef(t, status.Self.InEngine, "%q is not in in wireguard engine", client.Hostname())
for _ , peer := range status . Peer {
assert . NotEmptyf ( t , peer . HostName , "peer (%s) of %q does not have HostName set, likely missing Hostinfo" , peer . DNSName , client . Hostname ( ) )
assert . NotEmptyf ( t , peer . OS , "peer (%s) of %q does not have OS set, likely missing Hostinfo" , peer . DNSName , client . Hostname ( ) )
assert . NotEmptyf ( t , peer . Relay , "peer (%s) of %q does not have a relay, likely missing Hostinfo/Netinfo" , peer . DNSName , client . Hostname ( ) )
assert . NotEmptyf ( t , peer . TailscaleIPs , "peer (%s) of %q does not have Tailscale IPs" , peer . DNSName , client . Hostname ( ) )
// This seem to not appear until version 1.56
if peer . AllowedIPs != nil {
assert . NotEmptyf ( t , peer . AllowedIPs , "peer (%s) of %q does not have any allowed IPs" , peer . DNSName , client . Hostname ( ) )
}
// Addrs does not seem to appear in the status from peers.
// assert.NotEmptyf(t, peer.Addrs, "peer (%s) of %q does not have any endpoints", peer.DNSName, client.Hostname())
assert . Truef ( t , peer . Online , "peer (%s) of %q is not online" , peer . DNSName , client . Hostname ( ) )
assert . Truef ( t , peer . InNetworkMap , "peer (%s) of %q is not in network map" , peer . DNSName , client . Hostname ( ) )
assert . Truef ( t , peer . InMagicSock , "peer (%s) of %q is not tracked by magicsock" , peer . DNSName , client . Hostname ( ) )
// TODO(kradalby): InEngine is only true when a proper tunnel is set up,
// there might be some interesting stuff to test here in the future.
// assert.Truef(t, peer.InEngine, "peer (%s) of %q is not in wireguard engine", peer.DNSName, client.Hostname())
}
}
func assertValidNetcheck ( t * testing . T , client TailscaleClient ) {
t . Helper ( )
report , err := client . Netcheck ( )
if err != nil {
t . Fatalf ( "getting status for %q: %s" , client . Hostname ( ) , err )
}
assert . NotEqualf ( t , 0 , report . PreferredDERP , "%q does not have a DERP relay" , client . Hostname ( ) )
}
2023-04-13 15:09:09 -06:00
func isSelfClient ( client TailscaleClient , addr string ) bool {
if addr == client . Hostname ( ) {
return true
}
ips , err := client . IPs ( )
if err != nil {
return false
}
for _ , ip := range ips {
if ip . String ( ) == addr {
return true
}
}
return false
}
2023-09-10 02:00:12 -06:00
func isCI ( ) bool {
if _ , ok := os . LookupEnv ( "CI" ) ; ok {
return true
}
if _ , ok := os . LookupEnv ( "GITHUB_RUN_ID" ) ; ok {
return true
}
return false
}
func dockertestMaxWait ( ) time . Duration {
2024-02-08 23:26:41 -07:00
wait := 120 * time . Second //nolint
2023-09-10 02:00:12 -06:00
if isCI ( ) {
wait = 300 * time . Second //nolint
}
return wait
}
// func dockertestCommandTimeout() time.Duration {
// timeout := 10 * time.Second //nolint
//
// if isCI() {
// timeout = 60 * time.Second //nolint
// }
//
// return timeout
// }
2024-05-19 15:49:27 -06:00
// pingAllNegativeHelper is intended to have 1 or more nodes timing out from the ping,
2023-02-02 08:05:52 -07:00
// it counts failures instead of successes.
// func pingAllNegativeHelper(t *testing.T, clients []TailscaleClient, addrs []string) int {
// t.Helper()
// failures := 0
//
// timeout := 100
// count := 3
//
// for _, client := range clients {
// for _, addr := range addrs {
// err := client.Ping(
// addr,
// tsic.WithPingTimeout(time.Duration(timeout)*time.Millisecond),
// tsic.WithPingCount(count),
// )
// if err != nil {
// failures++
// }
// }
// }
//
// return failures
// }
2023-04-16 04:26:35 -06:00
// // findPeerByIP takes an IP and a map of peers from status.Peer, and returns a *ipnstate.PeerStatus
// // if there is a peer with the given IP. If no peer is found, nil is returned.
// func findPeerByIP(
// ip netip.Addr,
// peers map[key.NodePublic]*ipnstate.PeerStatus,
// ) *ipnstate.PeerStatus {
// for _, peer := range peers {
// for _, peerIP := range peer.TailscaleIPs {
// if ip == peerIP {
// return peer
// }
// }
// }
//
// return nil
// }
//
// // findPeerByHostname takes a hostname and a map of peers from status.Peer, and returns a *ipnstate.PeerStatus
// // if there is a peer with the given hostname. If no peer is found, nil is returned.
// func findPeerByHostname(
// hostname string,
// peers map[key.NodePublic]*ipnstate.PeerStatus,
// ) *ipnstate.PeerStatus {
// for _, peer := range peers {
// if hostname == peer.HostName {
// return peer
// }
// }
//
// return nil
// }