Get integration test netmap from watch-ipn command (#1729)

hscontrol/poll.go

@@ -212,15 +212,9 @@ func (h *Headscale) handlePoll(
 			return
 		}
 
-		// TODO(kradalby): Figure out why patch changes does
-		// not show up in output from `tailscale debug netmap`.
-		// stateUpdate := types.StateUpdate{
-		// 	Type:          types.StatePeerChangedPatch,
-		// 	ChangePatches: []*tailcfg.PeerChange{&change},
-		// }
 		stateUpdate := types.StateUpdate{
-			Type:        types.StatePeerChanged,
+			Type:          types.StatePeerChangedPatch,
-			ChangeNodes: types.Nodes{node},
+			ChangePatches: []*tailcfg.PeerChange{&change},
 		}
 		if stateUpdate.Valid() {
 			ctx := types.NotifyCtx(context.Background(), "poll-nodeupdate-peers-patch", node.Hostname)

hscontrol/util/util.go

@@ -0,0 +1,13 @@
+package util
+
+import "tailscale.com/util/cmpver"
+
+func TailscaleVersionNewerOrEqual(minimum, toCheck string) bool {
+	if cmpver.Compare(minimum, toCheck) <= 0 ||
+		toCheck == "unstable" ||
+		toCheck == "head" {
+		return true
+	}
+
+	return false
+}

hscontrol/util/util_test.go

@@ -0,0 +1,95 @@
+package util
+
+import "testing"
+
+func TestTailscaleVersionNewerOrEqual(t *testing.T) {
+	type args struct {
+		minimum string
+		toCheck string
+	}
+	tests := []struct {
+		name string
+		args args
+		want bool
+	}{
+		{
+			name: "is-equal",
+			args: args{
+				minimum: "1.56",
+				toCheck: "1.56",
+			},
+			want: true,
+		},
+		{
+			name: "is-newer-head",
+			args: args{
+				minimum: "1.56",
+				toCheck: "head",
+			},
+			want: true,
+		},
+		{
+			name: "is-newer-unstable",
+			args: args{
+				minimum: "1.56",
+				toCheck: "unstable",
+			},
+			want: true,
+		},
+		{
+			name: "is-newer-patch",
+			args: args{
+				minimum: "1.56.1",
+				toCheck: "1.56.1",
+			},
+			want: true,
+		},
+		{
+			name: "is-older-patch-same-minor",
+			args: args{
+				minimum: "1.56.1",
+				toCheck: "1.56.0",
+			},
+			want: false,
+		},
+		{
+			name: "is-older-unstable",
+			args: args{
+				minimum: "1.56",
+				toCheck: "1.55",
+			},
+			want: false,
+		},
+		{
+			name: "is-older-one-stable",
+			args: args{
+				minimum: "1.56",
+				toCheck: "1.54",
+			},
+			want: false,
+		},
+		{
+			name: "is-older-five-stable",
+			args: args{
+				minimum: "1.56",
+				toCheck: "1.46",
+			},
+			want: false,
+		},
+		{
+			name: "is-older-patch",
+			args: args{
+				minimum: "1.56",
+				toCheck: "1.48.1",
+			},
+			want: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := TailscaleVersionNewerOrEqual(tt.args.minimum, tt.args.toCheck); got != tt.want {
+				t.Errorf("TailscaleVersionNewerThan() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

integration/embedded_derp_test.go

@@ -70,8 +70,6 @@ func TestDERPServerScenario(t *testing.T) {
 	err = scenario.WaitForTailscaleSync()
 	assertNoErrSync(t, err)
 
-	assertClientsState(t, allClients)
-
 	allHostnames, err := scenario.ListTailscaleClientsFQDNs()
 	assertNoErrListFQDN(t, err)
 

integration/tsic/tsic.go

@@ -1,9 +1,11 @@
 package tsic
 
 import (
+	"context"
 	"encoding/json"
 	"errors"
 	"fmt"
+	"io"
 	"log"
 	"net/netip"
 	"net/url"
@@ -16,6 +18,7 @@ import (
 	"github.com/juanfont/headscale/integration/integrationutil"
 	"github.com/ory/dockertest/v3"
 	"github.com/ory/dockertest/v3/docker"
+	"tailscale.com/ipn"
 	"tailscale.com/ipn/ipnstate"
 	"tailscale.com/net/netcheck"
 	"tailscale.com/types/netmap"
@@ -522,27 +525,122 @@ func (t *TailscaleInContainer) Status() (*ipnstate.Status, error) {
 }
 
 // Netmap returns the current Netmap (netmap.NetworkMap) of the Tailscale instance.
-// Only works with Tailscale 1.56.1 and newer.
+// Only works with Tailscale 1.56 and newer.
+// Panics if version is lower then minimum.
+// func (t *TailscaleInContainer) Netmap() (*netmap.NetworkMap, error) {
+// 	if !util.TailscaleVersionNewerOrEqual("1.56", t.version) {
+// 		panic(fmt.Sprintf("tsic.Netmap() called with unsupported version: %s", t.version))
+// 	}
+
+// 	command := []string{
+// 		"tailscale",
+// 		"debug",
+// 		"netmap",
+// 	}
+
+// 	result, stderr, err := t.Execute(command)
+// 	if err != nil {
+// 		fmt.Printf("stderr: %s\n", stderr)
+// 		return nil, fmt.Errorf("failed to execute tailscale debug netmap command: %w", err)
+// 	}
+
+// 	var nm netmap.NetworkMap
+// 	err = json.Unmarshal([]byte(result), &nm)
+// 	if err != nil {
+// 		return nil, fmt.Errorf("failed to unmarshal tailscale netmap: %w", err)
+// 	}
+
+// 	return &nm, err
+// }
+
+// Netmap returns the current Netmap (netmap.NetworkMap) of the Tailscale instance.
+// This implementation is based on getting the netmap from `tailscale debug watch-ipn`
+// as there seem to be some weirdness omitting endpoint and DERP info if we use
+// Patch updates.
+// This implementation works on all supported versions.
 func (t *TailscaleInContainer) Netmap() (*netmap.NetworkMap, error) {
-	command := []string{
+	// watch-ipn will only give an update if something is happening,
-		"tailscale",
+	// since we send keep alives, the worst case for this should be
-		"debug",
+	// 1 minute, but set a slightly more conservative time.
-		"netmap",
+	ctx, _ := context.WithTimeout(context.Background(), 3*time.Minute)
-	}
 
-	result, stderr, err := t.Execute(command)
+	notify, err := t.watchIPN(ctx)
 	if err != nil {
-		fmt.Printf("stderr: %s\n", stderr)
+		return nil, err
-		return nil, fmt.Errorf("failed to execute tailscale debug netmap command: %w", err)
 	}
 
-	var nm netmap.NetworkMap
+	if notify.NetMap == nil {
-	err = json.Unmarshal([]byte(result), &nm)
+		return nil, fmt.Errorf("no netmap present in ipn.Notify")
-	if err != nil {
-		return nil, fmt.Errorf("failed to unmarshal tailscale netmap: %w", err)
 	}
 
-	return &nm, err
+	return notify.NetMap, nil
+}
+
+// watchIPN watches `tailscale debug watch-ipn` for a ipn.Notify object until
+// it gets one that has a netmap.NetworkMap.
+func (t *TailscaleInContainer) watchIPN(ctx context.Context) (*ipn.Notify, error) {
+	pr, pw := io.Pipe()
+
+	type result struct {
+		notify *ipn.Notify
+		err    error
+	}
+	resultChan := make(chan result, 1)
+
+	// There is no good way to kill the goroutine with watch-ipn,
+	// so make a nice func to send a kill command to issue when
+	// we are done.
+	killWatcher := func() {
+		stdout, stderr, err := t.Execute([]string{
+			"/bin/sh", "-c", `kill $(ps aux | grep "tailscale debug watch-ipn" | grep -v grep | awk '{print $1}') || true`,
+		})
+		if err != nil {
+			log.Printf("failed to kill tailscale watcher, \nstdout: %s\nstderr: %s\nerr: %s", stdout, stderr, err)
+		}
+	}
+
+	go func() {
+		_, _ = t.container.Exec(
+			// Prior to 1.56, the initial "Connected." message was printed to stdout,
+			// filter out with grep.
+			[]string{"/bin/sh", "-c", `tailscale debug watch-ipn | grep -v "Connected."`},
+			dockertest.ExecOptions{
+				// The interesting output is sent to stdout, so ignore stderr.
+				StdOut: pw,
+				// StdErr: pw,
+			},
+		)
+	}()
+
+	go func() {
+		decoder := json.NewDecoder(pr)
+		for decoder.More() {
+			var notify ipn.Notify
+			if err := decoder.Decode(&notify); err != nil {
+				resultChan <- result{nil, fmt.Errorf("parse notify: %w", err)}
+			}
+
+			if notify.NetMap != nil {
+				resultChan <- result{&notify, nil}
+			}
+		}
+	}()
+
+	select {
+	case <-ctx.Done():
+		killWatcher()
+
+		return nil, ctx.Err()
+
+	case result := <-resultChan:
+		killWatcher()
+
+		if result.err != nil {
+			return nil, result.err
+		}
+
+		return result.notify, nil
+	}
 }
 
 // Netcheck returns the current Netcheck Report (netcheck.Report) of the Tailscale instance.

integration/utils.go

@@ -3,12 +3,12 @@ package integration
 import (
 	"os"
 	"strings"
+	"sync"
 	"testing"
 	"time"
 
 	"github.com/juanfont/headscale/integration/tsic"
 	"github.com/stretchr/testify/assert"
-	"tailscale.com/util/cmpver"
 )
 
 const (
@@ -127,11 +127,21 @@ func pingDerpAllHelper(t *testing.T, clients []TailscaleClient, addrs []string)
 func assertClientsState(t *testing.T, clients []TailscaleClient) {
 	t.Helper()
 
+	var wg sync.WaitGroup
+
 	for _, client := range clients {
-		assertValidStatus(t, client)
+		wg.Add(1)
-		assertValidNetmap(t, client)
+		c := client // Avoid loop pointer
-		assertValidNetcheck(t, client)
+		go func() {
+			defer wg.Done()
+			assertValidStatus(t, c)
+			assertValidNetcheck(t, c)
+			assertValidNetmap(t, c)
+		}()
 	}
+
+	t.Logf("waiting for client state checks to finish")
+	wg.Wait()
 }
 
 // assertValidNetmap asserts that the netmap of a client has all
@@ -144,11 +154,13 @@ func assertClientsState(t *testing.T, clients []TailscaleClient) {
 func assertValidNetmap(t *testing.T, client TailscaleClient) {
 	t.Helper()
 
-	if cmpver.Compare("1.56.1", client.Version()) <= 0 ||
+	// if !util.TailscaleVersionNewerOrEqual("1.56", client.Version()) {
-		!strings.Contains(client.Hostname(), "unstable") ||
+	// 	t.Logf("%q has version %q, skipping netmap check...", client.Hostname(), client.Version())
-		!strings.Contains(client.Hostname(), "head") {
+
-		return
+	// 	return
-	}
+	// }
+
+	t.Logf("Checking netmap of %q", client.Hostname())
 
 	netmap, err := client.Netmap()
 	if err != nil {
@@ -177,7 +189,7 @@ func assertValidNetmap(t *testing.T, client TailscaleClient) {
 			assert.LessOrEqualf(t, 3, peer.Hostinfo().Services().Len(), "peer (%s) of %q does not have enough services, got: %v", peer.ComputedName(), client.Hostname(), peer.Hostinfo().Services())
 
 			// Netinfo is not always set
-			assert.Truef(t, hi.NetInfo().Valid(), "peer (%s) of %q does not have NetInfo", peer.ComputedName(), client.Hostname())
+			// assert.Truef(t, hi.NetInfo().Valid(), "peer (%s) of %q does not have NetInfo", peer.ComputedName(), client.Hostname())
 			if ni := hi.NetInfo(); ni.Valid() {
 				assert.NotEqualf(t, 0, ni.PreferredDERP(), "peer (%s) has no home DERP in %q's netmap, got: %s", peer.ComputedName(), client.Hostname(), peer.Hostinfo().NetInfo().PreferredDERP())
 			}