Add go profiling flag, and enable on integration tests (#1382)

.github/workflows/test-integration-v2-TestACLAllowStarDst.yaml

@@ -55,3 +55,9 @@ jobs:
         with:
           name: logs
           path: "control_logs/*.log"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: pprof
+          path: "control_logs/*.pprof.tar"

.github/workflows/test-integration-v2-TestACLAllowUser80Dst.yaml

@@ -55,3 +55,9 @@ jobs:
         with:
           name: logs
           path: "control_logs/*.log"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: pprof
+          path: "control_logs/*.pprof.tar"

.github/workflows/test-integration-v2-TestACLAllowUserDst.yaml

@@ -55,3 +55,9 @@ jobs:
         with:
           name: logs
           path: "control_logs/*.log"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: pprof
+          path: "control_logs/*.pprof.tar"

.github/workflows/test-integration-v2-TestACLDenyAllPort80.yaml

@@ -55,3 +55,9 @@ jobs:
         with:
           name: logs
           path: "control_logs/*.log"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: pprof
+          path: "control_logs/*.pprof.tar"

.github/workflows/test-integration-v2-TestACLDevice1CanAccessDevice2.yaml

@@ -55,3 +55,9 @@ jobs:
         with:
           name: logs
           path: "control_logs/*.log"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: pprof
+          path: "control_logs/*.pprof.tar"

.github/workflows/test-integration-v2-TestACLHostsInNetMapTable.yaml

@@ -55,3 +55,9 @@ jobs:
         with:
           name: logs
           path: "control_logs/*.log"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: pprof
+          path: "control_logs/*.pprof.tar"

.github/workflows/test-integration-v2-TestACLNamedHostsCanReach.yaml

@@ -55,3 +55,9 @@ jobs:
         with:
           name: logs
           path: "control_logs/*.log"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: pprof
+          path: "control_logs/*.pprof.tar"

.github/workflows/test-integration-v2-TestACLNamedHostsCanReachBySubnet.yaml

@@ -55,3 +55,9 @@ jobs:
         with:
           name: logs
           path: "control_logs/*.log"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: pprof
+          path: "control_logs/*.pprof.tar"

.github/workflows/test-integration-v2-TestAuthKeyLogoutAndRelogin.yaml

@@ -55,3 +55,9 @@ jobs:
         with:
           name: logs
           path: "control_logs/*.log"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: pprof
+          path: "control_logs/*.pprof.tar"

.github/workflows/test-integration-v2-TestAuthWebFlowAuthenticationPingAll.yaml

@@ -55,3 +55,9 @@ jobs:
         with:
           name: logs
           path: "control_logs/*.log"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: pprof
+          path: "control_logs/*.pprof.tar"

.github/workflows/test-integration-v2-TestAuthWebFlowLogoutAndRelogin.yaml

@@ -55,3 +55,9 @@ jobs:
         with:
           name: logs
           path: "control_logs/*.log"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: pprof
+          path: "control_logs/*.pprof.tar"

.github/workflows/test-integration-v2-TestCreateTailscale.yaml

@@ -55,3 +55,9 @@ jobs:
         with:
           name: logs
           path: "control_logs/*.log"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: pprof
+          path: "control_logs/*.pprof.tar"

.github/workflows/test-integration-v2-TestDERPServerScenario.yaml

@@ -55,3 +55,9 @@ jobs:
         with:
           name: logs
           path: "control_logs/*.log"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: pprof
+          path: "control_logs/*.pprof.tar"

.github/workflows/test-integration-v2-TestEnablingRoutes.yaml

@@ -55,3 +55,9 @@ jobs:
         with:
           name: logs
           path: "control_logs/*.log"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: pprof
+          path: "control_logs/*.pprof.tar"

.github/workflows/test-integration-v2-TestEphemeral.yaml

@@ -55,3 +55,9 @@ jobs:
         with:
           name: logs
           path: "control_logs/*.log"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: pprof
+          path: "control_logs/*.pprof.tar"

.github/workflows/test-integration-v2-TestExpireNode.yaml

@@ -55,3 +55,9 @@ jobs:
         with:
           name: logs
           path: "control_logs/*.log"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: pprof
+          path: "control_logs/*.pprof.tar"

.github/workflows/test-integration-v2-TestHeadscale.yaml

@@ -55,3 +55,9 @@ jobs:
         with:
           name: logs
           path: "control_logs/*.log"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: pprof
+          path: "control_logs/*.pprof.tar"

.github/workflows/test-integration-v2-TestOIDCAuthenticationPingAll.yaml

@@ -55,3 +55,9 @@ jobs:
         with:
           name: logs
           path: "control_logs/*.log"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: pprof
+          path: "control_logs/*.pprof.tar"

.github/workflows/test-integration-v2-TestOIDCExpireNodesBasedOnTokenExpiry.yaml

@@ -55,3 +55,9 @@ jobs:
         with:
           name: logs
           path: "control_logs/*.log"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: pprof
+          path: "control_logs/*.pprof.tar"

.github/workflows/test-integration-v2-TestPingAllByHostname.yaml

@@ -55,3 +55,9 @@ jobs:
         with:
           name: logs
           path: "control_logs/*.log"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: pprof
+          path: "control_logs/*.pprof.tar"

.github/workflows/test-integration-v2-TestPingAllByIP.yaml

@@ -55,3 +55,9 @@ jobs:
         with:
           name: logs
           path: "control_logs/*.log"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: pprof
+          path: "control_logs/*.pprof.tar"

.github/workflows/test-integration-v2-TestPreAuthKeyCommand.yaml

@@ -55,3 +55,9 @@ jobs:
         with:
           name: logs
           path: "control_logs/*.log"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: pprof
+          path: "control_logs/*.pprof.tar"

.github/workflows/test-integration-v2-TestPreAuthKeyCommandReusableEphemeral.yaml

@@ -55,3 +55,9 @@ jobs:
         with:
           name: logs
           path: "control_logs/*.log"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: pprof
+          path: "control_logs/*.pprof.tar"

.github/workflows/test-integration-v2-TestPreAuthKeyCommandWithoutExpiry.yaml

@@ -55,3 +55,9 @@ jobs:
         with:
           name: logs
           path: "control_logs/*.log"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: pprof
+          path: "control_logs/*.pprof.tar"

.github/workflows/test-integration-v2-TestResolveMagicDNS.yaml

@@ -55,3 +55,9 @@ jobs:
         with:
           name: logs
           path: "control_logs/*.log"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: pprof
+          path: "control_logs/*.pprof.tar"

.github/workflows/test-integration-v2-TestSSHIsBlockedInACL.yaml

@@ -55,3 +55,9 @@ jobs:
         with:
           name: logs
           path: "control_logs/*.log"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: pprof
+          path: "control_logs/*.pprof.tar"

.github/workflows/test-integration-v2-TestSSHMultipleUsersAllToAll.yaml

@@ -55,3 +55,9 @@ jobs:
         with:
           name: logs
           path: "control_logs/*.log"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: pprof
+          path: "control_logs/*.pprof.tar"

.github/workflows/test-integration-v2-TestSSHNoSSHConfigured.yaml

@@ -55,3 +55,9 @@ jobs:
         with:
           name: logs
           path: "control_logs/*.log"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: pprof
+          path: "control_logs/*.pprof.tar"

.github/workflows/test-integration-v2-TestSSHOneUserAllToAll.yaml

@@ -55,3 +55,9 @@ jobs:
         with:
           name: logs
           path: "control_logs/*.log"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: pprof
+          path: "control_logs/*.pprof.tar"

.github/workflows/test-integration-v2-TestSSUserOnlyIsolation.yaml

@@ -55,3 +55,9 @@ jobs:
         with:
           name: logs
           path: "control_logs/*.log"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: pprof
+          path: "control_logs/*.pprof.tar"

.github/workflows/test-integration-v2-TestTaildrop.yaml

@@ -55,3 +55,9 @@ jobs:
         with:
           name: logs
           path: "control_logs/*.log"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: pprof
+          path: "control_logs/*.pprof.tar"

.github/workflows/test-integration-v2-TestTailscaleNodesJoiningHeadcale.yaml

@@ -55,3 +55,9 @@ jobs:
         with:
           name: logs
           path: "control_logs/*.log"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: pprof
+          path: "control_logs/*.pprof.tar"

.github/workflows/test-integration-v2-TestUserCommand.yaml

@@ -55,3 +55,9 @@ jobs:
         with:
           name: logs
           path: "control_logs/*.log"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: pprof
+          path: "control_logs/*.pprof.tar"

CHANGELOG.md

@@ -4,6 +4,9 @@
 
 ### Changes
 
+- Add environment flags to enable pprof (profiling) [#1382](https://github.com/juanfont/headscale/pull/1382)
+  - Profiles are continously generated in our integration tests.
+
 ## 0.22.1 (2023-04-20)
 
 ### Changes

app.go

@@ -820,7 +820,6 @@ func (h *Headscale) Serve() error {
 
 				// And we're done:
 				cancel()
-				os.Exit(0)
 			}
 		}
 	}

cmd/gh-action-integration-generator/main.go

@@ -76,6 +76,12 @@ jobs:
         with:
           name: logs
           path: "control_logs/*.log"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: pprof
+          path: "control_logs/*.pprof.tar"
 `),
 	)
 )

cmd/headscale/headscale.go

@@ -6,11 +6,25 @@ import (
 
 	"github.com/efekarakus/termcolor"
 	"github.com/juanfont/headscale/cmd/headscale/cli"
+	"github.com/pkg/profile"
 	"github.com/rs/zerolog"
 	"github.com/rs/zerolog/log"
 )
 
 func main() {
+	if _, enableProfile := os.LookupEnv("HEADSCALE_PROFILING_ENABLED"); enableProfile {
+		if profilePath, ok := os.LookupEnv("HEADSCALE_PROFILING_PATH"); ok {
+			err := os.MkdirAll(profilePath, os.ModePerm)
+			if err != nil {
+				log.Fatal().Err(err).Msg("failed to create profiling directory")
+			}
+
+			defer profile.Start(profile.ProfilePath(profilePath)).Stop()
+		} else {
+			defer profile.Start().Stop()
+		}
+	}
+
 	var colors bool
 	switch l := termcolor.SupportLevel(os.Stderr); l {
 	case termcolor.Level16M:

flake.nix

@@ -36,7 +36,7 @@
 
             # When updating go.mod or go.sum, a new sha will need to be calculated,
             # update this if you have a mismatch after doing a change to thos files.
-            vendorSha256 = "sha256-lqMmTPLlnm4dGs3O+hS0LmR5LTWP5ICqMSkNnB+XXLI=";
+            vendorSha256 = "sha256-5a3SR77vYp8AsVFyFhBQtHn1I2kdlph+RuAkuT6hPIs=";
 
             ldflags = [ "-s" "-w" "-X github.com/juanfont/headscale/cmd/headscale/cli.Version=v${version}" ];
           };
@@ -100,6 +100,10 @@
             nfpm
             gotestsum
 
+            # 'dot' is needed for pprof graphs
+            # go tool pprof -http=: <source>
+            graphviz
+
             # Protobuf dependencies
             protobuf
             protoc-gen-go

go.mod

@@ -64,6 +64,7 @@ require (
 	github.com/docker/go-connections v0.4.0 // indirect
 	github.com/docker/go-units v0.5.0 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
+	github.com/felixge/fgprof v0.9.3 // indirect
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
 	github.com/fxamacker/cbor/v2 v2.4.0 // indirect
 	github.com/glebarez/go-sqlite v1.20.3 // indirect
@@ -75,6 +76,7 @@ require (
 	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/google/go-github v17.0.0+incompatible // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
+	github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 // indirect
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
 	github.com/google/uuid v1.3.0 // indirect
 	github.com/gookit/color v1.5.3 // indirect
@@ -111,6 +113,7 @@ require (
 	github.com/opencontainers/runc v1.1.4 // indirect
 	github.com/pelletier/go-toml/v2 v2.0.6 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pkg/profile v1.7.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_model v0.3.0 // indirect
 	github.com/prometheus/procfs v0.9.0 // indirect

go.sum

@@ -129,6 +129,8 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m
 github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po=
 github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/felixge/fgprof v0.9.3 h1:VvyZxILNuCiUCSXtPtYmmtGvb65nqXh2QFWc0Wpf2/g=
+github.com/felixge/fgprof v0.9.3/go.mod h1:RdbpDgzqYVh/T9fPELJyV7EYJuHB55UTEULNun8eiPw=
 github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k=
 github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE=
 github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
@@ -238,7 +240,9 @@ github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hf
 github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/pprof v0.0.0-20211214055906-6f57359322fd/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg=
 github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ=
+github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=
 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
@@ -272,6 +276,7 @@ github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec h1:qv2VnGeEQHchGaZ/u
 github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec/go.mod h1:Q48J4R4DvxnHolD5P8pOtXigYlRuPLGl6moFx3ulM68=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
+github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w=
 github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk=
 github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
@@ -384,6 +389,8 @@ github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsK
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/profile v1.7.0 h1:hnbDkaNWPCLMO9wGLdBFTIZvzDrDfBM2072E1S9gJkA=
+github.com/pkg/profile v1.7.0/go.mod h1:8Uer0jas47ZQMJ7VD+OHknK4YDY07LPUC6dEvqDjvNo=
 github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
@@ -669,6 +676,7 @@ golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211013075003-97ac67df715c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

integration/control.go

@@ -8,6 +8,7 @@ import (
 type ControlServer interface {
 	Shutdown() error
 	SaveLog(string) error
+	SaveProfile(string) error
 	Execute(command []string) (string, error)
 	ConnectToNetwork(network *dockertest.Network) error
 	GetHealthEndpoint() string

integration/hsic/hsic.go

@@ -16,6 +16,9 @@ import (
 	"net"
 	"net/http"
 	"net/url"
+	"os"
+	"path"
+	"strings"
 	"time"
 
 	"github.com/davecgh/go-spew/spew"
@@ -205,7 +208,10 @@ func New(
 		ContextDir: dockerContextPath,
 	}
 
-	env := []string{}
+	env := []string{
+		"HEADSCALE_PROFILING_ENABLED=1",
+		"HEADSCALE_PROFILING_PATH=/tmp/profile",
+	}
 	for key, value := range hsic.env {
 		env = append(env, fmt.Sprintf("%s=%s", key, value))
 	}
@@ -219,7 +225,7 @@ func New(
 		// Cmd:          []string{"headscale", "serve"},
 		// TODO(kradalby): Get rid of this hack, we currently need to give us some
 		// to inject the headscale configuration further down.
-		Entrypoint: []string{"/bin/bash", "-c", "/bin/sleep 3 ; headscale serve"},
+		Entrypoint: []string{"/bin/bash", "-c", "/bin/sleep 3 ; headscale serve ; /bin/sleep 30"},
 		Env:        env,
 	}
 
@@ -305,6 +311,33 @@ func (t *HeadscaleInContainer) hasTLS() bool {
 
 // Shutdown stops and cleans up the Headscale container.
 func (t *HeadscaleInContainer) Shutdown() error {
+	err := t.SaveLog("/tmp/control")
+	if err != nil {
+		log.Printf(
+			"Failed to save log from control: %s",
+			fmt.Errorf("failed to save log from control: %w", err),
+		)
+	}
+
+	// Send a interrupt signal to the "headscale" process inside the container
+	// allowing it to shut down gracefully and flush the profile to disk.
+	// The container will live for a bit longer due to the sleep at the end.
+	err = t.SendInterrupt()
+	if err != nil {
+		log.Printf(
+			"Failed to send graceful interrupt to control: %s",
+			fmt.Errorf("failed to send graceful interrupt to control: %w", err),
+		)
+	}
+
+	err = t.SaveProfile("/tmp/control")
+	if err != nil {
+		log.Printf(
+			"Failed to save profile from control: %s",
+			fmt.Errorf("failed to save profile from control: %w", err),
+		)
+	}
+
 	return t.pool.Purge(t.container)
 }
 
@@ -314,6 +347,24 @@ func (t *HeadscaleInContainer) SaveLog(path string) error {
 	return dockertestutil.SaveLog(t.pool, t.container, path)
 }
 
+func (t *HeadscaleInContainer) SaveProfile(savePath string) error {
+	tarFile, err := t.FetchPath("/tmp/profile")
+	if err != nil {
+		return err
+	}
+
+	err = os.WriteFile(
+		path.Join(savePath, t.hostname+".pprof.tar"),
+		tarFile,
+		os.ModePerm,
+	)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 // Execute runs a command inside the Headscale container and returns the
 // result of stdout as a string.
 func (t *HeadscaleInContainer) Execute(
@@ -498,6 +549,26 @@ func (t *HeadscaleInContainer) WriteFile(path string, data []byte) error {
 	return integrationutil.WriteFileToContainer(t.pool, t.container, path, data)
 }
 
+// FetchPath gets a path from inside the Headscale container and returns a tar
+// file as byte array.
+func (t *HeadscaleInContainer) FetchPath(path string) ([]byte, error) {
+	return integrationutil.FetchPathFromContainer(t.pool, t.container, path)
+}
+
+func (t *HeadscaleInContainer) SendInterrupt() error {
+	pid, err := t.Execute([]string{"pidof", "headscale"})
+	if err != nil {
+		return err
+	}
+
+	_, err = t.Execute([]string{"kill", "-2", strings.Trim(pid, "'\n")})
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 // nolint
 func createCertificate(hostname string) ([]byte, []byte, error) {
 	// From:

integration/integrationutil/util.go

@@ -72,3 +72,24 @@ func WriteFileToContainer(
 
 	return nil
 }
+
+func FetchPathFromContainer(
+	pool *dockertest.Pool,
+	container *dockertest.Resource,
+	path string,
+) ([]byte, error) {
+	buf := bytes.NewBuffer([]byte{})
+
+	err := pool.Client.DownloadFromContainer(
+		container.Container.ID,
+		docker.DownloadFromContainerOptions{
+			OutputStream: buf,
+			Path:         path,
+		},
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	return buf.Bytes(), nil
+}

integration/scenario.go

@@ -149,15 +149,7 @@ func NewScenario() (*Scenario, error) {
 // environment running the tests.
 func (s *Scenario) Shutdown() error {
 	s.controlServers.Range(func(_ string, control ControlServer) bool {
-		err := control.SaveLog("/tmp/control")
+		err := control.Shutdown()
-		if err != nil {
-			log.Printf(
-				"Failed to save log from control: %s",
-				fmt.Errorf("failed to save log from control: %w", err),
-			)
-		}
-
-		err = control.Shutdown()
 		if err != nil {
 			log.Printf(
 				"Failed to shut down control: %s",