From 0b4f59b82b158afc3533fa16b4660eb48f99781d Mon Sep 17 00:00:00 2001 From: Kristoffer Dalby Date: Tue, 31 May 2022 10:57:20 +0200 Subject: [PATCH 1/9] Improve signal handling This commit starts to wire up better signal handling, it starts with handling shutdown a bit better, using the graceful shutdown for all the listeners we use. It also adds the initial switch case for handling config and acl reload, which is to be implemented. --- app.go | 55 ++++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 42 insertions(+), 13 deletions(-) diff --git a/app.go b/app.go index a78a6b0..cc4728d 100644 --- a/app.go +++ b/app.go @@ -568,19 +568,6 @@ func (h *Headscale) Serve() error { return fmt.Errorf("failed change permission of gRPC socket: %w", err) } - // Handle common process-killing signals so we can gracefully shut down: - sigc := make(chan os.Signal, 1) - signal.Notify(sigc, os.Interrupt, syscall.SIGTERM) - go func(c chan os.Signal) { - // Wait for a SIGINT or SIGKILL: - sig := <-c - log.Printf("Caught signal %s: shutting down.", sig) - // Stop listening (and unlink the socket if unix type): - socketListener.Close() - // And we're done: - os.Exit(0) - }(sigc) - grpcGatewayMux := runtime.NewServeMux() // Make the grpc-gateway connect to grpc over socket @@ -725,6 +712,48 @@ func (h *Headscale) Serve() error { log.Info(). Msgf("listening and serving metrics on: %s", h.cfg.MetricsAddr) + // Handle common process-killing signals so we can gracefully shut down: + sigc := make(chan os.Signal, 1) + signal.Notify(sigc, + syscall.SIGHUP, + syscall.SIGINT, + syscall.SIGTERM, + syscall.SIGQUIT, + syscall.SIGHUP) + go func(c chan os.Signal) { + // Wait for a SIGINT or SIGKILL: + sig := <-c + switch sig { + case syscall.SIGHUP: + log.Info(). + Str("signal", sig.String()). + Msg("Received SIGHUP, reloading ACL and Config") + + // TODO(kradalby): Reload config on SIGHUP + + default: + log.Info(). + Str("signal", sig.String()). + Msg("Received signal to stop, shutting down gracefully") + + // Gracefully shut down servers + promHTTPServer.Shutdown(ctx) + httpServer.Shutdown(ctx) + grpcSocket.GracefulStop() + + // Close network listeners + promHTTPListener.Close() + httpListener.Close() + grpcGatewayConn.Close() + + // Stop listening (and unlink the socket if unix type): + socketListener.Close() + + // And we're done: + os.Exit(0) + } + }(sigc) + return errorGroup.Wait() } From 3e078f04946f0bf58471727c78d9bffa03352c99 Mon Sep 17 00:00:00 2001 From: Kristoffer Dalby Date: Tue, 31 May 2022 12:49:13 +0200 Subject: [PATCH 2/9] Fix logtail config function name --- cmd/headscale/cli/utils.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/headscale/cli/utils.go b/cmd/headscale/cli/utils.go index b1a5d4f..e47677c 100644 --- a/cmd/headscale/cli/utils.go +++ b/cmd/headscale/cli/utils.go @@ -169,7 +169,7 @@ func GetDERPConfig() headscale.DERPConfig { } } -func GetLogConfig() headscale.LogTailConfig { +func GetLogTailConfig() headscale.LogTailConfig { enabled := viper.GetBool("logtail.enabled") return headscale.LogTailConfig{ @@ -280,7 +280,7 @@ func absPath(path string) string { func getHeadscaleConfig() headscale.Config { dnsConfig, baseDomain := GetDNSConfig() derpConfig := GetDERPConfig() - logConfig := GetLogConfig() + logConfig := GetLogTailConfig() configuredPrefixes := viper.GetStringSlice("ip_prefixes") parsedPrefixes := make([]netaddr.IPPrefix, 0, len(configuredPrefixes)+1) From 5bfae22c8f52019e530c43b9d7bfe4f64ed3b9ef Mon Sep 17 00:00:00 2001 From: Kristoffer Dalby Date: Tue, 31 May 2022 12:49:41 +0200 Subject: [PATCH 3/9] Make config get function global --- cmd/headscale/cli/utils.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/headscale/cli/utils.go b/cmd/headscale/cli/utils.go index e47677c..797fba1 100644 --- a/cmd/headscale/cli/utils.go +++ b/cmd/headscale/cli/utils.go @@ -277,7 +277,7 @@ func absPath(path string) string { return path } -func getHeadscaleConfig() headscale.Config { +func GetHeadscaleConfig() headscale.Config { dnsConfig, baseDomain := GetDNSConfig() derpConfig := GetDERPConfig() logConfig := GetLogTailConfig() @@ -416,7 +416,7 @@ func getHeadscaleApp() (*headscale.Headscale, error) { return nil, err } - cfg := getHeadscaleConfig() + cfg := GetHeadscaleConfig() app, err := headscale.NewHeadscale(cfg) if err != nil { @@ -440,7 +440,7 @@ func getHeadscaleApp() (*headscale.Headscale, error) { } func getHeadscaleCLIClient() (context.Context, v1.HeadscaleServiceClient, *grpc.ClientConn, context.CancelFunc) { - cfg := getHeadscaleConfig() + cfg := GetHeadscaleConfig() log.Debug(). Dur("timeout", cfg.CLI.Timeout). From 24e4787a64d05e93c0028b6055c2fb77791d11f1 Mon Sep 17 00:00:00 2001 From: Kristoffer Dalby Date: Tue, 31 May 2022 12:50:16 +0200 Subject: [PATCH 4/9] Make ACL policy part of the config struct --- app.go | 6 ++++++ cmd/headscale/cli/utils.go | 10 ++++++++++ 2 files changed, 16 insertions(+) diff --git a/app.go b/app.go index cc4728d..cc867a4 100644 --- a/app.go +++ b/app.go @@ -116,6 +116,8 @@ type Config struct { LogTail LogTailConfig CLI CLIConfig + + ACL ACLConfig } type OIDCConfig struct { @@ -152,6 +154,10 @@ type CLIConfig struct { Insecure bool } +type ACLConfig struct { + PolicyPath string +} + // Headscale represents the base app of the service. type Headscale struct { cfg Config diff --git a/cmd/headscale/cli/utils.go b/cmd/headscale/cli/utils.go index 797fba1..2c0eb0a 100644 --- a/cmd/headscale/cli/utils.go +++ b/cmd/headscale/cli/utils.go @@ -177,6 +177,14 @@ func GetLogTailConfig() headscale.LogTailConfig { } } +func GetACLConfig() headscale.ACLConfig { + policyPath := viper.GetString("acl_policy_path") + + return headscale.ACLConfig{ + PolicyPath: policyPath, + } +} + func GetDNSConfig() (*tailcfg.DNSConfig, string) { if viper.IsSet("dns_config") { dnsConfig := &tailcfg.DNSConfig{} @@ -397,6 +405,8 @@ func GetHeadscaleConfig() headscale.Config { Timeout: viper.GetDuration("cli.timeout"), Insecure: viper.GetBool("cli.insecure"), }, + + ACL: GetACLConfig(), } } From 6b1482daee03482c90ac91392efbcc0a68f4b97b Mon Sep 17 00:00:00 2001 From: Kristoffer Dalby Date: Tue, 31 May 2022 12:50:43 +0200 Subject: [PATCH 5/9] Use config object instead of viper for policy path --- cmd/headscale/cli/utils.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/headscale/cli/utils.go b/cmd/headscale/cli/utils.go index 2c0eb0a..154b5ac 100644 --- a/cmd/headscale/cli/utils.go +++ b/cmd/headscale/cli/utils.go @@ -435,8 +435,8 @@ func getHeadscaleApp() (*headscale.Headscale, error) { // We are doing this here, as in the future could be cool to have it also hot-reload - if viper.GetString("acl_policy_path") != "" { - aclPath := absPath(viper.GetString("acl_policy_path")) + if cfg.ACL.PolicyPath != "" { + aclPath := absPath(cfg.ACL.PolicyPath) err = app.LoadACLPolicy(aclPath) if err != nil { log.Fatal(). From 06129277ed2e45a101fb5223e1e8152ef3165e0c Mon Sep 17 00:00:00 2001 From: Kristoffer Dalby Date: Tue, 31 May 2022 12:52:06 +0200 Subject: [PATCH 6/9] Rename abspath function to describe what it does --- cmd/headscale/cli/utils.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/cmd/headscale/cli/utils.go b/cmd/headscale/cli/utils.go index 154b5ac..d8d0987 100644 --- a/cmd/headscale/cli/utils.go +++ b/cmd/headscale/cli/utils.go @@ -272,7 +272,7 @@ func GetDNSConfig() (*tailcfg.DNSConfig, string) { return nil, "" } -func absPath(path string) string { +func AbsolutePathFromConfigPath(path string) string { // If a relative path is provided, prefix it with the the directory where // the config file was found. if (path != "") && !strings.HasPrefix(path, string(os.PathSeparator)) { @@ -350,7 +350,7 @@ func GetHeadscaleConfig() headscale.Config { GRPCAllowInsecure: viper.GetBool("grpc_allow_insecure"), IPPrefixes: prefixes, - PrivateKeyPath: absPath(viper.GetString("private_key_path")), + PrivateKeyPath: AbsolutePathFromConfigPath(viper.GetString("private_key_path")), BaseDomain: baseDomain, DERP: derpConfig, @@ -360,7 +360,7 @@ func GetHeadscaleConfig() headscale.Config { ), DBtype: viper.GetString("db_type"), - DBpath: absPath(viper.GetString("db_path")), + DBpath: AbsolutePathFromConfigPath(viper.GetString("db_path")), DBhost: viper.GetString("db_host"), DBport: viper.GetInt("db_port"), DBname: viper.GetString("db_name"), @@ -369,13 +369,13 @@ func GetHeadscaleConfig() headscale.Config { TLSLetsEncryptHostname: viper.GetString("tls_letsencrypt_hostname"), TLSLetsEncryptListen: viper.GetString("tls_letsencrypt_listen"), - TLSLetsEncryptCacheDir: absPath( + TLSLetsEncryptCacheDir: AbsolutePathFromConfigPath( viper.GetString("tls_letsencrypt_cache_dir"), ), TLSLetsEncryptChallengeType: viper.GetString("tls_letsencrypt_challenge_type"), - TLSCertPath: absPath(viper.GetString("tls_cert_path")), - TLSKeyPath: absPath(viper.GetString("tls_key_path")), + TLSCertPath: AbsolutePathFromConfigPath(viper.GetString("tls_cert_path")), + TLSKeyPath: AbsolutePathFromConfigPath(viper.GetString("tls_key_path")), TLSClientAuthMode: tlsClientAuthMode, DNSConfig: dnsConfig, @@ -436,7 +436,7 @@ func getHeadscaleApp() (*headscale.Headscale, error) { // We are doing this here, as in the future could be cool to have it also hot-reload if cfg.ACL.PolicyPath != "" { - aclPath := absPath(cfg.ACL.PolicyPath) + aclPath := AbsolutePathFromConfigPath(cfg.ACL.PolicyPath) err = app.LoadACLPolicy(aclPath) if err != nil { log.Fatal(). From 36dca3516a36e29bdb17a408c7d730728d8c0c6a Mon Sep 17 00:00:00 2001 From: Kristoffer Dalby Date: Tue, 31 May 2022 12:57:48 +0200 Subject: [PATCH 7/9] Move Abspath function to headscale utils --- cmd/headscale/cli/utils.go | 26 ++++++-------------------- utils.go | 16 ++++++++++++++++ 2 files changed, 22 insertions(+), 20 deletions(-) diff --git a/cmd/headscale/cli/utils.go b/cmd/headscale/cli/utils.go index d8d0987..af4391a 100644 --- a/cmd/headscale/cli/utils.go +++ b/cmd/headscale/cli/utils.go @@ -9,7 +9,6 @@ import ( "io/fs" "net/url" "os" - "path/filepath" "reflect" "strconv" "strings" @@ -272,19 +271,6 @@ func GetDNSConfig() (*tailcfg.DNSConfig, string) { return nil, "" } -func AbsolutePathFromConfigPath(path string) string { - // If a relative path is provided, prefix it with the the directory where - // the config file was found. - if (path != "") && !strings.HasPrefix(path, string(os.PathSeparator)) { - dir, _ := filepath.Split(viper.ConfigFileUsed()) - if dir != "" { - path = filepath.Join(dir, path) - } - } - - return path -} - func GetHeadscaleConfig() headscale.Config { dnsConfig, baseDomain := GetDNSConfig() derpConfig := GetDERPConfig() @@ -350,7 +336,7 @@ func GetHeadscaleConfig() headscale.Config { GRPCAllowInsecure: viper.GetBool("grpc_allow_insecure"), IPPrefixes: prefixes, - PrivateKeyPath: AbsolutePathFromConfigPath(viper.GetString("private_key_path")), + PrivateKeyPath: headscale.AbsolutePathFromConfigPath(viper.GetString("private_key_path")), BaseDomain: baseDomain, DERP: derpConfig, @@ -360,7 +346,7 @@ func GetHeadscaleConfig() headscale.Config { ), DBtype: viper.GetString("db_type"), - DBpath: AbsolutePathFromConfigPath(viper.GetString("db_path")), + DBpath: headscale.AbsolutePathFromConfigPath(viper.GetString("db_path")), DBhost: viper.GetString("db_host"), DBport: viper.GetInt("db_port"), DBname: viper.GetString("db_name"), @@ -369,13 +355,13 @@ func GetHeadscaleConfig() headscale.Config { TLSLetsEncryptHostname: viper.GetString("tls_letsencrypt_hostname"), TLSLetsEncryptListen: viper.GetString("tls_letsencrypt_listen"), - TLSLetsEncryptCacheDir: AbsolutePathFromConfigPath( + TLSLetsEncryptCacheDir: headscale.AbsolutePathFromConfigPath( viper.GetString("tls_letsencrypt_cache_dir"), ), TLSLetsEncryptChallengeType: viper.GetString("tls_letsencrypt_challenge_type"), - TLSCertPath: AbsolutePathFromConfigPath(viper.GetString("tls_cert_path")), - TLSKeyPath: AbsolutePathFromConfigPath(viper.GetString("tls_key_path")), + TLSCertPath: headscale.AbsolutePathFromConfigPath(viper.GetString("tls_cert_path")), + TLSKeyPath: headscale.AbsolutePathFromConfigPath(viper.GetString("tls_key_path")), TLSClientAuthMode: tlsClientAuthMode, DNSConfig: dnsConfig, @@ -436,7 +422,7 @@ func getHeadscaleApp() (*headscale.Headscale, error) { // We are doing this here, as in the future could be cool to have it also hot-reload if cfg.ACL.PolicyPath != "" { - aclPath := AbsolutePathFromConfigPath(cfg.ACL.PolicyPath) + aclPath := headscale.AbsolutePathFromConfigPath(cfg.ACL.PolicyPath) err = app.LoadACLPolicy(aclPath) if err != nil { log.Fatal(). diff --git a/utils.go b/utils.go index e7a7a73..6dddf4c 100644 --- a/utils.go +++ b/utils.go @@ -12,10 +12,13 @@ import ( "encoding/json" "fmt" "net" + "os" + "path/filepath" "reflect" "strings" "github.com/rs/zerolog/log" + "github.com/spf13/viper" "inet.af/netaddr" "tailscale.com/tailcfg" "tailscale.com/types/key" @@ -334,3 +337,16 @@ func IsStringInSlice(slice []string, str string) bool { return false } + +func AbsolutePathFromConfigPath(path string) string { + // If a relative path is provided, prefix it with the the directory where + // the config file was found. + if (path != "") && !strings.HasPrefix(path, string(os.PathSeparator)) { + dir, _ := filepath.Split(viper.ConfigFileUsed()) + if dir != "" { + path = filepath.Join(dir, path) + } + } + + return path +} From 2feed18b2838719bcd730b3dc8c2fb02831c424e Mon Sep 17 00:00:00 2001 From: Kristoffer Dalby Date: Tue, 31 May 2022 13:02:23 +0200 Subject: [PATCH 8/9] Support reloading ACLs with SIGHUP Also continously listen for signals, not just once. --- app.go | 59 +++++++++++++++++++++++++++++++++++----------------------- 1 file changed, 36 insertions(+), 23 deletions(-) diff --git a/app.go b/app.go index cc867a4..054fd17 100644 --- a/app.go +++ b/app.go @@ -728,35 +728,48 @@ func (h *Headscale) Serve() error { syscall.SIGHUP) go func(c chan os.Signal) { // Wait for a SIGINT or SIGKILL: - sig := <-c - switch sig { - case syscall.SIGHUP: - log.Info(). - Str("signal", sig.String()). - Msg("Received SIGHUP, reloading ACL and Config") + for { + sig := <-c + switch sig { + case syscall.SIGHUP: + log.Info(). + Str("signal", sig.String()). + Msg("Received SIGHUP, reloading ACL and Config") - // TODO(kradalby): Reload config on SIGHUP + // TODO(kradalby): Reload config on SIGHUP - default: - log.Info(). - Str("signal", sig.String()). - Msg("Received signal to stop, shutting down gracefully") + if h.cfg.ACL.PolicyPath != "" { + aclPath := AbsolutePathFromConfigPath(h.cfg.ACL.PolicyPath) + err := h.LoadACLPolicy(aclPath) + if err != nil { + log.Error().Err(err).Msg("Failed to reload ACL policy") + } + log.Info(). + Str("path", aclPath). + Msg("ACL policy successfully reloaded") + } - // Gracefully shut down servers - promHTTPServer.Shutdown(ctx) - httpServer.Shutdown(ctx) - grpcSocket.GracefulStop() + default: + log.Info(). + Str("signal", sig.String()). + Msg("Received signal to stop, shutting down gracefully") - // Close network listeners - promHTTPListener.Close() - httpListener.Close() - grpcGatewayConn.Close() + // Gracefully shut down servers + promHTTPServer.Shutdown(ctx) + httpServer.Shutdown(ctx) + grpcSocket.GracefulStop() - // Stop listening (and unlink the socket if unix type): - socketListener.Close() + // Close network listeners + promHTTPListener.Close() + httpListener.Close() + grpcGatewayConn.Close() - // And we're done: - os.Exit(0) + // Stop listening (and unlink the socket if unix type): + socketListener.Close() + + // And we're done: + os.Exit(0) + } } }(sigc) From 6f32b80b2b910dc59bb65846a4a73ef5cba43226 Mon Sep 17 00:00:00 2001 From: Kristoffer Dalby Date: Tue, 31 May 2022 14:30:11 +0200 Subject: [PATCH 9/9] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ba6dc30..bd93f70 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ - Add option to enable/disable logtail (Tailscale's logging infrastructure) [#596](https://github.com/juanfont/headscale/pull/596) - This change disables the logs by default - Use [Prometheus]'s duration parser, supporting days (`d`), weeks (`w`) and years (`y`) [#598](https://github.com/juanfont/headscale/pull/598) +- Add support for reloading ACLs with SIGHUP [#601](https://github.com/juanfont/headscale/pull/601) ## 0.15.0 (2022-03-20)