myriation/headscale
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity

✨ feat: add pqsql configs for open and idle connections (#1583)

Browse source 
When Postgres is used as the backing database for headscale,
it does not set a limit on maximum open and idle connections
which leads to hundreds of open connections to the Postgres
server.

This commit introduces the configuration variables to set those
values and also sets default while opening a new postgres connection.
This commit is contained in:
Pallab Pain 2024-02-09 22:04:28 +05:30 committed by GitHub
parent 91bb85e7d2
commit 9047c09871
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 287 additions and 242 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
CHANGELOG.md
View file

@ -46,6 +46,7 @@ after improving the test harness as part of adopting [#1460](https://github.com/
- Added the possibility to manually create a DERP-map entry which can be customized, instead of automatically creating it. [#1565](https://github.com/juanfont/headscale/pull/1565) - Added the possibility to manually create a DERP-map entry which can be customized, instead of automatically creating it. [#1565](https://github.com/juanfont/headscale/pull/1565)
- Change the structure of database configuration, see [config-example.yaml](./config-example.yaml) for the new structure. [#1700](https://github.com/juanfont/headscale/pull/1700) - Change the structure of database configuration, see [config-example.yaml](./config-example.yaml) for the new structure. [#1700](https://github.com/juanfont/headscale/pull/1700)
- Old structure is now considered deprecated and will be removed in the future. - Old structure is now considered deprecated and will be removed in the future.
- Adds additional configuration for PostgreSQL for setting max open, idle conection and idle connection lifetime.
## 0.22.3 (2023-05-12) ## 0.22.3 (2023-05-12)

3
config-example.yaml
View file

@ -153,6 +153,9 @@ database:
# name: headscale # name: headscale
# user: foo # user: foo
# pass: bar # pass: bar
# max_open_conns: 10
# max_idle_conns: 10
# conn_max_idle_time_secs: 3600
# # If other 'sslmode' is required instead of 'require(true)' and 'disabled(false)', set the 'sslmode' you need # # If other 'sslmode' is required instead of 'require(true)' and 'disabled(false)', set the 'sslmode' you need
# # in the 'db_ssl' field. Refers to https://www.postgresql.org/docs/current/libpq-ssl.html Table 34.1. # # in the 'db_ssl' field. Refers to https://www.postgresql.org/docs/current/libpq-ssl.html Table 34.1.

478
hscontrol/db/db.go
View file

@ -12,13 +12,14 @@ import (
"github.com/glebarez/sqlite" "github.com/glebarez/sqlite"
"github.com/go-gormigrate/gormigrate/v2" "github.com/go-gormigrate/gormigrate/v2"
"github.com/juanfont/headscale/hscontrol/notifier"
"github.com/juanfont/headscale/hscontrol/types"
"github.com/juanfont/headscale/hscontrol/util"
"github.com/rs/zerolog/log" "github.com/rs/zerolog/log"
"gorm.io/driver/postgres" "gorm.io/driver/postgres"
"gorm.io/gorm" "gorm.io/gorm"
"gorm.io/gorm/logger" "gorm.io/gorm/logger"
"github.com/juanfont/headscale/hscontrol/notifier"
"github.com/juanfont/headscale/hscontrol/types"
"github.com/juanfont/headscale/hscontrol/util"
) )
var errDatabaseNotSupported = errors.New("database type not supported") var errDatabaseNotSupported = errors.New("database type not supported")
@ -50,259 +51,273 @@ func NewHeadscaleDatabase(
return nil, err return nil, err
} }
migrations := gormigrate.New(dbConn, gormigrate.DefaultOptions, []*gormigrate.Migration{ migrations := gormigrate.New(
// New migrations should be added as transactions at the end of this list. dbConn,
// The initial commit here is quite messy, completely out of order and gormigrate.DefaultOptions,
// has no versioning and is the tech debt of not having versioned migrations []*gormigrate.Migration{
// prior to this point. This first migration is all DB changes to bring a DB // New migrations should be added as transactions at the end of this list.
// up to 0.23.0. // The initial commit here is quite messy, completely out of order and
{ // has no versioning and is the tech debt of not having versioned migrations
ID: "202312101416", // prior to this point. This first migration is all DB changes to bring a DB
Migrate: func(tx *gorm.DB) error { // up to 0.23.0.
if cfg.Type == types.DatabasePostgres { {
tx.Exec(`create extension if not exists "uuid-ossp";`) ID: "202312101416",
} Migrate: func(tx *gorm.DB) error {
if cfg.Type == types.DatabasePostgres {
_ = tx.Migrator().RenameTable("namespaces", "users") tx.Exec(`create extension if not exists "uuid-ossp";`)
// the big rename from Machine to Node
_ = tx.Migrator().RenameTable("machines", "nodes")
_ = tx.Migrator().RenameColumn(&types.Route{}, "machine_id", "node_id")
err = tx.AutoMigrate(types.User{})
if err != nil {
return err
}
_ = tx.Migrator().RenameColumn(&types.Node{}, "namespace_id", "user_id")
_ = tx.Migrator().RenameColumn(&types.PreAuthKey{}, "namespace_id", "user_id")
_ = tx.Migrator().RenameColumn(&types.Node{}, "ip_address", "ip_addresses")
_ = tx.Migrator().RenameColumn(&types.Node{}, "name", "hostname")
// GivenName is used as the primary source of DNS names, make sure
// the field is populated and normalized if it was not when the
// node was registered.
_ = tx.Migrator().RenameColumn(&types.Node{}, "nickname", "given_name")
// If the Node table has a column for registered,
// find all occourences of "false" and drop them. Then
// remove the column.
if tx.Migrator().HasColumn(&types.Node{}, "registered") {
log.Info().
Msg(`Database has legacy "registered" column in node, removing...`)
nodes := types.Nodes{}
if err := tx.Not("registered").Find(&nodes).Error; err != nil {
log.Error().Err(err).Msg("Error accessing db")
} }
for _, node := range nodes { _ = tx.Migrator().RenameTable("namespaces", "users")
log.Info().
Str("node", node.Hostname).
Str("machine_key", node.MachineKey.ShortString()).
Msg("Deleting unregistered node")
if err := tx.Delete(&types.Node{}, node.ID).Error; err != nil {
log.Error().
Err(err).
Str("node", node.Hostname).
Str("machine_key", node.MachineKey.ShortString()).
Msg("Error deleting unregistered node")
}
}
err := tx.Migrator().DropColumn(&types.Node{}, "registered") // the big rename from Machine to Node
if err != nil { _ = tx.Migrator().RenameTable("machines", "nodes")
log.Error().Err(err).Msg("Error dropping registered column") _ = tx.Migrator().
} RenameColumn(&types.Route{}, "machine_id", "node_id")
}
err = tx.AutoMigrate(&types.Route{}) err = tx.AutoMigrate(types.User{})
if err != nil {
return err
}
err = tx.AutoMigrate(&types.Node{})
if err != nil {
return err
}
// Ensure all keys have correct prefixes
// https://github.com/tailscale/tailscale/blob/main/types/key/node.go#L35
type result struct {
ID uint64
MachineKey string
NodeKey string
DiscoKey string
}
var results []result
err = tx.Raw("SELECT id, node_key, machine_key, disco_key FROM nodes").
Find(&results).
Error
if err != nil {
return err
}
for _, node := range results {
mKey := node.MachineKey
if !strings.HasPrefix(node.MachineKey, "mkey:") {
mKey = "mkey:" + node.MachineKey
}
nKey := node.NodeKey
if !strings.HasPrefix(node.NodeKey, "nodekey:") {
nKey = "nodekey:" + node.NodeKey
}
dKey := node.DiscoKey
if !strings.HasPrefix(node.DiscoKey, "discokey:") {
dKey = "discokey:" + node.DiscoKey
}
err := tx.Exec(
"UPDATE nodes SET machine_key = @mKey, node_key = @nKey, disco_key = @dKey WHERE ID = @id",
sql.Named("mKey", mKey),
sql.Named("nKey", nKey),
sql.Named("dKey", dKey),
sql.Named("id", node.ID),
).Error
if err != nil { if err != nil {
return err return err
} }
}
if tx.Migrator().HasColumn(&types.Node{}, "enabled_routes") { _ = tx.Migrator().
log.Info(). RenameColumn(&types.Node{}, "namespace_id", "user_id")
Msgf("Database has legacy enabled_routes column in node, migrating...") _ = tx.Migrator().
RenameColumn(&types.PreAuthKey{}, "namespace_id", "user_id")
type NodeAux struct { _ = tx.Migrator().
ID uint64 RenameColumn(&types.Node{}, "ip_address", "ip_addresses")
EnabledRoutes types.IPPrefixes _ = tx.Migrator().RenameColumn(&types.Node{}, "name", "hostname")
}
nodesAux := []NodeAux{} // GivenName is used as the primary source of DNS names, make sure
err := tx.Table("nodes").Select("id, enabled_routes").Scan(&nodesAux).Error // the field is populated and normalized if it was not when the
if err != nil { // node was registered.
log.Fatal().Err(err).Msg("Error accessing db") _ = tx.Migrator().
} RenameColumn(&types.Node{}, "nickname", "given_name")
for _, node := range nodesAux {
for _, prefix := range node.EnabledRoutes { // If the Node table has a column for registered,
if err != nil { // find all occourences of "false" and drop them. Then
// remove the column.
if tx.Migrator().HasColumn(&types.Node{}, "registered") {
log.Info().
Msg(`Database has legacy "registered" column in node, removing...`)
nodes := types.Nodes{}
if err := tx.Not("registered").Find(&nodes).Error; err != nil {
log.Error().Err(err).Msg("Error accessing db")
}
for _, node := range nodes {
log.Info().
Str("node", node.Hostname).
Str("machine_key", node.MachineKey.ShortString()).
Msg("Deleting unregistered node")
if err := tx.Delete(&types.Node{}, node.ID).Error; err != nil {
log.Error(). log.Error().
Err(err). Err(err).
Str("enabled_route", prefix.String()). Str("node", node.Hostname).
Msg("Error parsing enabled_route") Str("machine_key", node.MachineKey.ShortString()).
Msg("Error deleting unregistered node")
continue
} }
}
err = tx.Preload("Node"). err := tx.Migrator().DropColumn(&types.Node{}, "registered")
Where("node_id = ? AND prefix = ?", node.ID, types.IPPrefix(prefix)). if err != nil {
First(&types.Route{}). log.Error().Err(err).Msg("Error dropping registered column")
Error }
if err == nil { }
log.Info().
Str("enabled_route", prefix.String()).
Msg("Route already migrated to new table, skipping")
continue err = tx.AutoMigrate(&types.Route{})
if err != nil {
return err
}
err = tx.AutoMigrate(&types.Node{})
if err != nil {
return err
}
// Ensure all keys have correct prefixes
// https://github.com/tailscale/tailscale/blob/main/types/key/node.go#L35
type result struct {
ID uint64
MachineKey string
NodeKey string
DiscoKey string
}
var results []result
err = tx.Raw("SELECT id, node_key, machine_key, disco_key FROM nodes").
Find(&results).
Error
if err != nil {
return err
}
for _, node := range results {
mKey := node.MachineKey
if !strings.HasPrefix(node.MachineKey, "mkey:") {
mKey = "mkey:" + node.MachineKey
}
nKey := node.NodeKey
if !strings.HasPrefix(node.NodeKey, "nodekey:") {
nKey = "nodekey:" + node.NodeKey
}
dKey := node.DiscoKey
if !strings.HasPrefix(node.DiscoKey, "discokey:") {
dKey = "discokey:" + node.DiscoKey
}
err := tx.Exec(
"UPDATE nodes SET machine_key = @mKey, node_key = @nKey, disco_key = @dKey WHERE ID = @id",
sql.Named("mKey", mKey),
sql.Named("nKey", nKey),
sql.Named("dKey", dKey),
sql.Named("id", node.ID),
).Error
if err != nil {
return err
}
}
if tx.Migrator().HasColumn(&types.Node{}, "enabled_routes") {
log.Info().
Msgf("Database has legacy enabled_routes column in node, migrating...")
type NodeAux struct {
ID uint64
EnabledRoutes types.IPPrefixes
}
nodesAux := []NodeAux{}
err := tx.Table("nodes").
Select("id, enabled_routes").
Scan(&nodesAux).
Error
if err != nil {
log.Fatal().Err(err).Msg("Error accessing db")
}
for _, node := range nodesAux {
for _, prefix := range node.EnabledRoutes {
if err != nil {
log.Error().
Err(err).
Str("enabled_route", prefix.String()).
Msg("Error parsing enabled_route")
continue
}
err = tx.Preload("Node").
Where("node_id = ? AND prefix = ?", node.ID, types.IPPrefix(prefix)).
First(&types.Route{}).
Error
if err == nil {
log.Info().
Str("enabled_route", prefix.String()).
Msg("Route already migrated to new table, skipping")
continue
}
route := types.Route{
NodeID: node.ID,
Advertised: true,
Enabled: true,
Prefix: types.IPPrefix(prefix),
}
if err := tx.Create(&route).Error; err != nil {
log.Error().Err(err).Msg("Error creating route")
} else {
log.Info().
Uint64("node_id", route.NodeID).
Str("prefix", prefix.String()).
Msg("Route migrated")
}
} }
}
route := types.Route{ err = tx.Migrator().DropColumn(&types.Node{}, "enabled_routes")
NodeID: node.ID, if err != nil {
Advertised: true, log.Error().
Enabled: true, Err(err).
Prefix: types.IPPrefix(prefix), Msg("Error dropping enabled_routes column")
} }
if err := tx.Create(&route).Error; err != nil { }
log.Error().Err(err).Msg("Error creating route")
} else { if tx.Migrator().HasColumn(&types.Node{}, "given_name") {
log.Info(). nodes := types.Nodes{}
Uint64("node_id", route.NodeID). if err := tx.Find(&nodes).Error; err != nil {
Str("prefix", prefix.String()). log.Error().Err(err).Msg("Error accessing db")
Msg("Route migrated") }
for item, node := range nodes {
if node.GivenName == "" {
normalizedHostname, err := util.NormalizeToFQDNRulesConfigFromViper(
node.Hostname,
)
if err != nil {
log.Error().
Caller().
Str("hostname", node.Hostname).
Err(err).
Msg("Failed to normalize node hostname in DB migration")
}
err = tx.Model(nodes[item]).Updates(types.Node{
GivenName: normalizedHostname,
}).Error
if err != nil {
log.Error().
Caller().
Str("hostname", node.Hostname).
Err(err).
Msg("Failed to save normalized node name in DB migration")
}
} }
} }
} }
err = tx.Migrator().DropColumn(&types.Node{}, "enabled_routes") err = tx.AutoMigrate(&KV{})
if err != nil { if err != nil {
log.Error().Err(err).Msg("Error dropping enabled_routes column") return err
}
}
if tx.Migrator().HasColumn(&types.Node{}, "given_name") {
nodes := types.Nodes{}
if err := tx.Find(&nodes).Error; err != nil {
log.Error().Err(err).Msg("Error accessing db")
} }
for item, node := range nodes { err = tx.AutoMigrate(&types.PreAuthKey{})
if node.GivenName == "" { if err != nil {
normalizedHostname, err := util.NormalizeToFQDNRulesConfigFromViper( return err
node.Hostname,
)
if err != nil {
log.Error().
Caller().
Str("hostname", node.Hostname).
Err(err).
Msg("Failed to normalize node hostname in DB migration")
}
err = tx.Model(nodes[item]).Updates(types.Node{
GivenName: normalizedHostname,
}).Error
if err != nil {
log.Error().
Caller().
Str("hostname", node.Hostname).
Err(err).
Msg("Failed to save normalized node name in DB migration")
}
}
} }
}
err = tx.AutoMigrate(&KV{}) err = tx.AutoMigrate(&types.PreAuthKeyACLTag{})
if err != nil { if err != nil {
return err return err
} }
err = tx.AutoMigrate(&types.PreAuthKey{}) _ = tx.Migrator().DropTable("shared_machines")
if err != nil {
return err
}
err = tx.AutoMigrate(&types.PreAuthKeyACLTag{}) err = tx.AutoMigrate(&types.APIKey{})
if err != nil { if err != nil {
return err return err
} }
_ = tx.Migrator().DropTable("shared_machines") return nil
},
err = tx.AutoMigrate(&types.APIKey{}) Rollback: func(tx *gorm.DB) error {
if err != nil { return nil
return err },
}
return nil
}, },
Rollback: func(tx *gorm.DB) error { {
return nil // drop key-value table, it is not used, and has not contained
// useful data for a long time or ever.
ID: "202312101430",
Migrate: func(tx *gorm.DB) error {
return tx.Migrator().DropTable("kvs")
},
Rollback: func(tx *gorm.DB) error {
return nil
},
}, },
}, },
{ )
// drop key-value table, it is not used, and has not contained
// useful data for a long time or ever.
ID: "202312101430",
Migrate: func(tx *gorm.DB) error {
return tx.Migrator().DropTable("kvs")
},
Rollback: func(tx *gorm.DB) error {
return nil
},
},
})
if err = migrations.Migrate(); err != nil { if err = migrations.Migrate(); err != nil {
log.Fatal().Err(err).Msgf("Migration failed: %v", err) log.Fatal().Err(err).Msgf("Migration failed: %v", err)
@ -319,7 +334,6 @@ func NewHeadscaleDatabase(
} }
func openDB(cfg types.DatabaseConfig) (*gorm.DB, error) { func openDB(cfg types.DatabaseConfig) (*gorm.DB, error) {
// TODO(kradalby): Integrate this with zerolog // TODO(kradalby): Integrate this with zerolog
var dbLogger logger.Interface var dbLogger logger.Interface
if cfg.Debug { if cfg.Debug {
@ -374,10 +388,22 @@ func openDB(cfg types.DatabaseConfig) (*gorm.DB, error) {
dbString += fmt.Sprintf(" password=%s", cfg.Postgres.Pass) dbString += fmt.Sprintf(" password=%s", cfg.Postgres.Pass)
} }
return gorm.Open(postgres.Open(dbString), &gorm.Config{ db, err := gorm.Open(postgres.Open(dbString), &gorm.Config{
DisableForeignKeyConstraintWhenMigrating: true, DisableForeignKeyConstraintWhenMigrating: true,
Logger: dbLogger, Logger: dbLogger,
}) })
if err != nil {
return nil, err
}
sqlDB, _ := db.DB()
sqlDB.SetMaxIdleConns(cfg.Postgres.MaxIdleConnections)
sqlDB.SetMaxOpenConns(cfg.Postgres.MaxOpenConnections)
sqlDB.SetConnMaxIdleTime(
time.Duration(cfg.Postgres.ConnMaxIdleTimeSecs) * time.Second,
)
return db, nil
} }
return nil, fmt.Errorf( return nil, fmt.Errorf(

47
hscontrol/types/config.go
View file

@ -11,7 +11,6 @@ import (
"time" "time"
"github.com/coreos/go-oidc/v3/oidc" "github.com/coreos/go-oidc/v3/oidc"
"github.com/juanfont/headscale/hscontrol/util"
"github.com/prometheus/common/model" "github.com/prometheus/common/model"
"github.com/rs/zerolog" "github.com/rs/zerolog"
"github.com/rs/zerolog/log" "github.com/rs/zerolog/log"
@ -20,6 +19,8 @@ import (
"tailscale.com/net/tsaddr" "tailscale.com/net/tsaddr"
"tailscale.com/tailcfg" "tailscale.com/tailcfg"
"tailscale.com/types/dnstype" "tailscale.com/types/dnstype"
"github.com/juanfont/headscale/hscontrol/util"
) )
const ( const (
@ -75,12 +76,15 @@ type SqliteConfig struct {
} }
type PostgresConfig struct { type PostgresConfig struct {
Host string Host string
Port int Port int
Name string Name string
User string User string
Pass string Pass string
Ssl string Ssl string
MaxOpenConnections int
MaxIdleConnections int
ConnMaxIdleTimeSecs int
} }
type DatabaseConfig struct { type DatabaseConfig struct {
@ -213,6 +217,9 @@ func LoadConfig(path string, isFile bool) error {
viper.SetDefault("db_ssl", false) viper.SetDefault("db_ssl", false)
viper.SetDefault("database.postgres.ssl", false) viper.SetDefault("database.postgres.ssl", false)
viper.SetDefault("database.postgres.max_open_conns", 10)
viper.SetDefault("database.postgres.max_idle_conns", 10)
viper.SetDefault("database.postgres.conn_max_idle_time_secs", 3600)
viper.SetDefault("oidc.scope", []string{oidc.ScopeOpenID, "profile", "email"}) viper.SetDefault("oidc.scope", []string{oidc.ScopeOpenID, "profile", "email"})
viper.SetDefault("oidc.strip_email_domain", true) viper.SetDefault("oidc.strip_email_domain", true)
@ -287,7 +294,7 @@ func LoadConfig(path string, isFile bool) error {
} }
if errorText != "" { if errorText != "" {
//nolint // nolint
return errors.New(strings.TrimSuffix(errorText, "\n")) return errors.New(strings.TrimSuffix(errorText, "\n"))
} else { } else {
return nil return nil
@ -429,22 +436,30 @@ func GetDatabaseConfig() DatabaseConfig {
case "sqlite": case "sqlite":
type_ = "sqlite3" type_ = "sqlite3"
default: default:
log.Fatal().Msgf("invalid database type %q, must be sqlite, sqlite3 or postgres", type_) log.Fatal().
Msgf("invalid database type %q, must be sqlite, sqlite3 or postgres", type_)
} }
return DatabaseConfig{ return DatabaseConfig{
Type: type_, Type: type_,
Debug: debug, Debug: debug,
Sqlite: SqliteConfig{ Sqlite: SqliteConfig{
Path: util.AbsolutePathFromConfigPath(viper.GetString("database.sqlite.path")), Path: util.AbsolutePathFromConfigPath(
viper.GetString("database.sqlite.path"),
),
}, },
Postgres: PostgresConfig{ Postgres: PostgresConfig{
Host: viper.GetString("database.postgres.host"), Host: viper.GetString("database.postgres.host"),
Port: viper.GetInt("database.postgres.port"), Port: viper.GetInt("database.postgres.port"),
Name: viper.GetString("database.postgres.name"), Name: viper.GetString("database.postgres.name"),
User: viper.GetString("database.postgres.user"), User: viper.GetString("database.postgres.user"),
Pass: viper.GetString("database.postgres.pass"), Pass: viper.GetString("database.postgres.pass"),
Ssl: viper.GetString("database.postgres.ssl"), Ssl: viper.GetString("database.postgres.ssl"),
MaxOpenConnections: viper.GetInt("database.postgres.max_open_conns"),
MaxIdleConnections: viper.GetInt("database.postgres.max_idle_conns"),
ConnMaxIdleTimeSecs: viper.GetInt(
"database.postgres.conn_max_idle_time_secs",
),
}, },
} }
} }