Restructure database config (#1700)

CHANGELOG.md

@@ -34,16 +34,18 @@ after improving the test harness as part of adopting [#1460](https://github.com/
 
 ### Changes
 
-Use versioned migrations [#1644](https://github.com/juanfont/headscale/pull/1644)
-Make the OIDC callback page better [#1484](https://github.com/juanfont/headscale/pull/1484)
-SSH support [#1487](https://github.com/juanfont/headscale/pull/1487)
-State management has been improved [#1492](https://github.com/juanfont/headscale/pull/1492)
-Use error group handling to ensure tests actually pass [#1535](https://github.com/juanfont/headscale/pull/1535) based on [#1460](https://github.com/juanfont/headscale/pull/1460)
-Fix hang on SIGTERM [#1492](https://github.com/juanfont/headscale/pull/1492) taken from [#1480](https://github.com/juanfont/headscale/pull/1480)
-Send logs to stderr by default [#1524](https://github.com/juanfont/headscale/pull/1524)
-Fix [TS-2023-006](https://tailscale.com/security-bulletins/#ts-2023-006) security UPnP issue [#1563](https://github.com/juanfont/headscale/pull/1563)
-Turn off gRPC logging [#1640](https://github.com/juanfont/headscale/pull/1640) fixes [#1259](https://github.com/juanfont/headscale/issues/1259)
-Added the possibility to manually create a DERP-map entry which can be customized, instead of automatically creating it. [#1565](https://github.com/juanfont/headscale/pull/1565)
+- Use versioned migrations [#1644](https://github.com/juanfont/headscale/pull/1644)
+- Make the OIDC callback page better [#1484](https://github.com/juanfont/headscale/pull/1484)
+- SSH support [#1487](https://github.com/juanfont/headscale/pull/1487)
+- State management has been improved [#1492](https://github.com/juanfont/headscale/pull/1492)
+- Use error group handling to ensure tests actually pass [#1535](https://github.com/juanfont/headscale/pull/1535) based on [#1460](https://github.com/juanfont/headscale/pull/1460)
+- Fix hang on SIGTERM [#1492](https://github.com/juanfont/headscale/pull/1492) taken from [#1480](https://github.com/juanfont/headscale/pull/1480)
+- Send logs to stderr by default [#1524](https://github.com/juanfont/headscale/pull/1524)
+- Fix [TS-2023-006](https://tailscale.com/security-bulletins/#ts-2023-006) security UPnP issue [#1563](https://github.com/juanfont/headscale/pull/1563)
+- Turn off gRPC logging [#1640](https://github.com/juanfont/headscale/pull/1640) fixes [#1259](https://github.com/juanfont/headscale/issues/1259)
+- Added the possibility to manually create a DERP-map entry which can be customized, instead of automatically creating it. [#1565](https://github.com/juanfont/headscale/pull/1565)
+- Change the structure of database configuration, see [config-example.yaml](./config-example.yaml) for the new structure. [#1700](https://github.com/juanfont/headscale/pull/1700)
+  - Old structure is now considered deprecated and will be removed in the future.
 
 ## 0.22.3 (2023-05-12)
 

cmd/headscale/headscale_test.go

@@ -58,8 +58,10 @@ func (*Suite) TestConfigFileLoading(c *check.C) {
 	c.Assert(viper.GetString("server_url"), check.Equals, "http://127.0.0.1:8080")
 	c.Assert(viper.GetString("listen_addr"), check.Equals, "127.0.0.1:8080")
 	c.Assert(viper.GetString("metrics_listen_addr"), check.Equals, "127.0.0.1:9090")
-	c.Assert(viper.GetString("db_type"), check.Equals, "sqlite3")
+	c.Assert(viper.GetString("db_type"), check.Equals, "sqlite")
 	c.Assert(viper.GetString("db_path"), check.Equals, "/var/lib/headscale/db.sqlite")
+	c.Assert(viper.GetString("database.type"), check.Equals, "sqlite")
+	c.Assert(viper.GetString("database.sqlite.path"), check.Equals, "/var/lib/headscale/db.sqlite")
 	c.Assert(viper.GetString("tls_letsencrypt_hostname"), check.Equals, "")
 	c.Assert(viper.GetString("tls_letsencrypt_listen"), check.Equals, ":http")
 	c.Assert(viper.GetString("tls_letsencrypt_challenge_type"), check.Equals, "HTTP-01")
@@ -101,7 +103,7 @@ func (*Suite) TestConfigLoading(c *check.C) {
 	c.Assert(viper.GetString("server_url"), check.Equals, "http://127.0.0.1:8080")
 	c.Assert(viper.GetString("listen_addr"), check.Equals, "127.0.0.1:8080")
 	c.Assert(viper.GetString("metrics_listen_addr"), check.Equals, "127.0.0.1:9090")
-	c.Assert(viper.GetString("db_type"), check.Equals, "sqlite3")
+	c.Assert(viper.GetString("db_type"), check.Equals, "sqlite")
 	c.Assert(viper.GetString("db_path"), check.Equals, "/var/lib/headscale/db.sqlite")
 	c.Assert(viper.GetString("tls_letsencrypt_hostname"), check.Equals, "")
 	c.Assert(viper.GetString("tls_letsencrypt_listen"), check.Equals, ":http")

config-example.yaml

@@ -138,24 +138,25 @@ ephemeral_node_inactivity_timeout: 30m
 # In case of doubts, do not touch the default 10s.
 node_update_check_interval: 10s
 
-# SQLite config
-db_type: sqlite3
-
-# For production:
-db_path: /var/lib/headscale/db.sqlite
-
-# # Postgres config
-# If using a Unix socket to connect to Postgres, set the socket path in the 'host' field and leave 'port' blank.
-# db_type: postgres
-# db_host: localhost
-# db_port: 5432
-# db_name: headscale
-# db_user: foo
-# db_pass: bar
-
-# If other 'sslmode' is required instead of 'require(true)' and 'disabled(false)', set the 'sslmode' you need
-# in the 'db_ssl' field. Refers to https://www.postgresql.org/docs/current/libpq-ssl.html Table 34.1.
-# db_ssl: false
+database:
+  type: sqlite
+
+  # SQLite config
+  sqlite:
+    path: /var/lib/headscale/db.sqlite
+
+  # # Postgres config
+  # postgres:
+  #   # If using a Unix socket to connect to Postgres, set the socket path in the 'host' field and leave 'port' blank.
+  #   host: localhost
+  #   port: 5432
+  #   name: headscale
+  #   user: foo
+  #   pass: bar
+
+  #   # If other 'sslmode' is required instead of 'require(true)' and 'disabled(false)', set the 'sslmode' you need
+  #   # in the 'db_ssl' field. Refers to https://www.postgresql.org/docs/current/libpq-ssl.html Table 34.1.
+  #   ssl: false
 
 ### TLS configuration
 #

hscontrol/app.go

@@ -12,7 +12,6 @@ import (
 	"os"
 	"os/signal"
 	"runtime"
-	"strconv"
 	"strings"
 	"sync"
 	"syscall"
@@ -118,56 +117,22 @@ func NewHeadscale(cfg *types.Config) (*Headscale, error) {
 		return nil, fmt.Errorf("failed to read or create Noise protocol private key: %w", err)
 	}
 
-	var dbString string
-	switch cfg.DBtype {
-	case db.Postgres:
-		dbString = fmt.Sprintf(
-			"host=%s dbname=%s user=%s",
-			cfg.DBhost,
-			cfg.DBname,
-			cfg.DBuser,
-		)
-
-		if sslEnabled, err := strconv.ParseBool(cfg.DBssl); err == nil {
-			if !sslEnabled {
-				dbString += " sslmode=disable"
-			}
-		} else {
-			dbString += fmt.Sprintf(" sslmode=%s", cfg.DBssl)
-		}
-
-		if cfg.DBport != 0 {
-			dbString += fmt.Sprintf(" port=%d", cfg.DBport)
-		}
-
-		if cfg.DBpass != "" {
-			dbString += fmt.Sprintf(" password=%s", cfg.DBpass)
-		}
-	case db.Sqlite:
-		dbString = cfg.DBpath
-	default:
-		return nil, errUnsupportedDatabase
-	}
-
 	registrationCache := cache.New(
 		registerCacheExpiration,
 		registerCacheCleanup,
 	)
 
 	app := Headscale{
 		cfg:                cfg,
-		dbType:             cfg.DBtype,
-		dbString:           dbString,
 		noisePrivateKey:    noisePrivateKey,
 		registrationCache:  registrationCache,
 		pollNetMapStreamWG: sync.WaitGroup{},
 		nodeNotifier:       notifier.NewNotifier(),
 	}
 
 	database, err := db.NewHeadscaleDatabase(
-		cfg.DBtype,
-		dbString,
-		app.dbDebug,
+		cfg.Database,
+		app.nodeNotifier,
 		cfg.IPPrefixes,
 		cfg.BaseDomain)
 	if err != nil {
@@ -755,14 +720,16 @@ func (h *Headscale) Serve() error {
 
 	var tailsqlContext context.Context
 	if tailsqlEnabled {
-		if h.cfg.DBtype != db.Sqlite {
-			log.Fatal().Str("type", h.cfg.DBtype).Msgf("tailsql only support %q", db.Sqlite)
+		if h.cfg.Database.Type != types.DatabaseSqlite {
+			log.Fatal().
+				Str("type", h.cfg.Database.Type).
+				Msgf("tailsql only support %q", types.DatabaseSqlite)
 		}
 		if tailsqlTSKey == "" {
 			log.Fatal().Msg("tailsql requires TS_AUTHKEY to be set")
 		}
 		tailsqlContext = context.Background()
-		go runTailSQLService(ctx, util.TSLogfWrapper(), tailsqlStateDir, h.cfg.DBpath)
+		go runTailSQLService(ctx, util.TSLogfWrapper(), tailsqlStateDir, h.cfg.Database.Sqlite.Path)
 	}
 
 	// Handle common process-killing signals so we can gracefully shut down:

hscontrol/db/db.go

@@ -6,11 +6,13 @@ import (
 	"errors"
 	"fmt"
 	"net/netip"
+	"strconv"
 	"strings"
 	"time"
 
 	"github.com/glebarez/sqlite"
 	"github.com/go-gormigrate/gormigrate/v2"
+	"github.com/juanfont/headscale/hscontrol/notifier"
 	"github.com/juanfont/headscale/hscontrol/types"
 	"github.com/juanfont/headscale/hscontrol/util"
 	"github.com/rs/zerolog/log"
@@ -19,11 +21,6 @@ import (
 	"gorm.io/gorm/logger"
 )
 
-const (
-	Postgres = "postgres"
-	Sqlite   = "sqlite3"
-)
-
 var errDatabaseNotSupported = errors.New("database type not supported")
 
 // KV is a key-value store in a psql table. For future use...
@@ -43,12 +40,12 @@ type HSDatabase struct {
 // TODO(kradalby): assemble this struct from toptions or something typed
 // rather than arguments.
 func NewHeadscaleDatabase(
-	dbType, connectionAddr string,
-	debug bool,
+	cfg types.DatabaseConfig,
+	notifier *notifier.Notifier,
 	ipPrefixes []netip.Prefix,
 	baseDomain string,
 ) (*HSDatabase, error) {
-	dbConn, err := openDB(dbType, connectionAddr, debug)
+	dbConn, err := openDB(cfg)
 	if err != nil {
 		return nil, err
 	}
@@ -62,7 +59,7 @@ func NewHeadscaleDatabase(
 		{
 			ID: "202312101416",
 			Migrate: func(tx *gorm.DB) error {
-				if dbType == Postgres {
+				if cfg.Type == types.DatabasePostgres {
 					tx.Exec(`create extension if not exists "uuid-ossp";`)
 				}
 
@@ -321,20 +318,20 @@ func NewHeadscaleDatabase(
 	return &db, err
 }
 
-func openDB(dbType, connectionAddr string, debug bool) (*gorm.DB, error) {
-	log.Debug().Str("type", dbType).Str("connection", connectionAddr).Msg("opening database")
+func openDB(cfg types.DatabaseConfig) (*gorm.DB, error) {
 
+	// TODO(kradalby): Integrate this with zerolog
 	var dbLogger logger.Interface
-	if debug {
+	if cfg.Debug {
 		dbLogger = logger.Default
 	} else {
 		dbLogger = logger.Default.LogMode(logger.Silent)
 	}
 
-	switch dbType {
-	case Sqlite:
+	switch cfg.Type {
+	case types.DatabaseSqlite:
 		db, err := gorm.Open(
-			sqlite.Open(connectionAddr+"?_synchronous=1&_journal_mode=WAL"),
+			sqlite.Open(cfg.Sqlite.Path+"?_synchronous=1&_journal_mode=WAL"),
 			&gorm.Config{
 				DisableForeignKeyConstraintWhenMigrating: true,
 				Logger:                                   dbLogger,
@@ -353,16 +350,39 @@ func openDB(dbType, connectionAddr string, debug bool) (*gorm.DB, error) {
 
 		return db, err
 
-	case Postgres:
-		return gorm.Open(postgres.Open(connectionAddr), &gorm.Config{
+	case types.DatabasePostgres:
+		dbString := fmt.Sprintf(
+			"host=%s dbname=%s user=%s",
+			cfg.Postgres.Host,
+			cfg.Postgres.Name,
+			cfg.Postgres.User,
+		)
+
+		if sslEnabled, err := strconv.ParseBool(cfg.Postgres.Ssl); err == nil {
+			if !sslEnabled {
+				dbString += " sslmode=disable"
+			}
+		} else {
+			dbString += fmt.Sprintf(" sslmode=%s", cfg.Postgres.Ssl)
+		}
+
+		if cfg.Postgres.Port != 0 {
+			dbString += fmt.Sprintf(" port=%d", cfg.Postgres.Port)
+		}
+
+		if cfg.Postgres.Pass != "" {
+			dbString += fmt.Sprintf(" password=%s", cfg.Postgres.Pass)
+		}
+
+		return gorm.Open(postgres.Open(dbString), &gorm.Config{
 			DisableForeignKeyConstraintWhenMigrating: true,
 			Logger:                                   dbLogger,
 		})
 	}
 
 	return nil, fmt.Errorf(
 		"database of type %s is not supported: %w",
-		dbType,
+		cfg.Type,
 		errDatabaseNotSupported,
 	)
 }

hscontrol/db/routes_test.go

@@ -7,6 +7,7 @@ import (
 	"time"
 
 	"github.com/google/go-cmp/cmp"
+	"github.com/juanfont/headscale/hscontrol/notifier"
 	"github.com/juanfont/headscale/hscontrol/types"
 	"github.com/juanfont/headscale/hscontrol/util"
 	"github.com/stretchr/testify/assert"
@@ -654,9 +655,13 @@ func TestFailoverRoute(t *testing.T) {
 			assert.NoError(t, err)
 
 			db, err = NewHeadscaleDatabase(
-				"sqlite3",
-				tmpDir+"/headscale_test.db",
-				false,
+				types.DatabaseConfig{
+					Type: "sqlite3",
+					Sqlite: types.SqliteConfig{
+						Path: tmpDir + "/headscale_test.db",
+					},
+				},
+				notifier.NewNotifier(),
 				[]netip.Prefix{
 					netip.MustParsePrefix("10.27.0.0/23"),
 				},

hscontrol/db/suite_test.go

@@ -6,6 +6,8 @@ import (
 	"os"
 	"testing"
 
+	"github.com/juanfont/headscale/hscontrol/notifier"
+	"github.com/juanfont/headscale/hscontrol/types"
 	"gopkg.in/check.v1"
 )
 
@@ -44,9 +46,13 @@ func (s *Suite) ResetDB(c *check.C) {
 	log.Printf("database path: %s", tmpDir+"/headscale_test.db")
 
 	db, err = NewHeadscaleDatabase(
-		"sqlite3",
-		tmpDir+"/headscale_test.db",
-		false,
+		types.DatabaseConfig{
+			Type: "sqlite3",
+			Sqlite: types.SqliteConfig{
+				Path: tmpDir + "/headscale_test.db",
+			},
+		},
+		notifier.NewNotifier(),
 		[]netip.Prefix{
 			netip.MustParsePrefix("10.27.0.0/23"),
 		},

hscontrol/suite_test.go

@@ -41,8 +41,12 @@ func (s *Suite) ResetDB(c *check.C) {
 	}
 	cfg := types.Config{
 		NoisePrivateKeyPath: tmpDir + "/noise_private.key",
-		DBtype:              "sqlite3",
-		DBpath:              tmpDir + "/headscale_test.db",
+		Database: types.DatabaseConfig{
+			Type: "sqlite3",
+			Sqlite: types.SqliteConfig{
+				Path: tmpDir + "/headscale_test.db",
+			},
+		},
 		IPPrefixes: []netip.Prefix{
 			netip.MustParsePrefix("10.27.0.0/23"),
 		},

hscontrol/types/common.go

@@ -12,7 +12,11 @@ import (
 	"tailscale.com/tailcfg"
 )
 
-const SelfUpdateIdentifier = "self-update"
+const (
+	SelfUpdateIdentifier = "self-update"
+	DatabasePostgres     = "postgres"
+	DatabaseSqlite       = "sqlite3"
+)
 
 var ErrCannotParsePrefix = errors.New("cannot parse prefix")
 
@@ -154,7 +158,9 @@ func (su *StateUpdate) Valid() bool {
 		}
 	case StateSelfUpdate:
 		if su.ChangeNodes == nil || len(su.ChangeNodes) != 1 {
-			panic("Mandatory field ChangeNodes is not set for StateSelfUpdate or has more than one node")
+			panic(
+				"Mandatory field ChangeNodes is not set for StateSelfUpdate or has more than one node",
+			)
 		}
 	case StateDERPUpdated:
 		if su.DERPMap == nil {