v2.11: Add digest to represent configuration state (#4325)

Makes the server generate a digest representing the expanded version of the parsed configuration: ``` $ nats-server -c foo.conf -t nats-server: configuration file configs/foo.conf is valid (sha256:e14d9fd914ee4a83c481c6bf97b44d7bc23d8f475f7f30af0c4e96ab4e5ad00c) ``` It can be inspected via `/varz`: ``` "config_load_time": "2023-07-19T14:28:28.77128Z", "config_digest": "sha256:6a383afa5fa6e00262105fbf061cabf48db892d1aa677f491dbabe7894393413", "system_account": "$SYS" ``` And it is also logged at runtime making it possible to check whether the reload had side effects: ``` [1935] 2023/07/19 02:57:29.245020 [INF] Using configuration file: configs/foo.conf (sha256:6a383afa5fa6e00262105fbf061cabf48db892d1aa677f491dbabe7894393413) ... [1935] 2023/07/19 02:58:06.993874 [DBG] Trapped "hangup" signal ... [1935] 2023/07/19 02:58:06.996113 [INF] Reloaded server configuration (sha256:6a383afa5fa6e00262105fbf061cabf48db892d1aa677f491dbabe7894393413) ... [1935] 2023/07/19 02:58:32.895492 [INF] Reloaded server configuration (sha256:a622fb453f3464b041de668b82229aef155175824f3c3a6d126a33268d172987) [1935] 2023/07/19 02:58:44.615983 [DBG] Trapped "hangup" signal ... [1935] 2023/07/19 02:58:44.617870 [INF] Reloaded server configuration (sha256:6a383afa5fa6e00262105fbf061cabf48db892d1aa677f491dbabe7894393413) ``` - [x] Reloader tests - [x] Monitoring test - [x] Build is green in Travis CI - [X] You have certified that the contribution is your original work and that you license the work to the project under the [Apache 2 license](https://github.com/nats-io/nats-server/blob/main/LICENSE) --------- Signed-off-by: Waldemar Quevedo <[email protected]>
nats-io · Nov 28, 2024 · 6191305 · 6191305
1 parent 9c2061e
commit 6191305
Show file tree

Hide file tree

Showing 11 changed files with 264 additions and 16 deletions.
diff --git a/conf/parse.go b/conf/parse.go
@@ -26,6 +26,7 @@ package conf
 // see parse_test.go for more examples.
 
 import (
+	"crypto/sha256"
 	"encoding/json"
 	"fmt"
 	"os"
@@ -36,6 +37,8 @@ import (
 	"unicode"
 )
 
+const _EMPTY_ = ""
+
 type parser struct {
 	mapping map[string]any
 	lx      *lexer
@@ -108,6 +111,44 @@ func ParseFileWithChecks(fp string) (map[string]any, error) {
 	return p.mapping, nil
 }
 
+// cleanupUsedEnvVars will recursively remove all already used
+// environment variables which might be in the parsed tree.
+func cleanupUsedEnvVars(m map[string]any) {
+	for k, v := range m {
+		t := v.(*token)
+		if t.usedVariable {
+			delete(m, k)
+			continue
+		}
+		// Cleanup any other env var that is still in the map.
+		if tm, ok := t.value.(map[string]any); ok {
+			cleanupUsedEnvVars(tm)
+		}
+	}
+}
+
+// ParseFileWithChecksDigest returns the processed config and a digest
+// that represents the configuration.
+func ParseFileWithChecksDigest(fp string) (map[string]any, string, error) {
+	data, err := os.ReadFile(fp)
+	if err != nil {
+		return nil, _EMPTY_, err
+	}
+	p, err := parse(string(data), fp, true)
+	if err != nil {
+		return nil, _EMPTY_, err
+	}
+	// Filter out any environment variables before taking the digest.
+	cleanupUsedEnvVars(p.mapping)
+	digest := sha256.New()
+	e := json.NewEncoder(digest)
+	err = e.Encode(p.mapping)
+	if err != nil {
+		return nil, _EMPTY_, err
+	}
+	return p.mapping, fmt.Sprintf("sha256:%x", digest.Sum(nil)), nil
+}
+
 type token struct {
 	item         item
 	value        any

diff --git a/conf/parse_test.go b/conf/parse_test.go
@@ -874,3 +874,143 @@ func TestBlocks(t *testing.T) {
 		})
 	}
 }
+
+func TestParseDigest(t *testing.T) {
+	for _, test := range []struct {
+		input    string
+		includes map[string]string
+		digest   string
+	}{
+		{
+			`foo = bar`,
+			nil,
+			"sha256:226e49e13d16e5e8aa0d62e58cd63361bf097d3e2b2444aa3044334628a2e8de",
+		},
+		{
+			`# Comments and whitespace have no effect
+                        foo = bar
+                        `,
+			nil,
+			"sha256:226e49e13d16e5e8aa0d62e58cd63361bf097d3e2b2444aa3044334628a2e8de",
+		},
+		{
+			`# Syntax changes have no effect
+                        'foo': 'bar'
+                        `,
+			nil,
+			"sha256:226e49e13d16e5e8aa0d62e58cd63361bf097d3e2b2444aa3044334628a2e8de",
+		},
+		{
+			`# Syntax changes have no effect
+                        { 'foo': 'bar' }
+                        `,
+			nil,
+			"sha256:226e49e13d16e5e8aa0d62e58cd63361bf097d3e2b2444aa3044334628a2e8de",
+		},
+		{
+			`# substitutions
+                        BAR_USERS = { users = [ {user = "bar"} ]}
+                        hello = 'world'
+                        accounts {
+                          QUUX_USERS = [ { user: quux }]
+                          bar = $BAR_USERS
+                          quux = { users = $QUUX_USERS }
+                        }
+                        very { nested { env { VAR = 'NESTED', quux = $VAR }}}
+                        `,
+			nil,
+			"sha256:34f8faf3f269fe7509edc4742f20c8c4a7ad51fe21f8b361764314b533ac3ab5",
+		},
+		{
+			`# substitutions, same as previous one without env vars.
+                        hello = 'world'
+                        accounts {
+                          bar  = { users = [ { user = "bar" } ]}
+                          quux = { users = [ { user: quux   } ]}
+                        }
+                        very { nested { env { quux = 'NESTED' }}}
+                        `,
+			nil,
+			"sha256:34f8faf3f269fe7509edc4742f20c8c4a7ad51fe21f8b361764314b533ac3ab5",
+		},
+		{
+			`# substitutions
+                        BAR_USERS = { users = [ {user = "foo"} ]}
+                        bar = $BAR_USERS
+                        accounts {
+                          users = $BAR_USERS
+                        }
+                        `,
+			nil,
+			"sha256:f5d943b4ed22b80c6199203f8a7eaa8eb68ef7b2d46ef6b1b26f05e21f8beb13",
+		},
+		{
+			`# substitutions
+                        bar = { users = [ {user = "foo"} ]}
+                        accounts {
+                          users = { users = [ {user = "foo"} ]}
+                        }
+                        `,
+			nil,
+			"sha256:f5d943b4ed22b80c6199203f8a7eaa8eb68ef7b2d46ef6b1b26f05e21f8beb13",
+		},
+		{
+			`# includes
+			accounts {
+                          foo { include 'foo.conf'}
+                          bar { users = [{user = "bar"}] }
+                          quux { include 'quux.conf'}
+                        }
+                        `,
+			map[string]string{
+				"foo.conf":  ` users = [{user = "foo"}]`,
+				"quux.conf": ` users = [{user = "quux"}]`,
+			},
+			"sha256:e72d70c91b64b0f880f86decb95ec2600cbdcf8bdcd2355fce5ebc54a84a77e9",
+		},
+		{
+			`# includes
+			accounts {
+                          foo { include 'foo.conf'}
+                          bar { include 'bar.conf'}
+                          quux { include 'quux.conf'}
+                        }
+                        `,
+			map[string]string{
+				"foo.conf":  ` users = [{user = "foo"}]`,
+				"bar.conf":  ` users = [{user = "bar"}]`,
+				"quux.conf": ` users = [{user = "quux"}]`,
+			},
+			"sha256:e72d70c91b64b0f880f86decb95ec2600cbdcf8bdcd2355fce5ebc54a84a77e9",
+		},
+	} {
+		t.Run("", func(t *testing.T) {
+			sdir := t.TempDir()
+			f, err := os.CreateTemp(sdir, "nats.conf-")
+			if err != nil {
+				t.Fatal(err)
+			}
+			if err := os.WriteFile(f.Name(), []byte(test.input), 066); err != nil {
+				t.Error(err)
+			}
+			if test.includes != nil {
+				for includeFile, contents := range test.includes {
+					inf, err := os.Create(filepath.Join(sdir, includeFile))
+					if err != nil {
+						t.Fatal(err)
+					}
+					if err := os.WriteFile(inf.Name(), []byte(contents), 066); err != nil {
+						t.Error(err)
+					}
+				}
+			}
+			_, digest, err := ParseFileWithChecksDigest(f.Name())
+			if err != nil {
+				t.Fatal(err)
+			}
+			if digest != test.digest {
+				t.Errorf("\ngot: %s\nexpected: %s", digest, test.digest)
+			}
+		})
+	}
+}
diff --git a/main.go b/main.go
@@ -110,7 +110,7 @@ func main() {
 	if err != nil {
 		server.PrintAndDie(fmt.Sprintf("%s: %s", exe, err))
 	} else if opts.CheckConfig {
-		fmt.Fprintf(os.Stderr, "%s: configuration file %s is valid\n", exe, opts.ConfigFile)
+		fmt.Fprintf(os.Stderr, "%s: configuration file %s is valid (%s)\n", exe, opts.ConfigFile, opts.ConfigDigest())
 		os.Exit(0)
 	}
 
@@ -120,7 +120,7 @@ func main() {
 		server.PrintAndDie(fmt.Sprintf("%s: %s", exe, err))
 	}
 
-	// Configure the logger based on the flags
+	// Configure the logger based on the flags.
 	s.ConfigureLogger()
 
 	// Start things up. Block here until done.

diff --git a/server/monitor.go b/server/monitor.go
@@ -1217,6 +1217,7 @@ type Varz struct {
 	Subscriptions         uint32                 `json:"subscriptions"`
 	HTTPReqStats          map[string]uint64      `json:"http_req_stats"`
 	ConfigLoadTime        time.Time              `json:"config_load_time"`
+	ConfigDigest          string                 `json:"config_digest"`
 	Tags                  jwt.TagList            `json:"tags,omitempty"`
 	TrustedOperatorsJwt   []string               `json:"trusted_operators_jwt,omitempty"`
 	TrustedOperatorsClaim []*jwt.OperatorClaims  `json:"trusted_operators_claim,omitempty"`
@@ -1671,6 +1672,7 @@ func (s *Server) updateVarzConfigReloadableFields(v *Varz) {
 	v.TLSTimeout = opts.TLSTimeout
 	v.WriteDeadline = opts.WriteDeadline
 	v.ConfigLoadTime = s.configTime.UTC()
+	v.ConfigDigest = opts.configDigest
 	// Update route URLs if applicable
 	if s.varzUpdateRouteURLs {
 		v.Cluster.URLs = urlsToStrings(opts.Routes)

diff --git a/server/opts.go b/server/opts.go
@@ -449,6 +449,9 @@ type Options struct {
 
 	// Used to mark that we had a top level authorization block.
 	authBlockDefined bool
+
+	// configDigest represents the state of configuration.
+	configDigest string
 }
 
 // WebsocketOpts are options for websocket
@@ -891,10 +894,11 @@ func (o *Options) ProcessConfigFile(configFile string) error {
 	if configFile == _EMPTY_ {
 		return nil
 	}
-	m, err := conf.ParseFileWithChecks(configFile)
+	m, digest, err := conf.ParseFileWithChecksDigest(configFile)
 	if err != nil {
 		return err
 	}
+	o.configDigest = digest
 
 	return o.processConfigFile(configFile, m)
 }
@@ -910,6 +914,11 @@ func (o *Options) ProcessConfigString(data string) error {
 	return o.processConfigFile(_EMPTY_, m)
 }
 
+// ConfigDigest returns the digest representing the configuration.
+func (o *Options) ConfigDigest() string {
+	return o.configDigest
+}
+
 func (o *Options) processConfigFile(configFile string, m map[string]any) error {
 	// Collect all errors and warnings and report them all together.
 	errors := make([]error, 0)

diff --git a/server/opts_test.go b/server/opts_test.go
@@ -121,6 +121,7 @@ func TestConfigFile(t *testing.T) {
 		LameDuckDuration:      4 * time.Minute,
 		ConnectErrorReports:   86400,
 		ReconnectErrorReports: 5,
+		configDigest:          "sha256:314adbd9997c1183f028f5b620362daa45893da76bac746136bfb48b2fd14996",
 		authBlockDefined:      true,
 	}
 
@@ -142,6 +143,7 @@ func TestTLSConfigFile(t *testing.T) {
 		AuthTimeout:      1.0,
 		TLSTimeout:       2.0,
 		authBlockDefined: true,
+		configDigest:     "sha256:cec986d37d7c09c86916d1ba4990cea1b8dadd49c86f9f782b455b47d07c2ac8",
 	}
 	opts, err := ProcessConfigFile("./configs/tls.conf")
 	if err != nil {
@@ -287,6 +289,7 @@ func TestMergeOverrides(t *testing.T) {
 		ConnectErrorReports:   86400,
 		ReconnectErrorReports: 5,
 		authBlockDefined:      true,
+		configDigest:          "sha256:314adbd9997c1183f028f5b620362daa45893da76bac746136bfb48b2fd14996",
 	}
 	fopts, err := ProcessConfigFile("./configs/test.conf")
 	if err != nil {
@@ -362,8 +365,9 @@ func TestRouteFlagOverride(t *testing.T) {
 			Password:    "top_secret",
 			AuthTimeout: 0.5,
 		},
-		Routes:    []*url.URL{rurl},
-		RoutesStr: routeFlag,
+		Routes:       []*url.URL{rurl},
+		RoutesStr:    routeFlag,
+		configDigest: "sha256:fe3c13f82637723989a9bbd0ad6d064b95d48971666af440d4196d9c0d3af979",
 	}
 
 	fopts, err := ProcessConfigFile("./configs/srv_a.conf")
@@ -403,7 +407,8 @@ func TestClusterFlagsOverride(t *testing.T) {
 			Password:    "top_secret",
 			AuthTimeout: 0.5,
 		},
-		Routes: []*url.URL{rurl},
+		Routes:       []*url.URL{rurl},
+		configDigest: "sha256:fe3c13f82637723989a9bbd0ad6d064b95d48971666af440d4196d9c0d3af979",
 	}
 
 	fopts, err := ProcessConfigFile("./configs/srv_a.conf")
@@ -438,8 +443,9 @@ func TestRouteFlagOverrideWithMultiple(t *testing.T) {
 			Password:    "top_secret",
 			AuthTimeout: 0.5,
 		},
-		Routes:    rurls,
-		RoutesStr: routeFlag,
+		Routes:       rurls,
+		RoutesStr:    routeFlag,
+		configDigest: "sha256:fe3c13f82637723989a9bbd0ad6d064b95d48971666af440d4196d9c0d3af979",
 	}
 
 	fopts, err := ProcessConfigFile("./configs/srv_a.conf")

diff --git a/server/reload.go b/server/reload.go
@@ -994,6 +994,13 @@ func (s *Server) Reload() error {
 		// TODO: Dump previous good config to a .bak file?
 		return err
 	}
+
+	// Use the digest from the configuration to detect whether unnecessary to apply reload.
+	if s.getOpts().ConfigDigest() != "" && newOpts.ConfigDigest() == s.getOpts().ConfigDigest() {
+		s.Noticef("Config reload skipped. No changes detected.")
+		return nil
+	}
+
 	return s.ReloadOptions(newOpts)
 }
 
@@ -1627,6 +1634,10 @@ func (s *Server) diffOptions(newOpts *Options) ([]option, error) {
 			if new != old {
 				diffOpts = append(diffOpts, &profBlockRateReload{newValue: new})
 			}
+		case "configdigest":
+			// skip changes in config digest, this is handled already while
+			// processing the config.
+			continue
 		default:
 			// TODO(ik): Implement String() on those options to have a nice print.
 			// %v is difficult to figure what's what, %+v print private fields and
@@ -1774,8 +1785,11 @@ func (s *Server) applyOptions(ctx *reloadContext, opts []option) {
 	if err := s.reloadOCSP(); err != nil {
 		s.Warnf("Can't restart OCSP features: %v", err)
 	}
-
-	s.Noticef("Reloaded server configuration")
+	var cd string
+	if newOpts.configDigest != "" {
+		cd = fmt.Sprintf("(%s)", newOpts.configDigest)
+	}
+	s.Noticef("Reloaded server configuration %s", cd)
 }
 
 // This will send a reset to the internal send loop.