Backport of keyring: support external KMS for key encryption key (KEK) into release/1.8.x

.changelog/23580.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+keyring: Added support for encrypting the keyring via Vault transit or external KMS
+```

command/agent/agent.go

@@ -614,6 +614,8 @@ func convertServerConfig(agentConfig *Config) (*nomad.Config, error) {
 
 	conf.Reporting = agentConfig.Reporting
 
+	conf.KEKProviderConfigs = agentConfig.KEKProviders
+
 	return conf, nil
 }
 

command/agent/config.go

@@ -190,6 +190,9 @@ type Config struct {
 	// Reporting is used to enable go census reporting
 	Reporting *config.ReportingConfig `hcl:"reporting,block"`
 
+	// KEKProviders are used to wrap the Nomad keyring
+	KEKProviders []*structs.KEKProviderConfig `hcl:"keyring"`
+
 	// ExtraKeysHCL is used by hcl to surface unexpected keys
 	ExtraKeysHCL []string `hcl:",unusedKeys" json:"-"`
 }
@@ -1453,6 +1456,7 @@ func DefaultConfig() *Config {
 		DisableUpdateCheck: pointer.Of(false),
 		Limits:             config.DefaultLimits(),
 		Reporting:          config.DefaultReporting(),
+		KEKProviders:       []*structs.KEKProviderConfig{},
 	}
 
 	return cfg
@@ -1678,6 +1682,8 @@ func (c *Config) Merge(b *Config) *Config {
 
 	result.Limits = c.Limits.Merge(b.Limits)
 
+	result.KEKProviders = mergeKEKProviderConfigs(result.KEKProviders, b.KEKProviders)
+
 	return &result
 }
 
@@ -1749,6 +1755,40 @@ func mergeConsulConfigs(left, right []*config.ConsulConfig) []*config.ConsulConf
 	return results
 }
 
+func mergeKEKProviderConfigs(left, right []*structs.KEKProviderConfig) []*structs.KEKProviderConfig {
+	if len(left) == 0 {
+		return right
+	}
+	if len(right) == 0 {
+		return left
+	}
+	results := []*structs.KEKProviderConfig{}
+	doMerge := func(dstConfigs, srcConfigs []*structs.KEKProviderConfig) []*structs.KEKProviderConfig {
+		for _, src := range srcConfigs {
+			var found bool
+			for i, dst := range dstConfigs {
+				if dst.Provider == src.Provider && dst.Name == src.Name {
+					dstConfigs[i] = dst.Merge(src)
+					found = true
+					break
+				}
+			}
+			if !found {
+				dstConfigs = append(dstConfigs, src)
+			}
+		}
+		return dstConfigs
+	}
+
+	results = doMerge(results, left)
+	results = doMerge(results, right)
+	sort.Slice(results, func(i, j int) bool {
+		return results[i].ID() < results[j].ID()
+	})
+
+	return results
+}
+
 // Copy returns a deep copy safe for mutation.
 func (c *Config) Copy() *Config {
 	if c == nil {
@@ -1782,6 +1822,7 @@ func (c *Config) Copy() *Config {
 	nc.Limits = c.Limits.Copy()
 	nc.Audit = c.Audit.Copy()
 	nc.Reporting = c.Reporting.Copy()
+	nc.KEKProviders = helper.CopySlice(c.KEKProviders)
 	nc.ExtraKeysHCL = slices.Clone(c.ExtraKeysHCL)
 	return &nc
 }

command/agent/config_parse.go

@@ -10,6 +10,7 @@ import (
 	"os"
 	"path/filepath"
 	"slices"
+	"sort"
 	"time"
 
 	"github.com/hashicorp/hcl"
@@ -92,6 +93,13 @@ func ParseConfigFile(path string) (*Config, error) {
 		}
 	}
 
+	matches = list.Filter("keyring")
+	if len(matches.Items) > 0 {
+		if err := parseKeyringConfigs(c, matches); err != nil {
+			return nil, fmt.Errorf("error parsing 'keyring': %w", err)
+		}
+	}
+
 	// convert strings to time.Durations
 	tds := []durationConversionMap{
 		{"gc_interval", &c.Client.GCInterval, &c.Client.GCIntervalHCL, nil},
@@ -330,6 +338,11 @@ func extraKeys(c *Config) error {
 		helper.RemoveEqualFold(&c.ExtraKeysHCL, "telemetry")
 	}
 
+	helper.RemoveEqualFold(&c.ExtraKeysHCL, "keyring")
+	for _, provider := range c.KEKProviders {
+		helper.RemoveEqualFold(&c.ExtraKeysHCL, provider.Provider)
+	}
+
 	// Remove reporting extra keys
 	c.ExtraKeysHCL = slices.DeleteFunc(c.ExtraKeysHCL, func(s string) bool { return s == "license" })
 
@@ -522,3 +535,46 @@ func parseConsuls(c *Config, list *ast.ObjectList) error {
 
 	return nil
 }
+
+// parseKeyringConfigs parses the keyring blocks. At this point we have a list
+// of ast.Nodes and a KEKProviderConfig for each one. The KEKProviderConfig has
+// the unknown fields (provider-specific config) but not their values. So we
+// decode the ast.Node into a map and then read out the values for the unknown
+// fields. The results get added to the KEKProviderConfig's Config field
+func parseKeyringConfigs(c *Config, keyringBlocks *ast.ObjectList) error {
+	if len(keyringBlocks.Items) == 0 {
+		return nil
+	}
+
+	for idx, obj := range keyringBlocks.Items {
+		provider := c.KEKProviders[idx]
+		if len(provider.ExtraKeysHCL) == 0 {
+			continue
+		}
+
+		provider.Config = map[string]string{}
+
+		var m map[string]interface{}
+		if err := hcl.DecodeObject(&m, obj.Val); err != nil {
+			return err
+		}
+
+		for _, extraKey := range provider.ExtraKeysHCL {
+			val, ok := m[extraKey].(string)
+			if !ok {
+				return fmt.Errorf("failed to decode key %q to string", extraKey)
+			}
+			provider.Config[extraKey] = val
+		}
+
+		// clear the extra keys for these blocks because we've already handled
+		// them and don't want them to bubble up to the caller
+		provider.ExtraKeysHCL = nil
+	}
+
+	sort.Slice(c.KEKProviders, func(i, j int) bool {
+		return c.KEKProviders[i].ID() < c.KEKProviders[j].ID()
+	})
+
+	return nil
+}

command/agent/config_parse_test.go

@@ -348,6 +348,20 @@ var basicConfig = &Config{
 			Enabled: pointer.Of(true),
 		},
 	},
+	KEKProviders: []*structs.KEKProviderConfig{
+		{
+			Provider: "aead",
+			Active:   false,
+		},
+		{
+			Provider: "awskms",
+			Active:   true,
+			Config: map[string]string{
+				"region":     "us-east-1",
+				"kms_key_id": "alias/kms-nomad-keyring",
+			},
+		},
+	},
 }
 
 var pluginConfig = &Config{
@@ -481,6 +495,7 @@ func TestConfig_ParseMerge(t *testing.T) {
 	must.NoError(t, err)
 
 	actual, err := ParseConfigFile(path)
+	must.NoError(t, err)
 
 	// The Vault connection retry interval is an internal only configuration
 	// option, and therefore needs to be added here to ensure the test passes.
@@ -548,6 +563,7 @@ func TestConfig_Parse(t *testing.T) {
 			}
 			actual = oldDefault.Merge(actual)
 
+			must.Eq(t, tc.Result.KEKProviders, actual.KEKProviders)
 			must.Eq(t, tc.Result, removeHelperAttributes(actual))
 		})
 	}
@@ -751,6 +767,16 @@ var sample0 = &Config{
 		CleanupDeadServers: pointer.Of(true),
 	},
 	Reporting: config.DefaultReporting(),
+	KEKProviders: []*structs.KEKProviderConfig{
+		{
+			Provider: "awskms",
+			Active:   true,
+			Config: map[string]string{
+				"region":     "us-east-1",
+				"kms_key_id": "alias/kms-nomad-keyring",
+			},
+		},
+	},
 }
 
 func TestConfig_ParseSample0(t *testing.T) {
@@ -869,6 +895,20 @@ var sample1 = &Config{
 	Reporting: &config.ReportingConfig{
 		&config.LicenseReportingConfig{},
 	},
+	KEKProviders: []*structs.KEKProviderConfig{
+		{
+			Provider: "aead",
+			Active:   false,
+		},
+		{
+			Provider: "awskms",
+			Active:   true,
+			Config: map[string]string{
+				"region":     "us-east-1",
+				"kms_key_id": "alias/kms-nomad-keyring",
+			},
+		},
+	},
 }
 
 func TestConfig_ParseDir(t *testing.T) {

command/agent/config_test.go

@@ -1742,3 +1742,111 @@ func TestParseMultipleIPTemplates(t *testing.T) {
 		})
 	}
 }
+
+// this test makes sure Consul configs with and without WI merging happens
+// correctly; here to assure we don't introduce regressions
+func Test_mergeConsulConfigs(t *testing.T) {
+	ci.Parallel(t)
+
+	c0 := &Config{
+		Consuls: []*config.ConsulConfig{
+			{
+				Token:                "foo",
+				AllowUnauthenticated: pointer.Of(true),
+			},
+		},
+	}
+
+	c1 := &Config{
+		Consuls: []*config.ConsulConfig{
+			{
+				ServiceIdentity: &config.WorkloadIdentityConfig{
+					Audience: []string{"consul.io"},
+					TTL:      pointer.Of(time.Hour),
+				},
+				TaskIdentity: &config.WorkloadIdentityConfig{
+					Audience: []string{"consul.io"},
+					TTL:      pointer.Of(time.Hour),
+				},
+			},
+		},
+	}
+
+	result := c0.Merge(c1)
+
+	must.Eq(t, c1.Consuls[0].ServiceIdentity, result.Consuls[0].ServiceIdentity)
+	must.Eq(t, c1.Consuls[0].TaskIdentity, result.Consuls[0].TaskIdentity)
+	must.Eq(t, c0.Consuls[0].Token, result.Consuls[0].Token)
+	must.Eq(t, c0.Consuls[0].AllowUnauthenticated, result.Consuls[0].AllowUnauthenticated)
+}
+
+func Test_mergeKEKProviderConfigs(t *testing.T) {
+	ci.Parallel(t)
+
+	left := []*structs.KEKProviderConfig{
+		{
+			// incomplete config with name
+			Provider: "awskms",
+			Name:     "foo",
+			Active:   true,
+			Config: map[string]string{
+				"region":     "us-east-1",
+				"access_key": "AKIAIOSFODNN7EXAMPLE",
+			},
+		},
+		{
+			// empty config
+			Provider: "aead",
+		},
+	}
+	right := []*structs.KEKProviderConfig{
+		{
+			// same awskms.foo provider with fields to merge
+			Provider: "awskms",
+			Name:     "foo",
+			Active:   false,
+			Config: map[string]string{
+				"access_key": "AKIAIOSXABCD7EXAMPLE",
+				"secret_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
+				"kms_key_id": "19ec80b0-dfdd-4d97-8164-c6examplekey",
+			},
+		},
+		{
+			// same awskms provider, different name
+			Provider: "awskms",
+			Name:     "bar",
+			Active:   false,
+			Config: map[string]string{
+				"region":     "us-east-1",
+				"access_key": "AKIAIOSFODNN7EXAMPLE",
+			},
+		},
+	}
+
+	result := mergeKEKProviderConfigs(left, right)
+	must.Eq(t, []*structs.KEKProviderConfig{
+		{
+			Provider: "aead",
+		},
+		{
+			Provider: "awskms",
+			Name:     "bar",
+			Active:   false,
+			Config: map[string]string{
+				"region":     "us-east-1",
+				"access_key": "AKIAIOSFODNN7EXAMPLE",
+			},
+		},
+		{
+			Provider: "awskms",
+			Name:     "foo",
+			Active:   false, // should be flipped
+			Config: map[string]string{
+				"region":     "us-east-1",
+				"access_key": "AKIAIOSXABCD7EXAMPLE",                     // override
+				"secret_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", // added
+				"kms_key_id": "19ec80b0-dfdd-4d97-8164-c6examplekey",     // added
+			},
+		},
+	}, result)
+}

command/agent/testdata/basic.hcl

@@ -349,3 +349,13 @@ reporting {
     enabled = true
   }
 }
+
+keyring "awskms" {
+  active     = true
+  region     = "us-east-1"
+  kms_key_id = "alias/kms-nomad-keyring"
+}
+
+keyring "aead" {
+  active = false
+}

command/agent/testdata/basic.json

@@ -204,6 +204,14 @@
       "Access-Control-Allow-Origin": "*"
     }
   ],
+  "keyring": {
+    "awskms": {
+      "active": true,
+      "region": "us-east-1",
+      "kms_key_id": "alias/kms-nomad-keyring"
+    },
+    "aead": {}
+  },
   "leave_on_interrupt": true,
   "leave_on_terminate": true,
   "log_file": "/var/log/nomad.log",

command/agent/testdata/sample0.json

@@ -48,6 +48,13 @@
   "data_dir": "/opt/data/nomad/data",
   "datacenter": "dc1",
   "enable_syslog": true,
+  "keyring": {
+    "awskms": {
+      "active": true,
+      "region": "us-east-1",
+      "kms_key_id": "alias/kms-nomad-keyring"
+    }
+  },
   "leave_on_interrupt": true,
   "leave_on_terminate": true,
   "log_level": "INFO",