From 8a62a3e0196966196dfb6a4d78fcef273c28879a Mon Sep 17 00:00:00 2001 From: Tim Gross Date: Thu, 18 Jul 2024 09:42:28 -0400 Subject: [PATCH 1/2] keyring: support external KMS for key encryption key (KEK) (#23580) In Nomad 1.4.0, we shipped support for encrypted Variables and signed Workload Identities, but the key material is protected only by a AEAD encrypting the KEK. Add support for Vault transit encryption and external KMS from major cloud providers. The servers call out to the external service to decrypt each key in the on-disk keystore. Ref: https://hashicorp.atlassian.net/browse/NET-10334 Fixes: https://github.com/hashicorp/nomad/issues/14852 --- .changelog/23580.txt | 3 + command/agent/agent.go | 2 + command/agent/config.go | 41 +++ command/agent/config_parse.go | 56 +++++ command/agent/config_parse_test.go | 40 +++ command/agent/config_test.go | 108 ++++++++ command/agent/testdata/basic.hcl | 10 + command/agent/testdata/basic.json | 8 + command/agent/testdata/sample0.json | 7 + command/agent/testdata/sample1/sample0.json | 7 + command/agent/testdata/sample1/sample1.json | 5 +- go.mod | 22 +- go.sum | 57 ++++- nomad/config.go | 4 + nomad/encrypter.go | 260 +++++++++++++++----- nomad/encrypter_ce.go | 42 ++++ nomad/encrypter_test.go | 107 +++++++- nomad/keyring_endpoint.go | 7 +- nomad/structs/keyring.go | 80 +++++- 19 files changed, 768 insertions(+), 98 deletions(-) create mode 100644 .changelog/23580.txt create mode 100644 nomad/encrypter_ce.go diff --git a/.changelog/23580.txt b/.changelog/23580.txt new file mode 100644 index 00000000000..7ebd2ef3bbe --- /dev/null +++ b/.changelog/23580.txt @@ -0,0 +1,3 @@ +```release-note:improvement +keyring: Added support for encrypting the keyring via Vault transit or external KMS +``` diff --git a/command/agent/agent.go b/command/agent/agent.go index c8f3fc31bee..6c645d7df63 100644 --- a/command/agent/agent.go +++ b/command/agent/agent.go @@ -614,6 +614,8 @@ func convertServerConfig(agentConfig *Config) (*nomad.Config, error) { conf.Reporting = agentConfig.Reporting + conf.KEKProviderConfigs = agentConfig.KEKProviders + return conf, nil } diff --git a/command/agent/config.go b/command/agent/config.go index ef80e7718cc..98cf4787047 100644 --- a/command/agent/config.go +++ b/command/agent/config.go @@ -190,6 +190,9 @@ type Config struct { // Reporting is used to enable go census reporting Reporting *config.ReportingConfig `hcl:"reporting,block"` + // KEKProviders are used to wrap the Nomad keyring + KEKProviders []*structs.KEKProviderConfig `hcl:"keyring"` + // ExtraKeysHCL is used by hcl to surface unexpected keys ExtraKeysHCL []string `hcl:",unusedKeys" json:"-"` } @@ -1453,6 +1456,7 @@ func DefaultConfig() *Config { DisableUpdateCheck: pointer.Of(false), Limits: config.DefaultLimits(), Reporting: config.DefaultReporting(), + KEKProviders: []*structs.KEKProviderConfig{}, } return cfg @@ -1678,6 +1682,8 @@ func (c *Config) Merge(b *Config) *Config { result.Limits = c.Limits.Merge(b.Limits) + result.KEKProviders = mergeKEKProviderConfigs(result.KEKProviders, b.KEKProviders) + return &result } @@ -1749,6 +1755,40 @@ func mergeConsulConfigs(left, right []*config.ConsulConfig) []*config.ConsulConf return results } +func mergeKEKProviderConfigs(left, right []*structs.KEKProviderConfig) []*structs.KEKProviderConfig { + if len(left) == 0 { + return right + } + if len(right) == 0 { + return left + } + results := []*structs.KEKProviderConfig{} + doMerge := func(dstConfigs, srcConfigs []*structs.KEKProviderConfig) []*structs.KEKProviderConfig { + for _, src := range srcConfigs { + var found bool + for i, dst := range dstConfigs { + if dst.Provider == src.Provider && dst.Name == src.Name { + dstConfigs[i] = dst.Merge(src) + found = true + break + } + } + if !found { + dstConfigs = append(dstConfigs, src) + } + } + return dstConfigs + } + + results = doMerge(results, left) + results = doMerge(results, right) + sort.Slice(results, func(i, j int) bool { + return results[i].ID() < results[j].ID() + }) + + return results +} + // Copy returns a deep copy safe for mutation. func (c *Config) Copy() *Config { if c == nil { @@ -1782,6 +1822,7 @@ func (c *Config) Copy() *Config { nc.Limits = c.Limits.Copy() nc.Audit = c.Audit.Copy() nc.Reporting = c.Reporting.Copy() + nc.KEKProviders = helper.CopySlice(c.KEKProviders) nc.ExtraKeysHCL = slices.Clone(c.ExtraKeysHCL) return &nc } diff --git a/command/agent/config_parse.go b/command/agent/config_parse.go index 0ca36170446..6aab5d2e0d1 100644 --- a/command/agent/config_parse.go +++ b/command/agent/config_parse.go @@ -10,6 +10,7 @@ import ( "os" "path/filepath" "slices" + "sort" "time" "github.com/hashicorp/hcl" @@ -92,6 +93,13 @@ func ParseConfigFile(path string) (*Config, error) { } } + matches = list.Filter("keyring") + if len(matches.Items) > 0 { + if err := parseKeyringConfigs(c, matches); err != nil { + return nil, fmt.Errorf("error parsing 'keyring': %w", err) + } + } + // convert strings to time.Durations tds := []durationConversionMap{ {"gc_interval", &c.Client.GCInterval, &c.Client.GCIntervalHCL, nil}, @@ -330,6 +338,11 @@ func extraKeys(c *Config) error { helper.RemoveEqualFold(&c.ExtraKeysHCL, "telemetry") } + helper.RemoveEqualFold(&c.ExtraKeysHCL, "keyring") + for _, provider := range c.KEKProviders { + helper.RemoveEqualFold(&c.ExtraKeysHCL, provider.Provider) + } + // Remove reporting extra keys c.ExtraKeysHCL = slices.DeleteFunc(c.ExtraKeysHCL, func(s string) bool { return s == "license" }) @@ -522,3 +535,46 @@ func parseConsuls(c *Config, list *ast.ObjectList) error { return nil } + +// parseKeyringConfigs parses the keyring blocks. At this point we have a list +// of ast.Nodes and a KEKProviderConfig for each one. The KEKProviderConfig has +// the unknown fields (provider-specific config) but not their values. So we +// decode the ast.Node into a map and then read out the values for the unknown +// fields. The results get added to the KEKProviderConfig's Config field +func parseKeyringConfigs(c *Config, keyringBlocks *ast.ObjectList) error { + if len(keyringBlocks.Items) == 0 { + return nil + } + + for idx, obj := range keyringBlocks.Items { + provider := c.KEKProviders[idx] + if len(provider.ExtraKeysHCL) == 0 { + continue + } + + provider.Config = map[string]string{} + + var m map[string]interface{} + if err := hcl.DecodeObject(&m, obj.Val); err != nil { + return err + } + + for _, extraKey := range provider.ExtraKeysHCL { + val, ok := m[extraKey].(string) + if !ok { + return fmt.Errorf("failed to decode key %q to string", extraKey) + } + provider.Config[extraKey] = val + } + + // clear the extra keys for these blocks because we've already handled + // them and don't want them to bubble up to the caller + provider.ExtraKeysHCL = nil + } + + sort.Slice(c.KEKProviders, func(i, j int) bool { + return c.KEKProviders[i].ID() < c.KEKProviders[j].ID() + }) + + return nil +} diff --git a/command/agent/config_parse_test.go b/command/agent/config_parse_test.go index 0a128c0cefb..7b8808751d2 100644 --- a/command/agent/config_parse_test.go +++ b/command/agent/config_parse_test.go @@ -348,6 +348,20 @@ var basicConfig = &Config{ Enabled: pointer.Of(true), }, }, + KEKProviders: []*structs.KEKProviderConfig{ + { + Provider: "aead", + Active: false, + }, + { + Provider: "awskms", + Active: true, + Config: map[string]string{ + "region": "us-east-1", + "kms_key_id": "alias/kms-nomad-keyring", + }, + }, + }, } var pluginConfig = &Config{ @@ -481,6 +495,7 @@ func TestConfig_ParseMerge(t *testing.T) { must.NoError(t, err) actual, err := ParseConfigFile(path) + must.NoError(t, err) // The Vault connection retry interval is an internal only configuration // option, and therefore needs to be added here to ensure the test passes. @@ -548,6 +563,7 @@ func TestConfig_Parse(t *testing.T) { } actual = oldDefault.Merge(actual) + must.Eq(t, tc.Result.KEKProviders, actual.KEKProviders) must.Eq(t, tc.Result, removeHelperAttributes(actual)) }) } @@ -751,6 +767,16 @@ var sample0 = &Config{ CleanupDeadServers: pointer.Of(true), }, Reporting: config.DefaultReporting(), + KEKProviders: []*structs.KEKProviderConfig{ + { + Provider: "awskms", + Active: true, + Config: map[string]string{ + "region": "us-east-1", + "kms_key_id": "alias/kms-nomad-keyring", + }, + }, + }, } func TestConfig_ParseSample0(t *testing.T) { @@ -869,6 +895,20 @@ var sample1 = &Config{ Reporting: &config.ReportingConfig{ &config.LicenseReportingConfig{}, }, + KEKProviders: []*structs.KEKProviderConfig{ + { + Provider: "aead", + Active: false, + }, + { + Provider: "awskms", + Active: true, + Config: map[string]string{ + "region": "us-east-1", + "kms_key_id": "alias/kms-nomad-keyring", + }, + }, + }, } func TestConfig_ParseDir(t *testing.T) { diff --git a/command/agent/config_test.go b/command/agent/config_test.go index f8507e1f1d4..9debdc3a369 100644 --- a/command/agent/config_test.go +++ b/command/agent/config_test.go @@ -1742,3 +1742,111 @@ func TestParseMultipleIPTemplates(t *testing.T) { }) } } + +// this test makes sure Consul configs with and without WI merging happens +// correctly; here to assure we don't introduce regressions +func Test_mergeConsulConfigs(t *testing.T) { + ci.Parallel(t) + + c0 := &Config{ + Consuls: []*config.ConsulConfig{ + { + Token: "foo", + AllowUnauthenticated: pointer.Of(true), + }, + }, + } + + c1 := &Config{ + Consuls: []*config.ConsulConfig{ + { + ServiceIdentity: &config.WorkloadIdentityConfig{ + Audience: []string{"consul.io"}, + TTL: pointer.Of(time.Hour), + }, + TaskIdentity: &config.WorkloadIdentityConfig{ + Audience: []string{"consul.io"}, + TTL: pointer.Of(time.Hour), + }, + }, + }, + } + + result := c0.Merge(c1) + + must.Eq(t, c1.Consuls[0].ServiceIdentity, result.Consuls[0].ServiceIdentity) + must.Eq(t, c1.Consuls[0].TaskIdentity, result.Consuls[0].TaskIdentity) + must.Eq(t, c0.Consuls[0].Token, result.Consuls[0].Token) + must.Eq(t, c0.Consuls[0].AllowUnauthenticated, result.Consuls[0].AllowUnauthenticated) +} + +func Test_mergeKEKProviderConfigs(t *testing.T) { + ci.Parallel(t) + + left := []*structs.KEKProviderConfig{ + { + // incomplete config with name + Provider: "awskms", + Name: "foo", + Active: true, + Config: map[string]string{ + "region": "us-east-1", + "access_key": "AKIAIOSFODNN7EXAMPLE", + }, + }, + { + // empty config + Provider: "aead", + }, + } + right := []*structs.KEKProviderConfig{ + { + // same awskms.foo provider with fields to merge + Provider: "awskms", + Name: "foo", + Active: false, + Config: map[string]string{ + "access_key": "AKIAIOSXABCD7EXAMPLE", + "secret_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + "kms_key_id": "19ec80b0-dfdd-4d97-8164-c6examplekey", + }, + }, + { + // same awskms provider, different name + Provider: "awskms", + Name: "bar", + Active: false, + Config: map[string]string{ + "region": "us-east-1", + "access_key": "AKIAIOSFODNN7EXAMPLE", + }, + }, + } + + result := mergeKEKProviderConfigs(left, right) + must.Eq(t, []*structs.KEKProviderConfig{ + { + Provider: "aead", + }, + { + Provider: "awskms", + Name: "bar", + Active: false, + Config: map[string]string{ + "region": "us-east-1", + "access_key": "AKIAIOSFODNN7EXAMPLE", + }, + }, + { + Provider: "awskms", + Name: "foo", + Active: false, // should be flipped + Config: map[string]string{ + "region": "us-east-1", + "access_key": "AKIAIOSXABCD7EXAMPLE", // override + "secret_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", // added + "kms_key_id": "19ec80b0-dfdd-4d97-8164-c6examplekey", // added + }, + }, + }, result) +} diff --git a/command/agent/testdata/basic.hcl b/command/agent/testdata/basic.hcl index 4ee1f70d770..2ca7ab1de47 100644 --- a/command/agent/testdata/basic.hcl +++ b/command/agent/testdata/basic.hcl @@ -349,3 +349,13 @@ reporting { enabled = true } } + +keyring "awskms" { + active = true + region = "us-east-1" + kms_key_id = "alias/kms-nomad-keyring" +} + +keyring "aead" { + active = false +} diff --git a/command/agent/testdata/basic.json b/command/agent/testdata/basic.json index bc29b189719..2727e924409 100644 --- a/command/agent/testdata/basic.json +++ b/command/agent/testdata/basic.json @@ -204,6 +204,14 @@ "Access-Control-Allow-Origin": "*" } ], + "keyring": { + "awskms": { + "active": true, + "region": "us-east-1", + "kms_key_id": "alias/kms-nomad-keyring" + }, + "aead": {} + }, "leave_on_interrupt": true, "leave_on_terminate": true, "log_file": "/var/log/nomad.log", diff --git a/command/agent/testdata/sample0.json b/command/agent/testdata/sample0.json index cc7736893b3..a836f93c00e 100644 --- a/command/agent/testdata/sample0.json +++ b/command/agent/testdata/sample0.json @@ -48,6 +48,13 @@ "data_dir": "/opt/data/nomad/data", "datacenter": "dc1", "enable_syslog": true, + "keyring": { + "awskms": { + "active": true, + "region": "us-east-1", + "kms_key_id": "alias/kms-nomad-keyring" + } + }, "leave_on_interrupt": true, "leave_on_terminate": true, "log_level": "INFO", diff --git a/command/agent/testdata/sample1/sample0.json b/command/agent/testdata/sample1/sample0.json index a806ea9098a..1a23c7378f6 100644 --- a/command/agent/testdata/sample1/sample0.json +++ b/command/agent/testdata/sample1/sample0.json @@ -12,6 +12,13 @@ "data_dir": "/opt/data/nomad/data", "datacenter": "dc1", "enable_syslog": true, + "keyring": { + "awskms": { + "active": true, + "region": "us-east-1", + "kms_key_id": "alias/kms-nomad-keyring" + } + }, "leave_on_interrupt": true, "leave_on_terminate": true, "log_level": "INFO", diff --git a/command/agent/testdata/sample1/sample1.json b/command/agent/testdata/sample1/sample1.json index 8f83354d45f..f147acfb67b 100644 --- a/command/agent/testdata/sample1/sample1.json +++ b/command/agent/testdata/sample1/sample1.json @@ -11,13 +11,16 @@ "data_dir": "/opt/data/nomad/data", "datacenter": "dc1", "enable_syslog": true, + "keyring": { + "aead": {} + }, "leave_on_interrupt": true, "leave_on_terminate": true, "log_level": "INFO", "region": "global", "server": { "bootstrap_expect": 3, - "enabled": true, + "enabled": true }, "syslog_facility": "LOCAL0", "telemetry": { diff --git a/go.mod b/go.mod index f0e0af406b5..28bf9eaba07 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/Microsoft/go-winio v0.6.1 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e github.com/armon/go-metrics v0.5.3 - github.com/aws/aws-sdk-go v1.44.184 + github.com/aws/aws-sdk-go v1.44.210 github.com/brianvoe/gofakeit/v6 v6.20.1 github.com/container-storage-interface/spec v1.7.0 github.com/containerd/go-cni v1.1.9 @@ -58,6 +58,10 @@ require ( github.com/hashicorp/go-hclog v1.6.3 github.com/hashicorp/go-immutable-radix/v2 v2.1.0 github.com/hashicorp/go-kms-wrapping/v2 v2.0.15 + github.com/hashicorp/go-kms-wrapping/wrappers/awskms/v2 v2.0.9 + github.com/hashicorp/go-kms-wrapping/wrappers/azurekeyvault/v2 v2.0.11 + github.com/hashicorp/go-kms-wrapping/wrappers/gcpckms/v2 v2.0.12 + github.com/hashicorp/go-kms-wrapping/wrappers/transit/v2 v2.0.11 github.com/hashicorp/go-memdb v1.3.4 github.com/hashicorp/go-msgpack/v2 v2.1.2 github.com/hashicorp/go-multierror v1.1.1 @@ -139,13 +143,19 @@ require ( cloud.google.com/go/compute v1.23.0 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect cloud.google.com/go/iam v1.1.1 // indirect + cloud.google.com/go/kms v1.15.0 // indirect cloud.google.com/go/storage v1.30.1 // indirect dario.cat/mergo v1.0.0 // indirect github.com/Azure/azure-sdk-for-go v56.3.0+incompatible // indirect + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/keyvault/azkeys v0.10.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 // indirect github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect - github.com/Azure/go-autorest/autorest v0.11.20 // indirect - github.com/Azure/go-autorest/autorest/adal v0.9.15 // indirect + github.com/Azure/go-autorest/autorest v0.11.29 // indirect + github.com/Azure/go-autorest/autorest/adal v0.9.22 // indirect github.com/Azure/go-autorest/autorest/azure/auth v0.5.1 // indirect github.com/Azure/go-autorest/autorest/azure/cli v0.4.0 // indirect github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect @@ -153,6 +163,7 @@ require ( github.com/Azure/go-autorest/autorest/validation v0.3.0 // indirect github.com/Azure/go-autorest/logger v0.2.1 // indirect github.com/Azure/go-autorest/tracing v0.6.0 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1 // indirect github.com/BurntSushi/toml v1.3.2 // indirect github.com/DataDog/datadog-go v3.2.0+incompatible // indirect github.com/Masterminds/goutils v1.1.1 // indirect @@ -196,7 +207,7 @@ require ( github.com/godbus/dbus/v5 v5.1.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/gojuno/minimock/v3 v3.0.6 // indirect - github.com/golang-jwt/jwt/v4 v4.4.3 // indirect + github.com/golang-jwt/jwt/v4 v4.5.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/google/btree v1.1.2 // indirect github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135 // indirect @@ -213,6 +224,7 @@ require ( github.com/hashicorp/go-retryablehttp v0.7.7 // indirect github.com/hashicorp/go-rootcerts v1.0.2 // indirect github.com/hashicorp/go-safetemp v1.0.0 // indirect + github.com/hashicorp/go-secure-stdlib/awsutil v0.1.6 // indirect github.com/hashicorp/go-secure-stdlib/parseutil v0.1.7 // indirect github.com/hashicorp/go-secure-stdlib/reloadutil v0.1.1 // indirect github.com/hashicorp/go-secure-stdlib/tlsutil v0.1.2 // indirect @@ -228,6 +240,7 @@ require ( github.com/joyent/triton-go v0.0.0-20190112182421-51ffac552869 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.16.0 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect github.com/linode/linodego v0.7.1 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/mattn/go-isatty v0.0.20 // indirect @@ -250,6 +263,7 @@ require ( github.com/opencontainers/image-spec v1.1.0 // indirect github.com/opencontainers/selinux v1.11.0 // indirect github.com/packethost/packngo v0.1.1-0.20180711074735-b9cb5096f54c // indirect + github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect diff --git a/go.sum b/go.sum index e740e2b75cd..1a383501f99 100644 --- a/go.sum +++ b/go.sum @@ -111,6 +111,8 @@ cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp cloud.google.com/go/iam v0.5.0/go.mod h1:wPU9Vt0P4UmCux7mqtRu6jcpPAb74cP1fh50J3QpkUc= cloud.google.com/go/iam v1.1.1 h1:lW7fzj15aVIXYHREOqjRBV9PsH0Z6u8Y46a1YGvQP4Y= cloud.google.com/go/iam v1.1.1/go.mod h1:A5avdyVL2tCppe4unb0951eI9jreack+RJ0/d+KUZOU= +cloud.google.com/go/kms v1.15.0 h1:xYl5WEaSekKYN5gGRyhjvZKM22GVBBCzegGNVPy+aIs= +cloud.google.com/go/kms v1.15.0/go.mod h1:c9J991h5DTl+kg7gi3MYomh12YEENGrf48ee/N/2CDM= cloud.google.com/go/language v1.4.0/go.mod h1:F9dRpNFQmJbkaop6g0JhSBXCNlO90e1KWx5iDdxbWic= cloud.google.com/go/language v1.6.0/go.mod h1:6dJ8t3B+lUYfStgls25GusK04NLh3eDLQnWM3mdEbhI= cloud.google.com/go/lifesciences v0.5.0/go.mod h1:3oIKy8ycWGPUyZDR/8RNnTOYevhaMLqh5vLUXs9zvT8= @@ -192,6 +194,16 @@ github.com/AdaLogics/go-fuzz-headers v0.0.0-20210715213245-6c3934b029d8/go.mod h github.com/Azure/azure-sdk-for-go v44.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/azure-sdk-for-go v56.3.0+incompatible h1:DmhwMrUIvpeoTDiWRDtNHqelNUd3Og8JCkrLHQK795c= github.com/Azure/azure-sdk-for-go v56.3.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.0 h1:fb8kj/Dh4CSwgsOzHeZY4Xh68cFVbzXx+ONXGMY//4w= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.0/go.mod h1:uReU2sSxZExRPBAg3qKzmAucSi51+SP1OhohieR821Q= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0 h1:BMAjVKJM0U/CYF27gA0ZMmXGkOcvfFtD0oHVZ1TIPRI= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0/go.mod h1:1fXstnBMas5kzG+S3q8UoJcmyU6nUeunJcMDHcRYHhs= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.0 h1:d81/ng9rET2YqdVkVwkb6EXeRrLJIwyGnJcAlAWKwhs= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.0/go.mod h1:s4kgfzA0covAXNicZHDMN58jExvcng2mC/DepXiF1EI= +github.com/Azure/azure-sdk-for-go/sdk/keyvault/azkeys v0.10.0 h1:m/sWOGCREuSBqg2htVQTBY8nOZpyajYztF0vUvSZTuM= +github.com/Azure/azure-sdk-for-go/sdk/keyvault/azkeys v0.10.0/go.mod h1:Pu5Zksi2KrU7LPbZbNINx6fuVrUp/ffvpxdDj+i8LeE= +github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 h1:FbH3BbSb4bvGluTesZZ+ttN/MDsnMmQP36OSnDuSXqw= +github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1/go.mod h1:9V2j0jn9jDEkCkv8w/bKTNppX/d0FVA1ud77xCIP4KA= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= @@ -199,14 +211,14 @@ github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSW github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= github.com/Azure/go-autorest/autorest v0.11.0/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw= github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= -github.com/Azure/go-autorest/autorest v0.11.20 h1:s8H1PbCZSqg/DH7JMlOz6YMig6htWLNPsjDdlLqCx3M= -github.com/Azure/go-autorest/autorest v0.11.20/go.mod h1:o3tqFY+QR40VOlk+pV4d77mORO64jOXSgEnPQgLK6JY= +github.com/Azure/go-autorest/autorest v0.11.29 h1:I4+HL/JDvErx2LjyzaVxllw2lRDB5/BT2Bm4g20iqYw= +github.com/Azure/go-autorest/autorest v0.11.29/go.mod h1:ZtEzC4Jy2JDrZLxvWs8LrBWEBycl1hbT1eknI8MtfAs= github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= github.com/Azure/go-autorest/autorest/adal v0.9.0/go.mod h1:/c022QCutn2P7uY+/oQWWNcK9YU+MH96NgK+jErpbcg= github.com/Azure/go-autorest/autorest/adal v0.9.2/go.mod h1:/3SMAM86bP6wC9Ev35peQDUeqFZBMH07vvUOmg4z/fE= github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= -github.com/Azure/go-autorest/autorest/adal v0.9.15 h1:X+p2GF0GWyOiSmqohIaEeuNFNDY4I4EOlVuUQvFdWMk= -github.com/Azure/go-autorest/autorest/adal v0.9.15/go.mod h1:tGMin8I49Yij6AQ+rvV+Xa/zwxYQB5hmsd6DkfAx2+A= +github.com/Azure/go-autorest/autorest/adal v0.9.22 h1:/GblQdIudfEM3AWWZ0mrYJQSd7JS4S/Mbzh6F0ov0Xc= +github.com/Azure/go-autorest/autorest/adal v0.9.22/go.mod h1:XuAbAEUv2Tta//+voMI038TrJBqjKam0me7qR+L8Cmk= github.com/Azure/go-autorest/autorest/azure/auth v0.5.0/go.mod h1:QRTvSZQpxqm8mSErhnbI+tANIBAKP7B+UIE2z4ypUO0= github.com/Azure/go-autorest/autorest/azure/auth v0.5.1 h1:bvUhZciHydpBxBmCheUgxxbSwJy7xcfjkUsjUcqSojc= github.com/Azure/go-autorest/autorest/azure/auth v0.5.1/go.mod h1:ea90/jvmnAwDrSooLH4sRIehEPtG/EPUXavDh31MnA4= @@ -218,8 +230,9 @@ github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSY github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= github.com/Azure/go-autorest/autorest/mocks v0.4.0/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= -github.com/Azure/go-autorest/autorest/mocks v0.4.1 h1:K0laFcLE6VLTOwNgSxaGbUcLPuGXlNkbVvq4cW4nIHk= github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/autorest/mocks v0.4.2 h1:PGN4EDXnuQbojHbU0UWoNvmu9AGVwYHG9/fkDYhtAfw= +github.com/Azure/go-autorest/autorest/mocks v0.4.2/go.mod h1:Vy7OitM9Kei0i1Oj+LvyAWMXJHeKH1MVlzFugfVrmyU= github.com/Azure/go-autorest/autorest/to v0.4.0 h1:oXVqrxakqqV1UZdSazDOPOLvOIz+XA683u8EctwboHk= github.com/Azure/go-autorest/autorest/to v0.4.0/go.mod h1:fE8iZBn7LQR7zH/9XU2NcPR4o9jEImooCeWJcYV/zLE= github.com/Azure/go-autorest/autorest/validation v0.3.0 h1:3I9AAI63HfcLtphd9g39ruUwRI+Ca+z/f36KHPFRUss= @@ -231,6 +244,8 @@ github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZ github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= +github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1 h1:WpB/QDNLpMw72xHJc34BNNykqSOeEJDAWkhf0u12/Jk= +github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= @@ -282,9 +297,10 @@ github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj github.com/armon/go-radix v1.0.0 h1:F4z6KzEeeQIMeLFa97iZU6vupzoecKdU5TX24SNppXI= github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/aws/aws-sdk-go v1.25.41/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go v1.30.27/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/aws/aws-sdk-go v1.44.122/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= -github.com/aws/aws-sdk-go v1.44.184 h1:/MggyE66rOImXJKl1HqhLQITvWvqIV7w1Q4MaG6FHUo= -github.com/aws/aws-sdk-go v1.44.184/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go v1.44.210 h1:/cqRMHSSgzLEKILIDGwhaX2hiIpyRurw7MRy6aaSufg= +github.com/aws/aws-sdk-go v1.44.210/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= @@ -374,8 +390,9 @@ github.com/dimchansky/utfbom v1.1.0 h1:FcM3g+nofKgUteL8dm/UpdRXNC9KmADgTpLKsu0TR github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8= github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK2OFGvA0= github.com/distribution/reference v0.5.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= -github.com/dnaeon/go-vcr v1.0.1 h1:r8L/HqC0Hje5AXMu1ooW8oyQyOFv4GxqpL0nRP7SLLY= github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= +github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= +github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= github.com/docker/cli v24.0.6+incompatible h1:fF+XCQCgJjjQNIMjzaSmiKJSCcfcXb3TWTcc7GAneOY= github.com/docker/cli v24.0.6+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk= @@ -453,6 +470,7 @@ github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+ github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= +github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-test/deep v1.0.2/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= @@ -469,8 +487,8 @@ github.com/gojuno/minimock/v3 v3.0.4/go.mod h1:HqeqnwV8mAABn3pO5hqF+RE7gjA0jsN8c github.com/gojuno/minimock/v3 v3.0.6 h1:YqHcVR10x2ZvswPK8Ix5yk+hMpspdQ3ckSpkOzyF85I= github.com/gojuno/minimock/v3 v3.0.6/go.mod h1:v61ZjAKHr+WnEkND63nQPCZ/DTfQgJdvbCi3IuoMblY= github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= -github.com/golang-jwt/jwt/v4 v4.4.3 h1:Hxl6lhQFj4AnOX6MLrsCb/+7tCj7DxP7VA+2rDIq5AU= -github.com/golang-jwt/jwt/v4 v4.4.3/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= +github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-jwt/jwt/v5 v5.0.0 h1:1n1XNM9hk7O9mnQoNBGolZvzebBQ7p93ULHRc28XJUE= github.com/golang-jwt/jwt/v5 v5.0.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -653,6 +671,14 @@ github.com/hashicorp/go-immutable-radix/v2 v2.1.0 h1:CUW5RYIcysz+D3B+l1mDeXrQ7fU github.com/hashicorp/go-immutable-radix/v2 v2.1.0/go.mod h1:hgdqLXA4f6NIjRVisM1TJ9aOJVNRqKZj+xDGF6m7PBw= github.com/hashicorp/go-kms-wrapping/v2 v2.0.15 h1:f3+/VbanXOmVAaDBKwRiVmeL7EX340a4YmaTItMF4Xs= github.com/hashicorp/go-kms-wrapping/v2 v2.0.15/go.mod h1:0dWtzl2ilqKpavgM3id/kFK9L3tjo6fS4OhbVPSYpnQ= +github.com/hashicorp/go-kms-wrapping/wrappers/awskms/v2 v2.0.9 h1:qdxeZvDMRGZ3YSE4Oz0Pp7WUSUn5S6cWZguEOkEVL50= +github.com/hashicorp/go-kms-wrapping/wrappers/awskms/v2 v2.0.9/go.mod h1:DcXbvVpgNWbxGmxgmu3QN64bEydMu14Cpe34RRR30HY= +github.com/hashicorp/go-kms-wrapping/wrappers/azurekeyvault/v2 v2.0.11 h1:/7SKkYIhA8cr3l8m1EKT6Q90bPoSVqqVBuQ6HgoMIkw= +github.com/hashicorp/go-kms-wrapping/wrappers/azurekeyvault/v2 v2.0.11/go.mod h1:LepS5s6ESGE0qQMpYaui5lX+mQYeiYiy06VzwWRioO8= +github.com/hashicorp/go-kms-wrapping/wrappers/gcpckms/v2 v2.0.12 h1:PCqWzT/Hii0KL07JsBZ3lJbv/wx02IAHYlhWQq8rxRY= +github.com/hashicorp/go-kms-wrapping/wrappers/gcpckms/v2 v2.0.12/go.mod h1:HSaOaX/lv3ShCdilUYbOTPnSvmoZ9xtQhgw+8hYcZkg= +github.com/hashicorp/go-kms-wrapping/wrappers/transit/v2 v2.0.11 h1:hdzSrDJ0CgHgGFx+1toaf7Z5bmQ2EYaFQ/dtWNXxu1I= +github.com/hashicorp/go-kms-wrapping/wrappers/transit/v2 v2.0.11/go.mod h1:ywjP17x2t88pT3GA8gCc2vEH1vhvU1R9d5XwRQ0d7PQ= github.com/hashicorp/go-memdb v1.3.4 h1:XSL3NR682X/cVk2IeV0d70N4DZ9ljI885xAEU8IoK3c= github.com/hashicorp/go-memdb v1.3.4/go.mod h1:uBTr1oQbtuMgd1SSGoR8YV27eT3sBHbYiNm53bMpgSg= github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= @@ -675,6 +701,8 @@ github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5O github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= github.com/hashicorp/go-safetemp v1.0.0 h1:2HR189eFNrjHQyENnQMMpCiBAsRxzbTMIgBhEyExpmo= github.com/hashicorp/go-safetemp v1.0.0/go.mod h1:oaerMy3BhqiTbVye6QuFhFtIceqFoDHxNAB65b+Rj1I= +github.com/hashicorp/go-secure-stdlib/awsutil v0.1.6 h1:W9WN8p6moV1fjKLkeqEgkAMu5rauy9QeYDAmIaPuuiA= +github.com/hashicorp/go-secure-stdlib/awsutil v0.1.6/go.mod h1:MpCPSPGLDILGb4JMm94/mMi3YysIqsXzGCzkEZjcjXg= github.com/hashicorp/go-secure-stdlib/listenerutil v0.1.4 h1:6ajbq64FhrIJZ6prrff3upVVDil4yfCrnSKwTH0HIPE= github.com/hashicorp/go-secure-stdlib/listenerutil v0.1.4/go.mod h1:myX7XYMJRIP4PLHtYJiKMTJcKOX0M5ZJNwP0iw+l3uw= github.com/hashicorp/go-secure-stdlib/parseutil v0.1.1/go.mod h1:QmrqtbKuxxSWTN3ETMPuB+VtEiBJ/A9XhoYGv8E1uD8= @@ -768,6 +796,7 @@ github.com/jefferai/isbadcipher v0.0.0-20190226160619-51d2077c035f/go.mod h1:3J2 github.com/jhump/protoreflect v1.15.1 h1:HUMERORf3I3ZdX05WaQ6MIpd/NJ434hTp5YiKgfCL6c= github.com/jhump/protoreflect v1.15.1/go.mod h1:jD/2GMKKE6OqX8qTjhADU1e6DShO+gavG9e0Q693nKo= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= @@ -798,12 +827,15 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxv github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/linode/linodego v0.7.1 h1:4WZmMpSA2NRwlPZcc0+4Gyn7rr99Evk9bnr0B3gXRKE= github.com/linode/linodego v0.7.1/go.mod h1:ga11n3ivecUrPCHN0rANxKmfWBJVkOXfLMZinAbj2sY= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= @@ -940,6 +972,8 @@ github.com/packethost/packngo v0.1.1-0.20180711074735-b9cb5096f54c/go.mod h1:otz github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= +github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU= +github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -983,6 +1017,7 @@ github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= @@ -1119,6 +1154,7 @@ golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= @@ -1340,6 +1376,7 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/nomad/config.go b/nomad/config.go index 149d1375943..551bfd23adf 100644 --- a/nomad/config.go +++ b/nomad/config.go @@ -434,6 +434,9 @@ type Config struct { // If this is not configured the /.well-known/openid-configuration endpoint // will not be available. OIDCIssuer string + + // KEKProviders are used to wrap the Nomad keyring + KEKProviderConfigs []*structs.KEKProviderConfig } func (c *Config) Copy() *Config { @@ -462,6 +465,7 @@ func (c *Config) Copy() *Config { nc.AutopilotConfig = c.AutopilotConfig.Copy() nc.LicenseConfig = c.LicenseConfig.Copy() nc.SearchConfig = c.SearchConfig.Copy() + nc.KEKProviderConfigs = helper.CopySlice(c.KEKProviderConfigs) return &nc } diff --git a/nomad/encrypter.go b/nomad/encrypter.go index 9264612f74d..f39a2855bd7 100644 --- a/nomad/encrypter.go +++ b/nomad/encrypter.go @@ -11,6 +11,7 @@ import ( "crypto/rsa" "crypto/x509" "encoding/json" + "errors" "fmt" "io/fs" "os" @@ -21,15 +22,21 @@ import ( "github.com/go-jose/go-jose/v3" "github.com/go-jose/go-jose/v3/jwt" + "github.com/hashicorp/go-hclog" log "github.com/hashicorp/go-hclog" kms "github.com/hashicorp/go-kms-wrapping/v2" + wrapping "github.com/hashicorp/go-kms-wrapping/v2" "github.com/hashicorp/go-kms-wrapping/v2/aead" - "golang.org/x/time/rate" - + "github.com/hashicorp/go-kms-wrapping/wrappers/awskms/v2" + "github.com/hashicorp/go-kms-wrapping/wrappers/azurekeyvault/v2" + "github.com/hashicorp/go-kms-wrapping/wrappers/gcpckms/v2" + "github.com/hashicorp/go-kms-wrapping/wrappers/transit/v2" "github.com/hashicorp/nomad/helper" "github.com/hashicorp/nomad/helper/crypto" "github.com/hashicorp/nomad/helper/joseutil" "github.com/hashicorp/nomad/nomad/structs" + "github.com/hashicorp/nomad/nomad/structs/config" + "golang.org/x/time/rate" ) const nomadKeystoreExtension = ".nks.json" @@ -43,8 +50,10 @@ var _ claimSigner = &Encrypter{} // Encrypter is the keyring for encrypting variables and signing workload // identities. type Encrypter struct { - srv *Server - keystorePath string + srv *Server + log hclog.Logger + providerConfigs map[string]*structs.KEKProviderConfig + keystorePath string // issuer is the OIDC Issuer to use for workload identities if configured issuer string @@ -70,25 +79,67 @@ type keyset struct { func NewEncrypter(srv *Server, keystorePath string) (*Encrypter, error) { encrypter := &Encrypter{ - srv: srv, - keystorePath: keystorePath, - keyring: make(map[string]*keyset), - issuer: srv.GetConfig().OIDCIssuer, + srv: srv, + log: srv.logger.With("keyring"), + keystorePath: keystorePath, + keyring: make(map[string]*keyset), + issuer: srv.GetConfig().OIDCIssuer, + providerConfigs: map[string]*structs.KEKProviderConfig{}, } - err := encrypter.loadKeystore() + providerConfigs, err := getProviderConfigs(srv) + if err != nil { + return nil, err + } + encrypter.providerConfigs = providerConfigs + + err = encrypter.loadKeystore() if err != nil { return nil, err } return encrypter, nil } +// fallbackVaultConfig allows the transit provider to fallback to using the +// default Vault cluster's configuration block, instead of repeating those +// fields +func fallbackVaultConfig(provider *structs.KEKProviderConfig, vaultcfg *config.VaultConfig) { + + setFallback := func(key, fallback, env string) { + if provider.Config == nil { + provider.Config = map[string]string{} + } + if _, ok := provider.Config[key]; !ok { + if fallback != "" { + provider.Config[key] = fallback + } else { + provider.Config[key] = os.Getenv(env) + } + } + } + + setFallback("address", vaultcfg.Addr, "VAULT_ADDR") + setFallback("token", vaultcfg.Token, "VAULT_TOKEN") + setFallback("tls_ca_cert", vaultcfg.TLSCaPath, "VAULT_CACERT") + setFallback("tls_client_cert", vaultcfg.TLSCertFile, "VAULT_CLIENT_CERT") + setFallback("tls_client_key", vaultcfg.TLSKeyFile, "VAULT_CLIENT_KEY") + setFallback("tls_server_name", vaultcfg.TLSServerName, "VAULT_TLS_SERVER_NAME") + + skipVerify := "" + if vaultcfg.TLSSkipVerify != nil { + skipVerify = fmt.Sprintf("%v", *vaultcfg.TLSSkipVerify) + } + setFallback("tls_skip_verify", skipVerify, "VAULT_SKIP_VERIFY") +} + func (e *Encrypter) loadKeystore() error { if err := os.MkdirAll(e.keystorePath, 0o700); err != nil { return err } + keyErrors := map[string]error{} + return filepath.Walk(e.keystorePath, func(path string, info fs.FileInfo, err error) error { if err != nil { return fmt.Errorf("could not read path %s from keystore: %v", path, err) @@ -103,13 +154,22 @@ func (e *Encrypter) loadKeystore() error { if !strings.HasSuffix(path, nomadKeystoreExtension) { return nil } - id := strings.TrimSuffix(filepath.Base(path), nomadKeystoreExtension) + idWithIndex := strings.TrimSuffix(filepath.Base(path), nomadKeystoreExtension) + id, _, _ := strings.Cut(idWithIndex, ".") if !helper.IsUUID(id) { return nil } + e.lock.RLock() + _, ok := e.keyring[id] + e.lock.RUnlock() + if ok { + return nil // already loaded this key from another file + } + key, err := e.loadKeyFromStore(path) if err != nil { + keyErrors[id] = err return fmt.Errorf("could not load key file %s from keystore: %w", path, err) } if key.Meta.KeyID != id { @@ -120,6 +180,10 @@ func (e *Encrypter) loadKeystore() error { if err != nil { return fmt.Errorf("could not add key file %s to keystore: %w", path, err) } + + // we loaded this key from at least one KEK configuration, so clear any + // error from a previous file that we couldn't read from + delete(keyErrors, id) return nil }) } @@ -337,16 +401,16 @@ func (e *Encrypter) addCipher(rootKey *structs.RootKey) error { return nil } -// GetKey retrieves the key material by ID from the keyring -func (e *Encrypter) GetKey(keyID string) ([]byte, []byte, error) { +// GetKey retrieves the key material by ID from the keyring. +func (e *Encrypter) GetKey(keyID string) (*structs.RootKey, error) { e.lock.RLock() defer e.lock.RUnlock() keyset, err := e.keysetByIDLocked(keyID) if err != nil { - return nil, nil, err + return nil, err } - return keyset.rootKey.Key, keyset.rootKey.RSAKey, nil + return keyset.rootKey, nil } // activeKeySetLocked returns the keyset that belongs to the key marked as @@ -384,47 +448,73 @@ func (e *Encrypter) RemoveKey(keyID string) error { return nil } -// saveKeyToStore serializes a root key to the on-disk keystore. -func (e *Encrypter) saveKeyToStore(rootKey *structs.RootKey) error { - - kek, err := crypto.Bytes(32) - if err != nil { - return fmt.Errorf("failed to generate key wrapper key: %w", err) +func (e *Encrypter) encryptDEK(rootKey *structs.RootKey, provider *structs.KEKProviderConfig) (*structs.KeyEncryptionKeyWrapper, error) { + if provider == nil { + panic("can't encrypt DEK without a provider") } - wrapper, err := e.newKMSWrapper(rootKey.Meta.KeyID, kek) + var kek []byte + var err error + if provider.Provider == string(structs.KEKProviderAEAD) || provider.Provider == "" { + kek, err = crypto.Bytes(32) + if err != nil { + return nil, fmt.Errorf("failed to generate key wrapper key: %w", err) + } + } + wrapper, err := e.newKMSWrapper(provider, rootKey.Meta.KeyID, kek) if err != nil { - return fmt.Errorf("failed to create encryption wrapper: %w", err) + return nil, fmt.Errorf("unable to create key wrapper: %w", err) } + rootBlob, err := wrapper.Encrypt(e.srv.shutdownCtx, rootKey.Key) if err != nil { - return fmt.Errorf("failed to encrypt root key: %w", err) + return nil, fmt.Errorf("failed to encrypt root key: %w", err) } - kekWrapper := &structs.KeyEncryptionKeyWrapper{ - Meta: rootKey.Meta, - EncryptedDataEncryptionKey: rootBlob.Ciphertext, - KeyEncryptionKey: kek, + Meta: rootKey.Meta, + KeyEncryptionKey: kek, + Provider: provider.Provider, + ProviderID: provider.ID(), + WrappedDataEncryptionKey: rootBlob, } // Only keysets created after 1.7.0 will contain an RSA key. if len(rootKey.RSAKey) > 0 { rsaBlob, err := wrapper.Encrypt(e.srv.shutdownCtx, rootKey.RSAKey) if err != nil { - return fmt.Errorf("failed to encrypt rsa key: %w", err) + return nil, fmt.Errorf("failed to encrypt rsa key: %w", err) } - kekWrapper.EncryptedRSAKey = rsaBlob.Ciphertext + kekWrapper.WrappedRSAKey = rsaBlob } - buf, err := json.Marshal(kekWrapper) - if err != nil { - return err - } + return kekWrapper, nil +} - path := filepath.Join(e.keystorePath, rootKey.Meta.KeyID+nomadKeystoreExtension) - err = os.WriteFile(path, buf, 0o600) - if err != nil { - return err +// saveKeyToStore serializes a root key to the on-disk keystore. +func (e *Encrypter) saveKeyToStore(rootKey *structs.RootKey) error { + + for _, provider := range e.providerConfigs { + if !provider.Active { + continue + } + kekWrapper, err := e.encryptDEK(rootKey, provider) + if err != nil { + return err + } + + buf, err := json.Marshal(kekWrapper) + if err != nil { + return err + } + + filename := fmt.Sprintf("%s.%s%s", + rootKey.Meta.KeyID, provider.ID(), nomadKeystoreExtension) + path := filepath.Join(e.keystorePath, filename) + err = os.WriteFile(path, buf, 0o600) + if err != nil { + return err + } } + return nil } @@ -446,29 +536,47 @@ func (e *Encrypter) loadKeyFromStore(path string) (*structs.RootKey, error) { return nil, err } + if kekWrapper.ProviderID == "" { + kekWrapper.ProviderID = string(structs.KEKProviderAEAD) + } + provider, ok := e.providerConfigs[kekWrapper.ProviderID] + if !ok { + return nil, fmt.Errorf("no such provider %q configured", kekWrapper.ProviderID) + } + // the errors that bubble up from this library can be a bit opaque, so make // sure we wrap them with as much context as possible - wrapper, err := e.newKMSWrapper(meta.KeyID, kekWrapper.KeyEncryptionKey) + wrapper, err := e.newKMSWrapper(provider, meta.KeyID, kekWrapper.KeyEncryptionKey) if err != nil { - return nil, fmt.Errorf("unable to create key wrapper cipher: %w", err) + return nil, fmt.Errorf("unable to create key wrapper: %w", err) } - key, err := wrapper.Decrypt(e.srv.shutdownCtx, &kms.BlobInfo{ - Ciphertext: kekWrapper.EncryptedDataEncryptionKey, - }) + wrappedDEK := kekWrapper.WrappedDataEncryptionKey + if wrappedDEK == nil { + // older KEK wrapper versions with AEAD-only have the key material in a + // different field + wrappedDEK = &wrapping.BlobInfo{Ciphertext: kekWrapper.EncryptedDataEncryptionKey} + } + key, err := wrapper.Decrypt(e.srv.shutdownCtx, wrappedDEK) if err != nil { - return nil, fmt.Errorf("unable to decrypt wrapped root key: %w", err) + return nil, fmt.Errorf("%w (root key): %w", ErrDecryptFailed, err) } // Decrypt RSAKey for Workload Identity JWT signing if one exists. Prior to // 1.7 an ed25519 key derived from the root key was used instead of an RSA // key. var rsaKey []byte - if len(kekWrapper.EncryptedRSAKey) > 0 { - rsaKey, err = wrapper.Decrypt(e.srv.shutdownCtx, &kms.BlobInfo{ - Ciphertext: kekWrapper.EncryptedRSAKey, - }) + if kekWrapper.WrappedRSAKey != nil { + rsaKey, err = wrapper.Decrypt(e.srv.shutdownCtx, kekWrapper.WrappedRSAKey) if err != nil { - return nil, fmt.Errorf("unable to decrypt wrapped rsa key: %w", err) + return nil, fmt.Errorf("%w (rsa key): %w", ErrDecryptFailed, err) + } + } else if len(kekWrapper.EncryptedRSAKey) > 0 { + // older KEK wrapper versions with AEAD-only have the key material in a + // different field + rsaKey, err = wrapper.Decrypt(e.srv.shutdownCtx, &wrapping.BlobInfo{ + Ciphertext: kekWrapper.EncryptedRSAKey}) + if err != nil { + return nil, fmt.Errorf("%w (rsa key): %w", ErrDecryptFailed, err) } } @@ -479,6 +587,8 @@ func (e *Encrypter) loadKeyFromStore(path string) (*structs.RootKey, error) { }, nil } +var ErrDecryptFailed = errors.New("unable to decrypt wrapped key") + // GetPublicKey returns the public signing key for the requested key id or an // error if the key could not be found. func (e *Encrypter) GetPublicKey(keyID string) (*structs.KeyringPublicKey, error) { @@ -508,19 +618,44 @@ func (e *Encrypter) GetPublicKey(keyID string) (*structs.KeyringPublicKey, error } // newKMSWrapper returns a go-kms-wrapping interface the caller can use to -// encrypt the RootKey with a key encryption key (KEK). This is a bit of -// security theatre for local on-disk key material, but gives us a shim for -// external KMS providers in the future. -func (e *Encrypter) newKMSWrapper(keyID string, kek []byte) (kms.Wrapper, error) { - wrapper := aead.NewWrapper() - wrapper.SetConfig(context.Background(), - aead.WithAeadType(kms.AeadTypeAesGcm), - aead.WithHashType(kms.HashTypeSha256), - kms.WithKeyId(keyID), - ) - err := wrapper.SetAesGcmKeyBytes(kek) - if err != nil { - return nil, err +// encrypt the RootKey with a key encryption key (KEK). +func (e *Encrypter) newKMSWrapper(provider *structs.KEKProviderConfig, keyID string, kek []byte) (kms.Wrapper, error) { + var wrapper kms.Wrapper + + // note: adding support for another provider from go-kms-wrapping is a + // matter of adding the dependency and another case here, but the remaining + // third-party providers add significantly to binary size + + switch provider.Provider { + case structs.KEKProviderAWSKMS: + wrapper = awskms.NewWrapper() + case structs.KEKProviderAzureKeyVault: + wrapper = azurekeyvault.NewWrapper() + case structs.KEKProviderGCPCloudKMS: + wrapper = gcpckms.NewWrapper() + case structs.KEKProviderVaultTransit: + wrapper = transit.NewWrapper() + + default: // "aead" + wrapper := aead.NewWrapper() + wrapper.SetConfig(context.Background(), + aead.WithAeadType(kms.AeadTypeAesGcm), + aead.WithHashType(kms.HashTypeSha256), + kms.WithKeyId(keyID), + ) + err := wrapper.SetAesGcmKeyBytes(kek) + if err != nil { + return nil, err + } + return wrapper, nil + } + + config, ok := e.providerConfigs[provider.ID()] + if ok { + _, err := wrapper.SetConfig(context.Background(), wrapping.WithConfigMap(config.Config)) + if err != nil { + return nil, err + } } return wrapper, nil } @@ -582,7 +717,7 @@ func (krr *KeyringReplicator) run(ctx context.Context) { } keyMeta := raw.(*structs.RootKeyMeta) - if key, _, err := krr.encrypter.GetKey(keyMeta.KeyID); err == nil && len(key) > 0 { + if key, err := krr.encrypter.GetKey(keyMeta.KeyID); err == nil && len(key.Key) > 0 { // the key material is immutable so if we've already got it // we can move on to the next key continue @@ -595,6 +730,7 @@ func (krr *KeyringReplicator) run(ctx context.Context) { // prevent this case from sending excessive RPCs krr.logger.Error(err.Error(), "key", keyMeta.KeyID) } + } } } diff --git a/nomad/encrypter_ce.go b/nomad/encrypter_ce.go new file mode 100644 index 00000000000..211f960f17c --- /dev/null +++ b/nomad/encrypter_ce.go @@ -0,0 +1,42 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +//go:build !ent +// +build !ent + +package nomad + +import ( + "fmt" + + "github.com/hashicorp/nomad/nomad/structs" +) + +func getProviderConfigs(srv *Server) (map[string]*structs.KEKProviderConfig, error) { + providerConfigs := map[string]*structs.KEKProviderConfig{} + config := srv.GetConfig() + var active int + for _, provider := range config.KEKProviderConfigs { + if provider.Active { + active++ + } + if provider.Provider == structs.KEKProviderVaultTransit { + fallbackVaultConfig(provider, config.GetDefaultVault()) + } + + providerConfigs[provider.ID()] = provider + } + if active > 1 { + return nil, fmt.Errorf( + "only one server.keyring can be active in Nomad Community Edition") + } + + if len(srv.config.KEKProviderConfigs) == 0 { + providerConfigs[string(structs.KEKProviderAEAD)] = &structs.KEKProviderConfig{ + Provider: string(structs.KEKProviderAEAD), + Active: true, + } + } + + return providerConfigs, nil +} diff --git a/nomad/encrypter_test.go b/nomad/encrypter_test.go index a126f468357..e3227daf325 100644 --- a/nomad/encrypter_test.go +++ b/nomad/encrypter_test.go @@ -9,6 +9,7 @@ import ( "encoding/base64" "encoding/json" "fmt" + "maps" "os" "path/filepath" "testing" @@ -22,9 +23,12 @@ import ( "github.com/stretchr/testify/require" "github.com/hashicorp/nomad/ci" + "github.com/hashicorp/nomad/helper/pointer" + "github.com/hashicorp/nomad/helper/testlog" "github.com/hashicorp/nomad/helper/uuid" "github.com/hashicorp/nomad/nomad/mock" "github.com/hashicorp/nomad/nomad/structs" + "github.com/hashicorp/nomad/nomad/structs/config" "github.com/hashicorp/nomad/testutil" ) @@ -51,10 +55,10 @@ func (s *mockSigner) SignClaims(c *structs.IdentityClaims) (token, keyID string, func TestEncrypter_LoadSave(t *testing.T) { ci.Parallel(t) - srv, cleanupSrv := TestServer(t, func(c *Config) { - c.NumSchedulers = 0 - }) - t.Cleanup(cleanupSrv) + srv := &Server{ + logger: testlog.HCLogger(t), + config: &Config{}, + } tmpDir := t.TempDir() encrypter, err := NewEncrypter(srv, tmpDir) @@ -73,7 +77,7 @@ func TestEncrypter_LoadSave(t *testing.T) { // startup code path gotKey, err := encrypter.loadKeyFromStore( - filepath.Join(tmpDir, key.Meta.KeyID+".nks.json")) + filepath.Join(tmpDir, key.Meta.KeyID+".aead.nks.json")) must.NoError(t, err) must.NoError(t, encrypter.addCipher(gotKey)) must.Greater(t, 0, len(gotKey.RSAKey)) @@ -84,6 +88,33 @@ func TestEncrypter_LoadSave(t *testing.T) { must.Greater(t, 0, len(active.rootKey.RSAKey)) }) } + + t.Run("legacy aead wrapper", func(t *testing.T) { + key, err := structs.NewRootKey(structs.EncryptionAlgorithmAES256GCM) + must.NoError(t, err) + + // create a wrapper file identical to those before we had external KMS + kekWrapper, err := encrypter.encryptDEK(key, &structs.KEKProviderConfig{}) + kekWrapper.Provider = "" + kekWrapper.ProviderID = "" + kekWrapper.EncryptedDataEncryptionKey = kekWrapper.WrappedDataEncryptionKey.Ciphertext + kekWrapper.EncryptedRSAKey = kekWrapper.WrappedRSAKey.Ciphertext + kekWrapper.WrappedDataEncryptionKey = nil + kekWrapper.WrappedRSAKey = nil + + buf, err := json.Marshal(kekWrapper) + must.NoError(t, err) + + path := filepath.Join(tmpDir, key.Meta.KeyID+".nks.json") + err = os.WriteFile(path, buf, 0o600) + must.NoError(t, err) + + gotKey, err := encrypter.loadKeyFromStore(path) + must.NoError(t, err) + must.NoError(t, encrypter.addCipher(gotKey)) + must.Greater(t, 0, len(gotKey.RSAKey)) + }) + } // TestEncrypter_Restore exercises the entire reload of a keystore, @@ -253,7 +284,7 @@ func TestEncrypter_KeyringReplication(t *testing.T) { keyID1 := listResp.Keys[0].KeyID keyPath := filepath.Join(leader.GetConfig().DataDir, "keystore", - keyID1+nomadKeystoreExtension) + keyID1+".aead.nks.json") _, err := os.Stat(keyPath) must.NoError(t, err, must.Sprint("expected key to be found in leader keystore")) @@ -264,7 +295,7 @@ func TestEncrypter_KeyringReplication(t *testing.T) { return func() bool { for _, srv := range servers { keyPath := filepath.Join(srv.GetConfig().DataDir, "keystore", - keyID+nomadKeystoreExtension) + keyID+".aead.nks.json") if _, err := os.Stat(keyPath); err != nil { return false } @@ -302,7 +333,7 @@ func TestEncrypter_KeyringReplication(t *testing.T) { must.NotNil(t, getResp.Key, must.Sprint("expected key to be found on leader")) keyPath = filepath.Join(leader.GetConfig().DataDir, "keystore", - keyID2+nomadKeystoreExtension) + keyID2+".aead.nks.json") _, err = os.Stat(keyPath) must.NoError(t, err, must.Sprint("expected key to be found in leader keystore")) @@ -639,3 +670,63 @@ func TestEncrypter_Upgrade17(t *testing.T) { _, err = srv.encrypter.VerifyClaim(oldRawJWT) must.NoError(t, err) } + +func TestEncrypter_TransitConfigFallback(t *testing.T) { + srv := &Server{ + logger: testlog.HCLogger(t), + config: &Config{ + VaultConfigs: map[string]*config.VaultConfig{structs.VaultDefaultCluster: { + Addr: "https://localhost:8203", + TLSCaPath: "/etc/certs/ca", + TLSCertFile: "/var/certs/vault.crt", + TLSKeyFile: "/var/certs/vault.key", + TLSSkipVerify: pointer.Of(true), + TLSServerName: "foo", + Token: "vault-token", + }}, + KEKProviderConfigs: []*structs.KEKProviderConfig{ + { + Provider: "transit", + Name: "no-fallback", + Config: map[string]string{ + "address": "https://localhost:8203", + "token": "vault-token", + "tls_ca_cert": "/etc/certs/ca", + "tls_client_cert": "/var/certs/vault.crt", + "tls_client_key": "/var/certs/vault.key", + "tls_server_name": "foo", + "tls_skip_verify": "true", + }, + }, + { + Provider: "transit", + Name: "fallback-to-vault-block", + }, + { + Provider: "transit", + Name: "fallback-to-env", + }, + }, + }, + } + + providers := srv.config.KEKProviderConfigs + expect := maps.Clone(providers[0].Config) + + fallbackVaultConfig(providers[0], srv.config.GetDefaultVault()) + must.Eq(t, expect, providers[0].Config, must.Sprint("expected no change")) + + fallbackVaultConfig(providers[1], srv.config.GetDefaultVault()) + must.Eq(t, expect, providers[1].Config, must.Sprint("expected fallback to vault block")) + + t.Setenv("VAULT_ADDR", "https://localhost:8203") + t.Setenv("VAULT_TOKEN", "vault-token") + t.Setenv("VAULT_CACERT", "/etc/certs/ca") + t.Setenv("VAULT_CLIENT_CERT", "/var/certs/vault.crt") + t.Setenv("VAULT_CLIENT_KEY", "/var/certs/vault.key") + t.Setenv("VAULT_TLS_SERVER_NAME", "foo") + t.Setenv("VAULT_SKIP_VERIFY", "true") + + fallbackVaultConfig(providers[2], &config.VaultConfig{}) + must.Eq(t, expect, providers[2].Config, must.Sprint("expected fallback to env")) +} diff --git a/nomad/keyring_endpoint.go b/nomad/keyring_endpoint.go index 65a8d02b38c..95302b63916 100644 --- a/nomad/keyring_endpoint.go +++ b/nomad/keyring_endpoint.go @@ -268,15 +268,10 @@ func (k *Keyring) Get(args *structs.KeyringGetRootKeyRequest, reply *structs.Key } // retrieve the key material from the keyring - key, rsaKey, err := k.encrypter.GetKey(keyMeta.KeyID) + rootKey, err := k.encrypter.GetKey(keyMeta.KeyID) if err != nil { return err } - rootKey := &structs.RootKey{ - Meta: keyMeta, - Key: key, - RSAKey: rsaKey, - } reply.Key = rootKey // Use the last index that affected the policy table diff --git a/nomad/structs/keyring.go b/nomad/structs/keyring.go index 103068716d8..67379d89a9b 100644 --- a/nomad/structs/keyring.go +++ b/nomad/structs/keyring.go @@ -9,10 +9,12 @@ import ( "crypto/rsa" "crypto/x509" "fmt" + "maps" "net/url" "time" "github.com/go-jose/go-jose/v3" + wrapping "github.com/hashicorp/go-kms-wrapping/v2" "github.com/hashicorp/nomad/helper" "github.com/hashicorp/nomad/helper/crypto" "github.com/hashicorp/nomad/helper/uuid" @@ -86,6 +88,59 @@ type RootKeyMeta struct { State RootKeyState } +// KEKProviderName enum are the built-in KEK providers. +type KEKProviderName string + +const ( + KEKProviderAEAD KEKProviderName = "aead" + KEKProviderAWSKMS = "awskms" + KEKProviderAzureKeyVault = "azurekeyvault" + KEKProviderGCPCloudKMS = "gcpckms" + KEKProviderVaultTransit = "transit" +) + +// KEKProviderConfig is the server configuration for an external KMS provider +// the server will use as a Key Encryption Key (KEK) for encrypting/decrypting +// the DEK. +type KEKProviderConfig struct { + Provider string `hcl:",key"` + Name string `hcl:"name"` + Active bool `hcl:"active"` + Config map[string]string `hcl:"-" json:"-"` + + // ExtraKeysHCL gets used by HCL to surface unknown keys. The parser will + // then read these keys to create the Config map, so that we don't need a + // nested "config" block/map in the config file + ExtraKeysHCL []string `hcl:",unusedKeys" json:"-"` +} + +func (c *KEKProviderConfig) Copy() *KEKProviderConfig { + return &KEKProviderConfig{ + Provider: c.Provider, + Active: c.Active, + Name: c.Name, + Config: maps.Clone(c.Config), + } +} + +// Merge is used to merge two configurations. Note that Provider and Name should +// always be identical before we merge. +func (c *KEKProviderConfig) Merge(o *KEKProviderConfig) *KEKProviderConfig { + result := c.Copy() + result.Active = o.Active + for k, v := range o.Config { + result.Config[k] = v + } + return result +} + +func (c *KEKProviderConfig) ID() string { + if c.Name == "" { + return c.Provider + } + return c.Provider + "." + c.Name +} + // RootKeyState enum describes the lifecycle of a root key. type RootKeyState string @@ -192,13 +247,24 @@ func (rkm *RootKeyMeta) Validate() error { } // KeyEncryptionKeyWrapper is the struct that gets serialized for the on-disk -// KMS wrapper. This struct includes the server-specific key-wrapping key and -// should never be sent over RPC. +// KMS wrapper. When using the AEAD provider, this struct includes the +// server-specific key-wrapping key. This struct should never be sent over RPC +// or written to Raft. type KeyEncryptionKeyWrapper struct { - Meta *RootKeyMeta - EncryptedDataEncryptionKey []byte `json:"DEK"` - EncryptedRSAKey []byte `json:"RSAKey"` - KeyEncryptionKey []byte `json:"KEK"` + Meta *RootKeyMeta + + Provider string `json:"Provider,omitempty"` + ProviderID string `json:"ProviderID,omitempty"` + WrappedDataEncryptionKey *wrapping.BlobInfo `json:"WrappedDEK,omitempty"` + WrappedRSAKey *wrapping.BlobInfo `json:"WrappedRSAKey,omitempty"` + KeyEncryptionKey []byte `json:"KEK,omitempty"` + + // These fields were used for AEAD before we added support for external + // KMS. The wrapped key returned from the go-kms-wrapper library includes + // the ciphertext but we need all the fields in order to decrypt. We'll + // leave these fields so we can load keys from older servers. + EncryptedDataEncryptionKey []byte `json:"DEK,omitempty"` + EncryptedRSAKey []byte `json:"RSAKey,omitempty"` } // EncryptionAlgorithm chooses which algorithm is used for @@ -261,7 +327,7 @@ type KeyringGetRootKeyResponse struct { // KeyringUpdateRootKeyMetaRequest is used internally for key // replication so that we have a request wrapper for writing the -// metadata to the FSM without including the key material +// metadata to the FSM without including the key material. type KeyringUpdateRootKeyMetaRequest struct { RootKeyMeta *RootKeyMeta Rekey bool From 003c19862cd8c2f145bf7eae20bc3136c2e81d63 Mon Sep 17 00:00:00 2001 From: Tim Gross Date: Thu, 18 Jul 2024 09:53:12 -0400 Subject: [PATCH 2/2] deps: update go-kms-wrapping and Azure SDK I'm pulling this out to a shared PR between the two, because it'll make backporting easier. Closes: #23621 Closes: #23589 --- go.mod | 8 ++++---- go.sum | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 28bf9eaba07..cb979ee1539 100644 --- a/go.mod +++ b/go.mod @@ -57,7 +57,7 @@ require ( github.com/hashicorp/go-getter v1.7.5 github.com/hashicorp/go-hclog v1.6.3 github.com/hashicorp/go-immutable-radix/v2 v2.1.0 - github.com/hashicorp/go-kms-wrapping/v2 v2.0.15 + github.com/hashicorp/go-kms-wrapping/v2 v2.0.16 github.com/hashicorp/go-kms-wrapping/wrappers/awskms/v2 v2.0.9 github.com/hashicorp/go-kms-wrapping/wrappers/azurekeyvault/v2 v2.0.11 github.com/hashicorp/go-kms-wrapping/wrappers/gcpckms/v2 v2.0.12 @@ -146,10 +146,10 @@ require ( cloud.google.com/go/kms v1.15.0 // indirect cloud.google.com/go/storage v1.30.1 // indirect dario.cat/mergo v1.0.0 // indirect - github.com/Azure/azure-sdk-for-go v56.3.0+incompatible // indirect - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.0 // indirect + github.com/Azure/azure-sdk-for-go v68.0.0+incompatible // indirect + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1 // indirect github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2 // indirect github.com/Azure/azure-sdk-for-go/sdk/keyvault/azkeys v0.10.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 // indirect github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect diff --git a/go.sum b/go.sum index 1a383501f99..0938bfba642 100644 --- a/go.sum +++ b/go.sum @@ -192,14 +192,14 @@ dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7 github.com/AdaLogics/go-fuzz-headers v0.0.0-20210715213245-6c3934b029d8 h1:V8krnnfGj4pV65YLUm3C0/8bl7V5Nry2Pwvy3ru/wLc= github.com/AdaLogics/go-fuzz-headers v0.0.0-20210715213245-6c3934b029d8/go.mod h1:CzsSbkDixRphAF5hS6wbMKq0eI6ccJRb7/A0M6JBnwg= github.com/Azure/azure-sdk-for-go v44.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/azure-sdk-for-go v56.3.0+incompatible h1:DmhwMrUIvpeoTDiWRDtNHqelNUd3Og8JCkrLHQK795c= -github.com/Azure/azure-sdk-for-go v56.3.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.0 h1:fb8kj/Dh4CSwgsOzHeZY4Xh68cFVbzXx+ONXGMY//4w= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.0/go.mod h1:uReU2sSxZExRPBAg3qKzmAucSi51+SP1OhohieR821Q= +github.com/Azure/azure-sdk-for-go v68.0.0+incompatible h1:fcYLmCpyNYRnvJbPerq7U0hS+6+I79yEDJBqVNcqUzU= +github.com/Azure/azure-sdk-for-go v68.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1 h1:E+OJmp2tPvt1W+amx48v1eqbjDYsgN+RzP4q16yV5eM= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1/go.mod h1:a6xsAQUZg+VsS3TJ05SRp524Hs4pZ/AeFSr5ENf0Yjo= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0 h1:BMAjVKJM0U/CYF27gA0ZMmXGkOcvfFtD0oHVZ1TIPRI= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0/go.mod h1:1fXstnBMas5kzG+S3q8UoJcmyU6nUeunJcMDHcRYHhs= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.0 h1:d81/ng9rET2YqdVkVwkb6EXeRrLJIwyGnJcAlAWKwhs= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.0/go.mod h1:s4kgfzA0covAXNicZHDMN58jExvcng2mC/DepXiF1EI= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2 h1:LqbJ/WzJUwBf8UiaSzgX7aMclParm9/5Vgp+TY51uBQ= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2/go.mod h1:yInRyqWXAuaPrgI7p70+lDDgh3mlBohis29jGMISnmc= github.com/Azure/azure-sdk-for-go/sdk/keyvault/azkeys v0.10.0 h1:m/sWOGCREuSBqg2htVQTBY8nOZpyajYztF0vUvSZTuM= github.com/Azure/azure-sdk-for-go/sdk/keyvault/azkeys v0.10.0/go.mod h1:Pu5Zksi2KrU7LPbZbNINx6fuVrUp/ffvpxdDj+i8LeE= github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 h1:FbH3BbSb4bvGluTesZZ+ttN/MDsnMmQP36OSnDuSXqw= @@ -669,8 +669,8 @@ github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJ github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-immutable-radix/v2 v2.1.0 h1:CUW5RYIcysz+D3B+l1mDeXrQ7fUvGGCwJfdASSzbrfo= github.com/hashicorp/go-immutable-radix/v2 v2.1.0/go.mod h1:hgdqLXA4f6NIjRVisM1TJ9aOJVNRqKZj+xDGF6m7PBw= -github.com/hashicorp/go-kms-wrapping/v2 v2.0.15 h1:f3+/VbanXOmVAaDBKwRiVmeL7EX340a4YmaTItMF4Xs= -github.com/hashicorp/go-kms-wrapping/v2 v2.0.15/go.mod h1:0dWtzl2ilqKpavgM3id/kFK9L3tjo6fS4OhbVPSYpnQ= +github.com/hashicorp/go-kms-wrapping/v2 v2.0.16 h1:WZeXfD26QMWYC35at25KgE021SF9L3u9UMHK8fJAdV0= +github.com/hashicorp/go-kms-wrapping/v2 v2.0.16/go.mod h1:ZiKZctjRTLEppuRwrttWkp71VYMbTTCkazK4xT7U/NQ= github.com/hashicorp/go-kms-wrapping/wrappers/awskms/v2 v2.0.9 h1:qdxeZvDMRGZ3YSE4Oz0Pp7WUSUn5S6cWZguEOkEVL50= github.com/hashicorp/go-kms-wrapping/wrappers/awskms/v2 v2.0.9/go.mod h1:DcXbvVpgNWbxGmxgmu3QN64bEydMu14Cpe34RRR30HY= github.com/hashicorp/go-kms-wrapping/wrappers/azurekeyvault/v2 v2.0.11 h1:/7SKkYIhA8cr3l8m1EKT6Q90bPoSVqqVBuQ6HgoMIkw=