-
Notifications
You must be signed in to change notification settings - Fork 54
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Update secret version hashing algorithm (#198)
Updates the version generation algorithm to include an HMAC key. On each mount request, the provider will try to read the HMAC key from a Kubernetes secret and race to create it if not found. This ensures each provider produces consistent versions, and also makes recovering from unexpected errors easy (an admin just deletes the secret) without introducing the complexity and overhead of leader elections. Also makes generating versions best-effort. If we can't use an HMAC key, we log a warning and revert to our pre-1.2.0 behaviour of not reporting a version at all, as consistency and reliability seem much more important than accurate version reporting. If versions thrash about unnecessarily it will cause lots of thrashing for any systems that observe the version.
- Loading branch information
Showing
15 changed files
with
424 additions
and
47 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,96 @@ | ||
| // Copyright (c) HashiCorp, Inc. | ||
| // SPDX-License-Identifier: MPL-2.0 | ||
|
|
||
| package hmac | ||
|
|
||
| import ( | ||
| "context" | ||
| "crypto/rand" | ||
| "errors" | ||
| "fmt" | ||
|
|
||
| corev1 "k8s.io/api/core/v1" | ||
| apierrors "k8s.io/apimachinery/pkg/api/errors" | ||
| metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" | ||
| "k8s.io/client-go/kubernetes" | ||
| ) | ||
|
|
||
| const ( | ||
| hmacKeyName = "key" | ||
| hmacKeyLength = 32 | ||
| ) | ||
|
|
||
| var errDeleteSecret = errors.New("delete the kubernetes secret to trigger an automatic regeneration") | ||
|
|
||
| func NewHMACGenerator(client kubernetes.Interface, secretSpec *corev1.Secret) *HMACGenerator { | ||
| return &HMACGenerator{ | ||
| client: client, | ||
| secretSpec: secretSpec, | ||
| } | ||
| } | ||
|
|
||
| type HMACGenerator struct { | ||
| client kubernetes.Interface | ||
| secretSpec *corev1.Secret | ||
| } | ||
|
|
||
| // GetOrCreateHMACKey will try to read an HMAC key from a Kubernetes secret and | ||
| // race with other pods to create it if not found. The HMAC key is persisted to | ||
| // a Kubernetes secret to ensure all pods are deterministically producing the | ||
| // same version hashes when given the same inputs. | ||
| func (g *HMACGenerator) GetOrCreateHMACKey(ctx context.Context) ([]byte, error) { | ||
| // Fast path - most of the time the secret will already be created. | ||
| secret, err := g.client.CoreV1().Secrets(g.secretSpec.Namespace).Get(ctx, g.secretSpec.Name, metav1.GetOptions{}) | ||
| if err == nil { | ||
| return hmacKeyFromSecret(secret) | ||
| } | ||
| if !apierrors.IsNotFound(err) { | ||
| return nil, err | ||
| } | ||
|
|
||
| // Secret not found. We'll join the race to create it. | ||
| hmacKeyCandidate := make([]byte, hmacKeyLength) | ||
| _, err = rand.Read(hmacKeyCandidate) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| // Make a copy of the secretSpec to avoid a data race. | ||
| secretSpec := *g.secretSpec | ||
| secretSpec.Data = map[string][]byte{ | ||
| hmacKeyName: hmacKeyCandidate, | ||
| } | ||
|
|
||
| var persistedHMACSecret *corev1.Secret | ||
|
|
||
| // Try to create first | ||
| persistedHMACSecret, err = g.client.CoreV1().Secrets(secretSpec.Namespace).Create(ctx, &secretSpec, metav1.CreateOptions{}) | ||
| switch { | ||
| case err == nil: | ||
| // We created the secret, nothing to handle. | ||
| case apierrors.IsAlreadyExists(err): | ||
| // We lost the race to create the secret. Read the existing secret instead. | ||
| persistedHMACSecret, err = g.client.CoreV1().Secrets(secretSpec.Namespace).Get(ctx, secretSpec.Name, metav1.GetOptions{}) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| default: | ||
| // Unexpected error case. | ||
| return nil, err | ||
| } | ||
|
|
||
| return hmacKeyFromSecret(persistedHMACSecret) | ||
| } | ||
|
|
||
| func hmacKeyFromSecret(secret *corev1.Secret) ([]byte, error) { | ||
| hmacKey, ok := secret.Data[hmacKeyName] | ||
| if !ok { | ||
| return nil, fmt.Errorf("expected secret %q to have a key %q; %w", secret.Name, hmacKeyName, errDeleteSecret) | ||
| } | ||
|
|
||
| if len(hmacKey) == 0 { | ||
| return nil, fmt.Errorf("expected secret %q to have a non-zero HMAC key; %w", secret.Name, errDeleteSecret) | ||
| } | ||
|
|
||
| return hmacKey, nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,110 @@ | ||
| package hmac | ||
|
|
||
| import ( | ||
| "context" | ||
| "strings" | ||
| "testing" | ||
|
|
||
| "github.com/stretchr/testify/assert" | ||
| "github.com/stretchr/testify/require" | ||
| corev1 "k8s.io/api/core/v1" | ||
| metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" | ||
| "k8s.io/apimachinery/pkg/runtime" | ||
| "k8s.io/apimachinery/pkg/runtime/schema" | ||
| "k8s.io/client-go/kubernetes/fake" | ||
| k8stesting "k8s.io/client-go/testing" | ||
| ) | ||
|
|
||
| const ( | ||
| secretName = "test-secret" | ||
| secretNamespace = "test-namespace" | ||
| ) | ||
|
|
||
| var secretSpec = &corev1.Secret{ | ||
| ObjectMeta: metav1.ObjectMeta{ | ||
| Name: secretName, | ||
| Namespace: secretNamespace, | ||
| }, | ||
| Data: map[string][]byte{ | ||
| hmacKeyName: []byte(strings.Repeat("a", 32)), | ||
| }, | ||
| } | ||
|
|
||
| func setup(t *testing.T) (*HMACGenerator, *fake.Clientset) { | ||
| client := fake.NewSimpleClientset() | ||
| return NewHMACGenerator(client, secretSpec), client | ||
| } | ||
|
|
||
| func TestGenerateSecretIfNoneExists(t *testing.T) { | ||
| gen, client := setup(t) | ||
|
|
||
| // Add counter functions. | ||
| createCount := countAPICalls(client, "create", "secrets") | ||
| getCount := countAPICalls(client, "get", "secrets") | ||
|
|
||
| // Get an HMAC key, which should create the k8s secret. | ||
| key, err := gen.GetOrCreateHMACKey(context.Background()) | ||
| require.NoError(t, err) | ||
| assert.Len(t, key, hmacKeyLength) | ||
| assert.Equal(t, 1, *createCount) | ||
| assert.Equal(t, 1, *getCount) | ||
| assert.NotEqual(t, string(secretSpec.Data[hmacKeyName]), string(key)) | ||
| assert.NotEmpty(t, string(key)) | ||
| } | ||
|
|
||
| func TestReadSecretIfAlreadyExists(t *testing.T) { | ||
| gen, client := setup(t) | ||
|
|
||
| ctx := context.Background() | ||
| _, err := client.CoreV1().Secrets(secretNamespace).Create(ctx, secretSpec, metav1.CreateOptions{}) | ||
| require.NoError(t, err) | ||
|
|
||
| // Add counter functions. | ||
| createCount := countAPICalls(client, "create", "secrets") | ||
| getCount := countAPICalls(client, "get", "secrets") | ||
|
|
||
| // Get an HMAC key, which should read the existing k8s secret. | ||
| key, err := gen.GetOrCreateHMACKey(ctx) | ||
| require.NoError(t, err) | ||
| assert.Len(t, key, hmacKeyLength) | ||
| assert.Equal(t, 0, *createCount) | ||
| assert.Equal(t, 1, *getCount) | ||
| assert.Equal(t, string(secretSpec.Data[hmacKeyName]), string(key)) | ||
| } | ||
|
|
||
| func TestGracefullyHandlesLosingTheRace(t *testing.T) { | ||
| gen, client := setup(t) | ||
|
|
||
| ctx := context.Background() | ||
|
|
||
| client.PrependReactor("create", "secrets", func(action k8stesting.Action) (handled bool, ret runtime.Object, err error) { | ||
| // Intercept the create call and create the secret just before. | ||
| err = client.Tracker().Create(schema.GroupVersionResource{ | ||
| Group: "", | ||
| Version: "v1", | ||
| Resource: "secrets", | ||
| }, secretSpec, secretNamespace) | ||
| require.NoError(t, err) | ||
| return false, nil, nil | ||
| }) | ||
| createCount := countAPICalls(client, "create", "secrets") | ||
| getCount := countAPICalls(client, "get", "secrets") | ||
|
|
||
| // Get an HMAC key, which should initially find no secret, and then lose the race for creating it. | ||
| key, err := gen.GetOrCreateHMACKey(ctx) | ||
| require.NoError(t, err) | ||
| assert.Len(t, key, hmacKeyLength) | ||
| assert.Equal(t, 1, *createCount) | ||
| assert.Equal(t, 2, *getCount) | ||
| assert.Equal(t, string(secretSpec.Data[hmacKeyName]), string(key)) | ||
| } | ||
|
|
||
| // Counts the number of times an API is called. | ||
| func countAPICalls(client *fake.Clientset, verb string, resource string) *int { | ||
| i := 0 | ||
| client.PrependReactor(verb, resource, func(_ k8stesting.Action) (handled bool, ret runtime.Object, err error) { | ||
| i++ | ||
| return false, nil, nil | ||
| }) | ||
| return &i | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.