F1707 secret ref interpolation (#31)

internal/provider/data_env_variables.go

@@ -54,6 +54,13 @@ func (*dataEnvVariables) Schema(ctx context.Context) *tfprotov5.Schema {
 			Computed:        true,
 			Sensitive:       true,
 		},
+		{
+			Name:            "secret_refs",
+			Type:            tftypes.Map{ElementType: tftypes.String},
+			Description:     "Map of environment variables that refer to an existing secret key for their values.",
+			DescriptionKind: tfprotov5.StringKindMarkdown,
+			Computed:        true,
+		},
 	}
 
 	return &tfprotov5.Schema{
@@ -104,8 +111,22 @@ func (d *dataEnvVariables) Read(ctx context.Context, config map[string]tftypes.V
 	envVariables := copyMap(inputEnvVariables)
 	secrets := copyMap(inputSecrets)
 
-	regexPattern := "{{\\s?%s\\s?}}"
+	// if any env variables contain secret refs, pull them out
+	secretRefs := make(map[string]tftypes.Value, 0)
+	refRegexPattern := "{{\\s*secret\\((.+)\\)\\s*}}"
+	regex := regexp.MustCompile(refRegexPattern)
+	for k, v := range envVariables {
+		result := regex.FindStringSubmatch(extractStringFromTfValue(v))
+		// if we found a proper match, the result will contain a slice
+		// with the full match as the first item and the capture group as the second
+		if len(result) > 1 {
+			ref := result[1]
+			secretRefs[k] = tftypes.NewValue(tftypes.String, ref)
+			delete(envVariables, k)
+		}
+	}
 
+	regexPattern := "{{\\s*%s\\s*}}"
 	// we are going to first loop through all the input secrets
 	//   find and replace this secret in all the rest of the env variables and secrets
 	for key, secret := range secrets {
@@ -157,7 +178,7 @@ func (d *dataEnvVariables) Read(ctx context.Context, config map[string]tftypes.V
 	}
 
 	// calculate the unique id for this data source based on a hash of the resulting env variables and secrets
-	id := d.HashFromValues(envVariables, secrets)
+	id := d.HashFromValues(envVariables, secrets, secretRefs)
 
 	tflog.Debug(ctx, "id", id)
 	tflog.Debug(ctx, "env_variables", envVariables)
@@ -169,17 +190,21 @@ func (d *dataEnvVariables) Read(ctx context.Context, config map[string]tftypes.V
 		"input_secrets":       tftypes.NewValue(tftypes.Map{ElementType: tftypes.String}, inputSecrets),
 		"env_variables":       tftypes.NewValue(tftypes.Map{ElementType: tftypes.String}, envVariables),
 		"secrets":             tftypes.NewValue(tftypes.Map{ElementType: tftypes.String}, secrets),
+		"secret_refs":         tftypes.NewValue(tftypes.Map{ElementType: tftypes.String}, secretRefs),
 	}, nil, nil
 }
 
-func (d *dataEnvVariables) HashFromValues(envVariables, secrets map[string]tftypes.Value) string {
+func (d *dataEnvVariables) HashFromValues(envVariables, secrets, secretRefs map[string]tftypes.Value) string {
 	hashString := ""
 	for k, v := range envVariables {
 		hashString += fmt.Sprintf("%s=%s;", k, extractStringFromTfValue(v))
 	}
 	for k, v := range secrets {
 		hashString += fmt.Sprintf("%s=%s;", k, extractStringFromTfValue(v))
 	}
+	for k, v := range secretRefs {
+		hashString += fmt.Sprintf("%s=%s;", k, extractStringFromTfValue(v))
+	}
 
 	sum := sha256.Sum256([]byte(hashString))
 	return fmt.Sprintf("%x", sum)

internal/provider/data_env_variables_test.go

@@ -8,15 +8,19 @@ import (
 )
 
 func TestDataVariables(t *testing.T) {
+	arn := "arn:aws:secretsmanager:us-east-1:0123456789012:secret:my_little_secret"
+
 	checks := resource.ComposeTestCheckFunc(
-		resource.TestCheckResourceAttr("data.ns_env_variables.this", "input_env_variables.%", "6"),
+		resource.TestCheckResourceAttr("data.ns_env_variables.this", "input_env_variables.%", "7"),
 		resource.TestCheckResourceAttr("data.ns_env_variables.this", "input_secrets.%", "1"),
 		resource.TestCheckResourceAttr("data.ns_env_variables.this", "env_variables.%", "5"),
 		resource.TestCheckResourceAttr("data.ns_env_variables.this", "env_variables.FEATURE_FLAG_0115", "true"),
 		resource.TestCheckResourceAttr("data.ns_env_variables.this", "env_variables.IDENTIFIER", "primary.acme-api.dev"),
 		resource.TestCheckResourceAttr("data.ns_env_variables.this", "secrets.%", "2"),
 		resource.TestCheckResourceAttr("data.ns_env_variables.this", "secrets.POSTGRES_URL", "postgres://user:pass@host:port/db"),
 		resource.TestCheckResourceAttr("data.ns_env_variables.this", "secrets.DATABASE_URL", "postgres://user:pass@host:port/db"),
+		resource.TestCheckResourceAttr("data.ns_env_variables.this", "secret_refs.%", "1"),
+		resource.TestCheckResourceAttr("data.ns_env_variables.this", "secret_refs.VAR_WITH_REF", arn),
 	)
 
 	t.Run("sets up attributes properly hard-coded", func(t *testing.T) {
@@ -32,12 +36,13 @@ data "ns_env_variables" "this" {
 		FEATURE_FLAG_0115 = "true"
 		DATABASE_URL = "{{POSTGRES_URL}}"
 		IDENTIFIER = "{{ NULLSTONE_STACK }}.{{ NULLSTONE_BLOCK }}.{{ NULLSTONE_ENV }}"
+		VAR_WITH_REF = "{{ secret(%s) }}"
 	}
 	input_secrets = {
 		POSTGRES_URL = "postgres://user:pass@host:port/db"
 	}
 }
-`)
+`, arn)
 		getNsConfig, _ := mockNs(nil)
 		getTfeConfig, _ := mockTfe(nil)
 

internal/provider/data_secret_keys.go

@@ -93,17 +93,30 @@ func (d *dataSecretKeys) Read(ctx context.Context, config map[string]tftypes.Val
 	tflog.Debug(ctx, "input_secret_keys", inputSecretKeys)
 
 	// make sure we copy these so our changes below don't affect the original values
-	secretKeys := copySet(inputSecretKeys)
+	envVariables := copyMap(inputEnvVariables)
+
+	// first pull out any env variables that contain refs to secrets in their interpolation
+	refRegexPattern := "{{\\s*secret\\((.+)\\)\\s*}}"
+	regex := regexp.MustCompile(refRegexPattern)
+	for k, v := range envVariables {
+		// if extractStringFromTfValue(v) contains the regex pattern, and the interpolation is set to `secret(arn)`
+		// then we need to skip this replacement and output the arn as a secret_ref
+		if regex.MatchString(extractStringFromTfValue(v)) {
+			delete(envVariables, k)
+		}
+	}
 
-	regexPattern := "{{\\s?%s\\s?}}"
+	// make sure we copy these so our changes below don't affect the original values
+	secretKeys := copySet(inputSecretKeys)
 
+	regexPattern := "{{\\s*%s\\s*}}"
 	// loop through and determine if any of the environment variables contain interpolation using any of the secret keys
 	//   if they do, add their keys to the final set of secret keys
 	added := map[string]bool{}
 	for _, secretKey := range inputSecretKeys {
 		regex := regexp.MustCompile(fmt.Sprintf(regexPattern, extractStringFromTfValue(secretKey)))
 		// first try and replace in the env variables
-		for k, v := range inputEnvVariables {
+		for k, v := range envVariables {
 			// don't add the key more than once, the "added" map keeps track of whether it has already been added
 			if added[k] {
 				continue

internal/provider/data_secret_keys_test.go

@@ -8,10 +8,16 @@ import (
 )
 
 func TestSecretKeys(t *testing.T) {
+	arn := "arn:aws:secretsmanager:us-east-1:0123456789012:secret:my_little_secret"
+
 	checks := resource.ComposeTestCheckFunc(
-		resource.TestCheckResourceAttr("data.ns_secret_keys.this", "input_env_variables.%", "7"),
+		resource.TestCheckResourceAttr("data.ns_secret_keys.this", "input_env_variables.%", "8"),
 		resource.TestCheckResourceAttr("data.ns_secret_keys.this", "input_secret_keys.#", "2"),
 		resource.TestCheckResourceAttr("data.ns_secret_keys.this", "secret_keys.#", "4"),
+		resource.TestCheckResourceAttr("data.ns_secret_keys.this", "secret_keys.0", "DATABASE_URL"),
+		resource.TestCheckResourceAttr("data.ns_secret_keys.this", "secret_keys.1", "DUPLICATE_TEST"),
+		resource.TestCheckResourceAttr("data.ns_secret_keys.this", "secret_keys.2", "POSTGRES_URL"),
+		resource.TestCheckResourceAttr("data.ns_secret_keys.this", "secret_keys.3", "SECRET_KEY_BASE"),
 	)
 
 	t.Run("sets up attributes properly hard-coded", func(t *testing.T) {
@@ -28,13 +34,14 @@ data "ns_secret_keys" "this" {
 		DATABASE_URL = "{{POSTGRES_URL}}"
 		IDENTIFIER = "{{ NULLSTONE_STACK }}.{{ NULLSTONE_BLOCK }}.{{ NULLSTONE_ENV }}"
 		DUPLICATE_TEST = "{{ SECRET_KEY_BASE }}{{ POSTGRES_URL }}"
+		VAR_WITH_REF = "{{ secret(%s) }}"
 	}
 	input_secret_keys = [
 		"POSTGRES_URL",
 		"SECRET_KEY_BASE"
 	]
 }
-`)
+`, arn)
 		getNsConfig, _ := mockNs(nil)
 		getTfeConfig, _ := mockTfe(nil)