workload identity: add support for extra claims config for Vault (#23675

)

Although we encourage users to use Vault roles, sometimes they're going to want
to assign policies based on entity and pre-create entities and aliases based on
claims. This allows them to use single default role (or at least small number of
them) that has a templated policy, but have an escape hatch from that.

When defining Vault entities the `user_claim` must be unique. When writing Vault
binding rules for use with Nomad workload identities the binding rule won't be
able to create a 1:1 mapping because the selector language allows accessing only
a single field. The `nomad_job_id` claim isn't sufficient to uniquely identify a
job because of namespaces. It's possible to create a JWT auth role with
`bound_claims` to avoid this becoming a security problem, but this doesn't allow
for correct accounting of user claims.

Add support for an `extra_claims` block on the server's `default_identity`
blocks for Vault. This allows a cluster administrator to add a custom claim on
all allocations. The values for these claims are interpolatable with a limited
subset of fields, similar to how we interpolate the task environment.

Fixes: #23510
Ref: https://hashicorp.atlassian.net/browse/NET-10372
Ref: https://hashicorp.atlassian.net/browse/NET-10387

e2e/vaultcompat/cluster_setup_test.go

@@ -36,7 +36,7 @@ func roleWID(policies []string) map[string]any {
 	return map[string]any{
 		"role_type":               "jwt",
 		"bound_audiences":         "vault.io",
-		"user_claim":              "/nomad_job_id",
+		"user_claim":              "/extra_claims/nomad_workload_id",
 		"user_claim_json_pointer": true,
 		"claim_mappings": map[string]any{
 			"nomad_namespace": "nomad_namespace",

e2e/vaultcompat/input/restricted_jwt.hcl

@@ -0,0 +1,38 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+job "restricted_jwt" {
+  type = "batch"
+
+  // Tasks in this group are expected to succeed and run to completion.
+  group "success" {
+    vault {}
+
+    count = 2
+
+    // Task default_identity uses the default workload identity injected by the
+    // server and the inherits the Vault configuration from the group.
+    task "authorized" {
+      driver = "raw_exec"
+
+      config {
+        command = "cat"
+        args    = ["${NOMAD_SECRETS_DIR}/secret.txt"]
+      }
+
+      // Vault has an alias that maps this job's nomad_workload_id to an entity
+      // with a policy that allows access to these secrets
+      template {
+        data        = <<EOF
+{{with secret "secret/data/restricted"}}{{.Data.data.secret}}{{end}}
+EOF
+        destination = "${NOMAD_SECRETS_DIR}/secret.txt"
+      }
+
+      restart {
+        attempts = 0
+        mode     = "fail"
+      }
+    }
+  }
+}

e2e/vaultcompat/run_ce_test.go

@@ -63,4 +63,5 @@ func testVaultJWT(t *testing.T, b build) {
 
 	// Run test job.
 	runJob(t, nc, "input/cat_jwt.hcl", "default", validateJWTAllocs)
+	runJob(t, nc, "input/restricted_jwt.hcl", "default", validateJWTAllocs)
 }

e2e/vaultcompat/vaultcompat_test.go

@@ -239,6 +239,20 @@ func setupVaultJWT(t *testing.T, vc *vaultapi.Client, jwksURL string) {
 	rolePath = fmt.Sprintf("auth/%s/role/nomad-restricted", jwtPath)
 	_, err = logical.Write(rolePath, roleWID([]string{"nomad-restricted"}))
 	must.NoError(t, err)
+
+	entityOut, err := logical.Write("identity/entity", map[string]any{
+		"name":     "default:restricted_jwt",
+		"policies": []string{"nomad-restricted"},
+	})
+	must.NoError(t, err)
+	entityID := entityOut.Data["id"]
+
+	_, err = logical.Write("identity/entity-alias", map[string]any{
+		"name":           "default:restricted_jwt",
+		"canonical_id":   entityID,
+		"mount_accessor": jwtAuthAccessor,
+	})
+	must.NoError(t, err)
 }
 
 func startNomad(t *testing.T, cb func(*testutil.TestServerConfig)) (func(), *nomadapi.Client) {
@@ -285,6 +299,9 @@ func configureNomadVaultJWT(vc *vaultapi.Client) func(*testutil.TestServerConfig
 			DefaultIdentity: &testutil.WorkloadIdentityConfig{
 				Audience: []string{"vault.io"},
 				TTL:      "10m",
+				ExtraClaims: map[string]string{
+					"nomad_workload_id": "${job.namespace}:${job.id}",
+				},
 			},
 
 			// Client configs.

nomad/alloc_endpoint.go

@@ -619,11 +619,23 @@ func (a *Alloc) signTasks(
 		}
 
 		widFound = true
-		claims := structs.NewIdentityClaimsBuilder(alloc.Job, alloc, &idReq.WIHandle, wid).
+		builder := structs.NewIdentityClaimsBuilder(alloc.Job, alloc, &idReq.WIHandle, wid).
 			WithTask(task).
-			WithConsul().
-			WithVault().
-			Build(now)
+			WithConsul()
+
+		var node *structs.Node
+		node, err = a.srv.State().NodeByID(nil, alloc.NodeID)
+		if err != nil {
+			return
+		}
+		builder.WithNode(node)
+
+		vaultCfg := a.srv.GetConfig().GetVaultForIdentity(wid)
+		if vaultCfg != nil && vaultCfg.DefaultIdentity != nil {
+			builder.WithVault(vaultCfg.DefaultIdentity.ExtraClaims)
+		}
+
+		claims := builder.Build(now)
 		err = a.signClaims(claims, idReq, reply)
 		break
 	}

nomad/config.go

@@ -9,6 +9,7 @@ import (
 	"os"
 	"runtime"
 	"slices"
+	"strings"
 	"time"
 
 	log "github.com/hashicorp/go-hclog"
@@ -503,6 +504,21 @@ func (c *Config) VaultIdentityConfig(cluster string) *structs.WorkloadIdentity {
 	return workloadIdentityFromConfig(conf.DefaultIdentity)
 }
 
+// GetVaultForIdentity reverses VaultIdentityConfig and finds the Vault
+// configuration that goes with a particular workload identity intended for
+// Vault
+func (c *Config) GetVaultForIdentity(wi *structs.WorkloadIdentity) *config.VaultConfig {
+	if !wi.IsVault() {
+		return nil
+	}
+	cluster := strings.TrimPrefix(wi.Name, structs.WorkloadIdentityVaultPrefix)
+	if cluster == "" {
+		return nil
+	}
+	conf := c.VaultConfigs[cluster]
+	return conf
+}
+
 func (c *Config) GetDefaultConsul() *config.ConsulConfig {
 	return c.ConsulConfigs[structs.ConsulDefaultCluster]
 }

nomad/structs/config/workload_id.go

@@ -4,6 +4,7 @@
 package config
 
 import (
+	"maps"
 	"slices"
 	"time"
 
@@ -37,6 +38,10 @@ type WorkloadIdentityConfig struct {
 	// this identity (eg the JWT "exp" claim).
 	TTL    *time.Duration `mapstructure:"-"`
 	TTLHCL string         `mapstructure:"ttl" json:"-"`
+
+	// ExtraClaims allows a WI configuration to carry extra claims configured by
+	// the cluster administrator. Note this field is not available on jobspecs.
+	ExtraClaims map[string]string `mapstructure:"extra_claims"`
 }
 
 func (wi *WorkloadIdentityConfig) Copy() *WorkloadIdentityConfig {
@@ -56,6 +61,7 @@ func (wi *WorkloadIdentityConfig) Copy() *WorkloadIdentityConfig {
 	if wi.TTL != nil {
 		nwi.TTL = pointer.Of(*wi.TTL)
 	}
+	nwi.ExtraClaims = maps.Clone(wi.ExtraClaims)
 
 	return nwi
 }
@@ -83,6 +89,9 @@ func (wi *WorkloadIdentityConfig) Equal(other *WorkloadIdentityConfig) bool {
 	if wi.TTLHCL != other.TTLHCL {
 		return false
 	}
+	if !maps.Equal(wi.ExtraClaims, other.ExtraClaims) {
+		return false
+	}
 
 	return true
 }
@@ -114,5 +123,12 @@ func (wi *WorkloadIdentityConfig) Merge(other *WorkloadIdentityConfig) *Workload
 		result.TTLHCL = other.TTLHCL
 	}
 
+	if wi.ExtraClaims == nil {
+		result.ExtraClaims = map[string]string{}
+	}
+	for k, v := range other.ExtraClaims {
+		result.ExtraClaims[k] = v
+	}
+
 	return result
 }

nomad/structs/workload_id.go

@@ -76,6 +76,10 @@ type IdentityClaims struct {
 	VaultNamespace  string `json:"vault_namespace,omitempty"`
 	VaultRole       string `json:"vault_role,omitempty"`
 
+	// ExtraClaims are added based on this identity's
+	// WorkloadIdentityConfiguration, controlled by server configuration
+	ExtraClaims map[string]string `json:"extra_claims,omitempty"`
+
 	jwt.Claims
 }
 
@@ -93,6 +97,8 @@ type IdentityClaimsBuilder struct {
 	serviceName string
 	consul      *Consul
 	vault       *Vault
+	node        *Node
+	extras      map[string]string
 }
 
 // NewIdentityClaimsBuilder returns an initialized IdentityClaimsBuilder for the
@@ -111,6 +117,7 @@ func NewIdentityClaimsBuilder(job *Job, alloc *Allocation, wihandle *WIHandle, w
 		wihandle: wihandle,
 		wid:      wid,
 		tg:       tg,
+		extras:   map[string]string{},
 	}
 }
 
@@ -125,11 +132,14 @@ func (b *IdentityClaimsBuilder) WithTask(task *Task) *IdentityClaimsBuilder {
 
 // WithVault adds the task's vault block to the builder context. This should
 // only be called after WithTask.
-func (b *IdentityClaimsBuilder) WithVault() *IdentityClaimsBuilder {
+func (b *IdentityClaimsBuilder) WithVault(extraClaims map[string]string) *IdentityClaimsBuilder {
 	if !b.wid.IsVault() || b.task == nil {
 		return b
 	}
 	b.vault = b.task.Vault
+	for k, v := range extraClaims {
+		b.extras[k] = v
+	}
 	return b
 }
 
@@ -162,11 +172,18 @@ func (b *IdentityClaimsBuilder) WithService(service *Service) *IdentityClaimsBui
 	return b
 }
 
+// WithNode add the allocation's node to the builder context.
+func (b *IdentityClaimsBuilder) WithNode(node *Node) *IdentityClaimsBuilder {
+	b.node = node
+	return b
+}
+
 // Build is the terminal method for the builder and sets all the derived values
 // on the claim. The claim ID is random (nondeterministic) so multiple calls
 // with the same values will not return equal claims by design. JWT IDs should
 // never collide.
 func (b *IdentityClaimsBuilder) Build(now time.Time) *IdentityClaims {
+	b.interpolate()
 
 	jwtnow := jwt.NewNumericDate(now.UTC())
 	claims := &IdentityClaims{
@@ -178,6 +195,7 @@ func (b *IdentityClaimsBuilder) Build(now time.Time) *IdentityClaims {
 			NotBefore: jwtnow,
 			IssuedAt:  jwtnow,
 		},
+		ExtraClaims: b.extras,
 	}
 	// If this is a child job, use the parent's ID
 	if b.job.ParentID != "" {
@@ -203,6 +221,40 @@ func (b *IdentityClaimsBuilder) Build(now time.Time) *IdentityClaims {
 	return claims
 }
 
+func strAttrGet[T any](x *T, fn func(x *T) string) string {
+	if x != nil {
+		return fn(x)
+	}
+	return ""
+}
+
+func (b *IdentityClaimsBuilder) interpolate() {
+	if len(b.extras) == 0 {
+		return
+	}
+	r := strings.NewReplacer(
+		// attributes that always exist
+		"${job.region}", b.job.Region,
+		"${job.namespace}", b.job.Namespace,
+		"${job.id}", b.job.ID,
+		"${job.node_pool}", b.job.NodePool,
+		"${group.name}", b.tg.Name,
+
+		// attributes that conditionally exist
+		"${node.id}", strAttrGet(b.node, func(n *Node) string { return n.ID }),
+		"${node.datacenter}", strAttrGet(b.node, func(n *Node) string { return n.Datacenter }),
+		"${node.pool}", strAttrGet(b.node, func(n *Node) string { return n.NodePool }),
+		"${node.class}", strAttrGet(b.node, func(n *Node) string { return n.NodeClass }),
+		"${task.name}", strAttrGet(b.task, func(t *Task) string { return t.Name }),
+		"${vault.cluster}", strAttrGet(b.vault, func(v *Vault) string { return v.Cluster }),
+		"${vault.namespace}", strAttrGet(b.vault, func(v *Vault) string { return v.Namespace }),
+		"${vault.role}", strAttrGet(b.vault, func(v *Vault) string { return v.Role }),
+	)
+	for k, v := range b.extras {
+		b.extras[k] = r.Replace(v)
+	}
+}
+
 // setSubject creates the standard subject claim for workload identities.
 func (claims *IdentityClaims) setSubject(job *Job, group, widentifier, id string) {
 	claims.Subject = strings.Join([]string{
@@ -260,6 +312,10 @@ type WorkloadIdentity struct {
 	// TTL is used to determine the expiration of the credentials created for
 	// this identity (eg the JWT "exp" claim).
 	TTL time.Duration
+
+	// Note: ExtraClaims is available on config/WorkloadIdentity but not
+	// available here on jobspecs because that might allow a job author to
+	// escalate their privileges if they know what claim mappings to expect.
 }
 
 // IsConsul returns true if the identity name starts with the standard prefix