From 3d08fe39dfef9bbf5b99cf256f929f3092c1015d Mon Sep 17 00:00:00 2001 From: Tom Proctor Date: Sun, 19 Jun 2022 00:58:16 +0100 Subject: [PATCH 1/8] WIP: Add Vault Agent sidecar to CSI Provider --- templates/_helpers.tpl | 10 ++++++ templates/csi-agent-configmap.yaml | 30 +++++++++++++++++ templates/csi-daemonset.yaml | 53 +++++++++++++++++++++++++++--- values.yaml | 16 ++++++++- 4 files changed, 103 insertions(+), 6 deletions(-) create mode 100644 templates/csi-agent-configmap.yaml diff --git a/templates/_helpers.tpl b/templates/_helpers.tpl index 9dc0dcbee..310cb7d88 100644 --- a/templates/_helpers.tpl +++ b/templates/_helpers.tpl @@ -676,6 +676,16 @@ Sets the container resources if the user has set any. {{ end }} {{- end -}} +{{/* +Sets the container resources for CSI's Agent sidecar if the user has set any. +*/}} +{{- define "csi.agent.resources" -}} + {{- if .Values.csi.agent.resources -}} + resources: +{{ toYaml .Values.csi.agent.resources | indent 12}} + {{ end }} +{{- end -}} + {{/* Sets extra CSI daemonset annotations */}} diff --git a/templates/csi-agent-configmap.yaml b/templates/csi-agent-configmap.yaml new file mode 100644 index 000000000..376a9bbc0 --- /dev/null +++ b/templates/csi-agent-configmap.yaml @@ -0,0 +1,30 @@ +{{- template "vault.csiEnabled" . -}} +{{- if and (.csiEnabled) (eq (.Values.csi.agent.enabled | toString) "true") -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "vault.fullname" . }}-csi-agent-config + namespace: {{ .Release.Namespace }} + labels: + helm.sh/chart: {{ include "vault.chart" . }} + app.kubernetes.io/name: {{ include "vault.name" . }}-csi-provider + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} +data: + config.hcl: | + vault { + {{- if .Values.global.externalVaultAddr }} + "address" = "{{ .Values.global.externalVaultAddr }}" + {{- else }} + "address" = "{{ include "vault.scheme" . }}://{{ template "vault.fullname" . }}.{{ .Release.Namespace }}.svc:{{ .Values.server.service.port }}" + {{- end }} + } + + cache {} + + listener "tcp" { + address = "127.0.0.1:8200" + tls_disable = true + # TODO: Add TLS for localhost communication + } +{{- end }} diff --git a/templates/csi-daemonset.yaml b/templates/csi-daemonset.yaml index fa72b3661..2e78d7a02 100644 --- a/templates/csi-daemonset.yaml +++ b/templates/csi-daemonset.yaml @@ -48,11 +48,14 @@ spec: - --endpoint=/provider/vault.sock - --debug={{ .Values.csi.debug }} {{- if .Values.csi.extraArgs }} - {{- toYaml .Values.csi.extraArgs | nindent 12 }} + {{- toYaml .Values.csi.extraArgs | nindent 12 }} {{- end }} env: - name: VAULT_ADDR - {{- if .Values.global.externalVaultAddr }} + {{- if eq (.Values.csi.agent.enabled | toString) "true" }} + value: "http://127.0.0.1:8200" + # TODO: Add TLS for localhost + {{- else if .Values.global.externalVaultAddr }} value: "{{ .Values.global.externalVaultAddr }}" {{- else }} value: {{ include "vault.scheme" . }}://{{ template "vault.fullname" . }}.{{ .Release.Namespace }}.svc:{{ .Values.server.service.port }} @@ -84,6 +87,41 @@ spec: periodSeconds: {{ .Values.csi.readinessProbe.periodSeconds }} successThreshold: {{ .Values.csi.readinessProbe.successThreshold }} timeoutSeconds: {{ .Values.csi.readinessProbe.timeoutSeconds }} + {{- if eq (.Values.csi.agent.enabled | toString) "true" }} + - name: vault-agent + image: hashicorp/vault:1.10.3 + imagePullPolicy: IfNotPresent + {{ template "csi.agent.resources" . }} + command: + - vault + args: + - agent + - -config=/etc/vault/config.hcl + {{- if .Values.csi.agent.extraArgs }} + {{- toYaml .Values.csi.agent.extraArgs | nindent 12 }} + {{- end }} + ports: + - containerPort: 8200 + env: + - name: VAULT_LOG_LEVEL + value: "{{ .Values.csi.agent.logLevel }}" + - name: VAULT_LOG_FORMAT + value: "{{ .Values.csi.agent.logFormat }}" + securityContext: + runAsNonRoot: true + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsUser: 100 + runAsGroup: 1000 + volumeMounts: + - name: agent-config + mountPath: /etc/vault/config.hcl + subPath: config.hcl + readOnly: true + {{- if .Values.csi.volumeMounts }} + {{- toYaml .Values.csi.volumeMounts | nindent 12 }} + {{- end }} + {{- end }} volumes: - name: providervol hostPath: @@ -91,8 +129,13 @@ spec: - name: mountpoint-dir hostPath: path: {{ .Values.csi.daemonSet.kubeletRootDir }}/pods - {{- if .Values.csi.volumes }} - {{- toYaml .Values.csi.volumes | nindent 8}} - {{- end }} + {{- if eq (.Values.csi.agent.enabled | toString) "true" }} + - name: agent-config + configMap: + name: {{ template "vault.fullname" . }}-csi-agent-config + {{- end }} + {{- if .Values.csi.volumes }} + {{- toYaml .Values.csi.volumes | nindent 8}} + {{- end }} {{- include "imagePullSecrets" . | nindent 6 }} {{- end }} diff --git a/values.yaml b/values.yaml index eb85183cb..dacafddd8 100644 --- a/values.yaml +++ b/values.yaml @@ -909,7 +909,21 @@ csi: # This should be a YAML map of the labels to apply to the csi provider pod extraLabels: {} - + agent: + enabled: true + extraArgs: [] + + logLevel: info + logFormat: normal + + resources: {} + # resources: + # requests: + # memory: 256Mi + # cpu: 250m + # limits: + # memory: 256Mi + # cpu: 250m # Priority class for csi pods priorityClassName: "" From 2dad2b80b2cb5ede1b845a27264792193d36e8f4 Mon Sep 17 00:00:00 2001 From: Tom Proctor Date: Wed, 5 Apr 2023 19:57:52 +0100 Subject: [PATCH 2/8] Use unix socket, update CSI versions, add acceptance test --- templates/csi-agent-configmap.yaml | 5 ++- templates/csi-daemonset.yaml | 20 +++++------ .../vault-kv-secretproviderclass.yaml | 3 +- test/acceptance/csi.bats | 34 +++++++++++++++---- values.yaml | 4 +-- 5 files changed, 42 insertions(+), 24 deletions(-) diff --git a/templates/csi-agent-configmap.yaml b/templates/csi-agent-configmap.yaml index 376a9bbc0..81b897421 100644 --- a/templates/csi-agent-configmap.yaml +++ b/templates/csi-agent-configmap.yaml @@ -22,9 +22,8 @@ data: cache {} - listener "tcp" { - address = "127.0.0.1:8200" + listener "unix" { + address = "/var/run/vault/agent.sock" tls_disable = true - # TODO: Add TLS for localhost communication } {{- end }} diff --git a/templates/csi-daemonset.yaml b/templates/csi-daemonset.yaml index f86286f25..dec58b79c 100644 --- a/templates/csi-daemonset.yaml +++ b/templates/csi-daemonset.yaml @@ -60,8 +60,7 @@ spec: env: - name: VAULT_ADDR {{- if eq (.Values.csi.agent.enabled | toString) "true" }} - value: "http://127.0.0.1:8200" - # TODO: Add TLS for localhost + value: "unix:///var/run/vault/agent.sock" {{- else if .Values.global.externalVaultAddr }} value: "{{ .Values.global.externalVaultAddr }}" {{- else }} @@ -70,9 +69,8 @@ spec: volumeMounts: - name: providervol mountPath: "/provider" - - name: mountpoint-dir - mountPath: {{ .Values.csi.daemonSet.kubeletRootDir }}/pods - mountPropagation: HostToContainer + - name: agent-unix-socket + mountPath: /var/run/vault {{- if .Values.csi.volumeMounts }} {{- toYaml .Values.csi.volumeMounts | nindent 12}} {{- end }} @@ -95,8 +93,8 @@ spec: successThreshold: {{ .Values.csi.readinessProbe.successThreshold }} timeoutSeconds: {{ .Values.csi.readinessProbe.timeoutSeconds }} {{- if eq (.Values.csi.agent.enabled | toString) "true" }} - - name: vault-agent - image: hashicorp/vault:1.10.3 + - name: {{ include "vault.name" . }}-agent + image: hashicorp/vault:1.13.1 imagePullPolicy: IfNotPresent {{ template "csi.agent.resources" . }} command: @@ -125,6 +123,8 @@ spec: mountPath: /etc/vault/config.hcl subPath: config.hcl readOnly: true + - name: agent-unix-socket + mountPath: /var/run/vault {{- if .Values.csi.volumeMounts }} {{- toYaml .Values.csi.volumeMounts | nindent 12 }} {{- end }} @@ -133,13 +133,13 @@ spec: - name: providervol hostPath: path: {{ .Values.csi.daemonSet.providersDir }} - - name: mountpoint-dir - hostPath: - path: {{ .Values.csi.daemonSet.kubeletRootDir }}/pods {{- if eq (.Values.csi.agent.enabled | toString) "true" }} - name: agent-config configMap: name: {{ template "vault.fullname" . }}-csi-agent-config + - name: agent-unix-socket + emptyDir: + medium: Memory {{- end }} {{- if .Values.csi.volumes }} {{- toYaml .Values.csi.volumes | nindent 8}} diff --git a/test/acceptance/csi-test/vault-kv-secretproviderclass.yaml b/test/acceptance/csi-test/vault-kv-secretproviderclass.yaml index b9470fe29..d52fab1bc 100644 --- a/test/acceptance/csi-test/vault-kv-secretproviderclass.yaml +++ b/test/acceptance/csi-test/vault-kv-secretproviderclass.yaml @@ -2,7 +2,7 @@ # SPDX-License-Identifier: MPL-2.0 # The "Hello World" Vault SecretProviderClass -apiVersion: secrets-store.csi.x-k8s.io/v1alpha1 +apiVersion: secrets-store.csi.x-k8s.io/v1 kind: SecretProviderClass metadata: name: vault-kv @@ -10,7 +10,6 @@ spec: provider: vault parameters: roleName: "kv-role" - vaultAddress: http://vault:8200 objects: | - objectName: "bar" secretPath: "secret/data/kv1" diff --git a/test/acceptance/csi.bats b/test/acceptance/csi.bats index ea164f7dd..33bfc90c9 100644 --- a/test/acceptance/csi.bats +++ b/test/acceptance/csi.bats @@ -9,19 +9,26 @@ load _helpers kubectl create namespace acceptance # Install Secrets Store CSI driver - CSI_DRIVER_VERSION=1.0.0 + # Configure it to pass in a JWT for the provider to use, and rotate secrets rapidly + # so we can see Agent's cache working. + CSI_DRIVER_VERSION=1.3.2 helm install secrets-store-csi-driver https://kubernetes-sigs.github.io/secrets-store-csi-driver/charts/secrets-store-csi-driver-${CSI_DRIVER_VERSION}.tgz?raw=true \ --wait --timeout=5m \ --namespace=acceptance \ --set linux.image.pullPolicy="IfNotPresent" \ - --set syncSecret.enabled=true + --set tokenRequests[0].audience="vault" \ + --set enableSecretRotation=true \ + --set rotationPollInterval=5s # Install Vault and Vault provider helm install vault \ --wait --timeout=5m \ --namespace=acceptance \ --set="server.dev.enabled=true" \ --set="csi.enabled=true" \ - --set="injector.enabled=false" . + --set="csi.debug=true" \ + --set="csi.agent.logLevel=debug" \ + --set="injector.enabled=false" \ + . kubectl --namespace=acceptance wait --for=condition=Ready --timeout=5m pod -l app.kubernetes.io/name=vault kubectl --namespace=acceptance wait --for=condition=Ready --timeout=5m pod -l app.kubernetes.io/name=vault-csi-provider @@ -29,10 +36,7 @@ load _helpers cat ./test/acceptance/csi-test/vault-policy.hcl | kubectl --namespace=acceptance exec -i vault-0 -- vault policy write kv-policy - kubectl --namespace=acceptance exec vault-0 -- vault auth enable kubernetes kubectl --namespace=acceptance exec vault-0 -- sh -c 'vault write auth/kubernetes/config \ - token_reviewer_jwt="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" \ - kubernetes_host="https://$KUBERNETES_PORT_443_TCP_ADDR:443" \ - kubernetes_ca_cert=@/var/run/secrets/kubernetes.io/serviceaccount/ca.crt \ - disable_iss_validation=true' + kubernetes_host="https://$KUBERNETES_PORT_443_TCP_ADDR:443"' kubectl --namespace=acceptance exec vault-0 -- vault write auth/kubernetes/role/kv-role \ bound_service_account_names=nginx \ bound_service_account_namespaces=acceptance \ @@ -46,6 +50,22 @@ load _helpers result=$(kubectl --namespace=acceptance exec nginx -- cat /mnt/secrets-store/bar) [[ "$result" == "hello1" ]] + + for i in $(seq 10); do + sleep 2 + if [ "$(kubectl --namespace=acceptance logs --tail=-1 -l "app.kubernetes.io/name=vault-csi-provider" -c vault-agent | grep "returning cached response: path=/v1/auth/kubernetes/login")" ]; then + echo "Agent returned a cached login response" + return + fi + + echo "Waiting for a cached response from Agent..." + done + + # Print the logs and fail the test + echo "Failed to find a log for a cached Agent response" + kubectl --namespace=acceptance logs --tail=-1 -l "app.kubernetes.io/name=vault-csi-provider" -c vault-agent + kubectl --namespace=acceptance logs --tail=-1 -l "app.kubernetes.io/name=vault-csi-provider" -c vault-csi-provider + exit 1 } # Clean up diff --git a/values.yaml b/values.yaml index 6fa8704a7..4d4bca946 100644 --- a/values.yaml +++ b/values.yaml @@ -993,7 +993,7 @@ csi: image: repository: "hashicorp/vault-csi-provider" - tag: "1.2.1" + tag: "1.3.0" pullPolicy: IfNotPresent # volumes is a list of volumes made available to all containers. These are rendered @@ -1062,7 +1062,7 @@ csi: extraArgs: [] logLevel: info - logFormat: normal + logFormat: standard resources: {} # resources: From 550c333ffb31e8ba1c05e88b6ea6fb151ccc7a70 Mon Sep 17 00:00:00 2001 From: Tom Proctor Date: Wed, 5 Apr 2023 20:04:53 +0100 Subject: [PATCH 3/8] Update schema --- values.schema.json | 20 ++++++++++++++++++++ values.yaml | 2 +- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/values.schema.json b/values.schema.json index c52c20088..aa85d323a 100644 --- a/values.schema.json +++ b/values.schema.json @@ -5,6 +5,26 @@ "csi": { "type": "object", "properties": { + "agent": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + }, + "extraArgs": { + "type": "array" + }, + "logFormat": { + "type": "string" + }, + "logLevel": { + "type": "string" + }, + "resources": { + "type": "object" + } + } + }, "daemonSet": { "type": "object", "properties": { diff --git a/values.yaml b/values.yaml index 4d4bca946..e11585f2f 100644 --- a/values.yaml +++ b/values.yaml @@ -1061,8 +1061,8 @@ csi: enabled: true extraArgs: [] - logLevel: info logFormat: standard + logLevel: info resources: {} # resources: From 74538004476f9ead284ffa534e03de00162273f1 Mon Sep 17 00:00:00 2001 From: Tom Proctor Date: Wed, 5 Apr 2023 20:15:20 +0100 Subject: [PATCH 4/8] Remove outdated tests, fix existing tests --- test/unit/csi-daemonset.bats | 33 ++------------------------------- 1 file changed, 2 insertions(+), 31 deletions(-) diff --git a/test/unit/csi-daemonset.bats b/test/unit/csi-daemonset.bats index 0da308b67..1f0f6542f 100644 --- a/test/unit/csi-daemonset.bats +++ b/test/unit/csi-daemonset.bats @@ -379,21 +379,6 @@ load _helpers [ "${actual}" = "/etc/kubernetes/secrets-store-csi-providers" ] } -@test "csi/daemonset: csi kubeletRootDir default" { - cd `chart_dir` - - # Test that it defines it - local object=$(helm template \ - --show-only templates/csi-daemonset.yaml \ - --set 'csi.enabled=true' \ - . | tee /dev/stderr | - yq -r '.spec.template.spec.volumes[] | select(.name == "mountpoint-dir")' | tee /dev/stderr) - - local actual=$(echo $object | - yq -r '.hostPath.path' | tee /dev/stderr) - [ "${actual}" = "/var/lib/kubelet/pods" ] -} - @test "csi/daemonset: csi providersDir override " { cd `chart_dir` @@ -410,22 +395,6 @@ load _helpers [ "${actual}" = "/alt/csi-prov-dir" ] } -@test "csi/daemonset: csi kubeletRootDir override" { - cd `chart_dir` - - # Test that it defines it - local object=$(helm template \ - --show-only templates/csi-daemonset.yaml \ - --set 'csi.enabled=true' \ - --set 'csi.daemonSet.kubeletRootDir=/alt/kubelet-root' \ - . | tee /dev/stderr | - yq -r '.spec.template.spec.volumes[] | select(.name == "mountpoint-dir")' | tee /dev/stderr) - - local actual=$(echo $object | - yq -r '.hostPath.path' | tee /dev/stderr) - [ "${actual}" = "/alt/kubelet-root/pods" ] -} - #-------------------------------------------------------------------- # volumeMounts @@ -569,6 +538,7 @@ load _helpers local object=$(helm template \ --show-only templates/csi-daemonset.yaml \ --set 'csi.enabled=true' \ + --set 'csi.agent.enabled=false' \ --release-name not-external-test \ --set 'injector.externalVaultAddr=http://vault-outside' \ . | tee /dev/stderr | @@ -584,6 +554,7 @@ load _helpers local object=$(helm template \ --show-only templates/csi-daemonset.yaml \ --set 'csi.enabled=true' \ + --set 'csi.agent.enabled=false' \ --set 'global.externalVaultAddr=http://vault-outside' \ . | tee /dev/stderr | yq -r '.spec.template.spec.containers[0].env' | tee /dev/stderr) From 0cf588b938424a2f2760be96f7e00cfbd6126771 Mon Sep 17 00:00:00 2001 From: Tom Proctor Date: Wed, 5 Apr 2023 20:41:47 +0100 Subject: [PATCH 5/8] Add unit test coverage --- templates/csi-agent-configmap.yaml | 2 +- test/unit/csi-agent-configmap.bats | 55 ++++++++++++++ test/unit/csi-daemonset.bats | 117 +++++++++++++++++++++++++++++ 3 files changed, 173 insertions(+), 1 deletion(-) create mode 100644 test/unit/csi-agent-configmap.bats diff --git a/templates/csi-agent-configmap.yaml b/templates/csi-agent-configmap.yaml index 81b897421..cb373f833 100644 --- a/templates/csi-agent-configmap.yaml +++ b/templates/csi-agent-configmap.yaml @@ -3,7 +3,7 @@ apiVersion: v1 kind: ConfigMap metadata: - name: {{ template "vault.fullname" . }}-csi-agent-config + name: {{ template "vault.fullname" . }}-csi-provider-agent-config namespace: {{ .Release.Namespace }} labels: helm.sh/chart: {{ include "vault.chart" . }} diff --git a/test/unit/csi-agent-configmap.bats b/test/unit/csi-agent-configmap.bats new file mode 100644 index 000000000..85136cf72 --- /dev/null +++ b/test/unit/csi-agent-configmap.bats @@ -0,0 +1,55 @@ +#!/usr/bin/env bats + +load _helpers + +@test "csi/Agent-ConfigMap: disabled by default" { + cd `chart_dir` + local actual=$( (helm template \ + --show-only templates/csi-agent-configmap.yaml \ + . || echo "---") | tee /dev/stderr | + yq 'length > 0' | tee /dev/stderr) + [ "${actual}" = "false" ] +} + +@test "csi/Agent-ConfigMap: enabled with csi.enabled" { + cd `chart_dir` + local actual=$(helm template \ + --show-only templates/csi-agent-configmap.yaml \ + --set 'csi.enabled=true' \ + . | tee /dev/stderr | + yq 'length > 0' | tee /dev/stderr) + [ "${actual}" = "true" ] +} + +@test "csi/Agent-ConfigMap: name" { + cd `chart_dir` + local actual=$(helm template \ + --show-only templates/csi-agent-configmap.yaml \ + --set "csi.enabled=true" \ + . | tee /dev/stderr | + yq -r '.metadata.name' | tee /dev/stderr) + [ "${actual}" = "release-name-vault-csi-provider-agent-config" ] +} + +@test "csi/Agent-ConfigMap: Vault addr not affected by injector setting" { + cd `chart_dir` + local actual=$(helm template \ + --show-only templates/csi-agent-configmap.yaml \ + --set "csi.enabled=true" \ + --release-name not-external-test \ + --set 'injector.externalVaultAddr=http://vault-outside' \ + . | tee /dev/stderr | + yq -r '.data["config.hcl"]' | tee /dev/stderr) + echo "${actual}" | grep "http://not-external-test-vault.default.svc:8200" +} + +@test "csi/Agent-ConfigMap: Vault addr correctly set for externalVaultAddr" { + cd `chart_dir` + local actual=$(helm template \ + --show-only templates/csi-agent-configmap.yaml \ + --set "csi.enabled=true" \ + --set 'global.externalVaultAddr=http://vault-outside' \ + . | tee /dev/stderr | + yq -r '.data["config.hcl"]' | tee /dev/stderr) + echo "${actual}" | grep "http://vault-outside" +} \ No newline at end of file diff --git a/test/unit/csi-daemonset.bats b/test/unit/csi-daemonset.bats index 1f0f6542f..fafb786b8 100644 --- a/test/unit/csi-daemonset.bats +++ b/test/unit/csi-daemonset.bats @@ -533,6 +533,33 @@ load _helpers [ "${actual}" = "14" ] } +@test "csi/daemonset: VAULT_ADDR defaults to Agent unix socket" { + cd `chart_dir` + local object=$(helm template \ + --show-only templates/csi-daemonset.yaml \ + --set 'csi.enabled=true' \ + . | tee /dev/stderr | + yq -r '.spec.template.spec.containers[0].env' | tee /dev/stderr) + + local value=$(echo $object | + yq -r 'map(select(.name=="VAULT_ADDR")) | .[] .value' | tee /dev/stderr) + [ "${value}" = "unix:///var/run/vault/agent.sock" ] +} + +@test "csi/daemonset: VAULT_ADDR remains pointed to Agent unix socket if external Vault" { + cd `chart_dir` + local object=$(helm template \ + --show-only templates/csi-daemonset.yaml \ + --set 'csi.enabled=true' \ + --set 'global.externalVaultAddr=http://vault-outside' \ + . | tee /dev/stderr | + yq -r '.spec.template.spec.containers[0].env' | tee /dev/stderr) + + local value=$(echo $object | + yq -r 'map(select(.name=="VAULT_ADDR")) | .[] .value' | tee /dev/stderr) + [ "${value}" = "unix:///var/run/vault/agent.sock" ] +} + @test "csi/daemonset: with only injector.externalVaultAddr" { cd `chart_dir` local object=$(helm template \ @@ -619,3 +646,93 @@ load _helpers yq -r '.spec.template.spec.containers[0].securityContext.foo' | tee /dev/stderr) [ "${actual}" = "bar" ] } + +#-------------------------------------------------------------------- +# Agent sidecar configurables + +@test "csi/daemonset: Agent sidecar enabled by default" { + cd `chart_dir` + local actual=$(helm template \ + --show-only templates/csi-daemonset.yaml \ + --set 'csi.enabled=true' \ + . | tee /dev/stderr | + yq -r '.spec.template.spec.containers | length' | tee /dev/stderr) + [ "${actual}" = "2" ] +} + +@test "csi/daemonset: Agent sidecar can pass extra args" { + cd `chart_dir` + local actual=$(helm template \ + --show-only templates/csi-daemonset.yaml \ + --set 'csi.enabled=true' \ + --set 'csi.agent.extraArgs[0]=-config=extra-config.hcl' \ + . | tee /dev/stderr | + yq -r '.spec.template.spec.containers[1].args[2]' | tee /dev/stderr) + [ "${actual}" = "-config=extra-config.hcl" ] +} + +@test "csi/daemonset: Agent log level settable" { + cd `chart_dir` + local object=$(helm template \ + --show-only templates/csi-daemonset.yaml \ + --set 'csi.enabled=true' \ + --set 'csi.agent.logLevel=error' \ + . | tee /dev/stderr | + yq -r '.spec.template.spec.containers[1].env' | tee /dev/stderr) + + local value=$(echo $object | + yq -r 'map(select(.name=="VAULT_LOG_LEVEL")) | .[] .value' | tee /dev/stderr) + [ "${value}" = "error" ] +} + +@test "csi/daemonset: Agent log format settable" { + cd `chart_dir` + local object=$(helm template \ + --show-only templates/csi-daemonset.yaml \ + --set 'csi.enabled=true' \ + --set 'csi.agent.logFormat=json' \ + . | tee /dev/stderr | + yq -r '.spec.template.spec.containers[1].env' | tee /dev/stderr) + + local value=$(echo $object | + yq -r 'map(select(.name=="VAULT_LOG_FORMAT")) | .[] .value' | tee /dev/stderr) + [ "${value}" = "json" ] +} + +@test "csi/daemonset: Agent default resources" { + cd `chart_dir` + local actual=$(helm template \ + --show-only templates/csi-daemonset.yaml \ + --set 'csi.enabled=true' \ + . | tee /dev/stderr | + yq -r '.spec.template.spec.containers[1].resources' | tee /dev/stderr) + [ "${actual}" = "null" ] +} + +@test "csi/daemonset: Agent custom resources" { + cd `chart_dir` + local object=$(helm template \ + --show-only templates/csi-daemonset.yaml \ + --set 'csi.enabled=true' \ + --set 'csi.agent.resources.requests.memory=256Mi' \ + --set 'csi.agent.resources.requests.cpu=250m' \ + --set 'csi.agent.resources.limits.memory=512Mi' \ + --set 'csi.agent.resources.limits.cpu=500m' \ + . | tee /dev/stderr | + yq -r '.spec.template.spec.containers[1].resources' | tee /dev/stderr) + local value=$(echo $object | + yq -r '.requests.memory' | tee /dev/stderr) + [ "${value}" = "256Mi" ] + + local value=$(echo $object | + yq -r '.requests.cpu' | tee /dev/stderr) + [ "${value}" = "250m" ] + + local value=$(echo $object | + yq -r '.limits.memory' | tee /dev/stderr) + [ "${value}" = "512Mi" ] + + local value=$(echo $object | + yq -r '.limits.cpu' | tee /dev/stderr) + [ "${value}" = "500m" ] +} \ No newline at end of file From f3d6b1a75292b2f7cdd78bf5c23d33e474b80785 Mon Sep 17 00:00:00 2001 From: Tom Proctor Date: Wed, 5 Apr 2023 20:56:45 +0100 Subject: [PATCH 6/8] Fix up other usage of config map name --- templates/csi-daemonset.yaml | 2 +- test/acceptance/csi.bats | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/templates/csi-daemonset.yaml b/templates/csi-daemonset.yaml index dec58b79c..e7d58cfe7 100644 --- a/templates/csi-daemonset.yaml +++ b/templates/csi-daemonset.yaml @@ -136,7 +136,7 @@ spec: {{- if eq (.Values.csi.agent.enabled | toString) "true" }} - name: agent-config configMap: - name: {{ template "vault.fullname" . }}-csi-agent-config + name: {{ template "vault.fullname" . }}-csi-provider-agent-config - name: agent-unix-socket emptyDir: medium: Memory diff --git a/test/acceptance/csi.bats b/test/acceptance/csi.bats index 33bfc90c9..2d7ba8de9 100644 --- a/test/acceptance/csi.bats +++ b/test/acceptance/csi.bats @@ -12,7 +12,9 @@ load _helpers # Configure it to pass in a JWT for the provider to use, and rotate secrets rapidly # so we can see Agent's cache working. CSI_DRIVER_VERSION=1.3.2 - helm install secrets-store-csi-driver https://kubernetes-sigs.github.io/secrets-store-csi-driver/charts/secrets-store-csi-driver-${CSI_DRIVER_VERSION}.tgz?raw=true \ + helm install secrets-store-csi-driver secrets-store-csi-driver \ + --repo https://kubernetes-sigs.github.io/secrets-store-csi-driver/charts \ + --version=$(CSI_DRIVER_VERSION) \ --wait --timeout=5m \ --namespace=acceptance \ --set linux.image.pullPolicy="IfNotPresent" \ From 6a59ec4ca3faf8971692deb4e53c8868f1c9a345 Mon Sep 17 00:00:00 2001 From: Tom Proctor Date: Thu, 6 Apr 2023 13:04:25 +0100 Subject: [PATCH 7/8] Fix volume mount bug, tests, configurable agent image --- templates/csi-daemonset.yaml | 6 ++++-- test/unit/csi-agent-configmap.bats | 10 ---------- test/unit/csi-daemonset.bats | 32 +++++++++++++++++++----------- values.schema.json | 14 +++++++++++++ values.yaml | 5 +++++ 5 files changed, 43 insertions(+), 24 deletions(-) diff --git a/templates/csi-daemonset.yaml b/templates/csi-daemonset.yaml index e7d58cfe7..0285a0cbb 100644 --- a/templates/csi-daemonset.yaml +++ b/templates/csi-daemonset.yaml @@ -69,8 +69,10 @@ spec: volumeMounts: - name: providervol mountPath: "/provider" + {{- if eq (.Values.csi.agent.enabled | toString) "true" }} - name: agent-unix-socket mountPath: /var/run/vault + {{- end }} {{- if .Values.csi.volumeMounts }} {{- toYaml .Values.csi.volumeMounts | nindent 12}} {{- end }} @@ -94,8 +96,8 @@ spec: timeoutSeconds: {{ .Values.csi.readinessProbe.timeoutSeconds }} {{- if eq (.Values.csi.agent.enabled | toString) "true" }} - name: {{ include "vault.name" . }}-agent - image: hashicorp/vault:1.13.1 - imagePullPolicy: IfNotPresent + image: "{{ .Values.csi.agent.image.repository }}:{{ .Values.csi.agent.image.tag }}" + imagePullPolicy: {{ .Values.csi.agent.image.pullPolicy }} {{ template "csi.agent.resources" . }} command: - vault diff --git a/test/unit/csi-agent-configmap.bats b/test/unit/csi-agent-configmap.bats index 85136cf72..4ae4a30b8 100644 --- a/test/unit/csi-agent-configmap.bats +++ b/test/unit/csi-agent-configmap.bats @@ -11,16 +11,6 @@ load _helpers [ "${actual}" = "false" ] } -@test "csi/Agent-ConfigMap: enabled with csi.enabled" { - cd `chart_dir` - local actual=$(helm template \ - --show-only templates/csi-agent-configmap.yaml \ - --set 'csi.enabled=true' \ - . | tee /dev/stderr | - yq 'length > 0' | tee /dev/stderr) - [ "${actual}" = "true" ] -} - @test "csi/Agent-ConfigMap: name" { cd `chart_dir` local actual=$(helm template \ diff --git a/test/unit/csi-daemonset.bats b/test/unit/csi-daemonset.bats index fafb786b8..76c74b37f 100644 --- a/test/unit/csi-daemonset.bats +++ b/test/unit/csi-daemonset.bats @@ -65,24 +65,32 @@ load _helpers } # Image -@test "csi/daemonset: image is configurable" { +@test "csi/daemonset: images are configurable" { cd `chart_dir` - local actual=$(helm template \ + local object=$(helm template \ --show-only templates/csi-daemonset.yaml \ --set "csi.enabled=true" \ - --set "csi.image.repository=SomeOtherImage" \ + --set "csi.image.repository=Image1" \ --set "csi.image.tag=0.0.1" \ + --set "csi.image.pullPolicy=PullPolicy1" \ + --set "csi.agent.image.repository=Image2" \ + --set "csi.agent.image.tag=0.0.2" \ + --set "csi.agent.image.pullPolicy=PullPolicy2" \ . | tee /dev/stderr | - yq -r '.spec.template.spec.containers[0].image' | tee /dev/stderr) - [ "${actual}" = "SomeOtherImage:0.0.1" ] + yq -r '.spec.template.spec.containers' | tee /dev/stderr) - local actual=$(helm template \ - --show-only templates/csi-daemonset.yaml \ - --set "csi.enabled=true" \ - --set "csi.image.pullPolicy=SomePullPolicy" \ - . | tee /dev/stderr | - yq -r '.spec.template.spec.containers[0].imagePullPolicy' | tee /dev/stderr) - [ "${actual}" = "SomePullPolicy" ] + local actual=$(echo $object | + yq -r '.[0].image' | tee /dev/stderr) + [ "${actual}" = "Image1:0.0.1" ] + local actual=$(echo $object | + yq -r '.[0].imagePullPolicy' | tee /dev/stderr) + [ "${actual}" = "PullPolicy1" ] + local actual=$(echo $object | + yq -r '.[1].image' | tee /dev/stderr) + [ "${actual}" = "Image2:0.0.2" ] + local actual=$(echo $object | + yq -r '.[1].imagePullPolicy' | tee /dev/stderr) + [ "${actual}" = "PullPolicy2" ] } @test "csi/daemonset: Custom imagePullSecrets" { diff --git a/values.schema.json b/values.schema.json index aa85d323a..44980e169 100644 --- a/values.schema.json +++ b/values.schema.json @@ -14,6 +14,20 @@ "extraArgs": { "type": "array" }, + "image": { + "type": "object", + "properties": { + "pullPolicy": { + "type": "string" + }, + "repository": { + "type": "string" + }, + "tag": { + "type": "string" + } + } + }, "logFormat": { "type": "string" }, diff --git a/values.yaml b/values.yaml index 86bd2e58b..75da40b28 100644 --- a/values.yaml +++ b/values.yaml @@ -1065,6 +1065,11 @@ csi: enabled: true extraArgs: [] + image: + repository: "hashicorp/vault" + tag: "1.13.1" + pullPolicy: IfNotPresent + logFormat: standard logLevel: info From c7f0ec1d16f04bdff4753f43b741413c077cc7b8 Mon Sep 17 00:00:00 2001 From: Tom Proctor Date: Thu, 6 Apr 2023 13:09:34 +0100 Subject: [PATCH 8/8] Add changelog note --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a1dbf082..823fb5640 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ Features: * server: New `extraPorts` option for adding ports to the Vault server statefulset [GH-841](https://github.com/hashicorp/vault-helm/pull/841) * server: Add configurable Port Number in readinessProbe and livenessProbe for the server-statefulset [GH-831](https://github.com/hashicorp/vault-helm/pull/831) * injector: Make livenessProbe and readinessProbe configurable and add configurable startupProbe [GH-852](https://github.com/hashicorp/vault-helm/pull/852) +* csi: Add an Agent sidecar to Vault CSI Provider pods to provide lease caching and renewals [GH-749](https://github.com/hashicorp/vault-helm/pull/749) ## 0.23.0 (November 28th, 2022)