🐛 fix bucket secret configuration

charts/operator-wandb/Chart.lock

@@ -51,4 +51,4 @@ dependencies:
   repository: file://../wandb-base
   version: 0.1.1
 digest: sha256:af1ce94a6c5f99cc9d9df3d10db200ab45b976fd24bd65f85d37e5a32517c5b8
-generated: "2024-12-02T09:48:06.166674-06:00"
+generated: "2024-12-05T11:05:33.779928+05:30"

charts/operator-wandb/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: operator-wandb
 description: A Helm chart for deploying W&B to Kubernetes
 type: application
-version: 0.20.0
+version: 0.20.1
 appVersion: 1.0.0
 icon: https://wandb.ai/logo.svg
 

charts/operator-wandb/charts/app/templates/_helpers.tpl

@@ -123,11 +123,13 @@ app deployments.
 {{- $bucket = printf "gs://%s/%s" $bucketValues.name (default "" $bucketValues.path) -}}
 {{- end -}}
 {{- if eq $bucketValues.provider "s3" -}}
-{{- if and $bucketValues.accessKey $bucketValues.secretKey -}}
-{{- $bucket = printf "s3://%s:%s@%s/%s" $bucketValues.accessKey $bucketValues.secretKey $bucketValues.name (default "" $bucketValues.path) -}}
-{{- else -}}
-{{- $bucket = printf "s3://%s/%s" $bucketValues.name (default "" $bucketValues.path) -}}
-{{- end -}}
+  {{- if .Values.global.bucket.bucketSecret.name }}
+    {{- $bucket = printf "s3://$(ACCESS_KEY):$(SECRET_KEY)@%s/%s" $bucketValues.name (default "" $bucketValues.path) -}}
+  {{- else if and $bucketValues.accessKey $bucketValues.secretKey }}
+    {{- $bucket = printf "s3://%s:%s@%s/%s" $bucketValues.accessKey $bucketValues.secretKey $bucketValues.name (default "" $bucketValues.path) -}}
+  {{- else }}
+    {{- $bucket = printf "s3://%s/%s" $bucketValues.name (default "" $bucketValues.path) -}}
+  {{- end }}
 {{- end -}}
 {{- trimSuffix "/" $bucket -}}
 {{- end -}}

charts/operator-wandb/charts/app/templates/deployment.yaml

@@ -136,7 +136,7 @@ spec:
                 secretKeyRef:
                   name: {{ include "wandb.redis.passwordSecret" . }}
                   optional: true
-                  key: {{ .Values.global.redis.secretKey }}
+                  key: {{ .Values.global.redis.passwordSecret.secretKey }}
             - name: REDIS_PORT
               value: "{{ include "wandb.redis.port" . }}"
             - name: REDIS_HOST
@@ -202,6 +202,30 @@ spec:
               value: "0.0.0.0"
             {{- end }}
             {{- end }}
+            {{- if or .Values.global.bucket.accessKey .Values.global.bucket.secretKey .Values.global.bucket.bucketSecret.name }}
+            {{ if .Values.global.bucket.bucketSecret.name }}
+            - name: ACCESS_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: "{{ include "wandb.bucket.secret" . }}"
+                  key: {{ .Values.global.bucket.bucketSecret.accessKeyName }}
+                  optional: true
+            {{- else if .Values.global.bucket.accessKey }}
+            - name: ACCESS_KEY
+              value: "{{ .Values.global.bucket.accessKey }}"
+            {{- end }}
+            {{- if .Values.global.bucket.bucketSecret.name }}
+            - name: SECRET_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: "{{ include "wandb.bucket.secret" . }}"
+                  key: {{ .Values.global.bucket.bucketSecret.secretAccessKeyName }}
+                  optional: true
+            {{- else if .Values.global.bucket.secretKey }}
+            - name: SECRET_KEY
+              value: "{{ .Values.global.bucket.secretKey }}"
+            {{- end }}
+            {{- end }}
             - name: BUCKET
               value: {{ include "app.bucket" . | quote}}
             - name: AWS_REGION
@@ -216,7 +240,7 @@ spec:
               valueFrom:
                 secretKeyRef:
                   name: "{{ include "wandb.bucket.secret" . }}"
-                  key: {{ .Values.global.bucket.accessKeyName }}
+                  key: {{ .Values.global.bucket.bucketSecret.accessKeyName }}
                   optional: true
             - name: GORILLA_CUSTOMER_SECRET_STORE_K8S_CONFIG_NAMESPACE
               valueFrom:

charts/operator-wandb/charts/flat-run-fields-updater/templates/_helpers.tpl

@@ -119,11 +119,13 @@ Create the name of the service account to use
 {{- $bucket = printf "gs://%s/%s" $bucketValues.name (default "" $bucketValues.path) -}}
 {{- end -}}
 {{- if eq $bucketValues.provider "s3" -}}
-{{- if and $bucketValues.accessKey $bucketValues.secretKey -}}
-{{- $bucket = printf "s3://%s:%s@%s/%s" $bucketValues.accessKey $bucketValues.secretKey $bucketValues.name (default "" $bucketValues.path) -}}
-{{- else -}}
-{{- $bucket = printf "s3://%s/%s" $bucketValues.name (default "" $bucketValues.path) -}}
-{{- end -}}
+  {{- if .Values.global.bucket.bucketSecret.name }}
+    {{- $bucket = printf "s3://$(ACCESS_KEY):$(SECRET_KEY)@%s/%s" $bucketValues.name (default "" $bucketValues.path) -}}
+  {{- else if and $bucketValues.accessKey $bucketValues.secretKey }}
+    {{- $bucket = printf "s3://%s:%s@%s/%s" $bucketValues.accessKey $bucketValues.secretKey $bucketValues.name (default "" $bucketValues.path) -}}
+  {{- else }}
+    {{- $bucket = printf "s3://%s/%s" $bucketValues.name (default "" $bucketValues.path) -}}
+  {{- end }}
 {{- end -}}
 {{- trimSuffix "/" $bucket -}}
 {{- end -}}

charts/operator-wandb/charts/flat-run-fields-updater/templates/deployment.yaml

@@ -75,7 +75,7 @@ spec:
               valueFrom:
                 secretKeyRef:
                   name: "{{ include "wandb.bucket.secret" . }}"
-                  key: {{ .Values.global.bucket.accessKeyName }}
+                  key: {{ .Values.global.bucket.bucketSecret.accessKeyName }}
                   optional: true
             - name: GORILLA_CUSTOMER_SECRET_STORE_K8S_CONFIG_NAMESPACE
               valueFrom:
@@ -137,6 +137,33 @@ spec:
                     "flatRunFieldsUpdater": {{ include "flat-run-fields-updater.runUpdateShadowQueue" . | quote }}
                   }
                 }
+            {{- end }}
+            {{- if or .Values.global.bucket.accessKey .Values.global.bucket.secretKey .Values.global.bucket.bucketSecret.name }}
+            {{ if .Values.global.bucket.bucketSecret.name }}
+            - name: ACCESS_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: "{{ include "wandb.bucket.secret" . }}"
+                  key: {{ .Values.global.bucket.bucketSecret.accessKeyName }}
+                  optional: true
+            {{- else if .Values.global.bucket.accessKey }}
+            - name: ACCESS_KEY
+              value: "{{ .Values.global.bucket.accessKey }}"
+            {{- end }}
+            {{- if .Values.global.bucket.bucketSecret.name }}
+            - name: SECRET_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: "{{ include "wandb.bucket.secret" . }}"
+                  key: {{ .Values.global.bucket.bucketSecret.secretAccessKeyName }}
+                  optional: true
+            {{- else if .Values.global.bucket.secretKey }}
+            - name: SECRET_KEY
+              value: "{{ .Values.global.bucket.secretKey }}"
+            {{- end }}
+            {{- end }}
+            - name: BUCKET
+              value: "{{ include "flat-run-fields-updater.bucket" .}}"
             - name: AWS_REGION
               value: {{ .Values.global.bucket.region | default .Values.global.defaultBucket.region }}
             - name: AWS_S3_KMS_ID

charts/operator-wandb/charts/parquet/templates/_helpers.tpl

@@ -123,11 +123,13 @@ app deployments.
 {{- $bucket = printf "gs://%s/%s" $bucketValues.name (default "" $bucketValues.path) -}}
 {{- end -}}
 {{- if eq $bucketValues.provider "s3" -}}
-{{- if and $bucketValues.accessKey $bucketValues.secretKey -}}
-{{- $bucket = printf "s3://%s:%s@%s/%s" $bucketValues.accessKey $bucketValues.secretKey $bucketValues.name (default "" $bucketValues.path) -}}
-{{- else -}}
-{{- $bucket = printf "s3://%s/%s" $bucketValues.name (default "" $bucketValues.path) -}}
-{{- end -}}
+  {{- if .Values.global.bucket.bucketSecret.name }}
+    {{- $bucket = printf "s3://$(ACCESS_KEY):$(SECRET_KEY)@%s/%s" $bucketValues.name (default "" $bucketValues.path) -}}
+  {{- else if and $bucketValues.accessKey $bucketValues.secretKey }}
+    {{- $bucket = printf "s3://%s:%s@%s/%s" $bucketValues.accessKey $bucketValues.secretKey $bucketValues.name (default "" $bucketValues.path) -}}
+  {{- else }}
+    {{- $bucket = printf "s3://%s/%s" $bucketValues.name (default "" $bucketValues.path) -}}
+  {{- end }}
 {{- end -}}
 {{- trimSuffix "/" $bucket -}}
 {{- end -}}

charts/operator-wandb/charts/parquet/templates/cron.yaml

@@ -130,7 +130,30 @@ spec:
 
                 - name: LOGGING_ENABLED
                   value: 'true'
-
+                {{- if or .Values.global.bucket.accessKey .Values.global.bucket.secretKey .Values.global.bucket.bucketSecret.name }}
+                {{ if .Values.global.bucket.bucketSecret.name }}
+                - name: ACCESS_KEY
+                  valueFrom:
+                    secretKeyRef:
+                      name: "{{ include "wandb.bucket.secret" . }}"
+                      key: {{ .Values.global.bucket.bucketSecret.accessKeyName }}
+                      optional: true
+                {{- else if .Values.global.bucket.accessKey }}
+                - name: ACCESS_KEY
+                  value: "{{ .Values.global.bucket.accessKey }}"
+                {{- end }}
+                {{- if .Values.global.bucket.bucketSecret.name }}
+                - name: SECRET_KEY
+                  valueFrom:
+                    secretKeyRef:
+                      name: "{{ include "wandb.bucket.secret" . }}"
+                      key: {{ .Values.global.bucket.bucketSecret.secretAccessKeyName }}
+                      optional: true
+                {{- else if .Values.global.bucket.secretKey }}
+                - name: SECRET_KEY
+                  value: "{{ .Values.global.bucket.secretKey }}"
+                {{- end }}
+                {{- end }}
                 - name: BUCKET
                   value: "{{ include "parquet.bucket" . }}"
                 - name: AWS_REGION
@@ -142,7 +165,7 @@ spec:
                   valueFrom:
                     secretKeyRef:
                       name: "{{ include "wandb.bucket.secret" . }}"
-                      key: {{ .Values.global.bucket.accessKeyName }}
+                      key: {{ .Values.global.bucket.bucketSecret.accessKeyName }}
                       optional: true
 
                 - name: G_HOST_IP

charts/operator-wandb/charts/parquet/templates/deployment.yaml

@@ -114,7 +114,6 @@ spec:
               value: "{{ include "parquet.redis" . | trim }}"
             - name: GORILLA_METADATA_CACHE
               value: "{{ include "parquet.redis" . | trim }}"
-
             {{- if .Values.global.executor.enabled }}
             - name: GORILLA_TASK_QUEUE
               value: "{{ include "app.redis" . | trim }}"
@@ -123,14 +122,36 @@ spec:
             - name: GORILLA_CLEAR_TASK_DEDUPE_KEY_ENABLED
               value: "false"
             {{- end }}
-
             - name: GORILLA_HISTORY_STORE
               value: {{ include "parquet.historyStore" . | quote }}
             - name: GORILLA_PARQUET_LIVE_HISTORY_STORE
               value: {{ include "parquet.liveHistoryStore" . | quote }}
             - name: GORILLA_FILE_STREAM_STORE_ADDRESS
               value: {{ include "parquet.fileStreamStore" . | quote }}
-
+            {{- if or .Values.global.bucket.accessKey .Values.global.bucket.secretKey .Values.global.bucket.bucketSecret.name }}
+            {{ if .Values.global.bucket.bucketSecret.name }}
+            - name: ACCESS_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: "{{ include "wandb.bucket.secret" . }}"
+                  key: {{ .Values.global.bucket.bucketSecret.accessKeyName }}
+                  optional: true
+            {{- else if .Values.global.bucket.accessKey }}
+            - name: ACCESS_KEY
+              value: "{{ .Values.global.bucket.accessKey }}"
+            {{- end }}
+            {{- if .Values.global.bucket.bucketSecret.name }}
+            - name: SECRET_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: "{{ include "wandb.bucket.secret" . }}"
+                  key: {{ .Values.global.bucket.bucketSecret.secretAccessKeyName }}
+                  optional: true
+            {{- else if .Values.global.bucket.secretKey }}
+            - name: SECRET_KEY
+              value: "{{ .Values.global.bucket.secretKey }}"
+            {{- end }}
+            {{- end }}
             - name: BUCKET
               value: "{{ include "parquet.bucket" . }}"
             - name: AWS_REGION
@@ -141,7 +162,7 @@ spec:
               valueFrom:
                 secretKeyRef:
                   name: "{{ include "wandb.bucket.secret" . }}"
-                  key: {{ .Values.global.bucket.accessKeyName }}
+                  key: {{ .Values.global.bucket.bucketSecret.accessKeyName }}
                   optional: true
 
             - name: G_HOST_IP

charts/operator-wandb/templates/_bucket.tpl

@@ -2,10 +2,8 @@
 Return the bucket credentials secret name
 */}}
 {{- define "wandb.bucket.secret" -}}
-{{- if .Values.global.bucket.secretName -}}
-  {{ .Values.global.bucket.secretName }}
-{{- else if .Values.global.defaultBucket.secretName -}}
-  {{ .Values.global.defaultBucket.secretName }}
+{{- if .Values.global.bucket.bucketSecret.name -}}
+  {{ .Values.global.bucket.bucketSecret.name }}
 {{- else }}
   {{- print .Release.Name "-bucket" -}}
 {{- end -}}

charts/operator-wandb/templates/_redis.tpl

@@ -2,8 +2,8 @@
 Return name of secret where redis information is stored
 */}}
 {{- define "wandb.redis.passwordSecret" -}}
-{{- if .Values.global.redis.secretName -}}
-  {{ .Values.global.redis.secretName }}
+{{- if .Values.global.redis.passwordSecret.name -}}
+  {{ .Values.global.redis.passwordSecret.name }}
 {{- else -}}
   {{- print .Release.Name "-redis" -}}
 {{- end -}}
@@ -46,7 +46,7 @@ Return the redis to url
 
 {{- define "wandb.redis.connectionString" -}}
 {{- $password := include "wandb.redis.password" . }}
-{{- if or $password .Values.global.redis.secretName }}
+{{- if or $password .Values.global.redis.passwordSecret.name }}
 redis://:$(REDIS_PASSWORD)@$(REDIS_HOST):$(REDIS_PORT)
 {{- else }}
 redis://$(REDIS_HOST):$(REDIS_PORT)

charts/operator-wandb/templates/bucket.yaml

@@ -1,4 +1,4 @@
-{{- if not .Values.global.bucket.secretName }}
+{{- if not .Values.global.bucket.bucketSecret.name }}
 apiVersion: v1
 kind: Secret
 metadata:

charts/operator-wandb/values.yaml

@@ -104,13 +104,13 @@ global:
     kmsKey: ""
     secretKey: ""
     accessKey: ""
-    secretName: ""
 
   # If specified the application will use this bucket for all storage operations, and will not be overridable by the user.
   bucket:
-    secretName: ""
-    accessKeyName: "ACCESS_KEY"
-    secretAccessKeyName: "SECRET_KEY"
+    bucketSecret:
+      name: ""
+      accessKeyName: "ACCESS_KEY"
+      secretAccessKeyName: "SECRET_KEY"
 
   redis:
     host: ""
@@ -119,8 +119,9 @@ global:
     parameters: {}
     caCert: ""
     # The name of the user supplied secret containing the password for the redis instance.
-    secretName: ""
-    secretKey: "REDIS_PASSWORD"
+    passwordSecret:
+      name: ""
+      secretKey: "REDIS_PASSWORD"
 
   bigtable:
     v3: