diff --git a/charts/operator-wandb/Chart.lock b/charts/operator-wandb/Chart.lock index 5fe45c51..6a7730ba 100644 --- a/charts/operator-wandb/Chart.lock +++ b/charts/operator-wandb/Chart.lock @@ -51,4 +51,4 @@ dependencies: repository: file://../wandb-base version: 0.1.1 digest: sha256:af1ce94a6c5f99cc9d9df3d10db200ab45b976fd24bd65f85d37e5a32517c5b8 -generated: "2024-12-02T09:48:06.166674-06:00" +generated: "2024-12-05T11:05:33.779928+05:30" diff --git a/charts/operator-wandb/Chart.yaml b/charts/operator-wandb/Chart.yaml index 85381df5..4acfa074 100644 --- a/charts/operator-wandb/Chart.yaml +++ b/charts/operator-wandb/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: operator-wandb description: A Helm chart for deploying W&B to Kubernetes type: application -version: 0.20.0 +version: 0.20.1 appVersion: 1.0.0 icon: https://wandb.ai/logo.svg diff --git a/charts/operator-wandb/charts/app/templates/_helpers.tpl b/charts/operator-wandb/charts/app/templates/_helpers.tpl index 1a81505f..122dbb28 100644 --- a/charts/operator-wandb/charts/app/templates/_helpers.tpl +++ b/charts/operator-wandb/charts/app/templates/_helpers.tpl @@ -123,11 +123,13 @@ app deployments. {{- $bucket = printf "gs://%s/%s" $bucketValues.name (default "" $bucketValues.path) -}} {{- end -}} {{- if eq $bucketValues.provider "s3" -}} -{{- if and $bucketValues.accessKey $bucketValues.secretKey -}} -{{- $bucket = printf "s3://%s:%s@%s/%s" $bucketValues.accessKey $bucketValues.secretKey $bucketValues.name (default "" $bucketValues.path) -}} -{{- else -}} -{{- $bucket = printf "s3://%s/%s" $bucketValues.name (default "" $bucketValues.path) -}} -{{- end -}} + {{- if .Values.global.bucket.bucketSecret.name }} + {{- $bucket = printf "s3://$(ACCESS_KEY):$(SECRET_KEY)@%s/%s" $bucketValues.name (default "" $bucketValues.path) -}} + {{- else if and $bucketValues.accessKey $bucketValues.secretKey }} + {{- $bucket = printf "s3://%s:%s@%s/%s" $bucketValues.accessKey $bucketValues.secretKey $bucketValues.name (default "" $bucketValues.path) -}} + {{- else }} + {{- $bucket = printf "s3://%s/%s" $bucketValues.name (default "" $bucketValues.path) -}} + {{- end }} {{- end -}} {{- trimSuffix "/" $bucket -}} {{- end -}} diff --git a/charts/operator-wandb/charts/app/templates/deployment.yaml b/charts/operator-wandb/charts/app/templates/deployment.yaml index 230e2ec4..70c688fd 100644 --- a/charts/operator-wandb/charts/app/templates/deployment.yaml +++ b/charts/operator-wandb/charts/app/templates/deployment.yaml @@ -136,7 +136,7 @@ spec: secretKeyRef: name: {{ include "wandb.redis.passwordSecret" . }} optional: true - key: {{ .Values.global.redis.secretKey }} + key: {{ .Values.global.redis.passwordSecret.secretKey }} - name: REDIS_PORT value: "{{ include "wandb.redis.port" . }}" - name: REDIS_HOST @@ -202,6 +202,30 @@ spec: value: "0.0.0.0" {{- end }} {{- end }} + {{- if or .Values.global.bucket.accessKey .Values.global.bucket.secretKey .Values.global.bucket.bucketSecret.name }} + {{ if .Values.global.bucket.bucketSecret.name }} + - name: ACCESS_KEY + valueFrom: + secretKeyRef: + name: "{{ include "wandb.bucket.secret" . }}" + key: {{ .Values.global.bucket.bucketSecret.accessKeyName }} + optional: true + {{- else if .Values.global.bucket.accessKey }} + - name: ACCESS_KEY + value: "{{ .Values.global.bucket.accessKey }}" + {{- end }} + {{- if .Values.global.bucket.bucketSecret.name }} + - name: SECRET_KEY + valueFrom: + secretKeyRef: + name: "{{ include "wandb.bucket.secret" . }}" + key: {{ .Values.global.bucket.bucketSecret.secretAccessKeyName }} + optional: true + {{- else if .Values.global.bucket.secretKey }} + - name: SECRET_KEY + value: "{{ .Values.global.bucket.secretKey }}" + {{- end }} + {{- end }} - name: BUCKET value: {{ include "app.bucket" . | quote}} - name: AWS_REGION @@ -216,7 +240,7 @@ spec: valueFrom: secretKeyRef: name: "{{ include "wandb.bucket.secret" . }}" - key: {{ .Values.global.bucket.accessKeyName }} + key: {{ .Values.global.bucket.bucketSecret.accessKeyName }} optional: true - name: GORILLA_CUSTOMER_SECRET_STORE_K8S_CONFIG_NAMESPACE valueFrom: diff --git a/charts/operator-wandb/charts/flat-run-fields-updater/templates/_helpers.tpl b/charts/operator-wandb/charts/flat-run-fields-updater/templates/_helpers.tpl index a4dfc8cb..12410ad8 100644 --- a/charts/operator-wandb/charts/flat-run-fields-updater/templates/_helpers.tpl +++ b/charts/operator-wandb/charts/flat-run-fields-updater/templates/_helpers.tpl @@ -119,11 +119,13 @@ Create the name of the service account to use {{- $bucket = printf "gs://%s/%s" $bucketValues.name (default "" $bucketValues.path) -}} {{- end -}} {{- if eq $bucketValues.provider "s3" -}} -{{- if and $bucketValues.accessKey $bucketValues.secretKey -}} -{{- $bucket = printf "s3://%s:%s@%s/%s" $bucketValues.accessKey $bucketValues.secretKey $bucketValues.name (default "" $bucketValues.path) -}} -{{- else -}} -{{- $bucket = printf "s3://%s/%s" $bucketValues.name (default "" $bucketValues.path) -}} -{{- end -}} + {{- if .Values.global.bucket.bucketSecret.name }} + {{- $bucket = printf "s3://$(ACCESS_KEY):$(SECRET_KEY)@%s/%s" $bucketValues.name (default "" $bucketValues.path) -}} + {{- else if and $bucketValues.accessKey $bucketValues.secretKey }} + {{- $bucket = printf "s3://%s:%s@%s/%s" $bucketValues.accessKey $bucketValues.secretKey $bucketValues.name (default "" $bucketValues.path) -}} + {{- else }} + {{- $bucket = printf "s3://%s/%s" $bucketValues.name (default "" $bucketValues.path) -}} + {{- end }} {{- end -}} {{- trimSuffix "/" $bucket -}} {{- end -}} diff --git a/charts/operator-wandb/charts/flat-run-fields-updater/templates/deployment.yaml b/charts/operator-wandb/charts/flat-run-fields-updater/templates/deployment.yaml index 161bf443..db371f9d 100644 --- a/charts/operator-wandb/charts/flat-run-fields-updater/templates/deployment.yaml +++ b/charts/operator-wandb/charts/flat-run-fields-updater/templates/deployment.yaml @@ -75,7 +75,7 @@ spec: valueFrom: secretKeyRef: name: "{{ include "wandb.bucket.secret" . }}" - key: {{ .Values.global.bucket.accessKeyName }} + key: {{ .Values.global.bucket.bucketSecret.accessKeyName }} optional: true - name: GORILLA_CUSTOMER_SECRET_STORE_K8S_CONFIG_NAMESPACE valueFrom: @@ -137,6 +137,33 @@ spec: "flatRunFieldsUpdater": {{ include "flat-run-fields-updater.runUpdateShadowQueue" . | quote }} } } + {{- end }} + {{- if or .Values.global.bucket.accessKey .Values.global.bucket.secretKey .Values.global.bucket.bucketSecret.name }} + {{ if .Values.global.bucket.bucketSecret.name }} + - name: ACCESS_KEY + valueFrom: + secretKeyRef: + name: "{{ include "wandb.bucket.secret" . }}" + key: {{ .Values.global.bucket.bucketSecret.accessKeyName }} + optional: true + {{- else if .Values.global.bucket.accessKey }} + - name: ACCESS_KEY + value: "{{ .Values.global.bucket.accessKey }}" + {{- end }} + {{- if .Values.global.bucket.bucketSecret.name }} + - name: SECRET_KEY + valueFrom: + secretKeyRef: + name: "{{ include "wandb.bucket.secret" . }}" + key: {{ .Values.global.bucket.bucketSecret.secretAccessKeyName }} + optional: true + {{- else if .Values.global.bucket.secretKey }} + - name: SECRET_KEY + value: "{{ .Values.global.bucket.secretKey }}" + {{- end }} + {{- end }} + - name: BUCKET + value: "{{ include "flat-run-fields-updater.bucket" .}}" - name: AWS_REGION value: {{ .Values.global.bucket.region | default .Values.global.defaultBucket.region }} - name: AWS_S3_KMS_ID diff --git a/charts/operator-wandb/charts/parquet/templates/_helpers.tpl b/charts/operator-wandb/charts/parquet/templates/_helpers.tpl index 6de10860..8e85be28 100644 --- a/charts/operator-wandb/charts/parquet/templates/_helpers.tpl +++ b/charts/operator-wandb/charts/parquet/templates/_helpers.tpl @@ -123,11 +123,13 @@ app deployments. {{- $bucket = printf "gs://%s/%s" $bucketValues.name (default "" $bucketValues.path) -}} {{- end -}} {{- if eq $bucketValues.provider "s3" -}} -{{- if and $bucketValues.accessKey $bucketValues.secretKey -}} -{{- $bucket = printf "s3://%s:%s@%s/%s" $bucketValues.accessKey $bucketValues.secretKey $bucketValues.name (default "" $bucketValues.path) -}} -{{- else -}} -{{- $bucket = printf "s3://%s/%s" $bucketValues.name (default "" $bucketValues.path) -}} -{{- end -}} + {{- if .Values.global.bucket.bucketSecret.name }} + {{- $bucket = printf "s3://$(ACCESS_KEY):$(SECRET_KEY)@%s/%s" $bucketValues.name (default "" $bucketValues.path) -}} + {{- else if and $bucketValues.accessKey $bucketValues.secretKey }} + {{- $bucket = printf "s3://%s:%s@%s/%s" $bucketValues.accessKey $bucketValues.secretKey $bucketValues.name (default "" $bucketValues.path) -}} + {{- else }} + {{- $bucket = printf "s3://%s/%s" $bucketValues.name (default "" $bucketValues.path) -}} + {{- end }} {{- end -}} {{- trimSuffix "/" $bucket -}} {{- end -}} diff --git a/charts/operator-wandb/charts/parquet/templates/cron.yaml b/charts/operator-wandb/charts/parquet/templates/cron.yaml index 85e0c55c..87d8bf2c 100644 --- a/charts/operator-wandb/charts/parquet/templates/cron.yaml +++ b/charts/operator-wandb/charts/parquet/templates/cron.yaml @@ -130,7 +130,30 @@ spec: - name: LOGGING_ENABLED value: 'true' - + {{- if or .Values.global.bucket.accessKey .Values.global.bucket.secretKey .Values.global.bucket.bucketSecret.name }} + {{ if .Values.global.bucket.bucketSecret.name }} + - name: ACCESS_KEY + valueFrom: + secretKeyRef: + name: "{{ include "wandb.bucket.secret" . }}" + key: {{ .Values.global.bucket.bucketSecret.accessKeyName }} + optional: true + {{- else if .Values.global.bucket.accessKey }} + - name: ACCESS_KEY + value: "{{ .Values.global.bucket.accessKey }}" + {{- end }} + {{- if .Values.global.bucket.bucketSecret.name }} + - name: SECRET_KEY + valueFrom: + secretKeyRef: + name: "{{ include "wandb.bucket.secret" . }}" + key: {{ .Values.global.bucket.bucketSecret.secretAccessKeyName }} + optional: true + {{- else if .Values.global.bucket.secretKey }} + - name: SECRET_KEY + value: "{{ .Values.global.bucket.secretKey }}" + {{- end }} + {{- end }} - name: BUCKET value: "{{ include "parquet.bucket" . }}" - name: AWS_REGION @@ -142,7 +165,7 @@ spec: valueFrom: secretKeyRef: name: "{{ include "wandb.bucket.secret" . }}" - key: {{ .Values.global.bucket.accessKeyName }} + key: {{ .Values.global.bucket.bucketSecret.accessKeyName }} optional: true - name: G_HOST_IP diff --git a/charts/operator-wandb/charts/parquet/templates/deployment.yaml b/charts/operator-wandb/charts/parquet/templates/deployment.yaml index b4002288..0f77eb41 100644 --- a/charts/operator-wandb/charts/parquet/templates/deployment.yaml +++ b/charts/operator-wandb/charts/parquet/templates/deployment.yaml @@ -114,7 +114,6 @@ spec: value: "{{ include "parquet.redis" . | trim }}" - name: GORILLA_METADATA_CACHE value: "{{ include "parquet.redis" . | trim }}" - {{- if .Values.global.executor.enabled }} - name: GORILLA_TASK_QUEUE value: "{{ include "app.redis" . | trim }}" @@ -123,14 +122,36 @@ spec: - name: GORILLA_CLEAR_TASK_DEDUPE_KEY_ENABLED value: "false" {{- end }} - - name: GORILLA_HISTORY_STORE value: {{ include "parquet.historyStore" . | quote }} - name: GORILLA_PARQUET_LIVE_HISTORY_STORE value: {{ include "parquet.liveHistoryStore" . | quote }} - name: GORILLA_FILE_STREAM_STORE_ADDRESS value: {{ include "parquet.fileStreamStore" . | quote }} - + {{- if or .Values.global.bucket.accessKey .Values.global.bucket.secretKey .Values.global.bucket.bucketSecret.name }} + {{ if .Values.global.bucket.bucketSecret.name }} + - name: ACCESS_KEY + valueFrom: + secretKeyRef: + name: "{{ include "wandb.bucket.secret" . }}" + key: {{ .Values.global.bucket.bucketSecret.accessKeyName }} + optional: true + {{- else if .Values.global.bucket.accessKey }} + - name: ACCESS_KEY + value: "{{ .Values.global.bucket.accessKey }}" + {{- end }} + {{- if .Values.global.bucket.bucketSecret.name }} + - name: SECRET_KEY + valueFrom: + secretKeyRef: + name: "{{ include "wandb.bucket.secret" . }}" + key: {{ .Values.global.bucket.bucketSecret.secretAccessKeyName }} + optional: true + {{- else if .Values.global.bucket.secretKey }} + - name: SECRET_KEY + value: "{{ .Values.global.bucket.secretKey }}" + {{- end }} + {{- end }} - name: BUCKET value: "{{ include "parquet.bucket" . }}" - name: AWS_REGION @@ -141,7 +162,7 @@ spec: valueFrom: secretKeyRef: name: "{{ include "wandb.bucket.secret" . }}" - key: {{ .Values.global.bucket.accessKeyName }} + key: {{ .Values.global.bucket.bucketSecret.accessKeyName }} optional: true - name: G_HOST_IP diff --git a/charts/operator-wandb/templates/_bucket.tpl b/charts/operator-wandb/templates/_bucket.tpl index 693c0432..5c7b6624 100644 --- a/charts/operator-wandb/templates/_bucket.tpl +++ b/charts/operator-wandb/templates/_bucket.tpl @@ -2,10 +2,8 @@ Return the bucket credentials secret name */}} {{- define "wandb.bucket.secret" -}} -{{- if .Values.global.bucket.secretName -}} - {{ .Values.global.bucket.secretName }} -{{- else if .Values.global.defaultBucket.secretName -}} - {{ .Values.global.defaultBucket.secretName }} +{{- if .Values.global.bucket.bucketSecret.name -}} + {{ .Values.global.bucket.bucketSecret.name }} {{- else }} {{- print .Release.Name "-bucket" -}} {{- end -}} diff --git a/charts/operator-wandb/templates/_redis.tpl b/charts/operator-wandb/templates/_redis.tpl index 5d27a5be..22d721eb 100644 --- a/charts/operator-wandb/templates/_redis.tpl +++ b/charts/operator-wandb/templates/_redis.tpl @@ -2,8 +2,8 @@ Return name of secret where redis information is stored */}} {{- define "wandb.redis.passwordSecret" -}} -{{- if .Values.global.redis.secretName -}} - {{ .Values.global.redis.secretName }} +{{- if .Values.global.redis.passwordSecret.name -}} + {{ .Values.global.redis.passwordSecret.name }} {{- else -}} {{- print .Release.Name "-redis" -}} {{- end -}} @@ -46,7 +46,7 @@ Return the redis to url {{- define "wandb.redis.connectionString" -}} {{- $password := include "wandb.redis.password" . }} -{{- if or $password .Values.global.redis.secretName }} +{{- if or $password .Values.global.redis.passwordSecret.name }} redis://:$(REDIS_PASSWORD)@$(REDIS_HOST):$(REDIS_PORT) {{- else }} redis://$(REDIS_HOST):$(REDIS_PORT) diff --git a/charts/operator-wandb/templates/bucket.yaml b/charts/operator-wandb/templates/bucket.yaml index 08ff39ac..d177e61f 100644 --- a/charts/operator-wandb/templates/bucket.yaml +++ b/charts/operator-wandb/templates/bucket.yaml @@ -1,4 +1,4 @@ -{{- if not .Values.global.bucket.secretName }} +{{- if not .Values.global.bucket.bucketSecret.name }} apiVersion: v1 kind: Secret metadata: diff --git a/charts/operator-wandb/values.yaml b/charts/operator-wandb/values.yaml index a118a324..6fd2c02d 100644 --- a/charts/operator-wandb/values.yaml +++ b/charts/operator-wandb/values.yaml @@ -104,13 +104,13 @@ global: kmsKey: "" secretKey: "" accessKey: "" - secretName: "" # If specified the application will use this bucket for all storage operations, and will not be overridable by the user. bucket: - secretName: "" - accessKeyName: "ACCESS_KEY" - secretAccessKeyName: "SECRET_KEY" + bucketSecret: + name: "" + accessKeyName: "ACCESS_KEY" + secretAccessKeyName: "SECRET_KEY" redis: host: "" @@ -119,8 +119,9 @@ global: parameters: {} caCert: "" # The name of the user supplied secret containing the password for the redis instance. - secretName: "" - secretKey: "REDIS_PASSWORD" + passwordSecret: + name: "" + secretKey: "REDIS_PASSWORD" bigtable: v3: