perf: increase parquet arrow buffer size (#340)

wandb · Feb 11, 2025 · 02d5a9a · 02d5a9a
1 parent bc47d47
commit 02d5a9a
Show file tree

Hide file tree

Showing 6 changed files with 14 additions and 1 deletion.
diff --git a/charts/operator-wandb/Chart.yaml b/charts/operator-wandb/Chart.yaml
@@ -2,7 +2,7 @@ apiVersion: v2
 name: operator-wandb
 description: A Helm chart for deploying W&B to Kubernetes
 type: application
-version: 0.25.3
+version: 0.25.4
 appVersion: 1.0.0
 icon: https://wandb.ai/logo.svg
 

diff --git a/charts/operator-wandb/charts/app/templates/deployment.yaml b/charts/operator-wandb/charts/app/templates/deployment.yaml
@@ -308,6 +308,9 @@ spec:
             - name: GORILLA_ARTIFACTS_GC_DELETE_FILES_NUM_WORKERS
               value: {{ .Values.artifactsGc.DeleteFilesNumWorkers | quote }}
 
+            - name: GORILLA_PARQUET_ARROW_BUFFER_SIZE
+              value: "2147483648" # 2GB
+
             {{- if .Values.global.executor.enabled }}
             - name: GORILLA_TASK_QUEUE
               value: "{{ include "app.redis" . | trim }}"

diff --git a/charts/operator-wandb/charts/executor/templates/deployment.yaml b/charts/operator-wandb/charts/executor/templates/deployment.yaml
@@ -157,6 +157,9 @@ spec:
             - name: GORILLA_FILE_STREAM_STORE_ADDRESS
               value: {{ include "executor.fileStreamStore" . | quote }}
 
+            - name: GORILLA_PARQUET_ARROW_BUFFER_SIZE
+              value: "2147483648" # 2GB
+
             - name: AZURE_STORAGE_KEY
               valueFrom:
                 secretKeyRef:

diff --git a/charts/operator-wandb/charts/parquet/templates/cron.yaml b/charts/operator-wandb/charts/parquet/templates/cron.yaml
@@ -126,6 +126,9 @@ spec:
                   value: "false"
                 {{- end }}
 
+                - name: GORILLA_PARQUET_ARROW_BUFFER_SIZE
+                  value: "2147483648" # 2GB
+
                 - name: WEAVE_SERVICE
                   value: "{{ .Release.Name }}-weave:9994"
                 - name: PARQUET_HOST

diff --git a/charts/operator-wandb/charts/parquet/templates/deployment.yaml b/charts/operator-wandb/charts/parquet/templates/deployment.yaml
@@ -187,6 +187,9 @@ spec:
               value: "true"
             {{- end }}
 
+            - name: GORILLA_PARQUET_ARROW_BUFFER_SIZE
+              value: "2147483648" # 2GB
+
             {{- include "parquet.extraEnv" (dict "global" .Values.global "local" .Values) | nindent 12 }}
             {{- include "wandb.extraEnvFrom" (dict "root" $ "local" .) | nindent 12 }}
 

diff --git a/charts/operator-wandb/templates/gorilla.yaml b/charts/operator-wandb/templates/gorilla.yaml
@@ -62,6 +62,7 @@ data:
   GORILLA_COLLECT_AUDIT_LOGS: "true"
   GORILLA_USE_PARQUET_HISTORY_STORE: "true"
   GORILLA_PARQUET_PORT: "8087"
+  GORILLA_PARQUET_ARROW_BUFFER_SIZE: "2147483648" # 2GB
   GORILLA_RUN_UPDATE_QUEUE_ADDR: "internal://"
   GORILLA_FILE_STORE_IS_PROXIED: "false"
   GORILLA_ACTIVITY_STORE_SERVE: "true"