Merge branch 'master' into 7onn/autoscale-gms

acryldata · Nov 28, 2024 · 46681b4 · 46681b4
2 parents 779b0a9 + 72138ab
commit 46681b4
Show file tree

Hide file tree

Showing 9 changed files with 83 additions and 19 deletions.
diff --git a/charts/datahub/Chart.yaml b/charts/datahub/Chart.yaml
@@ -4,7 +4,7 @@ description: A Helm chart for DataHub
 type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
-version: 0.4.35
+version: 0.4.38
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application.
 appVersion: 0.14.1
@@ -22,7 +22,7 @@ dependencies:
     repository: file://./subcharts/datahub-mae-consumer
     condition: global.datahub_standalone_consumers_enabled
   - name: datahub-mce-consumer
-    version: 0.2.166
+    version: 0.2.167
     repository: file://./subcharts/datahub-mce-consumer
     condition: global.datahub_standalone_consumers_enabled
   - name: datahub-ingestion-cron

diff --git a/charts/datahub/subcharts/datahub-gms/templates/deployment.yaml b/charts/datahub/subcharts/datahub-gms/templates/deployment.yaml
@@ -506,6 +506,12 @@ spec:
             - name: FORMS_HOOK_CONSUMER_GROUP_SUFFIX
               value: "{{ .consumerGroupSuffix }}"
             {{- end }}
+            {{- with .Values.global.datahub.strictUrnValidation }}
+            - name: STRICT_URN_VALIDATION_ENABLED
+              value: "{{ . }}"
+            {{- end }}
+            - name: MCP_CONSUMER_BATCH_ENABLED
+              value: {{ .Values.global.datahub.metadataChangeProposal.consumer.batch.enabled | quote }}
           {{- with .Values.extraEnvs }}
             {{- toYaml . | nindent 12 }}
           {{- end }}

diff --git a/charts/datahub/subcharts/datahub-gms/values.yaml b/charts/datahub/subcharts/datahub-gms/values.yaml
@@ -265,6 +265,9 @@ global:
     ## Enables diff mode for graph writes, uses a different code path that produces a diff from previous to next to write relationships instead of wholesale deleting edges and reading
     enableGraphDiffMode: true
 
+    ## Enable stricter URN validation logic
+    strictUrnValidation: false
+
     cache:
       search:
         ## Enable general search caching
@@ -284,6 +287,11 @@ global:
           ttlSeconds: 86400
           lightningThreshold: 300
 
+    metadataChangeProposal:
+      consumer:
+        batch:
+          enabled: false
+
   hostAliases:
     - ip: "192.168.0.104"
       hostnames:

diff --git a/charts/datahub/subcharts/datahub-mce-consumer/Chart.yaml b/charts/datahub/subcharts/datahub-mce-consumer/Chart.yaml
@@ -12,7 +12,7 @@ description: A Helm chart for Kubernetes
 type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
-version: 0.2.166
+version: 0.2.167
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application.
 appVersion: v0.14.1
diff --git a/charts/datahub/subcharts/datahub-mce-consumer/templates/deployment.yaml b/charts/datahub/subcharts/datahub-mce-consumer/templates/deployment.yaml
@@ -339,6 +339,12 @@ spec:
             {{- end }}
             {{- end }}
             {{- end }}
+            {{- with .Values.global.datahub.strictUrnValidation }}
+            - name: STRICT_URN_VALIDATION_ENABLED
+              value: "{{ . }}"
+            {{- end }}
+            - name: MCP_CONSUMER_BATCH_ENABLED
+              value: {{ .Values.global.datahub.metadataChangeProposal.consumer.batch.enabled | quote }}
           {{- with .Values.extraEnvs }}
             {{- toYaml . | nindent 12 }}
           {{- end }}

diff --git a/charts/datahub/subcharts/datahub-mce-consumer/values.yaml b/charts/datahub/subcharts/datahub-mce-consumer/values.yaml
@@ -184,6 +184,9 @@ global:
     ## Enables diff mode for graph writes, uses a different code path that produces a diff from previous to next to write relationships instead of wholesale deleting edges and reading
     enableGraphDiffMode: true
 
+    ## Enable stricter URN validation logic
+    strictUrnValidation: false
+
   hostAliases:
     - ip: "192.168.0.104"
       hostnames:

diff --git a/charts/datahub/templates/datahub-upgrade/_upgrade.tpl b/charts/datahub/templates/datahub-upgrade/_upgrade.tpl
@@ -192,19 +192,27 @@ Return the env variables for upgrade jobs
 datahubGC cron daily custom scheduling
 */}}
 {{- define "datahub.systemUpdate.datahubGC.dailyCronWindow" -}}
-{{- if .Values.datahubSystemUpdate.bootstrapMCPs.datahubGC.dailyCronWindow.enabled -}}
+{{- if hasKey (index .Values.datahubSystemUpdate.bootstrapMCPs.datahubGC.values "schedule" | default dict) "interval" -}}
 schedule:
-  interval: {{ printf "%d %s * * * " (mod (randNumeric 2) 60) (include "randomHourInRange" (list .Values.datahubSystemUpdate.bootstrapMCPs.datahubGC.dailyCronWindow.startHour .Values.datahubSystemUpdate.bootstrapMCPs.datahubGC.dailyCronWindow.endHour)) }}
+  interval: {{ .Values.datahubSystemUpdate.bootstrapMCPs.datahubGC.values.schedule.interval | quote }}
 {{- else }}
 schedule:
-  interval: {{ .Values.datahubSystemUpdate.bootstrapMCPs.datahubGC.values.schedule.interval | quote }}
+  interval: {{ printf "%d %s * * * " (mod (randNumeric 2) 60) (include "randomHourInRange" (list .Values.datahubSystemUpdate.bootstrapMCPs.datahubGC.dailyCronWindow.startHour .Values.datahubSystemUpdate.bootstrapMCPs.datahubGC.dailyCronWindow.endHour)) }}
 {{- end }}
 {{- end -}}
 
 {{/*
-datahubGC timezone
+  timezone
 */}}
-{{- define "datahub.systemUpdate.datahubGC.timezone" -}}
+{{- define "datahub.bootstrapMCPs.default.schedule.timezone" -}}
 schedule:
-  timezone: {{ .Values.global.datahub.timezone | default .Values.datahubSystemUpdate.bootstrapMCPs.datahubGC.values.schedule.timezone | quote }}
+  timezone: {{ .Values.global.datahub.timezone | quote }}
+{{- end -}}
+
+{{/*
+  default cli version
+*/}}
+{{- define "datahub.bootstrapMCPs.default.ingestion.version" -}}
+ingestion:
+  version: {{ .Values.global.datahub.managed_ingestion.defaultCliVersion | quote }}
 {{- end -}}
diff --git a/charts/datahub/templates/datahub-upgrade/datahub-system-update-job.yml b/charts/datahub/templates/datahub-upgrade/datahub-system-update-job.yml
@@ -135,13 +135,25 @@ spec:
               value: {{ . | quote }}
             {{- end }}
             {{- range $k, $v := .Values.datahubSystemUpdate.bootstrapMCPs }}
-            {{- $result := deepCopy $v.values }}
+            {{- if ne $k "default" }}
+            {{- $result := dict }}
+            {{- range $.Values.datahubSystemUpdate.bootstrapMCPs.default.value_configs }}
+              {{- $funcOutput := include . $ | fromYaml }}
+              {{- $result = include "deepMerge" (dict "dst" $result "src" $funcOutput) | fromYaml }}
+            {{- end }}
+            {{- $valuesCopy := deepCopy $v.values }}
+            {{- $result = include "deepMerge" (dict "dst" $result "src" $valuesCopy) | fromYaml }}
             {{- range $v.values_generated_configs }}
               {{- $funcOutput := include . $ | fromYaml }}
               {{- $result = include "deepMerge" (dict "dst" $result "src" $funcOutput) | fromYaml }}
             {{- end }}
             - name: {{ $v.values_env }}
               value: {{ $result | toJson | quote }}
+            {{- with $v.revision_env }}
+            - name: {{ . }}
+              value: '{"version":"{{ $.Values.global.datahub.version }}-{{ $result | toJson | sha256sum | trunc 7 }}"}'
+            {{- end }}
+            {{- end }}
             {{- end }}
           {{- with .Values.datahubSystemUpdate.extraEnvs }}
             {{- toYaml . | nindent 12 }}
@@ -306,13 +318,25 @@ spec:
               value: {{ . | quote }}
             {{- end }}
             {{- range $k, $v := .Values.datahubSystemUpdate.bootstrapMCPs }}
-            {{- $result := deepCopy $v.values }}
+            {{- if ne $k "default" }}
+            {{- $result := dict }}
+            {{- range $.Values.datahubSystemUpdate.bootstrapMCPs.default.value_configs }}
+              {{- $funcOutput := include . $ | fromYaml }}
+              {{- $result = include "deepMerge" (dict "dst" $result "src" $funcOutput) | fromYaml }}
+            {{- end }}
+            {{- $valuesCopy := deepCopy $v.values }}
+            {{- $result = include "deepMerge" (dict "dst" $result "src" $valuesCopy) | fromYaml }}
             {{- range $v.values_generated_configs }}
               {{- $funcOutput := include . $ | fromYaml }}
               {{- $result = include "deepMerge" (dict "dst" $result "src" $funcOutput) | fromYaml }}
             {{- end }}
             - name: {{ $v.values_env }}
               value: {{ $result | toJson | quote }}
+            {{- with $v.revision_env }}
+            - name: {{ . }}
+              value: '{"version":"{{ $.Values.global.datahub.version }}-{{ $result | toJson | sha256sum | trunc 7 }}"}'
+            {{- end }}
+            {{- end }}
             {{- end }}
           {{- with .Values.datahubSystemUpdate.extraEnvs }}
             {{- toYaml . | nindent 12 }}

diff --git a/charts/datahub/values.yaml b/charts/datahub/values.yaml
@@ -395,30 +395,36 @@ datahubSystemUpdate:
       # args: []
   # Depends on v0.14.2 or greater
   bootstrapMCPs:
+    default:
+      value_configs:
+        - "datahub.bootstrapMCPs.default.ingestion.version"
+        - "datahub.bootstrapMCPs.default.schedule.timezone"
     datahubGC:
       # For information about this recipe https://datahubproject.io/docs/0.14.0/generated/ingestion/sources/datahubgc/#install-the-plugin
-      # Overrides values.schedule.interval below with a jitter window using a generated config
+      # Generate values.schedule.interval with a jitter window using a generated config
       dailyCronWindow:
-        enabled: true
         startHour: 18
         endHour: 5
       # dynamic overrides, the output to each function is deepMerged with values
       values_generated_configs:
         - "datahub.systemUpdate.datahubGC.dailyCronWindow"
-        - "datahub.systemUpdate.datahubGC.timezone"
       # Environment variable containing the json value for the template mcp
       values_env: DATAHUB_GC_BOOTSTRAP_VALUES
+      revision_env: DATAHUB_GC_BOOTSTRAP_REVISION
       # Base values for the template mcp
       values:
         ingestion:
           name: datahub-gc
-        schedule:
-          # overridden by global.datahub.timezone
-          timezone: "UTC"
-          # overridden if dynamic.dailyCronWindow.enable is true
-          interval: "0 1 * * *"
+          # # override global.datahub.managed_ingestion.defaultCliVersion
+          # version: "0.14.1.7rc2"
+        # schedule:
+          # # override global.datahub.timezone
+          # timezone: "UTC"
+          # # override dailyCronWindow
+          # interval: "0 1 * * *"
         cleanup_expired_tokens: "false"
         truncate_indices: "true"
+        truncate_indices_retention_days: 30
         dataprocess_cleanup:
           retention_days: 30
           delete_empty_data_jobs: "true"
@@ -814,6 +820,9 @@ global:
     ## Enables diff mode for graph writes, uses a different code path that produces a diff from previous to next to write relationships instead of wholesale deleting edges and reading
     enableGraphDiffMode: true
 
+    ## Enable stricter URN validation logic
+    strictUrnValidation: false
+
     ## Values specific to the unified search and browse feature.
     search_and_browse:
       show_search_v2: true  # If on, show the new search filters experience as of v0.10.5