From 645f29cd3e423cd6ca1d427acb8751b4aa374f83 Mon Sep 17 00:00:00 2001
From: Jarek Potiuk <jarek@potiuk.com>
Date: Thu, 9 Feb 2023 11:18:45 +0100
Subject: [PATCH] Add time to live for finished jobs in Kubernetes

When you upgrade/change job in K8S that has been finished and not
manually removed, this leads to "Field is immutable" error.

This is a known kubernetes issue:

https://github.com/kubernetes/kubernetes/issues/89657

And there are some workarounds (manually removing the job for
example), but the only good solution is possible only in K8S 1.23+
with ttlSecondsAfterFinished set for the job, so that K8S can auto
 clean it.

This PR adds it conditionally for K8S >= 1.23

Fixes: #21943
---
 chart/templates/jobs/create-user-job.yaml      |  3 +++
 chart/templates/jobs/migrate-database-job.yaml |  3 +++
 chart/values.schema.json                       | 10 ++++++++++
 chart/values.yaml                              |  6 +++++-
 4 files changed, 21 insertions(+), 1 deletion(-)

diff --git a/chart/templates/jobs/create-user-job.yaml b/chart/templates/jobs/create-user-job.yaml
index a4bb2bde9f899..e3ff94ba77025 100644
--- a/chart/templates/jobs/create-user-job.yaml
+++ b/chart/templates/jobs/create-user-job.yaml
@@ -48,6 +48,9 @@ metadata:
     {{- $annotations | toYaml | nindent 4 }}
   {{- end }}
 spec:
+  {{- if semverCompare ">= 1.23.x" (include "kubeVersion" .) }}
+  ttlSecondsAfterFinished: {{ .Values.createUserJob.ttlSecondsAfterFinished }}
+  {{- end }}
   template:
     metadata:
       labels:
diff --git a/chart/templates/jobs/migrate-database-job.yaml b/chart/templates/jobs/migrate-database-job.yaml
index 67e61abf48dc0..8b7400e8daaa6 100644
--- a/chart/templates/jobs/migrate-database-job.yaml
+++ b/chart/templates/jobs/migrate-database-job.yaml
@@ -48,6 +48,9 @@ metadata:
     {{- $annotations | toYaml | nindent 4 }}
   {{- end }}
 spec:
+  {{- if semverCompare ">= 1.23.x" (include "kubeVersion" .) }}
+  ttlSecondsAfterFinished: {{ .Values.migrateDatabaseJob.ttlSecondsAfterFinished }}
+  {{- end }}
   template:
     metadata:
       labels:
diff --git a/chart/values.schema.json b/chart/values.schema.json
index 803dadda77c91..4c7474c94f984 100644
--- a/chart/values.schema.json
+++ b/chart/values.schema.json
@@ -2894,6 +2894,11 @@
                         }
                     }
                 },
+                "ttlSecondsAfterFinished": {
+                    "description": "Number of seconds to wait until the job is eligible for deletion (Kubernetes 1.23+)",
+                    "type": "integer",
+                    "default": 300
+                },
                 "extraContainers": {
                     "description": "Launch additional containers for the create user job pod",
                     "type": "array",
@@ -3112,6 +3117,11 @@
                         }
                     ]
                 },
+                "ttlSecondsAfterFinished": {
+                    "description": "Number of seconds to wait until the job is eligible for deletion (Kubernetes 1.23+)",
+                    "type": "integer",
+                    "default": 300
+                },
                 "extraContainers": {
                     "description": "Launch additional containers for the migrate database job pod",
                     "type": "array",
diff --git a/chart/values.yaml b/chart/values.yaml
index 60a957591744b..8e0a584595705 100644
--- a/chart/values.yaml
+++ b/chart/values.yaml
@@ -782,11 +782,12 @@ createUserJob:
     - "{{ .Values.webserver.defaultUser.lastName }}"
     - "-p"
     - "{{ .Values.webserver.defaultUser.password }}"
-
   # Annotations on the create user job pod
   annotations: {}
   # jobAnnotations are annotations on the create user job
   jobAnnotations: {}
+  # Number of seconds to wait until the job is eligible for deletion (Kubernetes 1.23+)
+  ttlSecondsAfterFinished: 300
 
   # Labels specific to createUserJob objects and pods
   labels: {}
@@ -870,6 +871,9 @@ migrateDatabaseJob:
     # Annotations to add to migrate database job kubernetes service account.
     annotations: {}
 
+  # Number of seconds to wait until the job is eligible for deletion (Kubernetes 1.23+)
+  ttlSecondsAfterFinished: 300
+
   resources: {}
   #  limits:
   #   cpu: 100m