Add tests cases for multiple executors in chart (apache#44424)

* add examples for tests with multiple executors * fix pod-launcher-rolebinding to handel multiple executors * fix hpa tests * remove deprecated test
HariGS-DB · Jan 16, 2025 · ec731df · ec731df
1 parent 5cf8090
commit ec731df
Show file tree

Hide file tree

Showing 10 changed files with 41 additions and 10 deletions.
diff --git a/chart/templates/rbac/pod-launcher-rolebinding.yaml b/chart/templates/rbac/pod-launcher-rolebinding.yaml
@@ -23,6 +23,7 @@
 {{- if and .Values.rbac.create .Values.allowPodLaunching }}
 {{- $schedulerLaunchExecutors := list "LocalExecutor" "LocalKubernetesExecutor" "KubernetesExecutor" "CeleryKubernetesExecutor" }}
 {{- $workerLaunchExecutors := list "CeleryExecutor" "LocalKubernetesExecutor" "KubernetesExecutor" "CeleryKubernetesExecutor" }}
+{{- $executors := split "," .Values.executor }}
 apiVersion: rbac.authorization.k8s.io/v1
 {{- if .Values.multiNamespaceMode }}
 kind: ClusterRoleBinding
@@ -57,14 +58,20 @@ roleRef:
   name: {{ include "airflow.fullname" . }}-pod-launcher-role
   {{- end }}
 subjects:
-  {{- if has .Values.executor $schedulerLaunchExecutors }}
+  {{- range $executor := $executors }}
+  {{- if has $executor $schedulerLaunchExecutors }}
   - kind: ServiceAccount
-    name: {{ include "scheduler.serviceAccountName" . }}
-    namespace: "{{ .Release.Namespace }}"
+    name: {{ include "scheduler.serviceAccountName" $ }}
+    namespace: "{{ $.Release.Namespace }}"
+    {{- break -}}
+  {{- end }}
   {{- end }}
-  {{- if has .Values.executor $workerLaunchExecutors }}
+  {{- range $executor := $executors }}
+  {{- if has $executor $workerLaunchExecutors }}
   - kind: ServiceAccount
-    name: {{ include "worker.serviceAccountName" . }}
-    namespace: "{{ .Release.Namespace }}"
+    name: {{ include "worker.serviceAccountName" $ }}
+    namespace: "{{ $.Release.Namespace }}"
+    {{- break -}}
+  {{- end }}
   {{- end }}
 {{- end }}
diff --git a/chart/templates/workers/worker-hpa.yaml b/chart/templates/workers/worker-hpa.yaml
@@ -20,7 +20,7 @@
 ################################
 ## Airflow Worker HPA
 #################################
-{{- if and (and (not .Values.workers.keda.enabled) .Values.workers.hpa.enabled) (has .Values.executor (list "CeleryExecutor" "CeleryKubernetesExecutor")) }}
+{{- if and (and (not .Values.workers.keda.enabled) .Values.workers.hpa.enabled) (or (contains "CeleryExecutor" .Values.executor) (contains "CeleryKubernetesExecutor" .Values.executor)) }}
 apiVersion: autoscaling/v2
 kind: HorizontalPodAutoscaler
 metadata:

diff --git a/helm_tests/airflow_aux/test_configmap.py b/helm_tests/airflow_aux/test_configmap.py
@@ -105,6 +105,8 @@ def test_kerberos_config_available_with_celery_executor(self):
             ("KubernetesExecutor", "2.0.0", True),
             ("CeleryExecutor", "1.10.11", False),
             ("CeleryExecutor", "2.0.0", False),
+            ("CeleryExecutor,KubernetesExecutor", "2.0.0", True),
+            ("CeleryExecutor,KubernetesExecutor", "1.10.11", False),
         ],
     )
     def test_pod_template_created(self, executor, af_version, should_be_created):

diff --git a/helm_tests/airflow_aux/test_pod_launcher_role.py b/helm_tests/airflow_aux/test_pod_launcher_role.py
@@ -33,6 +33,7 @@ class TestPodLauncher:
             ("CeleryExecutor", True, True, ["worker"]),
             ("LocalExecutor", True, True, ["scheduler"]),
             ("LocalExecutor", False, False, []),
+            ("CeleryExecutor,KubernetesExecutor", True, True, ["scheduler", "worker"]),
         ],
     )
     def test_pod_launcher_role(self, executor, rbac, allow, expected_accounts):

diff --git a/helm_tests/airflow_core/test_scheduler.py b/helm_tests/airflow_core/test_scheduler.py
@@ -32,10 +32,12 @@ class TestScheduler:
             ("CeleryExecutor", False, "Deployment"),
             ("CeleryExecutor", True, "Deployment"),
             ("CeleryKubernetesExecutor", True, "Deployment"),
+            ("CeleryExecutor,KubernetesExecutor", True, "Deployment"),
             ("KubernetesExecutor", True, "Deployment"),
             ("LocalKubernetesExecutor", False, "Deployment"),
             ("LocalKubernetesExecutor", True, "StatefulSet"),
             ("LocalExecutor", True, "StatefulSet"),
+            ("LocalExecutor,KubernetesExecutor", True, "StatefulSet"),
             ("LocalExecutor", False, "Deployment"),
         ],
     )
@@ -642,15 +644,23 @@ def test_airflow_local_settings(self):
             ("CeleryExecutor", False, {"rollingUpdate": {"partition": 0}}, None),
             ("CeleryExecutor", True, {"rollingUpdate": {"partition": 0}}, None),
             ("LocalKubernetesExecutor", False, {"rollingUpdate": {"partition": 0}}, None),
+            ("LocalExecutor,KubernetesExecutor", False, {"rollingUpdate": {"partition": 0}}, None),
             (
                 "LocalKubernetesExecutor",
                 True,
                 {"rollingUpdate": {"partition": 0}},
                 {"rollingUpdate": {"partition": 0}},
             ),
+            (
+                "LocalExecutor,KubernetesExecutor",
+                True,
+                {"rollingUpdate": {"partition": 0}},
+                {"rollingUpdate": {"partition": 0}},
+            ),
             ("LocalExecutor", False, {"rollingUpdate": {"partition": 0}}, None),
             ("LocalExecutor", True, {"rollingUpdate": {"partition": 0}}, {"rollingUpdate": {"partition": 0}}),
             ("LocalExecutor", True, None, None),
+            ("LocalExecutor,KubernetesExecutor", True, None, None),
         ],
     )
     def test_scheduler_update_strategy(

diff --git a/helm_tests/airflow_core/test_worker.py b/helm_tests/airflow_core/test_worker.py
@@ -1262,6 +1262,7 @@ def test_should_add_component_specific_labels(self):
             ("LocalExecutor", False),
             ("CeleryExecutor", True),
             ("CeleryKubernetesExecutor", True),
+            ("CeleryExecutor,KubernetesExecutor", True),
             ("KubernetesExecutor", True),
             ("LocalKubernetesExecutor", True),
         ],

diff --git a/helm_tests/other/test_flower.py b/helm_tests/other/test_flower.py
@@ -30,9 +30,11 @@ class TestFlowerDeployment:
         [
             ("CeleryExecutor", False, False),
             ("CeleryKubernetesExecutor", False, False),
+            ("CeleryExecutor,KubernetesExecutor", False, False),
             ("KubernetesExecutor", False, False),
             ("CeleryExecutor", True, True),
             ("CeleryKubernetesExecutor", True, True),
+            ("CeleryExecutor,KubernetesExecutor", True, True),
             ("KubernetesExecutor", True, False),
         ],
     )

diff --git a/helm_tests/other/test_hpa.py b/helm_tests/other/test_hpa.py
@@ -38,6 +38,7 @@ def test_hpa_disabled_by_default(self):
         [
             ("CeleryExecutor", True),
             ("CeleryKubernetesExecutor", True),
+            ("CeleryExecutor,KubernetesExecutor", True),
         ],
     )
     def test_hpa_enabled(self, executor, is_created):
@@ -78,7 +79,9 @@ def test_min_max_replicas(self, min_replicas, max_replicas):
         assert jmespath.search("spec.minReplicas", docs[0]) == 0 if min_replicas is None else min_replicas
         assert jmespath.search("spec.maxReplicas", docs[0]) == 5 if max_replicas is None else max_replicas
 
-    @pytest.mark.parametrize("executor", ["CeleryExecutor", "CeleryKubernetesExecutor"])
+    @pytest.mark.parametrize(
+        "executor", ["CeleryExecutor", "CeleryKubernetesExecutor", "CeleryExecutor,KubernetesExecutor"]
+    )
     def test_hpa_behavior(self, executor):
         """Verify HPA behavior."""
         expected_behavior = {

diff --git a/helm_tests/other/test_keda.py b/helm_tests/other/test_keda.py
@@ -38,6 +38,7 @@ def test_keda_disabled_by_default(self):
         [
             ("CeleryExecutor", True),
             ("CeleryKubernetesExecutor", True),
+            ("CeleryExecutor,KubernetesExecutor", True),
         ],
     )
     def test_keda_enabled(self, executor, is_created):
@@ -54,7 +55,9 @@ def test_keda_enabled(self, executor, is_created):
         else:
             assert docs == []
 
-    @pytest.mark.parametrize("executor", ["CeleryExecutor", "CeleryKubernetesExecutor"])
+    @pytest.mark.parametrize(
+        "executor", ["CeleryExecutor", "CeleryKubernetesExecutor", "CeleryExecutor,KubernetesExecutor"]
+    )
     def test_keda_advanced(self, executor):
         """Verify keda advanced config."""
         expected_advanced = {
@@ -99,6 +102,8 @@ def build_query(executor, concurrency=16, queue=None):
             ("CeleryExecutor", 16),
             ("CeleryKubernetesExecutor", 8),
             ("CeleryKubernetesExecutor", 16),
+            ("CeleryExecutor,KubernetesExecutor", 8),
+            ("CeleryExecutor,KubernetesExecutor", 16),
         ],
     )
     def test_keda_concurrency(self, executor, concurrency):

diff --git a/helm_tests/other/test_redis.py b/helm_tests/other/test_redis.py
@@ -36,7 +36,7 @@
 }
 SET_POSSIBLE_REDIS_OBJECT_KEYS = set(REDIS_OBJECTS.values())
 
-CELERY_EXECUTORS_PARAMS = ["CeleryExecutor", "CeleryKubernetesExecutor"]
+CELERY_EXECUTORS_PARAMS = ["CeleryExecutor", "CeleryKubernetesExecutor", "CeleryExecutor,KubernetesExecutor"]
 
 
 class TestRedis: