Merge pull request #34010 from bart0sh/PR060-generate-job-configs

generate DRA job configs from a Jinja template
kubernetes · Jan 9, 2025 · c94cb9d · c94cb9d
2 parents cb5e6cf + 15ae942
commit c94cb9d
Show file tree

Hide file tree

Showing 13 changed files with 692 additions and 338 deletions.
diff --git a/Makefile b/Makefile
@@ -50,6 +50,9 @@ clean:
 # update generated code
 #generate:
 #	hack/make-rules/update/generated.sh
+.PHONY: generate-jobs
+generate-jobs:
+	hack/make-rules/update/generated-jobs.sh
 # gofmt
 #gofmt:
 #	hack/make-rules/update/gofmt.sh
@@ -126,4 +129,7 @@ verify-boilerplate:
 .PHONY: verify-yamllint
 verify-yamllint:
 	hack/make-rules/verify/yamllint.sh
+.PHONY: verify-generated-jobs
+verify-generated-jobs:
+	hack/make-rules/verify/generated-jobs.sh
 #################################################################################
diff --git a/config/jobs/kubernetes/sig-node/dra-canary.yaml b/config/jobs/kubernetes/sig-node/dra-canary.yaml
@@ -95,7 +95,7 @@ presubmits:
             cpu: 2
             memory: 9Gi
 
-  - name: pull-kubernetes-node-e2e-cgrpv1-crio-dra-canary
+  - name: pull-kubernetes-node-e2e-crio-cgrpv1-dra-canary
     cluster: k8s-infra-prow-build
     skip_branches:
     - release-\d+\.\d+  # per-release image
@@ -147,7 +147,7 @@ presubmits:
             cpu: 2
             memory: 9Gi
 
-  - name: pull-kubernetes-node-e2e-cgrpv2-crio-dra-canary
+  - name: pull-kubernetes-node-e2e-crio-cgrpv2-dra-canary
     cluster: k8s-infra-prow-build
     skip_branches:
     - release-\d+\.\d+  # per-release image

diff --git a/...sig-node/dynamic-resource-allocation.yaml → config/jobs/kubernetes/sig-node/dra-ci.yaml b/...sig-node/dynamic-resource-allocation.yaml → config/jobs/kubernetes/sig-node/dra-ci.yaml
@@ -1,43 +1,43 @@
+# GENERATED FILE - DO NOT EDIT!
+#
+# Instead, modify dra.jinja and run `make generate-jobs`.
 periodics:
-  # This jobs runs e2e.test with a focus on tests for the Dynamic Resource Allocation feature (currently beta)
-  # on a kind cluster with containerd updated to a version with CDI support.
   - name: ci-kind-dra
     cluster: eks-prow-build-cluster
     interval: 6h
+    labels:
+      preset-service-account: "true"
+      preset-dind-enabled: "true"
+      preset-kind-volume-mounts: "true"
     annotations:
       testgrid-dashboards: sig-node-dynamic-resource-allocation
-      testgrid-tab-name: ci-kind-dra
       description: Runs E2E tests for Dynamic Resource Allocation beta features against a Kubernetes master cluster created with sigs.k8s.io/kind
-      testgrid-alert-email: [email protected]
+      testgrid-alert-email: [email protected], [email protected]
       fork-per-release: "true"
     decorate: true
     decoration_config:
-      timeout: 3h
-    labels:
-      preset-service-account: "true"
-      preset-dind-enabled: "true"
-      preset-kind-volume-mounts: "true"
+      timeout: 90m
     extra_refs:
     - org: kubernetes
       repo: kubernetes
       base_ref: master
       path_alias: k8s.io/kubernetes
+      workdir: true
     spec:
       containers:
       - image: gcr.io/k8s-staging-test-infra/kubekins-e2e:v20241230-3006692a6f-master
         command:
         - runner.sh
         args:
-        - /bin/sh
-        - -xc
-        - >
-          make WHAT="github.com/onsi/ginkgo/v2/ginkgo k8s.io/kubernetes/test/e2e/e2e.test" &&
-          curl -sSL https://kind.sigs.k8s.io/dl/latest/linux-amd64.tgz | tar xvfz - -C "${PATH%%:*}/" kind &&
-          kind build node-image --image=dra/node:latest . &&
-          trap 'kind export logs "${ARTIFACTS}/kind"; kind delete cluster' EXIT &&
-          kind create cluster --retain --config test/e2e/dra/kind.yaml --image dra/node:latest &&
+        - /bin/bash
+        - -xce
+        - |
+          make WHAT="github.com/onsi/ginkgo/v2/ginkgo k8s.io/kubernetes/test/e2e/e2e.test"
+          curl -sSL https://kind.sigs.k8s.io/dl/latest/linux-amd64.tgz | tar xvfz - -C "${PATH%%:*}/" kind
+          kind build node-image --image=dra/node:latest .
+          trap 'kind export logs "${ARTIFACTS}/kind"; kind delete cluster' EXIT
+          kind create cluster --retain --config test/e2e/dra/kind.yaml --image dra/node:latest
           KUBERNETES_PROVIDER=local KUBECONFIG=${HOME}/.kube/config GINKGO_PARALLEL_NODES=8 E2E_REPORT_DIR=${ARTIFACTS} GINKGO_TIMEOUT=2h30m hack/ginkgo-e2e.sh -ginkgo.label-filter='Feature: containsAny DynamicResourceAllocation && Feature: isSubsetOf { Beta, DynamicResourceAllocation } && !Flaky'
-
         # docker-in-docker needs privileged mode
         securityContext:
           privileged: true
@@ -49,41 +49,36 @@ periodics:
             cpu: 2
             memory: 9Gi
 
-  # This jobs runs e2e.test with a focus on tests for the Dynamic Resource Allocation feature (currently alpha, soon beta)
-  # on a kind cluster with containerd updated to a version with CDI support.
-  #
-  # Compared to ci-kind-dra, this one enables all DRA-related features.
   - name: ci-kind-dra-all
     cluster: eks-prow-build-cluster
     interval: 6h
+    labels:
+      preset-service-account: "true"
+      preset-dind-enabled: "true"
+      preset-kind-volume-mounts: "true"
     annotations:
       testgrid-dashboards: sig-node-dynamic-resource-allocation
-      testgrid-tab-name: ci-kind-dra-all
       description: Runs E2E tests for Dynamic Resource Allocation alpha and beta features against a Kubernetes master cluster created with sigs.k8s.io/kind
-      testgrid-alert-email: [email protected]
+      testgrid-alert-email: [email protected], [email protected]
       fork-per-release: "true"
     decorate: true
     decoration_config:
-      timeout: 3h
-    labels:
-      preset-service-account: "true"
-      preset-dind-enabled: "true"
-      preset-kind-volume-mounts: "true"
+      timeout: 90m
     extra_refs:
     - org: kubernetes
       repo: kubernetes
       base_ref: master
       path_alias: k8s.io/kubernetes
+      workdir: true
     spec:
       containers:
       - image: gcr.io/k8s-staging-test-infra/kubekins-e2e:v20241230-3006692a6f-master
         command:
         - runner.sh
         args:
         - /bin/bash
-        - -xc
+        - -xce
         - |
-          set -ex
           make WHAT="github.com/onsi/ginkgo/v2/ginkgo k8s.io/kubernetes/test/e2e/e2e.test"
           curl -sSL https://kind.sigs.k8s.io/dl/latest/linux-amd64.tgz | tar xvfz - -C "${PATH%%:*}/" kind
           kind build node-image --image=dra/node:latest .
@@ -94,7 +89,6 @@ periodics:
           # Those additional features are not in kind.yaml, but they can be added at the end.
           kind create cluster --retain --config <(cat test/e2e/dra/kind.yaml; for feature in ${features}; do echo "  ${feature}: true"; done) --image dra/node:latest
           KUBERNETES_PROVIDER=local KUBECONFIG=${HOME}/.kube/config GINKGO_PARALLEL_NODES=8 E2E_REPORT_DIR=${ARTIFACTS} GINKGO_TIMEOUT=1h hack/ginkgo-e2e.sh -ginkgo.label-filter="Feature: containsAny DynamicResourceAllocation && Feature: isSubsetOf { Alpha, Beta, DynamicResourceAllocation$(for feature in ${features}; do echo , ${feature}; done)} && !Flaky && !Slow"
-
         # docker-in-docker needs privileged mode
         securityContext:
           privileged: true
@@ -106,19 +100,17 @@ periodics:
             cpu: 2
             memory: 9Gi
 
-  # This job runs e2e_node.test with a focus on tests for the Dynamic Resource Allocation feature (currently beta)
-  - name: ci-node-e2e-cgrpv1-crio-dra
+  - name: ci-node-e2e-crio-cgrpv1-dra
     cluster: k8s-infra-prow-build
     interval: 6h
-    annotations:
-      testgrid-dashboards: sig-node-cri-o, sig-node-dynamic-resource-allocation
-      testgrid-tab-name: ci-node-e2e-cgrpv1-crio-dra
-      description: Runs E2E node tests for Dynamic Resource Allocation beta features with CRI-O using cgroup v1
-      testgrid-alert-email: [email protected],[email protected]
-      fork-per-release: "true"
     labels:
       preset-service-account: "true"
       preset-k8s-ssh: "true"
+    annotations:
+      testgrid-dashboards: sig-node-dynamic-resource-allocation, sig-node-cri-o
+      description: Runs E2E node tests for Dynamic Resource Allocation beta features with CRI-O using cgroup v1
+      testgrid-alert-email: [email protected], [email protected]
+      fork-per-release: "true"
     decorate: true
     decoration_config:
       timeout: 90m
@@ -145,7 +137,7 @@ periodics:
         - '--node-test-args=--feature-gates=DynamicResourceAllocation=true --service-feature-gates=DynamicResourceAllocation=true --runtime-config=api/beta=true --container-runtime-endpoint=unix:///var/run/crio/crio.sock --container-runtime-process-name=/usr/local/bin/crio --container-runtime-pid-file= --kubelet-flags="--cgroup-driver=systemd --cgroups-per-qos=true --cgroup-root=/ --runtime-cgroups=/system.slice/crio.service --kubelet-cgroups=/system.slice/kubelet.service" --extra-log="{\"name\": \"crio.log\", \"journalctl\": [\"-u\", \"crio\"]}"'
         - --node-tests=true
         - --provider=gce
-        - '--test_args=--timeout=1h --label-filter="Feature: containsAny DynamicResourceAllocation && Feature: isSubsetOf { Beta, DynamicResourceAllocation } && !Flaky"'
+        - '--test_args=--timeout=1h --label-filter="Feature: containsAny DynamicResourceAllocation && Feature: isSubsetOf { Beta, DynamicResourceAllocation } && !Flaky && !Slow"'
         - --timeout=65m
         - --node-args=--image-config-file=/home/prow/go/src/k8s.io/test-infra/jobs/e2e_node/crio/latest/image-config-cgroupv1-serial.yaml
         env:
@@ -161,19 +153,17 @@ periodics:
             cpu: 2
             memory: 9Gi
 
-  # This job runs e2e_node.test with a focus on tests for the Dynamic Resource Allocation feature (currently beta)
-  - name: ci-node-e2e-cgrpv2-crio-dra
+  - name: ci-node-e2e-crio-cgrpv2-dra
     cluster: k8s-infra-prow-build
     interval: 6h
-    annotations:
-      testgrid-dashboards: sig-node-cri-o, sig-node-dynamic-resource-allocation
-      description: Runs E2E node tests for Dynamic Resource Allocation beta features with CRI-O using cgroup v2
-      testgrid-tab-name: ci-node-e2e-cgrpv2-crio-dra
-      testgrid-alert-email: [email protected],[email protected]
-      fork-per-release: "true"
     labels:
       preset-service-account: "true"
       preset-k8s-ssh: "true"
+    annotations:
+      testgrid-dashboards: sig-node-dynamic-resource-allocation, sig-node-cri-o
+      description: Runs E2E node tests for Dynamic Resource Allocation beta features with CRI-O using cgroup v2
+      testgrid-alert-email: [email protected], [email protected]
+      fork-per-release: "true"
     decorate: true
     decoration_config:
       timeout: 90m
@@ -200,7 +190,7 @@ periodics:
         - '--node-test-args=--feature-gates=DynamicResourceAllocation=true --service-feature-gates=DynamicResourceAllocation=true --runtime-config=api/beta=true --container-runtime-endpoint=unix:///var/run/crio/crio.sock --container-runtime-process-name=/usr/local/bin/crio --container-runtime-pid-file= --kubelet-flags="--cgroup-driver=systemd --cgroups-per-qos=true --cgroup-root=/ --runtime-cgroups=/system.slice/crio.service --kubelet-cgroups=/system.slice/kubelet.service" --extra-log="{\"name\": \"crio.log\", \"journalctl\": [\"-u\", \"crio\"]}"'
         - --node-tests=true
         - --provider=gce
-        - '--test_args=--timeout=1h --label-filter="Feature: containsAny DynamicResourceAllocation && Feature: isSubsetOf { Beta, DynamicResourceAllocation } && !Flaky"'
+        - '--test_args=--timeout=1h --label-filter="Feature: containsAny DynamicResourceAllocation && Feature: isSubsetOf { Beta, DynamicResourceAllocation } && !Flaky && !Slow"'
         - --timeout=65m
         - --node-args=--image-config-file=/home/prow/go/src/k8s.io/test-infra/jobs/e2e_node/crio/latest/image-config-cgroupv2-serial.yaml
         env:
@@ -216,19 +206,17 @@ periodics:
             cpu: 2
             memory: 9Gi
 
-  # This job runs the same tests as ci-node-e2e-crio-dra with Containerd 1.7 runtime
   - name: ci-node-e2e-containerd-1-7-dra
     cluster: k8s-infra-prow-build
     interval: 6h
-    annotations:
-      testgrid-dashboards: sig-node-dynamic-resource-allocation
-      testgrid-tab-name: ci-node-e2e-containerd-1-7-dra
-      description: Runs E2E node tests for Dynamic Resource Allocation beta features with containerd
-      testgrid-alert-email: [email protected],[email protected]
-      fork-per-release: "true"
     labels:
       preset-service-account: "true"
       preset-k8s-ssh: "true"
+    annotations:
+      testgrid-dashboards: sig-node-dynamic-resource-allocation, sig-node-containerd
+      description: Runs E2E node tests for Dynamic Resource Allocation beta features with containerd
+      testgrid-alert-email: [email protected], [email protected]
+      fork-per-release: "true"
     decorate: true
     decoration_config:
       timeout: 90m
@@ -254,7 +242,7 @@ periodics:
         - '--node-test-args=--feature-gates=DynamicResourceAllocation=true --service-feature-gates=DynamicResourceAllocation=true --runtime-config=api/beta=true --container-runtime-endpoint=unix:///var/run/containerd/containerd.sock --container-runtime-process-name=/usr/local/bin/containerd --container-runtime-pid-file= --kubelet-flags="--cgroup-driver=systemd --cgroups-per-qos=true --cgroup-root=/ --runtime-cgroups=/system.slice/containerd.service --kubelet-cgroups=/system.slice/kubelet.service" --extra-log="{\"name\": \"containerd.log\", \"journalctl\": [\"-u\", \"containerd\"]}"'
         - --node-tests=true
         - --provider=gce
-        - '--test_args=--timeout=1h --label-filter="Feature: containsAny DynamicResourceAllocation && Feature: isSubsetOf { Beta, DynamicResourceAllocation } && !Flaky"'
+        - '--test_args=--timeout=1h --label-filter="Feature: containsAny DynamicResourceAllocation && Feature: isSubsetOf { Beta, DynamicResourceAllocation } && !Flaky && !Slow"'
         - --timeout=65m
         - --node-args=--image-config-file=/home/prow/go/src/k8s.io/test-infra/jobs/e2e_node/dra/image-config-containerd-1.7.yaml
         resources: