Improve kind e2e tests

There were 3 feature specific jobs running e2e tests with AntreaProxy disabled, AntreaProxy and proxyAll enabled, and AntreaPolicy disabled separately. Having a job for each feature is not extensible and requires maintenance efforts to configure the job along with the feature's lifecycle. There is a lot of redundancy between these jobs as unrelated tests ran repeatedly in them. There were other feature specific test cases enabling the features at runtime, which doesn't cover the scenario when two features are enabled at the same time. This patch removes some feature specific jobs and creates one job running tests with all features enabled, which can test all features' interaction. The patch also makes necessary changes to kind test script and manifest generating script to achieve it. Signed-off-by: Quan Tian <[email protected]>
antrea-io · Jun 28, 2022 · e3b7e0e · e3b7e0e
1 parent 40a952b
commit e3b7e0e
Show file tree

Hide file tree

Showing 19 changed files with 158 additions and 254 deletions.
diff --git a/.github/workflows/kind.yml b/.github/workflows/kind.yml
@@ -149,7 +149,7 @@ jobs:
       run: |
         mkdir log
         mkdir test-e2e-encap-no-proxy-coverage
-        ANTREA_LOG_DIR=$PWD/log ANTREA_COV_DIR=$PWD/test-e2e-encap-no-proxy-coverage ./ci/kind/test-e2e-kind.sh --encap-mode encap --no-proxy --coverage --skip mode-irrelevant
+        ANTREA_LOG_DIR=$PWD/log ANTREA_COV_DIR=$PWD/test-e2e-encap-no-proxy-coverage ./ci/kind/test-e2e-kind.sh --encap-mode encap --feature-gates AntreaProxy=false --coverage --skip mode-irrelevant
     - name: Tar coverage files
       run: tar -czf test-e2e-encap-no-proxy-coverage.tar.gz test-e2e-encap-no-proxy-coverage
     - name: Upload coverage for test-e2e-encap-no-proxy-coverage
@@ -177,10 +177,10 @@ jobs:
         path: log.tar.gz
         retention-days: 30
 
-  test-e2e-encap-proxy-all:
-    name: E2e tests on a Kind cluster on Linux with AntreaProxy all Service support
-    needs: [ build-antrea-coverage-image ]
-    runs-on: [ ubuntu-latest ]
+  test-e2e-encap-all-features-enabled:
+    name: E2e tests on a Kind cluster on Linux with all features enabled
+    needs: [build-antrea-coverage-image]
+    runs-on: [ubuntu-latest]
     steps:
       - name: Free disk space
         # https://github.com/actions/virtual-environments/issues/709
@@ -204,32 +204,33 @@ jobs:
       - name: Run e2e tests
         run: |
           mkdir log
-          mkdir test-e2e-encap-proxy-all-coverage
-          ANTREA_LOG_DIR=$PWD/log ANTREA_COV_DIR=$PWD/test-e2e-encap-proxy-all-coverage ./ci/kind/test-e2e-kind.sh --encap-mode encap --proxy-all --coverage --skip mode-irrelevant
+          mkdir test-e2e-encap-all-features-enabled-coverage
+          # Currently multicast tests require specific testbeds, exclude it for now.
+          ANTREA_LOG_DIR=$PWD/log ANTREA_COV_DIR=$PWD/test-e2e-encap-all-features-enabled-coverage ./ci/kind/test-e2e-kind.sh --encap-mode encap --coverage --feature-gates AllAlpha=true,AllBeta=true,Multicast=false --proxy-all
       - name: Tar coverage files
-        run: tar -czf test-e2e-encap-proxy-all-coverage.tar.gz test-e2e-encap-proxy-all-coverage
-      - name: Upload coverage for test-e2e-encap-proxy-all-coverage
+        run: tar -czf test-e2e-encap-all-features-enabled-coverage.tar.gz test-e2e-encap-all-features-enabled-coverage
+      - name: Upload coverage for test-e2e-encap-all-features-enabled-coverage
         uses: actions/upload-artifact@v3
         with:
-          name: test-e2e-encap-proxy-all-coverage
-          path: test-e2e-encap-proxy-all-coverage.tar.gz
+          name: test-e2e-encap-all-features-enabled-coverage
+          path: test-e2e-encap-all-features-enabled-coverage.tar.gz
           retention-days: 30
       - name: Codecov
         uses: codecov/codecov-action@v3
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           file: '*.cov.out*'
           flags: kind-e2e-tests
-          name: codecov-test-e2e-encap-proxy-all
-          directory: test-e2e-encap-proxy-all-coverage
+          name: codecov-test-e2e-encap-all-features-enabled
+          directory: test-e2e-encap-all-features-enabled-coverage
       - name: Tar log files
         if: ${{ failure() }}
         run: tar -czf log.tar.gz log
       - name: Upload test log
         uses: actions/upload-artifact@v3
         if: ${{ failure() }}
         with:
-          name: e2e-kind-encap-proxy-all.tar.gz
+          name: e2e-kind-encap-all-features-enabled.tar.gz
           path: log.tar.gz
           retention-days: 30
 
@@ -345,65 +346,6 @@ jobs:
         path: log.tar.gz
         retention-days: 30
 
-  # TODO: remove when https://github.com/antrea-io/antrea/issues/897 is fixed.
-  # In the mean time, we keep this test around to ensure that at least one Kind
-  # test uses a Geneve overlay.
-  test-e2e-encap-no-np:
-    name: E2e tests on a Kind cluster on Linux with Antrea-native policies disabled
-    needs: [build-antrea-coverage-image]
-    runs-on: [ubuntu-latest]
-    steps:
-    - name: Free disk space
-      # https://github.com/actions/virtual-environments/issues/709
-      run: |
-        sudo apt-get clean
-        df -h
-    - uses: actions/checkout@v3
-    - uses: actions/setup-go@v3
-      with:
-        go-version: 1.17
-    - name: Download Antrea images from previous jobs
-      uses: actions/download-artifact@v3
-    - name: Load Antrea image
-      run: |
-        docker load -i antrea-ubuntu-cov/antrea-ubuntu.tar
-    - name: Install Kind
-      run: |
-        curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-$(uname)-amd64
-        chmod +x ./kind
-        sudo mv kind /usr/local/bin
-    - name: Run e2e tests
-      run: |
-        mkdir log
-        mkdir test-e2e-encap-no-np-coverage
-        ANTREA_LOG_DIR=$PWD/log ANTREA_COV_DIR=$PWD/test-e2e-encap-no-np-coverage ./ci/kind/test-e2e-kind.sh --encap-mode encap --no-np --coverage --skip mode-irrelevant
-    - name: Tar coverage files
-      run: tar -czf test-e2e-encap-no-np-coverage.tar.gz test-e2e-encap-no-np-coverage
-    - name: Upload coverage for test-e2e-encap-no-np-coverage
-      uses: actions/upload-artifact@v3
-      with:
-        name: test-e2e-encap-no-np-coverage
-        path: test-e2e-encap-no-np-coverage.tar.gz
-        retention-days: 30
-    - name: Codecov
-      uses: codecov/codecov-action@v3
-      with:
-        token: ${{ secrets.CODECOV_TOKEN }}
-        file: '*.cov.out*'
-        flags: kind-e2e-tests
-        name: codecov-test-e2e-no-np-encap
-        directory: test-e2e-encap-no-np-coverage
-    - name: Tar log files
-      if: ${{ failure() }}
-      run: tar -czf log.tar.gz log
-    - name: Upload test log
-      uses: actions/upload-artifact@v3
-      if: ${{ failure() }}
-      with:
-        name: e2e-kind-encap-no-np.tar.gz
-        path: log.tar.gz
-        retention-days: 30
-
   test-e2e-flow-visibility:
     name: E2e tests on a Kind cluster on Linux for Flow Visibility
     needs: [build-antrea-coverage-image, build-flow-aggregator-coverage-image]
@@ -536,7 +478,7 @@ jobs:
   # yet.
   artifact-cleanup:
     name: Delete uploaded images
-    needs: [build-antrea-coverage-image, build-flow-aggregator-coverage-image, test-e2e-encap, test-e2e-encap-no-proxy, test-e2e-encap-proxy-all, test-e2e-noencap, test-e2e-hybrid, test-e2e-encap-no-np, test-netpol-tmp, validate-prometheus-metrics-doc, test-e2e-flow-visibility]
+    needs: [build-antrea-coverage-image, build-flow-aggregator-coverage-image, test-e2e-encap, test-e2e-encap-no-proxy, test-e2e-encap-all-features-enabled, test-e2e-noencap, test-e2e-hybrid, test-netpol-tmp, validate-prometheus-metrics-doc, test-e2e-flow-visibility]
     if: ${{ always() &&  (needs.build-antrea-coverage-image.result == 'success' || needs.build-flow-aggregator-coverage-image.result == 'success') }}
     runs-on: [ubuntu-latest]
     steps:

diff --git a/build/charts/antrea/conf/antrea-agent.conf b/build/charts/antrea/conf/antrea-agent.conf
@@ -1,5 +1,11 @@
 # FeatureGates is a map of feature names to bools that enable or disable experimental features.
 featureGates:
+# AllAlpha is a global toggle for alpha features. Per-feature key values override the default set by AllAlpha.
+{{- include "featureGate" (dict "featureGates" .Values.featureGates "name" "AllAlpha" "default" false) }}
+
+# AllBeta is a global toggle for beta features. Per-feature key values override the default set by AllBeta.
+{{- include "featureGate" (dict "featureGates" .Values.featureGates "name" "AllBeta" "default" false) }}
+
 # Enable AntreaProxy which provides ServiceLB for in-cluster Services in antrea-agent.
 # It should be enabled on Windows, otherwise NetworkPolicy will not take effect on
 # Service traffic.

diff --git a/build/charts/antrea/conf/antrea-controller.conf b/build/charts/antrea/conf/antrea-controller.conf
@@ -1,5 +1,11 @@
 # FeatureGates is a map of feature names to bools that enable or disable experimental features.
 featureGates:
+# AllAlpha is a global toggle for alpha features. Per-feature key values override the default set by AllAlpha.
+{{- include "featureGate" (dict "featureGates" .Values.featureGates "name" "AllAlpha" "default" false) }}
+
+# AllBeta is a global toggle for beta features. Per-feature key values override the default set by AllBeta.
+{{- include "featureGate" (dict "featureGates" .Values.featureGates "name" "AllBeta" "default" false) }}
+
 # Enable traceflow which provides packet tracing feature to diagnose network issue.
 {{- include "featureGate" (dict "featureGates" .Values.featureGates "name" "Traceflow" "default" true) }}
 

diff --git a/build/yamls/antrea-aks.yml b/build/yamls/antrea-aks.yml
@@ -2537,6 +2537,12 @@ data:
   antrea-agent.conf: |
     # FeatureGates is a map of feature names to bools that enable or disable experimental features.
     featureGates:
+    # AllAlpha is a global toggle for alpha features. Per-feature key values override the default set by AllAlpha.
+    #  AllAlpha: false
+
+    # AllBeta is a global toggle for beta features. Per-feature key values override the default set by AllBeta.
+    #  AllBeta: false
+
     # Enable AntreaProxy which provides ServiceLB for in-cluster Services in antrea-agent.
     # It should be enabled on Windows, otherwise NetworkPolicy will not take effect on
     # Service traffic.
@@ -2845,6 +2851,12 @@ data:
   antrea-controller.conf: |
     # FeatureGates is a map of feature names to bools that enable or disable experimental features.
     featureGates:
+    # AllAlpha is a global toggle for alpha features. Per-feature key values override the default set by AllAlpha.
+    #  AllAlpha: false
+
+    # AllBeta is a global toggle for beta features. Per-feature key values override the default set by AllBeta.
+    #  AllBeta: false
+
     # Enable traceflow which provides packet tracing feature to diagnose network issue.
     #  Traceflow: true
 
@@ -3676,7 +3688,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: a4e7182f54a10234001d8baeb7d950a5053ffb97b2f6951db516244ac3620cd8
+        checksum/config: 890f1364c9b89811375830c94fab2fa9f1957518351cc52c623e22b6964e5e75
       labels:
         app: antrea
         component: antrea-agent
@@ -3916,7 +3928,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: a4e7182f54a10234001d8baeb7d950a5053ffb97b2f6951db516244ac3620cd8
+        checksum/config: 890f1364c9b89811375830c94fab2fa9f1957518351cc52c623e22b6964e5e75
       labels:
         app: antrea
         component: antrea-controller

diff --git a/build/yamls/antrea-eks.yml b/build/yamls/antrea-eks.yml
@@ -2537,6 +2537,12 @@ data:
   antrea-agent.conf: |
     # FeatureGates is a map of feature names to bools that enable or disable experimental features.
     featureGates:
+    # AllAlpha is a global toggle for alpha features. Per-feature key values override the default set by AllAlpha.
+    #  AllAlpha: false
+
+    # AllBeta is a global toggle for beta features. Per-feature key values override the default set by AllBeta.
+    #  AllBeta: false
+
     # Enable AntreaProxy which provides ServiceLB for in-cluster Services in antrea-agent.
     # It should be enabled on Windows, otherwise NetworkPolicy will not take effect on
     # Service traffic.
@@ -2845,6 +2851,12 @@ data:
   antrea-controller.conf: |
     # FeatureGates is a map of feature names to bools that enable or disable experimental features.
     featureGates:
+    # AllAlpha is a global toggle for alpha features. Per-feature key values override the default set by AllAlpha.
+    #  AllAlpha: false
+
+    # AllBeta is a global toggle for beta features. Per-feature key values override the default set by AllBeta.
+    #  AllBeta: false
+
     # Enable traceflow which provides packet tracing feature to diagnose network issue.
     #  Traceflow: true
 
@@ -3676,7 +3688,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: a4e7182f54a10234001d8baeb7d950a5053ffb97b2f6951db516244ac3620cd8
+        checksum/config: 890f1364c9b89811375830c94fab2fa9f1957518351cc52c623e22b6964e5e75
       labels:
         app: antrea
         component: antrea-agent
@@ -3918,7 +3930,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: a4e7182f54a10234001d8baeb7d950a5053ffb97b2f6951db516244ac3620cd8
+        checksum/config: 890f1364c9b89811375830c94fab2fa9f1957518351cc52c623e22b6964e5e75
       labels:
         app: antrea
         component: antrea-controller

diff --git a/build/yamls/antrea-gke.yml b/build/yamls/antrea-gke.yml
@@ -2537,6 +2537,12 @@ data:
   antrea-agent.conf: |
     # FeatureGates is a map of feature names to bools that enable or disable experimental features.
     featureGates:
+    # AllAlpha is a global toggle for alpha features. Per-feature key values override the default set by AllAlpha.
+    #  AllAlpha: false
+
+    # AllBeta is a global toggle for beta features. Per-feature key values override the default set by AllBeta.
+    #  AllBeta: false
+
     # Enable AntreaProxy which provides ServiceLB for in-cluster Services in antrea-agent.
     # It should be enabled on Windows, otherwise NetworkPolicy will not take effect on
     # Service traffic.
@@ -2845,6 +2851,12 @@ data:
   antrea-controller.conf: |
     # FeatureGates is a map of feature names to bools that enable or disable experimental features.
     featureGates:
+    # AllAlpha is a global toggle for alpha features. Per-feature key values override the default set by AllAlpha.
+    #  AllAlpha: false
+
+    # AllBeta is a global toggle for beta features. Per-feature key values override the default set by AllBeta.
+    #  AllBeta: false
+
     # Enable traceflow which provides packet tracing feature to diagnose network issue.
     #  Traceflow: true
 
@@ -3676,7 +3688,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: afbe6d81852e79d710d0350ab5173f7d0e05b79c75744cffb94ce2294afc328c
+        checksum/config: cc8af4219d403a137ab87500ae0ab15b681fc635e41057b5623df6154443fddf
       labels:
         app: antrea
         component: antrea-agent
@@ -3916,7 +3928,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: afbe6d81852e79d710d0350ab5173f7d0e05b79c75744cffb94ce2294afc328c
+        checksum/config: cc8af4219d403a137ab87500ae0ab15b681fc635e41057b5623df6154443fddf
       labels:
         app: antrea
         component: antrea-controller

diff --git a/build/yamls/antrea-ipsec.yml b/build/yamls/antrea-ipsec.yml
@@ -2550,6 +2550,12 @@ data:
   antrea-agent.conf: |
     # FeatureGates is a map of feature names to bools that enable or disable experimental features.
     featureGates:
+    # AllAlpha is a global toggle for alpha features. Per-feature key values override the default set by AllAlpha.
+    #  AllAlpha: false
+
+    # AllBeta is a global toggle for beta features. Per-feature key values override the default set by AllBeta.
+    #  AllBeta: false
+
     # Enable AntreaProxy which provides ServiceLB for in-cluster Services in antrea-agent.
     # It should be enabled on Windows, otherwise NetworkPolicy will not take effect on
     # Service traffic.
@@ -2858,6 +2864,12 @@ data:
   antrea-controller.conf: |
     # FeatureGates is a map of feature names to bools that enable or disable experimental features.
     featureGates:
+    # AllAlpha is a global toggle for alpha features. Per-feature key values override the default set by AllAlpha.
+    #  AllAlpha: false
+
+    # AllBeta is a global toggle for beta features. Per-feature key values override the default set by AllBeta.
+    #  AllBeta: false
+
     # Enable traceflow which provides packet tracing feature to diagnose network issue.
     #  Traceflow: true
 
@@ -3689,7 +3701,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 626cd437985469e9b0e55e2faacbab203aee641d8a99ff9831c2bb3319f02e95
+        checksum/config: df5271a5c42a550d3f8e73fbe8e5fad8d178884fb74d81c7322128187546db86
         checksum/ipsec-secret: d0eb9c52d0cd4311b6d252a951126bf9bea27ec05590bed8a394f0f792dcb2a4
       labels:
         app: antrea
@@ -3975,7 +3987,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 626cd437985469e9b0e55e2faacbab203aee641d8a99ff9831c2bb3319f02e95
+        checksum/config: df5271a5c42a550d3f8e73fbe8e5fad8d178884fb74d81c7322128187546db86
       labels:
         app: antrea
         component: antrea-controller

diff --git a/build/yamls/antrea.yml b/build/yamls/antrea.yml
@@ -2537,6 +2537,12 @@ data:
   antrea-agent.conf: |
     # FeatureGates is a map of feature names to bools that enable or disable experimental features.
     featureGates:
+    # AllAlpha is a global toggle for alpha features. Per-feature key values override the default set by AllAlpha.
+    #  AllAlpha: false
+
+    # AllBeta is a global toggle for beta features. Per-feature key values override the default set by AllBeta.
+    #  AllBeta: false
+
     # Enable AntreaProxy which provides ServiceLB for in-cluster Services in antrea-agent.
     # It should be enabled on Windows, otherwise NetworkPolicy will not take effect on
     # Service traffic.
@@ -2845,6 +2851,12 @@ data:
   antrea-controller.conf: |
     # FeatureGates is a map of feature names to bools that enable or disable experimental features.
     featureGates:
+    # AllAlpha is a global toggle for alpha features. Per-feature key values override the default set by AllAlpha.
+    #  AllAlpha: false
+
+    # AllBeta is a global toggle for beta features. Per-feature key values override the default set by AllBeta.
+    #  AllBeta: false
+
     # Enable traceflow which provides packet tracing feature to diagnose network issue.
     #  Traceflow: true
 
@@ -3676,7 +3688,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 750097fa4ed60fda68fedf2f31b06a1a62dbbe5dc9a38e9c1f9ddc861a389401
+        checksum/config: 033b7f8c7b77a918ad1a90c3db034bbfc1df67de264a77f8aee6a035836b6812
       labels:
         app: antrea
         component: antrea-agent
@@ -3916,7 +3928,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 750097fa4ed60fda68fedf2f31b06a1a62dbbe5dc9a38e9c1f9ddc861a389401
+        checksum/config: 033b7f8c7b77a918ad1a90c3db034bbfc1df67de264a77f8aee6a035836b6812
       labels:
         app: antrea
         component: antrea-controller

diff --git a/ci/jenkins/test-vmc.sh b/ci/jenkins/test-vmc.sh
@@ -574,10 +574,10 @@ function run_conformance {
 
     if [[ "$TESTCASE" == "all-features-conformance" ]]; then
       if [[ "$COVERAGE" == true ]]; then
-        $GIT_CHECKOUT_DIR/hack/generate-manifest.sh --mode dev --all-features --coverage > $GIT_CHECKOUT_DIR/build/yamls/antrea-all-coverage.yml
+        $GIT_CHECKOUT_DIR/hack/generate-manifest.sh --mode dev --feature-gates AllAlpha=true,AllBeta=true --proxy-all --coverage > $GIT_CHECKOUT_DIR/build/yamls/antrea-all-coverage.yml
         antrea_yml="antrea-all-coverage.yml"
       else
-        $GIT_CHECKOUT_DIR/hack/generate-manifest.sh --mode dev --all-features --verbose-log > $GIT_CHECKOUT_DIR/build/yamls/antrea-all.yml
+        $GIT_CHECKOUT_DIR/hack/generate-manifest.sh --mode dev --feature-gates AllAlpha=true,AllBeta=true --proxy-all --verbose-log > $GIT_CHECKOUT_DIR/build/yamls/antrea-all.yml
         antrea_yml="antrea-all.yml"
       fi
     fi