[WIP Multicast] fix multicast e2e tests in kind cluster

Signed-off-by: ceclinux <[email protected]>
antrea-io · Jul 5, 2022 · 613eece · 613eece
1 parent 79b6480
commit 613eece
Show file tree

Hide file tree

Showing 6 changed files with 130 additions and 8 deletions.
diff --git a/.github/workflows/kind.yml b/.github/workflows/kind.yml
@@ -205,8 +205,7 @@ jobs:
         run: |
           mkdir log
           mkdir test-e2e-encap-all-features-enabled-coverage
-          # Currently multicast tests require specific testbeds, exclude it for now.
-          ANTREA_LOG_DIR=$PWD/log ANTREA_COV_DIR=$PWD/test-e2e-encap-all-features-enabled-coverage ./ci/kind/test-e2e-kind.sh --encap-mode encap --coverage --feature-gates AllAlpha=true,AllBeta=true,Multicast=false --proxy-all
+          ANTREA_LOG_DIR=$PWD/log ANTREA_COV_DIR=$PWD/test-e2e-encap-all-features-enabled-coverage ./ci/kind/test-e2e-kind.sh --encap-mode encap --coverage --feature-gates AllAlpha=true,AllBeta=true --proxy-all
       - name: Tar coverage files
         run: tar -czf test-e2e-encap-all-features-enabled-coverage.tar.gz test-e2e-encap-all-features-enabled-coverage
       - name: Upload coverage for test-e2e-encap-all-features-enabled-coverage
@@ -290,6 +289,120 @@ jobs:
         path: log.tar.gz
         retention-days: 30
 
+  test-multicast-e2e-noencap:
+    name: Multicast e2e tests on a Kind cluster on Linux (noEncap)
+    needs: [build-antrea-coverage-image]
+    runs-on: [ubuntu-latest]
+    steps:
+    - name: Free disk space
+      # https://github.com/actions/virtual-environments/issues/709
+      run: |
+        sudo apt-get clean
+        df -h
+    - uses: actions/checkout@v3
+    - uses: actions/setup-go@v3
+      with:
+        go-version: 1.17
+    - name: Download Antrea images from previous jobs
+      uses: actions/download-artifact@v3
+    - name: Load Antrea image
+      run: |
+        docker load -i antrea-ubuntu-cov/antrea-ubuntu.tar
+    - name: Install Kind
+      run: |
+        curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-$(uname)-amd64
+        chmod +x ./kind
+        sudo mv kind /usr/local/bin
+    - name: Run e2e tests
+      run: |
+        mkdir log
+        mkdir test-e2e-noencap-coverage
+        ANTREA_LOG_DIR=$PWD/log ANTREA_COV_DIR=$PWD/test-e2e-noencap-coverage ./ci/kind/test-e2e-kind.sh --run TestMulticast --encap-mode noEncap --coverage --feature-gates Multicast=true
+    - name: Tar coverage files
+      run: tar -czf test-e2e-noencap-coverage.tar.gz test-e2e-noencap-coverage
+    - name: Upload coverage for test-e2e-noencap-coverage
+      uses: actions/upload-artifact@v3
+      with:
+        name: test-e2e-noencap-coverage
+        path: test-e2e-noencap-coverage.tar.gz
+        retention-days: 30
+    - name: Codecov
+      uses: codecov/codecov-action@v3
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }}
+        file: '*.cov.out*'
+        flags: kind-e2e-tests
+        name: codecov-test-e2e-noencap
+        directory: test-e2e-noencap-coverage
+    - name: Tar log files
+      if: ${{ failure() }}
+      run: tar -czf log.tar.gz log
+    - name: Upload test log
+      uses: actions/upload-artifact@v3
+      if: ${{ failure() }}
+      with:
+        name: e2e-kind-noencap.tar.gz
+        path: log.tar.gz
+        retention-days: 30
+
+  test-multicast-e2e-encap:
+    name: Multicast e2e tests on a Kind cluster on Linux (Encap)
+    needs: [build-antrea-coverage-image]
+    runs-on: [ubuntu-latest]
+    steps:
+    - name: Free disk space
+      # https://github.com/actions/virtual-environments/issues/709
+      run: |
+        sudo apt-get clean
+        df -h
+    - uses: actions/checkout@v3
+    - uses: actions/setup-go@v3
+      with:
+        go-version: 1.17
+    - name: Download Antrea images from previous jobs
+      uses: actions/download-artifact@v3
+    - name: Load Antrea image
+      run: |
+        docker load -i antrea-ubuntu-cov/antrea-ubuntu.tar
+    - name: Install Kind
+      run: |
+        curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-$(uname)-amd64
+        chmod +x ./kind
+        sudo mv kind /usr/local/bin
+    - name: Run e2e tests
+      run: |
+        mkdir log
+        mkdir test-e2e-noencap-coverage
+        ANTREA_LOG_DIR=$PWD/log ANTREA_COV_DIR=$PWD/test-e2e-noencap-coverage ./ci/kind/test-e2e-kind.sh --encap-mode encap --run TestMulticast --coverage --feature-gates Multicast=true
+    - name: Tar coverage files
+      run: tar -czf test-e2e-noencap-coverage.tar.gz test-e2e-noencap-coverage
+    - name: Upload coverage for test-e2e-noencap-coverage
+      uses: actions/upload-artifact@v3
+      with:
+        name: test-e2e-noencap-coverage
+        path: test-e2e-noencap-coverage.tar.gz
+        retention-days: 30
+    - name: Codecov
+      uses: codecov/codecov-action@v3
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }}
+        file: '*.cov.out*'
+        flags: kind-e2e-tests
+        name: codecov-test-e2e-noencap
+        directory: test-e2e-noencap-coverage
+    - name: Tar log files
+      if: ${{ failure() }}
+      run: tar -czf log.tar.gz log
+    - name: Upload test log
+      uses: actions/upload-artifact@v3
+      if: ${{ failure() }}
+      with:
+        name: e2e-kind-noencap.tar.gz
+        path: log.tar.gz
+        retention-days: 30
+
+
+
   test-e2e-hybrid:
     name: E2e tests on a Kind cluster on Linux (hybrid)
     needs: [build-antrea-coverage-image]

diff --git a/ci/jenkins/test.sh b/ci/jenkins/test.sh
@@ -541,7 +541,7 @@ function run_e2e {
     if [[ $TESTBED_TYPE == "flexible-ipam" ]]; then
         go test -v antrea.io/antrea/test/e2e --logs-export-dir `pwd`/antrea-test-logs --provider remote -timeout=100m --prometheus --antrea-ipam
     else
-        go test -v antrea.io/antrea/test/e2e --logs-export-dir `pwd`/antrea-test-logs --provider remote -timeout=100m --prometheus
+        go test -run=TestMulticast -v antrea.io/antrea/test/e2e --logs-export-dir `pwd`/antrea-test-logs --provider remote -timeout=100m --prometheus
     fi
     if [[ "$?" != "0" ]]; then
         TEST_FAILURE=true

diff --git a/ci/kind/test-e2e-kind.sh b/ci/kind/test-e2e-kind.sh
@@ -138,7 +138,10 @@ trap "quit" INT EXIT
 
 manifest_args="$manifest_args --verbose-log"
 if [ -n "$feature_gates" ]; then
-  manifest_args="$manifest_args --feature-gates $feature_gates"
+    if [[ "$feature_gates" == *"Multicast=true"* ]] || ([[ "$feature_gates" == *"AllAlpha=true"* ]] && [[ "$feature_gates" != *"AllAlpha=false"* ]]); then
+        manifest_args="$manifest_args --extra-helm-values multicast.igmpQueryInterval=10s"
+    fi
+    manifest_args="$manifest_args --feature-gates $feature_gates"
 fi
 if $proxy_all; then
     manifest_args="$manifest_args --proxy-all"

diff --git a/test/e2e/framework.go b/test/e2e/framework.go
@@ -2165,7 +2165,7 @@ func (data *TestData) GetMulticastInterfaces(antreaNamespace string) ([]string,
 }
 
 func GetTransportInterface(data *TestData) (string, error) {
-	_, transportInterfaceUntrimmed, _, err := data.RunCommandOnNode(nodeName(0), fmt.Sprintf("ip -br addr show | grep %s | awk '{print $1}'", clusterInfo.nodes[0].ipv4Addr))
+	_, transportInterfaceUntrimmed, _, err := data.RunCommandOnNode(nodeName(0), fmt.Sprintf("/bin/sh -c ip -br addr show | grep %s | sed \"s/[ @].*//\"", clusterInfo.nodes[0].ipv4Addr))
 	if err != nil {
 		return "", err
 	}

diff --git a/test/e2e/multicast_test.go b/test/e2e/multicast_test.go
@@ -580,7 +580,7 @@ func runTestMulticastBetweenPods(t *testing.T, data *TestData, mc multicastTestc
 	if err := wait.Poll(3*time.Second, defaultTimeout, func() (bool, error) {
 		if !senderReady {
 			// Sender pods should add an outbound multicast route except running as HostNetwork.
-			_, mrouteResult, _, err := data.RunCommandOnNode(nodeName(mc.senderConfig.nodeIdx), fmt.Sprintf("ip mroute show to %s iif %s | grep '%s'", mc.group.String(), gatewayInterface, strings.Join(nodeMulticastInterfaces[mc.senderConfig.nodeIdx], " ")))
+			_, mrouteResult, _, err := data.RunCommandOnNode(nodeName(mc.senderConfig.nodeIdx), fmt.Sprintf("/bin/sh -c ip mroute show iif %s |grep \"%s)\"|grep '%s'", gatewayInterface, mc.group.String(), strings.Join(nodeMulticastInterfaces[mc.senderConfig.nodeIdx], " ")))
 			if err != nil {
 				return false, err
 			}
@@ -602,7 +602,7 @@ func runTestMulticastBetweenPods(t *testing.T, data *TestData, mc multicastTestc
 				continue
 			}
 			for _, receiverMulticastInterface := range nodeMulticastInterfaces[receiver.nodeIdx] {
-				_, mRouteResult, _, err := data.RunCommandOnNode(nodeName(receiver.nodeIdx), fmt.Sprintf("ip mroute show to %s iif %s ", mc.group.String(), receiverMulticastInterface))
+				_, mRouteResult, _, err := data.RunCommandOnNode(nodeName(receiver.nodeIdx), fmt.Sprintf("/bin/sh -c ip mroute show iif %s|grep \"%s)\"", receiverMulticastInterface, mc.group.String()))
 				if err != nil {
 					return false, err
 				}
@@ -617,7 +617,7 @@ func runTestMulticastBetweenPods(t *testing.T, data *TestData, mc multicastTestc
 						return false, nil
 					}
 				}
-				_, mAddrResult, _, err := data.RunCommandOnNode(nodeName(receiver.nodeIdx), fmt.Sprintf("ip maddr show %s | grep %s", receiverMulticastInterface, mc.group.String()))
+				_, mAddrResult, _, err := data.RunCommandOnNode(nodeName(receiver.nodeIdx), fmt.Sprintf("/bin/sh -c ip maddr show %s | grep %s", receiverMulticastInterface, mc.group.String()))
 				if err != nil {
 					return false, err
 				}

diff --git a/test/e2e/providers/exec/ssh.go b/test/e2e/providers/exec/ssh.go
@@ -54,6 +54,12 @@ func RunSSHCommand(host string, config *ssh.ClientConfig, cmd string, envs map[s
 	if stdin != "" {
 		session.Stdin = strings.NewReader(stdin)
 	}
+	if strings.Contains(cmd, "/bin/sh") {
+		// Just split in to "/bin/sh" "-c" and "actual_cmd"
+		// This is used to keep align with running command in docker.
+		splitCmd := strings.SplitN(cmd, " ", 3)
+		cmd = splitCmd[0] + " -c " + fmt.Sprintf("'%s'", splitCmd[2])
+	}
 
 	if err := session.Run(cmd); err != nil {
 		switch e := err.(type) {