From 613eece2605138cf875930957e51da9dc161b63d Mon Sep 17 00:00:00 2001 From: ceclinux Date: Tue, 28 Jun 2022 23:35:21 +0800 Subject: [PATCH] [WIP Multicast] fix multicast e2e tests in kind cluster Signed-off-by: ceclinux --- .github/workflows/kind.yml | 117 ++++++++++++++++++++++++++++++++- ci/jenkins/test.sh | 2 +- ci/kind/test-e2e-kind.sh | 5 +- test/e2e/framework.go | 2 +- test/e2e/multicast_test.go | 6 +- test/e2e/providers/exec/ssh.go | 6 ++ 6 files changed, 130 insertions(+), 8 deletions(-) diff --git a/.github/workflows/kind.yml b/.github/workflows/kind.yml index 9de62ee6323..090bf702e14 100644 --- a/.github/workflows/kind.yml +++ b/.github/workflows/kind.yml @@ -205,8 +205,7 @@ jobs: run: | mkdir log mkdir test-e2e-encap-all-features-enabled-coverage - # Currently multicast tests require specific testbeds, exclude it for now. - ANTREA_LOG_DIR=$PWD/log ANTREA_COV_DIR=$PWD/test-e2e-encap-all-features-enabled-coverage ./ci/kind/test-e2e-kind.sh --encap-mode encap --coverage --feature-gates AllAlpha=true,AllBeta=true,Multicast=false --proxy-all + ANTREA_LOG_DIR=$PWD/log ANTREA_COV_DIR=$PWD/test-e2e-encap-all-features-enabled-coverage ./ci/kind/test-e2e-kind.sh --encap-mode encap --coverage --feature-gates AllAlpha=true,AllBeta=true --proxy-all - name: Tar coverage files run: tar -czf test-e2e-encap-all-features-enabled-coverage.tar.gz test-e2e-encap-all-features-enabled-coverage - name: Upload coverage for test-e2e-encap-all-features-enabled-coverage @@ -290,6 +289,120 @@ jobs: path: log.tar.gz retention-days: 30 + test-multicast-e2e-noencap: + name: Multicast e2e tests on a Kind cluster on Linux (noEncap) + needs: [build-antrea-coverage-image] + runs-on: [ubuntu-latest] + steps: + - name: Free disk space + # https://github.com/actions/virtual-environments/issues/709 + run: | + sudo apt-get clean + df -h + - uses: actions/checkout@v3 + - uses: actions/setup-go@v3 + with: + go-version: 1.17 + - name: Download Antrea images from previous jobs + uses: actions/download-artifact@v3 + - name: Load Antrea image + run: | + docker load -i antrea-ubuntu-cov/antrea-ubuntu.tar + - name: Install Kind + run: | + curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-$(uname)-amd64 + chmod +x ./kind + sudo mv kind /usr/local/bin + - name: Run e2e tests + run: | + mkdir log + mkdir test-e2e-noencap-coverage + ANTREA_LOG_DIR=$PWD/log ANTREA_COV_DIR=$PWD/test-e2e-noencap-coverage ./ci/kind/test-e2e-kind.sh --run TestMulticast --encap-mode noEncap --coverage --feature-gates Multicast=true + - name: Tar coverage files + run: tar -czf test-e2e-noencap-coverage.tar.gz test-e2e-noencap-coverage + - name: Upload coverage for test-e2e-noencap-coverage + uses: actions/upload-artifact@v3 + with: + name: test-e2e-noencap-coverage + path: test-e2e-noencap-coverage.tar.gz + retention-days: 30 + - name: Codecov + uses: codecov/codecov-action@v3 + with: + token: ${{ secrets.CODECOV_TOKEN }} + file: '*.cov.out*' + flags: kind-e2e-tests + name: codecov-test-e2e-noencap + directory: test-e2e-noencap-coverage + - name: Tar log files + if: ${{ failure() }} + run: tar -czf log.tar.gz log + - name: Upload test log + uses: actions/upload-artifact@v3 + if: ${{ failure() }} + with: + name: e2e-kind-noencap.tar.gz + path: log.tar.gz + retention-days: 30 + + test-multicast-e2e-encap: + name: Multicast e2e tests on a Kind cluster on Linux (Encap) + needs: [build-antrea-coverage-image] + runs-on: [ubuntu-latest] + steps: + - name: Free disk space + # https://github.com/actions/virtual-environments/issues/709 + run: | + sudo apt-get clean + df -h + - uses: actions/checkout@v3 + - uses: actions/setup-go@v3 + with: + go-version: 1.17 + - name: Download Antrea images from previous jobs + uses: actions/download-artifact@v3 + - name: Load Antrea image + run: | + docker load -i antrea-ubuntu-cov/antrea-ubuntu.tar + - name: Install Kind + run: | + curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-$(uname)-amd64 + chmod +x ./kind + sudo mv kind /usr/local/bin + - name: Run e2e tests + run: | + mkdir log + mkdir test-e2e-noencap-coverage + ANTREA_LOG_DIR=$PWD/log ANTREA_COV_DIR=$PWD/test-e2e-noencap-coverage ./ci/kind/test-e2e-kind.sh --encap-mode encap --run TestMulticast --coverage --feature-gates Multicast=true + - name: Tar coverage files + run: tar -czf test-e2e-noencap-coverage.tar.gz test-e2e-noencap-coverage + - name: Upload coverage for test-e2e-noencap-coverage + uses: actions/upload-artifact@v3 + with: + name: test-e2e-noencap-coverage + path: test-e2e-noencap-coverage.tar.gz + retention-days: 30 + - name: Codecov + uses: codecov/codecov-action@v3 + with: + token: ${{ secrets.CODECOV_TOKEN }} + file: '*.cov.out*' + flags: kind-e2e-tests + name: codecov-test-e2e-noencap + directory: test-e2e-noencap-coverage + - name: Tar log files + if: ${{ failure() }} + run: tar -czf log.tar.gz log + - name: Upload test log + uses: actions/upload-artifact@v3 + if: ${{ failure() }} + with: + name: e2e-kind-noencap.tar.gz + path: log.tar.gz + retention-days: 30 + + + test-e2e-hybrid: name: E2e tests on a Kind cluster on Linux (hybrid) needs: [build-antrea-coverage-image] diff --git a/ci/jenkins/test.sh b/ci/jenkins/test.sh index 4efb8f988c6..9e683a0c6c5 100755 --- a/ci/jenkins/test.sh +++ b/ci/jenkins/test.sh @@ -541,7 +541,7 @@ function run_e2e { if [[ $TESTBED_TYPE == "flexible-ipam" ]]; then go test -v antrea.io/antrea/test/e2e --logs-export-dir `pwd`/antrea-test-logs --provider remote -timeout=100m --prometheus --antrea-ipam else - go test -v antrea.io/antrea/test/e2e --logs-export-dir `pwd`/antrea-test-logs --provider remote -timeout=100m --prometheus + go test -run=TestMulticast -v antrea.io/antrea/test/e2e --logs-export-dir `pwd`/antrea-test-logs --provider remote -timeout=100m --prometheus fi if [[ "$?" != "0" ]]; then TEST_FAILURE=true diff --git a/ci/kind/test-e2e-kind.sh b/ci/kind/test-e2e-kind.sh index a849969a852..c50d947a05c 100755 --- a/ci/kind/test-e2e-kind.sh +++ b/ci/kind/test-e2e-kind.sh @@ -138,7 +138,10 @@ trap "quit" INT EXIT manifest_args="$manifest_args --verbose-log" if [ -n "$feature_gates" ]; then - manifest_args="$manifest_args --feature-gates $feature_gates" + if [[ "$feature_gates" == *"Multicast=true"* ]] || ([[ "$feature_gates" == *"AllAlpha=true"* ]] && [[ "$feature_gates" != *"AllAlpha=false"* ]]); then + manifest_args="$manifest_args --extra-helm-values multicast.igmpQueryInterval=10s" + fi + manifest_args="$manifest_args --feature-gates $feature_gates" fi if $proxy_all; then manifest_args="$manifest_args --proxy-all" diff --git a/test/e2e/framework.go b/test/e2e/framework.go index ea5c97218c1..173b4b596c5 100644 --- a/test/e2e/framework.go +++ b/test/e2e/framework.go @@ -2165,7 +2165,7 @@ func (data *TestData) GetMulticastInterfaces(antreaNamespace string) ([]string, } func GetTransportInterface(data *TestData) (string, error) { - _, transportInterfaceUntrimmed, _, err := data.RunCommandOnNode(nodeName(0), fmt.Sprintf("ip -br addr show | grep %s | awk '{print $1}'", clusterInfo.nodes[0].ipv4Addr)) + _, transportInterfaceUntrimmed, _, err := data.RunCommandOnNode(nodeName(0), fmt.Sprintf("/bin/sh -c ip -br addr show | grep %s | sed \"s/[ @].*//\"", clusterInfo.nodes[0].ipv4Addr)) if err != nil { return "", err } diff --git a/test/e2e/multicast_test.go b/test/e2e/multicast_test.go index 136e9e43d6f..eb4e3b64bef 100644 --- a/test/e2e/multicast_test.go +++ b/test/e2e/multicast_test.go @@ -580,7 +580,7 @@ func runTestMulticastBetweenPods(t *testing.T, data *TestData, mc multicastTestc if err := wait.Poll(3*time.Second, defaultTimeout, func() (bool, error) { if !senderReady { // Sender pods should add an outbound multicast route except running as HostNetwork. - _, mrouteResult, _, err := data.RunCommandOnNode(nodeName(mc.senderConfig.nodeIdx), fmt.Sprintf("ip mroute show to %s iif %s | grep '%s'", mc.group.String(), gatewayInterface, strings.Join(nodeMulticastInterfaces[mc.senderConfig.nodeIdx], " "))) + _, mrouteResult, _, err := data.RunCommandOnNode(nodeName(mc.senderConfig.nodeIdx), fmt.Sprintf("/bin/sh -c ip mroute show iif %s |grep \"%s)\"|grep '%s'", gatewayInterface, mc.group.String(), strings.Join(nodeMulticastInterfaces[mc.senderConfig.nodeIdx], " "))) if err != nil { return false, err } @@ -602,7 +602,7 @@ func runTestMulticastBetweenPods(t *testing.T, data *TestData, mc multicastTestc continue } for _, receiverMulticastInterface := range nodeMulticastInterfaces[receiver.nodeIdx] { - _, mRouteResult, _, err := data.RunCommandOnNode(nodeName(receiver.nodeIdx), fmt.Sprintf("ip mroute show to %s iif %s ", mc.group.String(), receiverMulticastInterface)) + _, mRouteResult, _, err := data.RunCommandOnNode(nodeName(receiver.nodeIdx), fmt.Sprintf("/bin/sh -c ip mroute show iif %s|grep \"%s)\"", receiverMulticastInterface, mc.group.String())) if err != nil { return false, err } @@ -617,7 +617,7 @@ func runTestMulticastBetweenPods(t *testing.T, data *TestData, mc multicastTestc return false, nil } } - _, mAddrResult, _, err := data.RunCommandOnNode(nodeName(receiver.nodeIdx), fmt.Sprintf("ip maddr show %s | grep %s", receiverMulticastInterface, mc.group.String())) + _, mAddrResult, _, err := data.RunCommandOnNode(nodeName(receiver.nodeIdx), fmt.Sprintf("/bin/sh -c ip maddr show %s | grep %s", receiverMulticastInterface, mc.group.String())) if err != nil { return false, err } diff --git a/test/e2e/providers/exec/ssh.go b/test/e2e/providers/exec/ssh.go index 6701c1f29f1..1eeae2b3bae 100644 --- a/test/e2e/providers/exec/ssh.go +++ b/test/e2e/providers/exec/ssh.go @@ -54,6 +54,12 @@ func RunSSHCommand(host string, config *ssh.ClientConfig, cmd string, envs map[s if stdin != "" { session.Stdin = strings.NewReader(stdin) } + if strings.Contains(cmd, "/bin/sh") { + // Just split in to "/bin/sh" "-c" and "actual_cmd" + // This is used to keep align with running command in docker. + splitCmd := strings.SplitN(cmd, " ", 3) + cmd = splitCmd[0] + " -c " + fmt.Sprintf("'%s'", splitCmd[2]) + } if err := session.Run(cmd); err != nil { switch e := err.(type) {