[Multicast] use real external node in multicast e2e

ci/jenkins/external-hosts-config.yml

@@ -0,0 +1,8 @@
+externalHosts:
+  - sshIP: 10.1.10.10
+    name: antrea-multicast-external
+    interfaces:
+      - name: ens224
+        ipv4: 10.1.10.10
+        clusterInterfaces:
+          antrea-multicast-0-0: [ens224]

ci/jenkins/test.sh

@@ -53,13 +53,14 @@ _usage="Usage: $0 [--kubeconfig <KubeconfigSavePath>] [--workdir <HomePath>]
 
 Run K8s e2e community tests (Conformance & Network Policy) or Antrea e2e tests on a remote (Jenkins) Windows or Linux cluster.
 
-        --kubeconfig             Path of cluster kubeconfig.
-        --workdir                Home path for Go, vSphere information and antrea_logs during cluster setup. Default is $WORKDIR.
-        --testcase               Windows install OVS, Conformance and Network Policy or Antrea e2e testcases on a Windows or Linux cluster. It can also be flexible ipam or multicast e2e test.
-        --registry               The docker registry to use instead of dockerhub.
-        --proxyall               Enable proxyAll to test AntreaProxy.
-        --testbed-type           The testbed type to run tests. It can be flexible-ipam, jumper or legacy.
-        --ip-mode                IP mode for flexible-ipam e2e test. Default is $DEFAULT_IP_MODE. It can also be ipv6 or ds."
+        --kubeconfig                    Path of cluster kubeconfig.
+        --workdir                       Home path for Go, vSphere information and antrea_logs during cluster setup. Default is $WORKDIR.
+        --testcase                      Windows install OVS, Conformance and Network Policy or Antrea e2e testcases on a Windows or Linux cluster. It can also be flexible ipam or multicast e2e test.
+        --registry                      The docker registry to use instead of dockerhub.
+        --proxyall                      Enable proxyAll to test AntreaProxy.
+        --testbed-type                  The testbed type to run tests. It can be flexible-ipam, jumper or legacy.
+        --ip-mode                IP mode for flexible-ipam e2e test. Default is $DEFAULT_IP_MODE. It can also be ipv6 or ds.
+        --external-hosts-config-path    The path of external host configuration file."
 
 function print_usage {
     echoerr "$_usage"
@@ -78,6 +79,10 @@ case $key in
     KUBECONFIG_PATH="$2"
     shift 2
     ;;
+    --external-hosts-config-path)
+    EXTERNAL_HOSTS_CONFIG_PATH="$2"
+    shift 2
+    ;;
     --workdir)
     WORKDIR="$2"
     shift 2
@@ -120,6 +125,9 @@ if [[ "${IP_MODE}" != "${DEFAULT_IP_MODE}" && "${IP_MODE}" != "ipv6" && "${IP_MO
     echoerr "--ip-mode must be ipv4, ipv6 or ds"
     exit 1
 fi
+
+EXTERNAL_HOSTS_CONFIG_PATH="ci/jenkins/external-hosts-config.yml"
+
 if [[ "$WORKDIR" != "$DEFAULT_WORKDIR" && "$KUBECONFIG_PATH" == "$DEFAULT_KUBECONFIG_PATH" ]]; then
     KUBECONFIG_PATH=${WORKDIR}/.kube/config
 fi
@@ -516,6 +524,21 @@ function deliver_antrea {
     fi
 }
 
+function add_sshconfig_entry {
+    sshconfig_nodeip="$1"
+    sshconfig_nodename="$2"
+    cp ci/jenkins/ssh-config "${SSH_CONFIG_DST}.new"
+    sed -i "s/SSHCONFIGNODEIP/${sshconfig_nodeip}/g" "${SSH_CONFIG_DST}.new"
+    sed -i "s/SSHCONFIGNODENAME/${sshconfig_nodename}/g" "${SSH_CONFIG_DST}.new"
+    if [[ "${sshconfig_nodename}" =~ "win" ]]; then
+        sed -i "s/capv/administrator/g" "${SSH_CONFIG_DST}.new"
+    else
+        sed -i "s/capv/jenkins/g" "${SSH_CONFIG_DST}.new"
+    fi
+    echo "    IdentityFile ${WORKDIR}/.ssh/id_rsa" >> "${SSH_CONFIG_DST}.new"
+    cat "${SSH_CONFIG_DST}.new" >> "${SSH_CONFIG_DST}"
+}
+
 function generate_ssh_config {
     echo "=== Generate ssh-config ==="
     SSH_CONFIG_DST="${WORKDIR}/.ssh/config"
@@ -527,17 +550,14 @@ function generate_ssh_config {
         if [[ ! "${sshconfig_nodeip}" =~ ^[0-9]+(\.[0-9]+){3}$ ]];then
             sshconfig_nodeip="[${sshconfig_nodeip}]"
         fi
-        cp ci/jenkins/ssh-config "${SSH_CONFIG_DST}.new"
-        sed -i "s/SSHCONFIGNODEIP/${sshconfig_nodeip}/g" "${SSH_CONFIG_DST}.new"
-        sed -i "s/SSHCONFIGNODENAME/${sshconfig_nodename}/g" "${SSH_CONFIG_DST}.new"
-        if [[ "${sshconfig_nodename}" =~ "win" ]]; then
-            sed -i "s/capv/administrator/g" "${SSH_CONFIG_DST}.new"
-        else
-            sed -i "s/capv/jenkins/g" "${SSH_CONFIG_DST}.new"
-        fi
-        echo "    IdentityFile ${WORKDIR}/.ssh/id_rsa" >> "${SSH_CONFIG_DST}.new"
-        cat "${SSH_CONFIG_DST}.new" >> "${SSH_CONFIG_DST}"
+        add_sshconfig_entry "$sshconfig_nodeip" "${sshconfig_nodename}"
     done
+    if [[ -n ${EXTERNAL_HOSTS_CONFIG_PATH} ]]; then
+        yq -r '.externalHosts.[] | {.name:.sshIP}' "$EXTERNAL_HOSTS_CONFIG_PATH" | while IFS=' :' read -r ssh_hostname ssh_ip; do
+            echo "adding ssh config for external host with hostname:$ssh_hostname and IP:$ssh_ip"
+            add_sshconfig_entry "$ssh_ip" "$ssh_hostname"
+        done
+    fi
 }
 
 function run_e2e {
@@ -552,15 +572,17 @@ function run_e2e {
     mkdir -p "${WORKDIR}/.ssh"
     cp -f "${WORKDIR}/kube.conf" "${WORKDIR}/.kube/config"
     generate_ssh_config
-
+    if [[ -n ${EXTERNAL_HOSTS_CONFIG_PATH} ]]; then
+        EXTERNAL_HOSTS_CONFIG_PATH="../../${EXTERNAL_HOSTS_CONFIG_PATH}"
+    fi
     set +e
     mkdir -p `pwd`/antrea-test-logs
     # HACK: see https://github.com/antrea-io/antrea/issues/2292
     go mod edit -replace github.com/moby/spdystream=github.com/antoninbas/[email protected] && go mod tidy
     if [[ $TESTBED_TYPE == "flexible-ipam" ]]; then
-        go test -v antrea.io/antrea/test/e2e --logs-export-dir `pwd`/antrea-test-logs --provider remote -timeout=100m --prometheus --antrea-ipam
+        go test -v antrea.io/antrea/test/e2e --logs-export-dir `pwd`/antrea-test-logs --external-hosts-config-path "$EXTERNAL_HOSTS_CONFIG_PATH" --provider remote -timeout=100m --prometheus --antrea-ipam
     else
-        go test -v antrea.io/antrea/test/e2e --logs-export-dir `pwd`/antrea-test-logs --provider remote -timeout=100m --prometheus
+        go test -run=TestMulticast -v antrea.io/antrea/test/e2e --logs-export-dir `pwd`/antrea-test-logs --external-hosts-config-path "$EXTERNAL_HOSTS_CONFIG_PATH" --provider remote -timeout=20m --prometheus
     fi
     if [[ "$?" != "0" ]]; then
         TEST_FAILURE=true
@@ -614,10 +636,12 @@ function run_e2e_windows {
     mkdir -p "${WORKDIR}/.ssh"
     cp -f "${WORKDIR}/kube.conf" "${WORKDIR}/.kube/config"
     generate_ssh_config
-
+    if [[ -n ${EXTERNAL_HOSTS_CONFIG_PATH} ]]; then
+        EXTERNAL_HOSTS_CONFIG_PATH="../../${EXTERNAL_HOSTS_CONFIG_PATH}"
+    fi
     set +e
     mkdir -p `pwd`/antrea-test-logs
-    go test -v antrea.io/antrea/test/e2e --logs-export-dir `pwd`/antrea-test-logs --provider remote -timeout=50m --prometheus
+    go test -v antrea.io/antrea/test/e2e --logs-export-dir `pwd`/antrea-test-logs --external-hosts-config-path "$EXTERNAL_HOSTS_CONFIG_PATH" --provider remote -timeout=50m --prometheus
     if [[ "$?" != "0" ]]; then
         TEST_FAILURE=true
     fi

test/e2e/antreapolicy_test.go

@@ -2987,12 +2987,11 @@ func testACNPIGMPQuery(t *testing.T, data *TestData, acnpName, caseName, groupAd
 	testNamespace := data.testNamespace
 	mc := multicastTestcase{
 		name:            caseName,
-		senderConfig:    multicastTestPodConfig{nodeIdx: 0, isHostNetwork: false},
-		receiverConfigs: []multicastTestPodConfig{{1, false}},
+		receiverIndices: []int{1},
 		port:            3457,
 		group:           net.ParseIP(groupAddress),
 	}
-	senderName, _, cleanupFunc := createAndWaitForPod(t, data, data.createMcJoinPodOnNode, "test-sender-", nodeName(mc.senderConfig.nodeIdx), testNamespace, mc.senderConfig.isHostNetwork)
+	senderName, _, cleanupFunc := createAndWaitForPod(t, data, data.createMcJoinPodOnNode, "test-sender-", nodeName(0), testNamespace, false)
 	defer cleanupFunc()
 	var wg sync.WaitGroup
 	receiverNames, cleanupFuncs := setupReceivers(t, data, mc, mcjoinWaitTimeout, &wg)
@@ -3006,11 +3005,11 @@ func testACNPIGMPQuery(t *testing.T, data *TestData, acnpName, caseName, groupAd
 		data.RunCommandFromPod(testNamespace, senderName, mcjoinContainerName, sendMulticastCommand)
 	}()
 
-	tcpdumpName, _, cleanupFunc := createAndWaitForPod(t, data, data.createNetshootPodOnNode, "test-tcpdump-", nodeName(mc.receiverConfigs[0].nodeIdx), testNamespace, true)
+	tcpdumpName, _, cleanupFunc := createAndWaitForPod(t, data, data.createNetshootPodOnNode, "test-tcpdump-", nodeName(mc.receiverIndices[0]), testNamespace, true)
 	defer cleanupFunc()
 
 	queryGroupAddress := "224.0.0.1"
-	cmd, err := generatePacketCaptureCmd(t, data, 15, queryGroupAddress, nodeName(mc.receiverConfigs[0].nodeIdx), receiverNames[0])
+	cmd, err := generatePacketCaptureCmd(t, data, 15, queryGroupAddress, nodeName(mc.receiverIndices[0]), receiverNames[0])
 	if err != nil {
 		t.Fatalf("failed to call generateConnCheckCmd: %v", err)
 	}
@@ -3073,12 +3072,11 @@ func testACNPMulticastEgress(t *testing.T, data *TestData, acnpName, caseName, g
 	testNamespace := data.testNamespace
 	mc := multicastTestcase{
 		name:            caseName,
-		senderConfig:    multicastTestPodConfig{nodeIdx: 0, isHostNetwork: false},
-		receiverConfigs: []multicastTestPodConfig{{1, false}},
+		receiverIndices: []int{1},
 		port:            3457,
 		group:           net.ParseIP(groupAddress),
 	}
-	senderName, _, cleanupFunc := createAndWaitForPod(t, data, data.createMcJoinPodOnNode, "test-sender-", nodeName(mc.senderConfig.nodeIdx), testNamespace, mc.senderConfig.isHostNetwork)
+	senderName, _, cleanupFunc := createAndWaitForPod(t, data, data.createMcJoinPodOnNode, "test-sender-", nodeName(0), testNamespace, false)
 	defer cleanupFunc()
 	var wg sync.WaitGroup
 	receiverNames, cleanupFuncs := setupReceivers(t, data, mc, mcjoinWaitTimeout, &wg)
@@ -3093,9 +3091,9 @@ func testACNPMulticastEgress(t *testing.T, data *TestData, acnpName, caseName, g
 		data.RunCommandFromPod(testNamespace, senderName, mcjoinContainerName, sendMulticastCommand)
 	}()
 	// check if receiver can receive multicast packet
-	tcpdumpName, _, cleanupFunc := createAndWaitForPod(t, data, data.createNetshootPodOnNode, "test-tcpdump-", nodeName(mc.receiverConfigs[0].nodeIdx), testNamespace, true)
+	tcpdumpName, _, cleanupFunc := createAndWaitForPod(t, data, data.createNetshootPodOnNode, "test-tcpdump-", nodeName(mc.receiverIndices[0]), testNamespace, true)
 	defer cleanupFunc()
-	cmd, err := generatePacketCaptureCmd(t, data, 5, mc.group.String(), nodeName(mc.receiverConfigs[0].nodeIdx), receiverNames[0])
+	cmd, err := generatePacketCaptureCmd(t, data, 5, mc.group.String(), nodeName(mc.receiverIndices[0]), receiverNames[0])
 	if err != nil {
 		t.Fatalf("failed to call generateConnCheckCmd: %v", err)
 	}

test/e2e/fixtures.go

@@ -93,6 +93,12 @@ func skipIfNotIPv4Cluster(tb testing.TB) {
 	}
 }
 
+func skipIfNoExternalHosts(tb testing.TB) {
+	if len(externalHostInfo.hosts) == 0 {
+		tb.Skipf("Skipping test as it requires external hosts info but the external hosts info is not set")
+	}
+}
+
 func skipIfIPv6Cluster(tb testing.TB) {
 	if clusterInfo.podV6NetworkCIDR != "" {
 		tb.Skipf("Skipping test as it is not supported in IPv6 cluster")

test/e2e/framework.go

@@ -19,6 +19,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"math/rand"
 	"net"
 	"os"
@@ -154,6 +155,15 @@ type ClusterNode struct {
 	os               string
 }
 
+type ExternalHost struct {
+	Name       string                  `name`
+	Interfaces []ExternalHostInterface `interfaces`
+}
+
+type ExternalHostInterface struct {
+	Name string `name`
+}
+
 func (n ClusterNode) ip() string {
 	if n.ipv4Addr != "" {
 		return n.ipv4Addr
@@ -180,17 +190,28 @@ type ClusterInfo struct {
 
 var clusterInfo ClusterInfo
 
+type ExternalHostsConfig struct {
+	ExternalHosts []ExternalHost `externalHosts`
+}
+
+var externalHostInfo ExternalHostInfo
+
+type ExternalHostInfo struct {
+	hosts map[int]ExternalHost
+}
+
 type TestOptions struct {
-	providerName        string
-	providerConfigPath  string
-	logsExportDir       string
-	logsExportOnSuccess bool
-	withBench           bool
-	enableCoverage      bool
-	enableAntreaIPAM    bool
-	flowVisibility      bool
-	coverageDir         string
-	skipCases           string
+	providerName            string
+	providerConfigPath      string
+	logsExportDir           string
+	logsExportOnSuccess     bool
+	withBench               bool
+	enableCoverage          bool
+	enableAntreaIPAM        bool
+	flowVisibility          bool
+	coverageDir             string
+	skipCases               string
+	externalHostsConfigPath string
 }
 
 var testOptions TestOptions
@@ -359,6 +380,14 @@ func nodeName(idx int) string {
 	return node.name
 }
 
+func externalHostName(idx int) string {
+	host, ok := externalHostInfo.hosts[idx]
+	if !ok {
+		return ""
+	}
+	return host.Name
+}
+
 // nodeIP returns an empty string if there is no Node with the provided idx. If idx is 0, the IP
 // of the control-plane Node will be returned.
 func nodeIP(idx int) string {
@@ -425,6 +454,29 @@ func (data *TestData) RunCommandOnNodeExt(nodeName, cmd string, envs map[string]
 	return data.provider.RunCommandOnNodeExt(nodeName, cmd, envs, stdin, sudo)
 }
 
+func (data *TestData) collectExternalHostsInfo(path string) error {
+	externalHostInfo = ExternalHostInfo{}
+	externalHostInfo.hosts = make(map[int]ExternalHost)
+	filePath, _ := filepath.Abs(path)
+	yamlFile, err := ioutil.ReadFile(filePath)
+
+	if err != nil {
+		panic(err)
+	}
+
+	var config ExternalHostsConfig
+
+	err = yaml.Unmarshal(yamlFile, &config)
+	if err != nil {
+		panic(err)
+	}
+	for i, v := range config.ExternalHosts {
+		externalHostInfo.hosts[i] = v
+		fmt.Printf("The information of external host index %d is: %+v\n", i, externalHostInfo.hosts[i])
+	}
+	return nil
+}
+
 func (data *TestData) collectClusterInfo() error {
 	// retrieve K8s server version
 	// this needs to be done first, as there may be dependencies on the

test/e2e/main_test.go

@@ -75,6 +75,7 @@ func (tOptions *TestOptions) setupCoverage() func() {
 // testMain is meant to be called by TestMain and enables the use of defer statements.
 func testMain(m *testing.M) int {
 	flag.StringVar(&testOptions.providerName, "provider", "vagrant", "K8s test cluster provider")
+	flag.StringVar(&testOptions.externalHostsConfigPath, "external-hosts-config-path", "", "Path of external hosts config file")
 	flag.StringVar(&testOptions.providerConfigPath, "provider-cfg-path", "", "Optional config file for provider")
 	flag.StringVar(&testOptions.logsExportDir, "logs-export-dir", "", "Export directory for test logs")
 	flag.BoolVar(&testOptions.logsExportOnSuccess, "logs-export-on-success", false, "Export logs even when a test is successful")
@@ -105,6 +106,12 @@ func testMain(m *testing.M) int {
 	if err := testData.collectClusterInfo(); err != nil {
 		log.Fatalf("Error when collecting information about K8s cluster: %v", err)
 	}
+	if testOptions.externalHostsConfigPath != "" {
+		log.Println("Collecting external information about K8s cluster")
+		if err := testData.collectExternalHostsInfo(testOptions.externalHostsConfigPath); err != nil {
+			log.Fatalf("Error when collecting information about external hosts: %v", err)
+		}
+	}
 	if clusterInfo.podV4NetworkCIDR != "" {
 		log.Printf("Pod IPv4 network: '%s'", clusterInfo.podV4NetworkCIDR)
 	}