Add HostNetworking Test for PPSG in test agent

test/agent/cmd/networking/main.go

@@ -29,10 +29,13 @@ func main() {
 	var podNetworkingValidationInputString string
 	var shouldTestSetup bool
 	var shouldTestCleanup bool
+	// per pod security group
+	var ppsg bool
 
 	flag.StringVar(&podNetworkingValidationInputString, "pod-networking-validation-input", "", "json string containing the array of pods whose networking needs to be validated")
 	flag.BoolVar(&shouldTestCleanup, "test-cleanup", false, "bool flag when set to true tests that networking is teared down after pod has been deleted")
 	flag.BoolVar(&shouldTestSetup, "test-setup", false, "bool flag when set to true tests the networking is setup correctly after pod is running")
+	flag.BoolVar(&ppsg, "test-ppsg", false, "bool flag when set to true tests the networking setup for pods using security groups")
 
 	flag.Parse()
 
@@ -47,7 +50,19 @@ func main() {
 
 	log.Printf("list of pod against which test will be run %v", podNetworkingValidationInput.PodList)
 
-	if shouldTestSetup {
+	if ppsg && shouldTestSetup {
+		log.Print("testing networking is setup for pods using security groups")
+		err := tester.TestNetworkingSetupForPodsUsingSecurityGroup(podNetworkingValidationInput)
+		if err != nil {
+			log.Fatalf("found 1 or more pod setup validation failure: %v", err)
+		}
+	} else if ppsg && shouldTestCleanup {
+		log.Print("testing networking is teared down for pods using security groups")
+		err := tester.TestNetworkTearedDownForPodsUsingSecurityGroup(podNetworkingValidationInput)
+		if err != nil {
+			log.Fatalf("found 1 or more pod setup validation failure: %v", err)
+		}
+	} else if !ppsg && shouldTestSetup {
 		log.Print("testing networking is setup for regular pods")
 		err := tester.TestNetworkingSetupForRegularPod(podNetworkingValidationInput)
 		if err != nil {

test/agent/cmd/networking/tester/network.go

@@ -185,6 +185,133 @@ func TestNetworkingSetupForRegularPod(podNetworkingValidationInput input.PodNetw
 	return validationErrors
 }
 
+// TestNetworkingSetupForPods using security groups
+func TestNetworkingSetupForPodsUsingSecurityGroup(podNetworkingValidationInput input.PodNetworkingValidationInput) []error {
+	var validationErrors []error
+	var podIP net.IP
+	interfaceToVlanTableMap := make(map[string]int)
+	vlanTableToBranchENIMap := make(map[int]string)
+
+	ipFamily := netlink.FAMILY_V4
+	if podNetworkingValidationInput.IPFamily == "IPv6" {
+		ipFamily = netlink.FAMILY_V6
+	}
+
+	// Get the list of IP rules
+	ruleList, err := netlink.RuleList(ipFamily)
+	if err != nil {
+		log.Fatalf("failed to list ip rules %v", err)
+	}
+
+	for _, rule := range ruleList {
+		if strings.HasPrefix(rule.IifName, "vlan.eth") && rule.Table != 0 {
+			vlanTableToBranchENIMap[rule.Table] = rule.IifName
+		} else if rule.IifName != "" && rule.Table != 0 {
+			interfaceToVlanTableMap[rule.IifName] = rule.Table
+		}
+	}
+
+	for _, pod := range podNetworkingValidationInput.PodList {
+		podIP = net.ParseIP(pod.PodIPv4Address)
+		if podNetworkingValidationInput.IPFamily == "IPv6" {
+			podIP = net.ParseIP(pod.PodIPv6Address)
+		}
+
+		log.Printf("testing for Pod name: %s Namespace: %s, IP: %s",
+			pod.PodName, pod.PodNamespace, podIP)
+
+		// Get the veth pair for pod in host network namespace
+		hostVethName := getHostVethPairName(pod, podNetworkingValidationInput.VethPrefix)
+		link, err := netlink.LinkByName(hostVethName)
+		if err != nil {
+			validationErrors = append(validationErrors,
+				fmt.Errorf("failed to find netlink %s: %v", hostVethName, err))
+			continue
+		}
+
+		vlanTable := 0
+		if table, ok := interfaceToVlanTableMap[hostVethName]; !ok {
+			validationErrors = append(validationErrors,
+				fmt.Errorf("Missing Rule for pod: %s, podNamespace: %s, veth: %s", pod.PodName, pod.PodNamespace, hostVethName))
+		} else {
+			vlanTable = table
+		}
+
+		// Check if branch ENI exists for given pod
+		if branchENI, ok := vlanTableToBranchENIMap[vlanTable]; !ok {
+			validationErrors = append(validationErrors,
+				fmt.Errorf("Missing Branch ENI for pod: %s, podNamespace: %s", pod.PodName, pod.PodNamespace))
+		} else {
+			// Check if branchENI is in UP state
+			eniLink, err := netlink.LinkByName(branchENI)
+			if err != nil {
+				validationErrors = append(validationErrors,
+					fmt.Errorf("failed to find netlink %s: %v", branchENI, err))
+			} else {
+				isENILinkUp := strings.Contains(eniLink.Attrs().Flags.String(), "up")
+				if !isENILinkUp {
+					validationErrors = append(validationErrors,
+						fmt.Errorf("branch eni %s is not up %s", branchENI, link.Attrs().Flags.String()))
+				} else {
+					log.Printf("Found Branch ENI: %s for Pod: %s Namespace: %s, IP: %s in UP state", branchENI,
+						pod.PodName, pod.PodNamespace, podIP)
+				}
+			}
+		}
+
+		// Validate MTU value if it is set to true
+		if podNetworkingValidationInput.ValidateMTU {
+			if link.Attrs().MTU != podNetworkingValidationInput.MTU {
+				validationErrors = append(validationErrors,
+					fmt.Errorf("MTU value %v for pod: %s on veth pair: %s failed to match the expected value: %v", link.Attrs().MTU, pod.PodName, hostVethName, podNetworkingValidationInput.MTU))
+			} else {
+				log.Printf("Found Valid MTU value:%d for pod: %s on veth Pair: %s\n", link.Attrs().MTU, pod.PodName, hostVethName)
+			}
+		}
+
+		// Verify IP Link for the Pod is UP
+		isLinkUp := strings.Contains(link.Attrs().Flags.String(), "up")
+		if !isLinkUp {
+			validationErrors = append(validationErrors,
+				fmt.Errorf("veth pair on host side is not up %s", link.Attrs().Flags.String()))
+			continue
+		}
+
+		log.Printf("found veth pair %s in host network namespace in up state with index %d",
+			hostVethName, link.Attrs().Index)
+
+		routes, err := netlink.RouteListFiltered(ipFamily, &netlink.Route{
+			Table: vlanTable,
+		}, netlink.RT_FILTER_TABLE)
+
+		if err != nil {
+			validationErrors = append(validationErrors,
+				fmt.Errorf("Failed to list routes for table %d", vlanTable))
+		}
+
+		if len(routes) != 3 {
+			validationErrors = append(validationErrors,
+				fmt.Errorf("Some Routes missing for vlanTable: %d", vlanTable))
+		} else {
+			log.Printf("Found correct number(3) of routes in vlanTable: %d for podIP: %s", vlanTable, podIP)
+		}
+
+		routeExistsForPodIP := false
+		for _, route := range routes {
+			if route.Dst != nil && route.Dst.IP.String() == podIP.String() {
+				routeExistsForPodIP = true
+				break
+			}
+		}
+
+		if !routeExistsForPodIP {
+			validationErrors = append(validationErrors,
+				fmt.Errorf("Missing Route for PodIP: %s in Table: %d", podIP, vlanTable))
+		}
+	}
+	return validationErrors
+}
+
 // TestNetworkTearedDownForRegularPods test pod networking is correctly teared down by the CNI Plugin
 // The test assumes that the IP assigned to the older Pod is not assigned to a new Pod while this test
 // is being executed
@@ -270,6 +397,88 @@ func TestNetworkTearedDownForRegularPods(podNetworkingValidationInput input.PodN
 	return validationError
 }
 
+// TestNetworkingForPods using security groups is teared down correctly
+func TestNetworkTearedDownForPodsUsingSecurityGroup(podNetworkingValidationInput input.PodNetworkingValidationInput) []error {
+	var validationErrors []error
+	var podIP net.IP
+	interfaceToVlanTableMap := make(map[string]int)
+	vlanTableToBranchENIMap := make(map[int]string)
+
+	ipFamily := netlink.FAMILY_V4
+	if podNetworkingValidationInput.IPFamily == "IPv6" {
+		ipFamily = netlink.FAMILY_V6
+	}
+	// Get the list of IP rules
+	ruleList, err := netlink.RuleList(ipFamily)
+	if err != nil {
+		log.Fatalf("failed to list ip rules %v", err)
+	}
+
+	for _, rule := range ruleList {
+		if strings.HasPrefix(rule.IifName, "vlan.eth") && rule.Table != 0 {
+			vlanTableToBranchENIMap[rule.Table] = rule.IifName
+		} else if rule.IifName != "" && rule.Table != 0 {
+			interfaceToVlanTableMap[rule.IifName] = rule.Table
+		}
+	}
+
+	// Check if branchENI's are cleanup
+	if len(vlanTableToBranchENIMap) != 0 {
+		validationErrors = append(validationErrors,
+			fmt.Errorf("found leaked branch ENI"))
+		for _, eni := range vlanTableToBranchENIMap {
+			log.Printf("Leaked branch ENI: %s", eni)
+		}
+	}
+
+	for _, pod := range podNetworkingValidationInput.PodList {
+		podIP = net.ParseIP(pod.PodIPv4Address)
+		if podNetworkingValidationInput.IPFamily == "IPv6" {
+			podIP = net.ParseIP(pod.PodIPv6Address)
+		}
+
+		log.Printf("testing for Pod name: %s Namespace: %s, IP: %s",
+			pod.PodName, pod.PodNamespace, pod.PodIPv4Address)
+
+		// Make sure the veth pair doesn't exist anymore
+		hostVethName := getHostVethPairName(pod, podNetworkingValidationInput.VethPrefix)
+		link, err := netlink.LinkByName(hostVethName)
+		if err == nil {
+			// Found leaked veth pair
+			validationErrors = append(validationErrors,
+				fmt.Errorf("found an existing veth pair for the pod %s: %v", pod.PodName, link))
+
+			// check if vlanTable rule exists for leaked veth pair
+			if table, ok := interfaceToVlanTableMap[hostVethName]; ok {
+				validationErrors = append(validationErrors,
+					fmt.Errorf("Rule exists for pod: %s, podNamespace: %s, veth: %s with target vlanTable:%d", pod.PodName, pod.PodNamespace, hostVethName, table))
+
+				routes, err := netlink.RouteListFiltered(ipFamily, &netlink.Route{
+					Table: table,
+				}, netlink.RT_FILTER_TABLE)
+
+				if err != nil {
+					validationErrors = append(validationErrors,
+						fmt.Errorf("Failed to list routes for table %d", table))
+				} else {
+					if len(routes) != 0 {
+						validationErrors = append(validationErrors,
+							fmt.Errorf("Some Routes exist in vlanTable: %d for podIP: %s", table, podIP))
+					} else {
+						log.Printf("No Routes routes found in vlanTable: %d for podIP: %s", table, podIP)
+					}
+				}
+			}
+			continue
+		}
+		log.Printf("veth pair %s not found for the pod: %v", hostVethName, err)
+
+		log.Printf("no leaked resource found for the pod %s/%s", pod.PodNamespace, pod.PodName)
+	}
+
+	return validationErrors
+}
+
 func isRuleToOrFromIP(rule netlink.Rule, ip net.IP) bool {
 	if (rule.Src != nil && rule.Src.IP.Equal(ip)) ||
 		(rule.Dst != nil && rule.Dst.IP.Equal(ip)) {