From 4352a0d9f4b24cfd8aadbd8335aa2f36ca9cf3cf Mon Sep 17 00:00:00 2001 From: Chinmay Gadgil Date: Mon, 1 Nov 2021 13:33:07 -0700 Subject: [PATCH 1/2] Add HostNetworking Test for PPSG in test agent --- test/agent/cmd/networking/main.go | 17 +- test/agent/cmd/networking/tester/network.go | 172 ++++++++++++++++++++ 2 files changed, 188 insertions(+), 1 deletion(-) diff --git a/test/agent/cmd/networking/main.go b/test/agent/cmd/networking/main.go index d95c3ab833..5a002ea27a 100644 --- a/test/agent/cmd/networking/main.go +++ b/test/agent/cmd/networking/main.go @@ -29,10 +29,13 @@ func main() { var podNetworkingValidationInputString string var shouldTestSetup bool var shouldTestCleanup bool + // per pod security group + var ppsg bool flag.StringVar(&podNetworkingValidationInputString, "pod-networking-validation-input", "", "json string containing the array of pods whose networking needs to be validated") flag.BoolVar(&shouldTestCleanup, "test-cleanup", false, "bool flag when set to true tests that networking is teared down after pod has been deleted") flag.BoolVar(&shouldTestSetup, "test-setup", false, "bool flag when set to true tests the networking is setup correctly after pod is running") + flag.BoolVar(&ppsg, "test-ppsg", false, "bool flag when set to true tests the networking setup for pods using security groups") flag.Parse() @@ -47,7 +50,19 @@ func main() { log.Printf("list of pod against which test will be run %v", podNetworkingValidationInput.PodList) - if shouldTestSetup { + if ppsg && shouldTestSetup { + log.Print("testing networking is setup for pods using security groups") + err := tester.TestNetworkingSetupForPodsUsingSecurityGroup(podNetworkingValidationInput) + if err != nil { + log.Fatalf("found 1 or more pod setup validation failure: %v", err) + } + } else if ppsg && shouldTestCleanup { + log.Print("testing networking is teared down for pods using security groups") + err := tester.TestNetworkTearedDownForPodsUsingSecurityGroup(podNetworkingValidationInput) + if err != nil { + log.Fatalf("found 1 or more pod setup validation failure: %v", err) + } + } else if !ppsg && shouldTestSetup { log.Print("testing networking is setup for regular pods") err := tester.TestNetworkingSetupForRegularPod(podNetworkingValidationInput) if err != nil { diff --git a/test/agent/cmd/networking/tester/network.go b/test/agent/cmd/networking/tester/network.go index 685a98184e..d3a8bca59d 100644 --- a/test/agent/cmd/networking/tester/network.go +++ b/test/agent/cmd/networking/tester/network.go @@ -185,6 +185,108 @@ func TestNetworkingSetupForRegularPod(podNetworkingValidationInput input.PodNetw return validationErrors } +// TestNetworkingSetupForPods using security groups +func TestNetworkingSetupForPodsUsingSecurityGroup(podNetworkingValidationInput input.PodNetworkingValidationInput) []error { + var validationErrors []error + var podIP net.IP + interfaceToVlanTableMap := make(map[string]int) + + ipFamily := netlink.FAMILY_V4 + if podNetworkingValidationInput.IPFamily == "IPv6" { + ipFamily = netlink.FAMILY_V6 + } + + // Get the list of IP rules + ruleList, err := netlink.RuleList(ipFamily) + if err != nil { + log.Fatalf("failed to list ip rules %v", err) + } + + for _, rule := range ruleList { + if rule.IifName != "" && rule.Table != 0 { + interfaceToVlanTableMap[rule.IifName] = rule.Table + } + } + + for _, pod := range podNetworkingValidationInput.PodList { + podIP = net.ParseIP(pod.PodIPv4Address) + if podNetworkingValidationInput.IPFamily == "IPv6" { + podIP = net.ParseIP(pod.PodIPv6Address) + } + + log.Printf("testing for Pod name: %s Namespace: %s, IP: %s", + pod.PodName, pod.PodNamespace, podIP) + + // Get the veth pair for pod in host network namespace + hostVethName := getHostVethPairName(pod, podNetworkingValidationInput.VethPrefix) + link, err := netlink.LinkByName(hostVethName) + if err != nil { + validationErrors = append(validationErrors, + fmt.Errorf("failed to find netlink %s: %v", hostVethName, err)) + continue + } + + vlanTable := 0 + if table, ok := interfaceToVlanTableMap[hostVethName]; !ok { + validationErrors = append(validationErrors, + fmt.Errorf("Missing Rule for pod: %s, podNamespace: %s, veth: %s", pod.PodName, pod.PodNamespace, hostVethName)) + } else { + vlanTable = table + } + + // Validate MTU value if it is set to true + if podNetworkingValidationInput.ValidateMTU { + if link.Attrs().MTU != podNetworkingValidationInput.MTU { + validationErrors = append(validationErrors, + fmt.Errorf("MTU value %v for pod: %s on veth pair: %s failed to match the expected value: %v", link.Attrs().MTU, pod.PodName, hostVethName, podNetworkingValidationInput.MTU)) + } else { + log.Printf("Found Valid MTU value:%d for pod: %s on veth Pair: %s\n", link.Attrs().MTU, pod.PodName, hostVethName) + } + } + + // Verify IP Link for the Pod is UP + isLinkUp := strings.Contains(link.Attrs().Flags.String(), "up") + if !isLinkUp { + validationErrors = append(validationErrors, + fmt.Errorf("veth pair on host side is not up %s", link.Attrs().Flags.String())) + continue + } + + log.Printf("found veth pair %s in host network namespace in up state with index %d", + hostVethName, link.Attrs().Index) + + routes, err := netlink.RouteListFiltered(ipFamily, &netlink.Route{ + Table: vlanTable, + }, netlink.RT_FILTER_TABLE) + + if err != nil { + validationErrors = append(validationErrors, + fmt.Errorf("Failed to list routes for table %d", vlanTable)) + } + + if len(routes) != 3 { + validationErrors = append(validationErrors, + fmt.Errorf("Some Routes missing for vlanTable: %d", vlanTable)) + } else { + log.Printf("Found correct number(3) of routes in vlanTable: %d for podIP: %s", vlanTable, podIP) + } + + routeExistsForPodIP := false + for _, route := range routes { + if route.Dst != nil && route.Dst.IP.String() == podIP.String() { + routeExistsForPodIP = true + break + } + } + + if !routeExistsForPodIP { + validationErrors = append(validationErrors, + fmt.Errorf("Missing Route for PodIP: %s in Table: %d", podIP, vlanTable)) + } + } + return validationErrors +} + // TestNetworkTearedDownForRegularPods test pod networking is correctly teared down by the CNI Plugin // The test assumes that the IP assigned to the older Pod is not assigned to a new Pod while this test // is being executed @@ -270,6 +372,76 @@ func TestNetworkTearedDownForRegularPods(podNetworkingValidationInput input.PodN return validationError } +// TestNetworkingForPods using security groups is teared down correctly +func TestNetworkTearedDownForPodsUsingSecurityGroup(podNetworkingValidationInput input.PodNetworkingValidationInput) []error { + var validationErrors []error + var podIP net.IP + interfaceToVlanTableMap := make(map[string]int) + + ipFamily := netlink.FAMILY_V4 + if podNetworkingValidationInput.IPFamily == "IPv6" { + ipFamily = netlink.FAMILY_V6 + } + // Get the list of IP rules + ruleList, err := netlink.RuleList(ipFamily) + if err != nil { + log.Fatalf("failed to list ip rules %v", err) + } + + for _, rule := range ruleList { + if rule.IifName != "" && rule.Table != 0 { + interfaceToVlanTableMap[rule.IifName] = rule.Table + } + } + + for _, pod := range podNetworkingValidationInput.PodList { + podIP = net.ParseIP(pod.PodIPv4Address) + if podNetworkingValidationInput.IPFamily == "IPv6" { + podIP = net.ParseIP(pod.PodIPv6Address) + } + + log.Printf("testing for Pod name: %s Namespace: %s, IP: %s", + pod.PodName, pod.PodNamespace, pod.PodIPv4Address) + + // Make sure the veth pair doesn't exist anymore + hostVethName := getHostVethPairName(pod, podNetworkingValidationInput.VethPrefix) + link, err := netlink.LinkByName(hostVethName) + if err == nil { + // Found leaked veth pair + validationErrors = append(validationErrors, + fmt.Errorf("found an existing veth pair for the pod %s: %v", pod.PodName, link)) + + // check if vlanTable rule exists for leaked veth pair + if table, ok := interfaceToVlanTableMap[hostVethName]; ok { + validationErrors = append(validationErrors, + fmt.Errorf("Rule exists for pod: %s, podNamespace: %s, veth: %s with target vlanTable:%d", pod.PodName, pod.PodNamespace, hostVethName, table)) + + routes, err := netlink.RouteListFiltered(ipFamily, &netlink.Route{ + Table: table, + }, netlink.RT_FILTER_TABLE) + + if err != nil { + validationErrors = append(validationErrors, + fmt.Errorf("Failed to list routes for table %d", table)) + } else { + if len(routes) != 0 { + validationErrors = append(validationErrors, + fmt.Errorf("Some Routes exist in vlanTable: %d for podIP: %s", table, podIP)) + } else { + log.Printf("No Routes routes found in vlanTable: %d for podIP: %s", table, podIP) + } + } + } + continue + } + log.Printf("veth pair %s not found for the pod: %v", hostVethName, err) + + log.Printf("no leaked resource found for the pod %s/%s", pod.PodNamespace, pod.PodName) + } + + return validationErrors +} + func isRuleToOrFromIP(rule netlink.Rule, ip net.IP) bool { if (rule.Src != nil && rule.Src.IP.Equal(ip)) || (rule.Dst != nil && rule.Dst.IP.Equal(ip)) { From 599ef82221014b745fdcc965ad22bced94209bcf Mon Sep 17 00:00:00 2001 From: Chinmay Gadgil Date: Tue, 9 Nov 2021 06:59:09 -0800 Subject: [PATCH 2/2] Updated PPSG test to validate vlan.eth link --- test/agent/cmd/networking/tester/network.go | 41 ++++++++++++++++++++- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/test/agent/cmd/networking/tester/network.go b/test/agent/cmd/networking/tester/network.go index d3a8bca59d..b2c8c62559 100644 --- a/test/agent/cmd/networking/tester/network.go +++ b/test/agent/cmd/networking/tester/network.go @@ -190,6 +190,7 @@ func TestNetworkingSetupForPodsUsingSecurityGroup(podNetworkingValidationInput i var validationErrors []error var podIP net.IP interfaceToVlanTableMap := make(map[string]int) + vlanTableToBranchENIMap := make(map[int]string) ipFamily := netlink.FAMILY_V4 if podNetworkingValidationInput.IPFamily == "IPv6" { @@ -203,7 +204,9 @@ func TestNetworkingSetupForPodsUsingSecurityGroup(podNetworkingValidationInput i } for _, rule := range ruleList { - if rule.IifName != "" && rule.Table != 0 { + if strings.HasPrefix(rule.IifName, "vlan.eth") && rule.Table != 0 { + vlanTableToBranchENIMap[rule.Table] = rule.IifName + } else if rule.IifName != "" && rule.Table != 0 { interfaceToVlanTableMap[rule.IifName] = rule.Table } } @@ -234,6 +237,28 @@ func TestNetworkingSetupForPodsUsingSecurityGroup(podNetworkingValidationInput i vlanTable = table } + // Check if branch ENI exists for given pod + if branchENI, ok := vlanTableToBranchENIMap[vlanTable]; !ok { + validationErrors = append(validationErrors, + fmt.Errorf("Missing Branch ENI for pod: %s, podNamespace: %s", pod.PodName, pod.PodNamespace)) + } else { + // Check if branchENI is in UP state + eniLink, err := netlink.LinkByName(branchENI) + if err != nil { + validationErrors = append(validationErrors, + fmt.Errorf("failed to find netlink %s: %v", branchENI, err)) + } else { + isENILinkUp := strings.Contains(eniLink.Attrs().Flags.String(), "up") + if !isENILinkUp { + validationErrors = append(validationErrors, + fmt.Errorf("branch eni %s is not up %s", branchENI, link.Attrs().Flags.String())) + } else { + log.Printf("Found Branch ENI: %s for Pod: %s Namespace: %s, IP: %s in UP state", branchENI, + pod.PodName, pod.PodNamespace, podIP) + } + } + } + // Validate MTU value if it is set to true if podNetworkingValidationInput.ValidateMTU { if link.Attrs().MTU != podNetworkingValidationInput.MTU { @@ -377,6 +402,7 @@ func TestNetworkTearedDownForPodsUsingSecurityGroup(podNetworkingValidationInput var validationErrors []error var podIP net.IP interfaceToVlanTableMap := make(map[string]int) + vlanTableToBranchENIMap := make(map[int]string) ipFamily := netlink.FAMILY_V4 if podNetworkingValidationInput.IPFamily == "IPv6" { @@ -389,11 +415,22 @@ func TestNetworkTearedDownForPodsUsingSecurityGroup(podNetworkingValidationInput } for _, rule := range ruleList { - if rule.IifName != "" && rule.Table != 0 { + if strings.HasPrefix(rule.IifName, "vlan.eth") && rule.Table != 0 { + vlanTableToBranchENIMap[rule.Table] = rule.IifName + } else if rule.IifName != "" && rule.Table != 0 { interfaceToVlanTableMap[rule.IifName] = rule.Table } } + // Check if branchENI's are cleanup + if len(vlanTableToBranchENIMap) != 0 { + validationErrors = append(validationErrors, + fmt.Errorf("found leaked branch ENI")) + for _, eni := range vlanTableToBranchENIMap { + log.Printf("Leaked branch ENI: %s", eni) + } + } + for _, pod := range podNetworkingValidationInput.PodList { podIP = net.ParseIP(pod.PodIPv4Address) if podNetworkingValidationInput.IPFamily == "IPv6" {