Add pMTU probe checker

+ Restructure netutils code base
+ Move summary of all checks to be printed at the very end
+ Add pmtu discovery for external-ip and dst-ip checks

Caveats:
+ TODO: Check mtu of src iface and compare it to working pmtu as a
valid check.
+ TODO: New tool specifically for pmtu discovery given interface and
destination IP.

README.md

@@ -80,15 +80,17 @@ make k8snetlook-osx
 
 By having to initialize kubernetes client-set, the tool intrinsically performs API connectivity check via K8s-apiserver's VIP/External Loadbalancer in case of highly available k8s-apiserver clusters
 
-| Host Checks                                      | Pod Checks                                       |
-| ------------------------------------------------ | ------------------------------------------------ |
-| Default gateway connectivity (icmp)              | Default gateway connectivity (icmp)              |
-| K8s-apiserver ClusterIP check (https)            | K8s-apiserver ClusterIP check (https)            |
-| K8s-apiserver individual endpoints check (https) | K8s-apiserver individual endpoints check (https) |
-| K8s-apiserver health-check api (livez)           | Destination Pod IP connectivity (icmp)           |
-|                                                  | External IP connectivity (icmp)                  |
-|                                                  | K8s DNS name lookup check (kubernetes.local)     |
-|                                                  | K8s DNS name lookup for specific service check   |
+| Host Checks                                      | Pod Checks                                              |
+| ------------------------------------------------ | ------------------------------------------------------- |
+| Default gateway connectivity (icmp)              | Default gateway connectivity (icmp)                     |
+| K8s-apiserver ClusterIP check (https)            | K8s-apiserver ClusterIP check (https)                   |
+| K8s-apiserver individual endpoints check (https) | K8s-apiserver individual endpoints check (https)        |
+| K8s-apiserver health-check api (livez)           | Destination Pod IP connectivity (icmp)                  |
+|                                                  | External IP connectivity (icmp)                         |
+|                                                  | K8s DNS name lookup check (kubernetes.local)            |
+|                                                  | K8s DNS name lookup for specific service check          |
+|                                                  | Path MTU discovery between Src & Dst Pod (icmp)         |
+|                                                  | Path MTU discovery between Src Pod & External IP (icmp) |
 
 
 ## Contribute

cmd/k8snetlook/main.go

@@ -69,7 +69,11 @@ func main() {
 	k8snetlook.RunHostChecks()
 	if podDebugging == true {
 		k8snetlook.RunPodChecks()
+		fmt.Println("----------- Pod Checks Summary -----------")
+		fmt.Printf("Passed checks: %d/%d\n", k8snetlook.PassingPodChecks, k8snetlook.TotalPodChecks)
 	}
+	fmt.Println("----------- Host Checks Summary -----------")
+	fmt.Printf("Passed checks: %d/%d\n", k8snetlook.PassingHostChecks, k8snetlook.TotalHostChecks)
 }
 
 func validateArgs() {

go.mod

@@ -5,6 +5,7 @@ go 1.14
 require (
 	github.com/gogo/protobuf v1.3.1 // indirect
 	github.com/google/gofuzz v1.1.0 // indirect
+	github.com/google/gopacket v1.1.18
 	github.com/googleapis/gnostic v0.4.0 // indirect
 	github.com/imdario/mergo v0.3.10 // indirect
 	github.com/miekg/dns v1.1.31

go.sum

@@ -50,6 +50,8 @@ github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSN
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g=
 github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/gopacket v1.1.18 h1:lum7VRA9kdlvBi7/v2p7/zcbkduHaCH/SVVyurs7OpY=
+github.com/google/gopacket v1.1.18/go.mod h1:UdDNZ1OO62aGYVnPhxT1U6aI7ukYtA/kB8vaU0diBUM=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
@@ -153,6 +155,7 @@ golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190405154228-4b34438f7a67/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

k8snetlook/checkers.go

@@ -2,16 +2,19 @@ package k8snetlook
 
 import (
 	"fmt"
+	"net"
 	"net/http"
+
+	"github.com/sarun87/k8snetlook/netutils"
 )
 
 func RunGatewayConnectivityCheck(checkCounter *int) {
-	pass, err := sendRecvICMPMessage(Cfg.HostGatewayIP)
+	pass, err := netutils.SendRecvICMPMessage(Cfg.HostGatewayIP, 64, true)
 	if err != nil {
 		fmt.Printf("  (Failed) Error running RunGatewayConnectivityCheck. Error: %v\n", err)
 		return
 	}
-	if pass {
+	if pass == 0 {
 		*checkCounter++
 		fmt.Println("  (Passed) Gateway connectivity check completed successfully")
 	} else {
@@ -20,12 +23,12 @@ func RunGatewayConnectivityCheck(checkCounter *int) {
 }
 
 func RunDstConnectivityCheck(dstIP string, checkCounter *int) {
-	pass, err := sendRecvICMPMessage(dstIP)
+	pass, err := netutils.SendRecvICMPMessage(dstIP, 64, true)
 	if err != nil {
 		fmt.Printf("  (Failed) Error running connectivity check to %s. Error: %v\n", dstIP, err)
 		return
 	}
-	if pass {
+	if pass == 0 {
 		*checkCounter++
 		fmt.Printf("  (Passed) Connectivity check to destination %s completed successfully\n", dstIP)
 	} else {
@@ -38,7 +41,7 @@ func RunKubeAPIServiceIPConnectivityCheck(checkCounter *int) {
 	// HTTP 401 return code is a successful check
 	url := fmt.Sprintf("https://%s:%d", Cfg.KubeAPIService.IP, Cfg.KubeAPIService.Port)
 	var body []byte
-	responseCode, err := sendRecvHTTPMessage(url, "", &body)
+	responseCode, err := netutils.SendRecvHTTPMessage(url, "", &body)
 	if err != nil {
 		fmt.Printf("  (Failed) Error running RunKubeAPIServiceIPConnectivityCheck. Error: %v\n", err)
 		return
@@ -61,7 +64,7 @@ func RunKubeAPIEndpointIPConnectivityCheck(checkCounter *int) {
 		url := fmt.Sprintf("https://%s:%d", ep.IP, ep.Port)
 		fmt.Printf("  checking endpoint: %s ........", url)
 		var body []byte
-		responseCode, err := sendRecvHTTPMessage(url, "", &body)
+		responseCode, err := netutils.SendRecvHTTPMessage(url, "", &body)
 		if err != nil {
 			fmt.Printf("    failed connectivity check. Error: %v\n", err)
 			continue
@@ -89,7 +92,7 @@ func RunAPIServerHealthCheck(checkCounter *int) {
 		return
 	}
 	var body []byte
-	responseCode, err := sendRecvHTTPMessage(url, svcAccountToken, &body)
+	responseCode, err := netutils.SendRecvHTTPMessage(url, svcAccountToken, &body)
 	if err != nil {
 		fmt.Printf("    Unable to fetch api server check. Error: %v\n", err)
 		return
@@ -107,7 +110,7 @@ func RunK8sDNSLookupCheck(dnsServerIP, dstSvcName, dstSvcNamespace, dstSvcExpect
 	dnsServerURL := fmt.Sprintf("%s:53", dnsServerIP)
 	// TODO: Fetch domain information from cluster
 	svcfqdn := fmt.Sprintf("%s.%s.svc.cluster.local.", dstSvcName, dstSvcNamespace)
-	ips, err := runDNSLookupUsingCustomResolver(dnsServerURL, svcfqdn)
+	ips, err := netutils.RunDNSLookupUsingCustomResolver(dnsServerURL, svcfqdn)
 	if err != nil {
 		fmt.Printf("  (Failed) Unable to run dns lookup to %s, error: %v\n", svcfqdn, err)
 		return
@@ -123,3 +126,29 @@ func RunK8sDNSLookupCheck(dnsServerIP, dstSvcName, dstSvcNamespace, dstSvcExpect
 	fmt.Printf("  (Failed) Lookup of %s retured: %v, expected: %s\n", svcfqdn, ips, dstSvcExpectedIP)
 	return
 }
+
+func RunMTUProbeToDstIPCheck(dstIP string, checkCounter *int) {
+	supportedMTU, err := netutils.PMTUProbeToDestIP(dstIP)
+	if err != nil {
+		fmt.Printf("   (Failed) Unable to run pmtud for %s. Error: %v\n", dstIP, err)
+		return
+	}
+	fmt.Printf("   Maximum MTU that works for destination IP: %s is %d\n", dstIP, supportedMTU)
+	ifaces, err := net.Interfaces()
+	if err != nil {
+		fmt.Printf("   Unable to fetch network interfaces. Error: %v\n", err)
+		return
+	}
+	for _, iface := range ifaces {
+		// If loopback device, skip
+		if iface.Flags&net.FlagLoopback == net.FlagLoopback {
+			continue
+		}
+		if iface.MTU > supportedMTU {
+			fmt.Printf("  Iface %s has higher mtu than supported path mtu. Has: %d, should be less than %d\n", iface.Name, iface.MTU, supportedMTU)
+		}
+	}
+	*checkCounter++
+	// TODO: Check for the outgoing interface mtu and compare
+	fmt.Printf("   (Passed) MTU looks good.. Retured MTU: %d\n", supportedMTU)
+}

k8snetlook/host.go

@@ -5,25 +5,24 @@ import (
 )
 
 const (
-	totalHostChecks = 4
+	TotalHostChecks = 4
 )
 
 var (
-	passingHostChecks int
+	PassingHostChecks int
 )
 
 func RunHostChecks() {
 	fmt.Println("----------- Host Checks -----------")
 
 	fmt.Println("----> [From Host] Running default gateway connectivity check..")
-	RunGatewayConnectivityCheck(&passingHostChecks)
+	RunGatewayConnectivityCheck(&PassingHostChecks)
 	fmt.Println("----> [From Host] Running Kube service IP connectivity check..")
-	RunKubeAPIServiceIPConnectivityCheck(&passingHostChecks)
+	RunKubeAPIServiceIPConnectivityCheck(&PassingHostChecks)
 	fmt.Println("----> [From Host] Running Kube API Server Endpoint IP connectivity check..")
-	RunKubeAPIEndpointIPConnectivityCheck(&passingHostChecks)
+	RunKubeAPIEndpointIPConnectivityCheck(&PassingHostChecks)
 	fmt.Println("----> [From Host] Running Kube API Server health check..")
-	RunAPIServerHealthCheck(&passingHostChecks)
+	RunAPIServerHealthCheck(&PassingHostChecks)
 
-	fmt.Println("----------- Host Checks Summary -----------")
-	fmt.Printf("Passed checks: %d/%d\n", passingHostChecks, totalHostChecks)
+	fmt.Println("-----------------------------------")
 }

k8snetlook/init.go

@@ -5,6 +5,7 @@ import (
 	"os"
 	"strings"
 
+	"github.com/sarun87/k8snetlook/netutils"
 	"github.com/vishvananda/netns"
 )
 
@@ -56,7 +57,7 @@ func InitKubeClient(kubeconfigPath string) {
 func InitK8sInfo() {
 	Cfg.KubeAPIService = getServiceClusterIP("default", "kubernetes")
 	Cfg.KubeDNSService = getServiceClusterIP("kube-system", "kube-dns")
-	Cfg.HostGatewayIP = getHostGatewayIP()
+	Cfg.HostGatewayIP = netutils.GetHostGatewayIP()
 	Cfg.SrcPod.NsHandle = netns.NsHandle(-1)
 	if Cfg.SrcPod.Name != "" && Cfg.SrcPod.Namespace != "" {
 		Cfg.SrcPod.IP = getPodIPFromName(Cfg.SrcPod.Namespace, Cfg.SrcPod.Name)