Add DNS check from inside Pod net ns

README.md

@@ -36,6 +36,29 @@ k8snetlook pod -config /etc/kubernetes/admin.yaml -srcpodname nginx-sdvyx -srcpo
 * The binary is run on the host where the Pod with connectivity issues are present
 * If the tool isn't able to initialize k8s client using specified kubeconfig, the tool will fail (FUTURE? run other tests that don't need k8s information)
 
+## Download binary & run
+64-bit linux binary is available for download from the [Releases](https://github.com/sarun87/k8snetlook/releases) page.
+
+* download binary to a host
+```
+wget https://github.com/sarun87/k8snetlook/releases/download/v0.1/k8snetlook
+```
+* Make the downloaded file executable
+```
+chmod u+x k8snetlook
+```
+* Run tool using sudo or as root
+```
+./k8snetlook
+'host' or 'pod' subcommand expected
+
+usage: k8snetlook subcommand [sub-command-options] [-config path-to-kube-config]
+
+valid subcommands
+  pod       Debug Pod & host networking
+  host      Debug host networking only
+```
+
 ## How to build from source
 To build tool from source, run `make` as follows:
 ```
@@ -57,13 +80,16 @@ make k8snetlook-osx
 
 By having to initialize kubernetes client-set, the tool intrinsically performs API connectivity check via K8s-apiserver's VIP/External Loadbalancer in case of highly available k8s-apiserver clusters
 
-| Host Checks                                      | Pod Checks                                         |
-| ------------------------------------------------ | -------------------------------------------------- |
-| Default gateway connectivity (icmp)              | Default gateway connectivity (icmp)                |
-| K8s-apiserver ClusterIP check (https)            | K8s-apiserver ClusterIP check (https)              |
-| K8s-apiserver individual endpoints check (https) | K8s-apiserver individual endpoints check (https)   |
-| K8s-apiserver health-check api (livez)           | Destination Pod IP connectivity (icmp)             |
-|                                                  | External IP connectivity (icmp)                    |
+| Host Checks                                      | Pod Checks                                       |
+| ------------------------------------------------ | ------------------------------------------------ |
+| Default gateway connectivity (icmp)              | Default gateway connectivity (icmp)              |
+| K8s-apiserver ClusterIP check (https)            | K8s-apiserver ClusterIP check (https)            |
+| K8s-apiserver individual endpoints check (https) | K8s-apiserver individual endpoints check (https) |
+| K8s-apiserver health-check api (livez)           | Destination Pod IP connectivity (icmp)           |
+|                                                  | External IP connectivity (icmp)                  |
+|                                                  | K8s DNS name lookup check (kubernetes.local)     |
+|                                                  | K8s DNS name lookup for specific service check   |
+
 
 ## Contribute
 PRs welcome :)

go.mod

@@ -7,6 +7,7 @@ require (
 	github.com/google/gofuzz v1.1.0 // indirect
 	github.com/googleapis/gnostic v0.4.0 // indirect
 	github.com/imdario/mergo v0.3.10 // indirect
+	github.com/miekg/dns v1.1.31
 	github.com/stretchr/testify v1.6.1 // indirect
 	github.com/vishvananda/netlink v1.1.0
 	github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df

go.sum

@@ -77,6 +77,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/miekg/dns v1.1.31 h1:sJFOl9BgwbYAWOGEwr61FU28pqsBNdpRBnhGXtO06Oo=
+github.com/miekg/dns v1.1.31/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -113,13 +115,15 @@ go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899 h1:DZhuSZLsGlFL4CmhA8BcRA0mnthyA/nZ00AqCUo7vHg=
 golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
 golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -128,6 +132,8 @@ golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73r
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU=
 golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
@@ -140,6 +146,8 @@ golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -148,6 +156,7 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -165,6 +174,8 @@ golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGm
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508=

k8snetlook/checkers.go

@@ -102,3 +102,24 @@ func RunAPIServerHealthCheck(checkCounter *int) {
 		*checkCounter++
 	}
 }
+
+func RunK8sDNSLookupCheck(dnsServerIP, dstSvcName, dstSvcNamespace, dstSvcExpectedIP string, checkCounter *int) {
+	dnsServerURL := fmt.Sprintf("%s:53", dnsServerIP)
+	// TODO: Fetch domain information from cluster
+	svcfqdn := fmt.Sprintf("%s.%s.svc.cluster.local.", dstSvcName, dstSvcNamespace)
+	ips, err := runDNSLookupUsingCustomResolver(dnsServerURL, svcfqdn)
+	if err != nil {
+		fmt.Printf("  (Failed) Unable to run dns lookup to %s, error: %v\n", svcfqdn, err)
+		return
+	}
+	// Check if the resolved IP matches with the IP reported by K8s
+	for _, ip := range ips {
+		if ip == dstSvcExpectedIP {
+			*checkCounter++
+			fmt.Printf("  (Passed) dns lookup to %s returned: %v. Expected: %s\n", svcfqdn, ips, ip)
+			return
+		}
+	}
+	fmt.Printf("  (Failed) Lookup of %s retured: %v, expected: %s\n", svcfqdn, ips, dstSvcExpectedIP)
+	return
+}

k8snetlook/init.go

@@ -58,15 +58,15 @@ func InitK8sInfo() {
 	Cfg.KubeDNSService = getServiceClusterIP("kube-system", "kube-dns")
 	Cfg.HostGatewayIP = getHostGatewayIP()
 	Cfg.SrcPod.NsHandle = netns.NsHandle(-1)
-	if Cfg.SrcPod.Name != "" {
+	if Cfg.SrcPod.Name != "" && Cfg.SrcPod.Namespace != "" {
 		Cfg.SrcPod.IP = getPodIPFromName(Cfg.SrcPod.Namespace, Cfg.SrcPod.Name)
 		Cfg.SrcPod.NsHandle = getPodNetnsHandle(Cfg.SrcPod.Namespace, Cfg.SrcPod.Name)
 	}
 	Cfg.DstPod.NsHandle = netns.NsHandle(-1)
-	if Cfg.DstPod.Name != "" {
+	if Cfg.DstPod.Name != "" && Cfg.DstPod.Namespace != "" {
 		Cfg.DstPod.IP = getPodIPFromName(Cfg.DstPod.Namespace, Cfg.DstPod.Name)
 	}
-	if Cfg.DstSvc.Name != "" {
+	if Cfg.DstSvc.Name != "" && Cfg.DstSvc.Namespace != "" {
 		Cfg.DstSvc.SvcEndpoint = getServiceClusterIP(Cfg.DstSvc.Namespace, Cfg.DstSvc.Name)
 	}
 }

k8snetlook/netutils.go

@@ -2,17 +2,20 @@ package k8snetlook
 
 import (
 	"crypto/tls"
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"net"
 	"net/http"
 	"os"
 	"time"
 
-	"github.com/vishvananda/netlink"
 	"golang.org/x/net/icmp"
 	"golang.org/x/net/ipv4"
 	"golang.org/x/sys/unix"
+
+	"github.com/miekg/dns"
+	"github.com/vishvananda/netlink"
 )
 
 const (
@@ -104,3 +107,67 @@ func sendRecvHTTPMessage(url string, token string, body *[]byte) (int, error) {
 	res.Body.Close()
 	return res.StatusCode, nil
 }
+
+// Does not work, not sure why :(
+/*func runDNSLookupUsingCustomResolver(dnsFQDN) ([]net.IPAddr, error) {
+	// Create a custom resolver since /etc/resolv.conf is the host's configuration
+	// and not the pods config. This is because the below code is executed within
+	// the Pod netns but not the file system of the pod.
+	r := &net.Resolver{
+		PreferGo: true,
+		Dial: func(ctx context.Context, network, address string) (net.Conn, error) {
+			d := net.Dialer{
+				Timeout: time.Millisecond * time.Duration(10000),
+			}
+			//return net.Dial(network, net.JoinHostPort(dnsServerIP, "53"))
+			return d.DialContext(ctx, network, net.JoinHostPort(dnsServerIP, "53"))
+		},
+	}
+	// Lookup for svcname.svcnamespace eg: kubernetes.default
+	dnsFQDN := fmt.Sprintf("%s.%s", dstSvcName, dstSvcNamespace)
+	ips, err := r.LookupIPAddr(context.Background(), dnsFQDN)
+	if err != nil {
+		fmt.Printf("  (Failed) dns lookup to %s failed. Error: %v\n", dnsFQDN, err)
+		return nil, err
+	}
+	return ips, nil
+}
+*/
+
+// run dns lookup using github.com/miekg/dns
+// code referenced from: https://github.com/bogdanovich/dns_resolver
+// nameserver string format: "ip:port"
+// hostFQDN string format: "abc.def.ghi."
+func runDNSLookupUsingCustomResolver(nameserver, hostFQDN string) ([]string, error) {
+	// TODO: Add retries
+
+	// Create DNS Message with single question
+	msg := &dns.Msg{
+		MsgHdr: dns.MsgHdr{
+			Id:               dns.Id(),
+			RecursionDesired: true,
+		},
+		Question: []dns.Question{{Name: dns.Fqdn(hostFQDN), Qtype: dns.TypeA, Qclass: dns.ClassINET}},
+	}
+
+	// Send question to nameserver and wait for answer
+	in, err := dns.Exchange(msg, nameserver)
+	if err != nil {
+		return nil, err
+	}
+
+	result := []string{}
+
+	if in != nil && in.Rcode != dns.RcodeSuccess {
+		// Return error code
+		return result, errors.New(dns.RcodeToString[in.Rcode])
+	}
+
+	// Fetch IP Addresses in DNS Answer and return
+	for _, record := range in.Answer {
+		if t, ok := record.(*dns.A); ok {
+			result = append(result, t.A.String())
+		}
+	}
+	return result, err
+}

k8snetlook/pod.go

@@ -19,7 +19,7 @@ func RunPodChecks() {
 
 	fmt.Println("----------- Pod Checks -----------")
 
-	totalPodChecks = 3
+	totalPodChecks = 4
 	// Lock OS thread to prevent ns change
 	runtime.LockOSThread()
 	defer runtime.UnlockOSThread()
@@ -45,6 +45,9 @@ func RunPodChecks() {
 	RunKubeAPIEndpointIPConnectivityCheck(&passingPodChecks)
 	fmt.Println("----> [From SrcPod] Running default gateway connectivity check..")
 	RunGatewayConnectivityCheck(&passingPodChecks)
+	fmt.Println("----> [From SrcPod] Running DNS lookup test (kubernetes.default)..")
+	RunK8sDNSLookupCheck(Cfg.KubeDNSService.IP, "kubernetes", "default",
+		Cfg.KubeAPIService.IP, &passingPodChecks)
 
 	if Cfg.DstPod.IP != "" {
 		totalPodChecks++
@@ -58,6 +61,13 @@ func RunPodChecks() {
 		RunDstConnectivityCheck(Cfg.ExternalIP, &passingPodChecks)
 	}
 
+	if Cfg.DstSvc.SvcEndpoint.IP != "" {
+		totalPodChecks++
+		fmt.Println("----> [From SrcPod] Running DstSvc DNS lookup check..")
+		RunK8sDNSLookupCheck(Cfg.KubeDNSService.IP, Cfg.DstSvc.Name, Cfg.DstSvc.Namespace,
+			Cfg.DstSvc.SvcEndpoint.IP, &passingPodChecks)
+	}
+
 	// Change network ns back to host
 	netns.Set(hostNsHandle)