From 96a721ea2e6c0b38cb1e9f70abc3fb5ca598252a Mon Sep 17 00:00:00 2001
From: phillip-stephens <phillip@cs.stanford.edu>
Date: Tue, 26 Nov 2024 18:06:31 -0500
Subject: [PATCH 01/28] add domain name to root servers default

---
 examples/all_nameservers_lookup/main.go |  1 +
 src/zdns/conf.go                        | 52 ++++++++++++-------------
 2 files changed, 27 insertions(+), 26 deletions(-)
 create mode 100644 examples/all_nameservers_lookup/main.go

diff --git a/examples/all_nameservers_lookup/main.go b/examples/all_nameservers_lookup/main.go
new file mode 100644
index 00000000..798742c3
--- /dev/null
+++ b/examples/all_nameservers_lookup/main.go
@@ -0,0 +1 @@
+package all_nameservers_lookup
diff --git a/src/zdns/conf.go b/src/zdns/conf.go
index 5dff5c7e..d0d0f09c 100644
--- a/src/zdns/conf.go
+++ b/src/zdns/conf.go
@@ -52,35 +52,35 @@ const (
 )
 
 var RootServersV4 = []NameServer{
-	{IP: net.ParseIP("198.41.0.4"), Port: 53},     // A
-	{IP: net.ParseIP("170.247.170.2"), Port: 53},  // B - Changed several times, this is current as of July '24
-	{IP: net.ParseIP("192.33.4.12"), Port: 53},    // C
-	{IP: net.ParseIP("199.7.91.13"), Port: 53},    // D
-	{IP: net.ParseIP("192.203.230.10"), Port: 53}, // E
-	{IP: net.ParseIP("192.5.5.241"), Port: 53},    // F
-	{IP: net.ParseIP("192.112.36.4"), Port: 53},   // G
-	{IP: net.ParseIP("198.97.190.53"), Port: 53},  // H
-	{IP: net.ParseIP("192.36.148.17"), Port: 53},  // I
-	{IP: net.ParseIP("192.58.128.30"), Port: 53},  // J
-	{IP: net.ParseIP("193.0.14.129"), Port: 53},   // K
-	{IP: net.ParseIP("199.7.83.42"), Port: 53},    // L
-	{IP: net.ParseIP("202.12.27.33"), Port: 53},   // M
+	{IP: net.ParseIP("198.41.0.4"), Port: 53, DomainName: "a.root-servers.net"},     // A
+	{IP: net.ParseIP("170.247.170.2"), Port: 53, DomainName: "b.root-servers.net"},  // B - Changed several times, this is current as of July '24
+	{IP: net.ParseIP("192.33.4.12"), Port: 53, DomainName: "c.root-servers.net"},    // C
+	{IP: net.ParseIP("199.7.91.13"), Port: 53, DomainName: "d.root-servers.net"},    // D
+	{IP: net.ParseIP("192.203.230.10"), Port: 53, DomainName: "e.root-servers.net"}, // E
+	{IP: net.ParseIP("192.5.5.241"), Port: 53, DomainName: "f.root-servers.net"},    // F
+	{IP: net.ParseIP("192.112.36.4"), Port: 53, DomainName: "g.root-servers.net"},   // G
+	{IP: net.ParseIP("198.97.190.53"), Port: 53, DomainName: "h.root-servers.net"},  // H
+	{IP: net.ParseIP("192.36.148.17"), Port: 53, DomainName: "i.root-servers.net"},  // I
+	{IP: net.ParseIP("192.58.128.30"), Port: 53, DomainName: "j.root-servers.net"},  // J
+	{IP: net.ParseIP("193.0.14.129"), Port: 53, DomainName: "k.root-servers.net"},   // K
+	{IP: net.ParseIP("199.7.83.42"), Port: 53, DomainName: "l.root-servers.net"},    // L
+	{IP: net.ParseIP("202.12.27.33"), Port: 53, DomainName: "m.root-servers.net"},   // M
 }
 
 var RootServersV6 = []NameServer{
-	{IP: net.ParseIP("2001:503:ba3e::2:30"), Port: 53}, // A
-	{IP: net.ParseIP("2801:1b8:10::b"), Port: 53},      // B
-	{IP: net.ParseIP("2001:500:2::c"), Port: 53},       // C
-	{IP: net.ParseIP("2001:500:2d::d"), Port: 53},      // D
-	{IP: net.ParseIP("2001:500:a8::e"), Port: 53},      // E
-	{IP: net.ParseIP("2001:500:2f::f"), Port: 53},      // F
-	{IP: net.ParseIP("2001:500:12::d0d"), Port: 53},    // G
-	{IP: net.ParseIP("2001:500:1::53"), Port: 53},      // H
-	{IP: net.ParseIP("2001:7fe::53"), Port: 53},        // I
-	{IP: net.ParseIP("2001:503:c27::2:30"), Port: 53},  // J
-	{IP: net.ParseIP("2001:7fd::1"), Port: 53},         // K
-	{IP: net.ParseIP("2001:500:9f::42"), Port: 53},     // L
-	{IP: net.ParseIP("2001:dc3::35"), Port: 53},        // M
+	{IP: net.ParseIP("2001:503:ba3e::2:30"), Port: 53, DomainName: "a.root-servers.net"}, // A
+	{IP: net.ParseIP("2801:1b8:10::b"), Port: 53, DomainName: "b.root-servers.net"},      // B
+	{IP: net.ParseIP("2001:500:2::c"), Port: 53, DomainName: "c.root-servers.net"},       // C
+	{IP: net.ParseIP("2001:500:2d::d"), Port: 53, DomainName: "d.root-servers.net"},      // D
+	{IP: net.ParseIP("2001:500:a8::e"), Port: 53, DomainName: "e.root-servers.net"},      // E
+	{IP: net.ParseIP("2001:500:2f::f"), Port: 53, DomainName: "f.root-servers.net"},      // F
+	{IP: net.ParseIP("2001:500:12::d0d"), Port: 53, DomainName: "g.root-servers.net"},    // G
+	{IP: net.ParseIP("2001:500:1::53"), Port: 53, DomainName: "h.root-servers.net"},      // H
+	{IP: net.ParseIP("2001:7fe::53"), Port: 53, DomainName: "i.root-servers.net"},        // I
+	{IP: net.ParseIP("2001:503:c27::2:30"), Port: 53, DomainName: "j.root-servers.net"},  // J
+	{IP: net.ParseIP("2001:7fd::1"), Port: 53, DomainName: "k.root-servers.net"},         // K
+	{IP: net.ParseIP("2001:500:9f::42"), Port: 53, DomainName: "l.root-servers.net"},     // L
+	{IP: net.ParseIP("2001:dc3::35"), Port: 53, DomainName: "m.root-servers.net"},        // M
 }
 
 var DefaultExternalResolversV4 = []NameServer{

From 0461fe94fca5614b91eae5bd6d92984373950f19 Mon Sep 17 00:00:00 2001
From: phillip-stephens <phillip@cs.stanford.edu>
Date: Tue, 26 Nov 2024 18:07:26 -0500
Subject: [PATCH 02/28] start the all nameservers, for a single layer

---
 examples/all_nameservers_lookup/main.go |  64 ++++++++++++++-
 src/internal/util/util.go               |   4 +-
 src/zdns/lookup.go                      | 104 +++++++++++++-----------
 src/zdns/qa.go                          |   9 +-
 4 files changed, 126 insertions(+), 55 deletions(-)

diff --git a/examples/all_nameservers_lookup/main.go b/examples/all_nameservers_lookup/main.go
index 798742c3..c15c1cfc 100644
--- a/examples/all_nameservers_lookup/main.go
+++ b/examples/all_nameservers_lookup/main.go
@@ -1 +1,63 @@
-package all_nameservers_lookup
+/* ZDNS Copyright 2024 Regents of the University of Michigan
+*
+* Licensed under the Apache License, Version 2.0 (the "License"); you may not
+* use this file except in compliance with the License. You may obtain a copy
+* of the License at http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+* implied. See the License for the specific language governing
+* permissions and limitations under the License.
+ */
+
+package main
+
+import (
+	"context"
+	"net"
+
+	"github.com/miekg/dns"
+	log "github.com/sirupsen/logrus"
+
+	"github.com/zmap/zdns/examples/utils"
+	"github.com/zmap/zdns/src/zdns"
+)
+
+func main() {
+	// Perform the lookup
+	domain := "google.com"
+	dnsQuestion := &zdns.Question{Name: domain, Type: dns.TypeA, Class: dns.ClassINET}
+	resolver := initializeResolver()
+
+	log.Warn("\n\n This lookup just used the Cloudflare recursive resolver, let's run our own recursion.")
+	// Iterative Lookups start at the root nameservers and follow the chain of referrals to the authoritative nameservers.
+	result, _, status, err := resolver.LookupAllNameservers(context.Background(), dnsQuestion, 2)
+	if err != nil {
+		log.Fatal("Error looking up domain: ", err)
+	}
+	log.Warnf("Result: %v", result)
+	log.Warnf("Status: %v", status)
+	resolver.Close()
+}
+
+func initializeResolver() *zdns.Resolver {
+	localAddr, err := utils.GetLocalIPByConnecting()
+	if err != nil {
+		log.Fatal("Error getting local IP: ", err)
+	}
+	// Create a ResolverConfig object
+	resolverConfig := zdns.NewResolverConfig()
+	// Set any desired options on the ResolverConfig object
+	resolverConfig.LogLevel = log.InfoLevel
+	resolverConfig.LocalAddrsV4 = []net.IP{localAddr}
+	resolverConfig.ExternalNameServersV4 = []zdns.NameServer{{IP: net.ParseIP("1.1.1.1"), Port: 53}}
+	resolverConfig.RootNameServersV4 = zdns.RootServersV4
+	resolverConfig.IPVersionMode = zdns.IPv4Only
+	// Create a new Resolver object with the ResolverConfig object, it will retain all settings set on the ResolverConfig object
+	resolver, err := zdns.InitResolver(resolverConfig)
+	if err != nil {
+		log.Fatal("Error initializing resolver: ", err)
+	}
+	return resolver
+}
diff --git a/src/internal/util/util.go b/src/internal/util/util.go
index 094c8608..61316aa2 100644
--- a/src/internal/util/util.go
+++ b/src/internal/util/util.go
@@ -57,9 +57,9 @@ func IsStringValidDomainName(domain string) bool {
 }
 
 // HasCtxExpired checks if the context has expired. Common function used in various places.
-func HasCtxExpired(ctx *context.Context) bool {
+func HasCtxExpired(ctx context.Context) bool {
 	select {
-	case <-(*ctx).Done():
+	case <-(ctx).Done():
 		return true
 	default:
 		return false
diff --git a/src/zdns/lookup.go b/src/zdns/lookup.go
index c61b782f..3df290e3 100644
--- a/src/zdns/lookup.go
+++ b/src/zdns/lookup.go
@@ -121,7 +121,7 @@ func (r *Resolver) lookup(ctx context.Context, qWithMeta *QuestionWithMetadata,
 	var trace Trace
 	var status Status
 	var err error
-	if util.HasCtxExpired(&ctx) {
+	if util.HasCtxExpired(ctx) {
 		return res, trace, StatusTimeout, nil
 	}
 	if isIterative {
@@ -273,55 +273,65 @@ func isLookupComplete(originalName string, candidateSet map[string][]Answer, cNa
 	return false
 }
 
-// TODO - This is incomplete. We only lookup all nameservers for the initial name server lookup, then just send the DNS query to this set.
-// If we want to iteratively lookup all nameservers at each level of the query, we need to fix this.
-// Issue - https://github.com/zmap/zdns/issues/362
-func (r *Resolver) LookupAllNameservers(q *Question, nameServer *NameServer) (*CombinedResults, Trace, Status, error) {
-	var retv CombinedResults
-	var curServer string
-
-	// Lookup both ipv4 and ipv6 addresses of nameservers.
-	nsResults, nsTrace, nsStatus, nsError := r.DoNSLookup(q.Name, nameServer, false, true, true)
-
-	// Terminate early if nameserver lookup also failed
-	if nsStatus != StatusNoError {
-		return nil, nsTrace, nsStatus, nsError
+// LookupAllNameservers will send a query to all name servers at each level of DNS resolution. It starts at the root,
+// queries each NS, and then builds a de-duplicated list of the union of all responses. It repeats this process to the TLD
+// NS servers, etc.
+func (r *Resolver) LookupAllNameservers(ctx context.Context, q *Question, perNameServerRetriesLimit int) (*AllNameServersResult, Trace, Status, error) {
+	//retriesRemaining := r.retries // make a copy so we can decrement freely in the lookup process
+	////qWithMeta := &QuestionWithMetadata{
+	////	Q: *q,
+	////	RetriesRemaining: &retriesRemaining,
+	////}
+	retv := AllNameServersResult{
+		LayeredResponses: make(map[string][]ExtendedResult),
 	}
-	if nsResults == nil {
-		return nil, nsTrace, nsStatus, errors.New("no results from nameserver lookup")
+	var trace Trace
+	//trace := Trace{}
+	currentLayer := "."
+	layerResults, currTrace, err := r.queryAllNameServersInLayer(ctx, perNameServerRetriesLimit, q, r.rootNameServers)
+	trace = append(trace, currTrace...)
+	if err != nil {
+		return &retv, trace, StatusError, errors.Wrapf(err, "error encountered on layer %s", currentLayer)
+	} else {
+		retv.LayeredResponses[currentLayer] = layerResults
 	}
 
-	// fullTrace holds the complete trace including all lookups
-	var fullTrace = Trace{}
-	if nsTrace != nil {
-		fullTrace = append(fullTrace, nsTrace...)
-	}
-	for _, nserver := range nsResults.Servers {
-		// Use all the ipv4 and ipv6 addresses of each nameserver
-		nameserver := nserver.Name
-		ips := util.Concat(nserver.IPv4Addresses, nserver.IPv6Addresses)
-		for _, ip := range ips {
-			// construct the nameserver
-			res, trace, status, err := r.ExternalLookup(q, &NameServer{
-				IP:   net.ParseIP(ip),
-				Port: DefaultDNSPort,
-			})
-			if err != nil {
-				// log and move on
-				log.Errorf("lookup for domain %s to nameserver %s failed with error %s. Continueing to next nameserver", q.Name, curServer, err)
-				continue
-			}
+	return &retv, trace, StatusNoError, nil
+}
 
-			fullTrace = append(fullTrace, trace...)
-			extendedResult := ExtendedResult{
-				Res:        *res,
-				Status:     status,
-				Nameserver: nameserver,
+func (r *Resolver) queryAllNameServersInLayer(ctx context.Context, perNameServerRetriesLimit int, q *Question, currentNameServers []NameServer) ([]ExtendedResult, Trace, error) {
+	trace := make([]TraceStep, 0)
+	currentLayerResults := make([]ExtendedResult, 0, len(currentNameServers))
+	for _, nameServer := range currentNameServers {
+		var extResult *ExtendedResult
+		for retry := 0; retry < perNameServerRetriesLimit; retry++ {
+			if util.HasCtxExpired(ctx) {
+				return currentLayerResults, trace, errors.New("context expired")
 			}
-			retv.Results = append(retv.Results, extendedResult)
+			result, currTrace, status, err := r.ExternalLookup(q, &nameServer)
+			trace = append(trace, currTrace...)
+			if err == nil && status == StatusNoError {
+				extResult = &ExtendedResult{
+					Res:        *result,
+					Status:     status,
+					Nameserver: nameServer.DomainName,
+				}
+				// successful result, continue to next nameserver
+				break
+			}
+			if err != nil {
+				log.Debugf("LookupAllNameservers of name %s errored for %s: %v", q.Name, nameServer.IP.String(), err)
+			} else {
+				log.Debugf("LookupAllNameservers of name %s failed for %s: %v", q.Name, nameServer.IP.String(), status)
+			}
+		}
+		if extResult == nil {
+			log.Debugf("LookupAllNameservers of name %s against nameserver %s ran out of retries, continueing to next nameserver", q.Name, nameServer.IP.String())
+		} else {
+			currentLayerResults = append(currentLayerResults, *extResult)
 		}
 	}
-	return &retv, fullTrace, StatusNoError, nil
+	return currentLayerResults, trace, nil
 }
 
 func (r *Resolver) iterativeLookup(ctx context.Context, qWithMeta *QuestionWithMetadata, nameServers []NameServer,
@@ -331,7 +341,7 @@ func (r *Resolver) iterativeLookup(ctx context.Context, qWithMeta *QuestionWithM
 		return nil, trace, StatusError, errors.New("max recursion depth reached")
 	}
 	// check that context hasn't expired
-	if util.HasCtxExpired(&ctx) {
+	if util.HasCtxExpired(ctx) {
 		r.verboseLog(depth+1, "-> Context expired")
 		return nil, trace, StatusTimeout, nil
 	}
@@ -352,7 +362,7 @@ func (r *Resolver) iterativeLookup(ctx context.Context, qWithMeta *QuestionWithM
 		t.Try = getTryNumber(r.retries, *qWithMeta.RetriesRemaining)
 		trace = append(trace, t)
 	}
-	if status == StatusTimeout && util.HasCtxExpired(&iterationStepCtx) && !util.HasCtxExpired(&ctx) {
+	if status == StatusTimeout && util.HasCtxExpired(iterationStepCtx) && !util.HasCtxExpired(ctx) {
 		// ctx's have a deadline of the minimum of their deadline and their parent's
 		// retryingLookup doesn't disambiguate of whether the timeout was caused by the iteration timeout or the global timeout
 		// we'll disambiguate here by checking if the iteration context has expired but the global context hasn't
@@ -409,7 +419,7 @@ func (r *Resolver) cyclingLookup(ctx context.Context, qWithMeta *QuestionWithMet
 	var nameServer *NameServer
 
 	for *qWithMeta.RetriesRemaining >= 0 {
-		if util.HasCtxExpired(&ctx) {
+		if util.HasCtxExpired(ctx) {
 			return &SingleQueryResult{}, false, StatusTimeout, nil
 		}
 		// get random unqueried nameserver
@@ -882,7 +892,7 @@ func (r *Resolver) iterateOnAuthorities(ctx context.Context, qWithMeta *Question
 	newTrace := trace
 	nameServers := make([]NameServer, 0, len(result.Authorities))
 	for i, elem := range result.Authorities {
-		if util.HasCtxExpired(&ctx) {
+		if util.HasCtxExpired(ctx) {
 			return &SingleQueryResult{}, newTrace, StatusTimeout, nil
 		}
 		var ns *NameServer
diff --git a/src/zdns/qa.go b/src/zdns/qa.go
index 4e949a94..f0ded15f 100644
--- a/src/zdns/qa.go
+++ b/src/zdns/qa.go
@@ -79,7 +79,7 @@ type SingleQueryResult struct {
 	Additional         []interface{} `json:"additionals,omitempty" groups:"short,normal,long,trace"`
 	Authorities        []interface{} `json:"authorities,omitempty" groups:"short,normal,long,trace"`
 	Protocol           string        `json:"protocol" groups:"protocol,normal,long,trace"`
-	Resolver           string        `json:"resolver" groups:"resolver,normal,long,trace"`
+	Resolver           string        `json:"resolver" groups:"resolver,normal,long,trace"` // IP address
 	Flags              DNSFlags      `json:"flags" groups:"flags,long,trace"`
 	TLSServerHandshake interface{}   `json:"tls_handshake,omitempty" groups:"normal,long,trace"` // used for --tls and --https, JSON string of the TLS handshake
 }
@@ -87,12 +87,11 @@ type SingleQueryResult struct {
 type ExtendedResult struct {
 	Res        SingleQueryResult `json:"result,omitempty" groups:"short,normal,long,trace"`
 	Status     Status            `json:"status" groups:"short,normal,long,trace"`
-	Nameserver string            `json:"nameserver" groups:"short,normal,long,trace"`
-	Trace      Trace             `json:"trace,omitempty" groups:"trace"`
+	Nameserver string            `json:"nameserver" groups:"short,normal,long,trace"` // NS domain name
 }
 
-type CombinedResults struct {
-	Results []ExtendedResult `json:"results" groups:"short,normal,long,trace"`
+type AllNameServersResult struct {
+	LayeredResponses map[string][]ExtendedResult `json:"per_layer_responses" groups:"short,normal,long,trace"` //
 }
 
 type IPResult struct {

From aae77d068fa52cd605a251e22bf2f6a7c37992e7 Mon Sep 17 00:00:00 2001
From: phillip-stephens <phillip@cs.stanford.edu>
Date: Wed, 11 Dec 2024 12:32:22 -0500
Subject: [PATCH 03/28] working for google.com

---
 src/cli/modules.go |   2 +-
 src/zdns/lookup.go | 124 ++++++++++++++++++++++++++++++++++++++-------
 2 files changed, 108 insertions(+), 18 deletions(-)

diff --git a/src/cli/modules.go b/src/cli/modules.go
index ec267b07..e2e51885 100644
--- a/src/cli/modules.go
+++ b/src/cli/modules.go
@@ -167,7 +167,7 @@ func (lm *BasicLookupModule) NewFlags() interface{} {
 
 func (lm *BasicLookupModule) Lookup(resolver *zdns.Resolver, lookupName string, nameServer *zdns.NameServer) (interface{}, zdns.Trace, zdns.Status, error) {
 	if lm.LookupAllNameServers {
-		return resolver.LookupAllNameservers(&zdns.Question{Name: lookupName, Type: lm.DNSType, Class: lm.DNSClass}, nameServer)
+		return resolver.LookupAllNameservers(&zdns.Question{Name: lookupName, Type: lm.DNSType, Class: lm.DNSClass})
 	}
 	if lm.IsIterative {
 		return resolver.IterativeLookup(&zdns.Question{Name: lookupName, Type: lm.DNSType, Class: lm.DNSClass})
diff --git a/src/zdns/lookup.go b/src/zdns/lookup.go
index 3df290e3..c43dd6b4 100644
--- a/src/zdns/lookup.go
+++ b/src/zdns/lookup.go
@@ -276,37 +276,124 @@ func isLookupComplete(originalName string, candidateSet map[string][]Answer, cNa
 // LookupAllNameservers will send a query to all name servers at each level of DNS resolution. It starts at the root,
 // queries each NS, and then builds a de-duplicated list of the union of all responses. It repeats this process to the TLD
 // NS servers, etc.
-func (r *Resolver) LookupAllNameservers(ctx context.Context, q *Question, perNameServerRetriesLimit int) (*AllNameServersResult, Trace, Status, error) {
-	//retriesRemaining := r.retries // make a copy so we can decrement freely in the lookup process
-	////qWithMeta := &QuestionWithMetadata{
-	////	Q: *q,
-	////	RetriesRemaining: &retriesRemaining,
-	////}
+func (r *Resolver) LookupAllNameservers(q *Question) (*AllNameServersResult, Trace, Status, error) {
+	perNameServerRetriesLimit := 2
+	ctx, cancel := context.WithTimeout(context.Background(), r.timeout)
+	defer cancel()
 	retv := AllNameServersResult{
 		LayeredResponses: make(map[string][]ExtendedResult),
 	}
 	var trace Trace
-	//trace := Trace{}
 	currentLayer := "."
-	layerResults, currTrace, err := r.queryAllNameServersInLayer(ctx, perNameServerRetriesLimit, q, r.rootNameServers)
-	trace = append(trace, currTrace...)
-	if err != nil {
-		return &retv, trace, StatusError, errors.Wrapf(err, "error encountered on layer %s", currentLayer)
-	} else {
-		retv.LayeredResponses[currentLayer] = layerResults
-	}
+	isAuthoritative := false
+	var err error
+	currentLayerNameServers := r.rootNameServers
 
+	for isAuthoritative == false {
+		var layerResults []ExtendedResult
+		currTrace := make(Trace, 0)
+		layerResults, currTrace, isAuthoritative, err = r.queryAllNameServersInLayer(ctx, perNameServerRetriesLimit, q, currentLayerNameServers)
+		trace = append(trace, currTrace...)
+		if err != nil {
+			return &retv, trace, StatusError, errors.Wrapf(err, "error encountered on layer %s", currentLayer)
+		} else {
+			retv.LayeredResponses[currentLayer] = layerResults
+		}
+		if isAuthoritative {
+			return &retv, trace, StatusNoError, nil
+		}
+		// Set the next layer to query
+		currentLayer, err = nextAuthority(q.Name, currentLayer)
+		if err != nil {
+			return &retv, trace, StatusError, errors.Wrapf(err, "error determining next authority for layer %s", currentLayer)
+		}
+		currentLayerNameServers, err = r.extractNameServersFromLayerResults(layerResults)
+		if err != nil {
+			return &retv, trace, StatusError, errors.Wrapf(err, "error extracting nameservers from layer %s", currentLayer)
+		}
+	}
 	return &retv, trace, StatusNoError, nil
 }
 
-func (r *Resolver) queryAllNameServersInLayer(ctx context.Context, perNameServerRetriesLimit int, q *Question, currentNameServers []NameServer) ([]ExtendedResult, Trace, error) {
+// extractNameServersFromLayerResults
+// extracts unique nameservers from Additionals/Authorities. Uniques by nameserver name, not by IP
+func (r *Resolver) extractNameServersFromLayerResults(layerResults []ExtendedResult) ([]NameServer, error) {
+	type mapKey struct {
+		Type   uint16
+		Name   string
+		Answer string
+	}
+	uniqueAdditionals := make(map[mapKey]Answer)
+	uniqueAuthorities := make(map[mapKey]Answer)
+	for _, res := range layerResults {
+		if res.Status != StatusNoError {
+			continue
+		}
+		for _, ans := range res.Res.Additional {
+			if a, ok := ans.(Answer); ok {
+				uniqueAdditionals[mapKey{Type: a.RrType, Name: a.Name, Answer: a.Answer}] = a
+			}
+		}
+		for _, ans := range res.Res.Authorities {
+			if a, ok := ans.(Answer); ok {
+				uniqueAuthorities[mapKey{Type: a.RrType, Name: a.Name, Answer: a.Answer}] = a
+			}
+		}
+	}
+	// We have a map of unique additional and authority records. Now we need to extract the nameservers from them.
+	v4NameServers := make(map[string]NameServer)
+	v6NameServers := make(map[string]NameServer)
+	for _, authorities := range uniqueAuthorities {
+		if authorities.RrType == dns.TypeNS {
+			v4NameServers[strings.TrimSuffix(authorities.Answer, ".")] = NameServer{DomainName: authorities.Answer}
+			v6NameServers[strings.TrimSuffix(authorities.Answer, ".")] = NameServer{DomainName: authorities.Answer}
+		}
+	}
+	for _, additionals := range uniqueAdditionals {
+		if strings.HasSuffix(additionals.Name, ".") {
+			// TODO checking here to see if this is an issue
+			log.Fatalf("Name %s has a trailing dot", additionals.Name)
+		}
+		if additionals.RrType == dns.TypeA {
+			if ns, ok := v4NameServers[additionals.Name]; ok {
+				ns.IP = net.ParseIP(additionals.Answer)
+				v4NameServers[additionals.Name] = ns
+			}
+		}
+		if additionals.RrType == dns.TypeAAAA {
+			if ns, ok := v6NameServers[additionals.Name]; ok {
+				ns.IP = net.ParseIP(additionals.Answer)
+				v6NameServers[additionals.Name] = ns
+			}
+		}
+	}
+	// convert to slice
+	nameServersSlice := make([]NameServer, 0, len(v4NameServers))
+	if r.ipVersionMode != IPv6Only {
+		for _, ns := range v4NameServers {
+			nameServersSlice = append(nameServersSlice, ns)
+		}
+	}
+	if r.ipVersionMode != IPv4Only {
+		for _, ns := range v6NameServers {
+			nameServersSlice = append(nameServersSlice, ns)
+		}
+	}
+
+	return nameServersSlice, nil
+}
+
+// queryAllNameServersInLayer queries all nameservers in a given layer
+// Returns a slice of ExtendedResults from each NS, a Trace, whether any answer is authoritative, and an error if one occurs
+func (r *Resolver) queryAllNameServersInLayer(ctx context.Context, perNameServerRetriesLimit int, q *Question, currentNameServers []NameServer) ([]ExtendedResult, Trace, bool, error) {
 	trace := make([]TraceStep, 0)
 	currentLayerResults := make([]ExtendedResult, 0, len(currentNameServers))
+	isAuthoritative := false
 	for _, nameServer := range currentNameServers {
 		var extResult *ExtendedResult
 		for retry := 0; retry < perNameServerRetriesLimit; retry++ {
 			if util.HasCtxExpired(ctx) {
-				return currentLayerResults, trace, errors.New("context expired")
+				return currentLayerResults, trace, false, errors.New("context expired")
 			}
 			result, currTrace, status, err := r.ExternalLookup(q, &nameServer)
 			trace = append(trace, currTrace...)
@@ -316,6 +403,9 @@ func (r *Resolver) queryAllNameServersInLayer(ctx context.Context, perNameServer
 					Status:     status,
 					Nameserver: nameServer.DomainName,
 				}
+				if result.Flags.Authoritative {
+					isAuthoritative = true
+				}
 				// successful result, continue to next nameserver
 				break
 			}
@@ -331,7 +421,7 @@ func (r *Resolver) queryAllNameServersInLayer(ctx context.Context, perNameServer
 			currentLayerResults = append(currentLayerResults, *extResult)
 		}
 	}
-	return currentLayerResults, trace, nil
+	return currentLayerResults, trace, isAuthoritative, nil
 }
 
 func (r *Resolver) iterativeLookup(ctx context.Context, qWithMeta *QuestionWithMetadata, nameServers []NameServer,

From 6b90b7a862ca42d390136ceb857a1e2b4d3e3d38 Mon Sep 17 00:00:00 2001
From: phillip-stephens <phillip@cs.stanford.edu>
Date: Wed, 11 Dec 2024 17:18:20 -0500
Subject: [PATCH 04/28] example.com working

---
 src/zdns/lookup.go | 94 ++++++++++++++++++++++++++++++++++++++++++----
 1 file changed, 87 insertions(+), 7 deletions(-)

diff --git a/src/zdns/lookup.go b/src/zdns/lookup.go
index c43dd6b4..e71b9531 100644
--- a/src/zdns/lookup.go
+++ b/src/zdns/lookup.go
@@ -345,8 +345,8 @@ func (r *Resolver) extractNameServersFromLayerResults(layerResults []ExtendedRes
 	v6NameServers := make(map[string]NameServer)
 	for _, authorities := range uniqueAuthorities {
 		if authorities.RrType == dns.TypeNS {
-			v4NameServers[strings.TrimSuffix(authorities.Answer, ".")] = NameServer{DomainName: authorities.Answer}
-			v6NameServers[strings.TrimSuffix(authorities.Answer, ".")] = NameServer{DomainName: authorities.Answer}
+			v4NameServers[strings.TrimSuffix(authorities.Answer, ".")] = NameServer{DomainName: strings.TrimSuffix(authorities.Answer, ".")}
+			v6NameServers[strings.TrimSuffix(authorities.Answer, ".")] = NameServer{DomainName: strings.TrimSuffix(authorities.Answer, ".")}
 		}
 	}
 	for _, additionals := range uniqueAdditionals {
@@ -367,20 +367,91 @@ func (r *Resolver) extractNameServersFromLayerResults(layerResults []ExtendedRes
 			}
 		}
 	}
-	// convert to slice
-	nameServersSlice := make([]NameServer, 0, len(v4NameServers))
+	// dedupe
+
+	uniqNameServersSet := make(map[string]NameServer)
 	if r.ipVersionMode != IPv6Only {
 		for _, ns := range v4NameServers {
-			nameServersSlice = append(nameServersSlice, ns)
+			key := ns.DomainName + ns.IP.String()
+			if _, ok := uniqNameServersSet[key]; !ok {
+				uniqNameServersSet[key] = ns
+
+			}
 		}
 	}
 	if r.ipVersionMode != IPv4Only {
 		for _, ns := range v6NameServers {
-			nameServersSlice = append(nameServersSlice, ns)
+			key := ns.DomainName + ns.IP.String()
+			if _, ok := uniqNameServersSet[key]; !ok {
+				uniqNameServersSet[key] = ns
+			}
 		}
 	}
 
-	return nameServersSlice, nil
+	uniqNameServers := make([]NameServer, 0, len(uniqNameServersSet))
+	for _, ns := range uniqNameServersSet {
+		uniqNameServers = append(uniqNameServers, ns)
+	}
+
+	return uniqNameServers, nil
+}
+
+func (r *Resolver) populateNameServerIP(ctx context.Context, nameServer *NameServer) (Trace, error) {
+	if nameServer.IP != nil {
+		// already have an IP
+		return nil, nil
+	}
+	retries := r.retries
+	var q Question
+	if r.ipVersionMode == IPv4Only {
+		q = Question{dns.TypeA, dns.ClassINET, nameServer.DomainName}
+	} else if r.ipVersionMode == IPv6Only {
+		q = Question{dns.TypeAAAA, dns.ClassINET, nameServer.DomainName}
+	} else if r.iterationIPPreference == PreferIPv4 {
+		q = Question{dns.TypeA, dns.ClassINET, nameServer.DomainName}
+	} else {
+		q = Question{dns.TypeAAAA, dns.ClassINET, nameServer.DomainName}
+	}
+	res, nsTrace, status, err := r.followingLookup(ctx, &QuestionWithMetadata{
+		Q:                q,
+		RetriesRemaining: &retries,
+	}, r.rootNameServers, true)
+	if err == nil && status == StatusNoError {
+		for _, ans := range res.Answers {
+			if a, ok := ans.(Answer); ok {
+				if a.RrType == q.Type {
+					nameServer.IP = net.ParseIP(a.Answer)
+					return nsTrace, nil
+				}
+			}
+		}
+	}
+	// if we get here, we couldn't find an IP for the nameserver, let's try with the other A/AAAA if we can
+	if r.ipVersionMode == IPv4OrIPv6 {
+		if q.Type == dns.TypeA {
+			q.Type = dns.TypeAAAA
+		} else {
+			q.Type = dns.TypeA
+		}
+		res, nsTrace, status, err = r.followingLookup(ctx, &QuestionWithMetadata{
+			Q:                q,
+			RetriesRemaining: &retries,
+		}, r.rootNameServers, true)
+		if err == nil && status == StatusNoError {
+			for _, ans := range res.Answers {
+				if a, ok := ans.(Answer); ok {
+					if a.RrType == q.Type {
+						nameServer.IP = net.ParseIP(a.Answer)
+						return nsTrace, nil
+					}
+				}
+			}
+		}
+	}
+	if err != nil {
+		return nil, errors.Wrapf(err, "could not find IP for nameserver: %s", nameServer.DomainName)
+	}
+	return nil, errors.Errorf("could not find IP for nameserver: %s", nameServer.DomainName)
 }
 
 // queryAllNameServersInLayer queries all nameservers in a given layer
@@ -395,6 +466,15 @@ func (r *Resolver) queryAllNameServersInLayer(ctx context.Context, perNameServer
 			if util.HasCtxExpired(ctx) {
 				return currentLayerResults, trace, false, errors.New("context expired")
 			}
+			if nameServer.IP == nil {
+				nsTrace, err := r.populateNameServerIP(ctx, &nameServer)
+				if err != nil {
+					log.Debugf("LookupAllNameservers of name %s errored for %s: %v", q.Name, nameServer.DomainName, err)
+					continue
+				}
+				trace = append(trace, nsTrace...)
+				// we've populated NS IP, we can proceed
+			}
 			result, currTrace, status, err := r.ExternalLookup(q, &nameServer)
 			trace = append(trace, currTrace...)
 			if err == nil && status == StatusNoError {

From 1ee507c201028906a4ae278db92dd2fc674e405a Mon Sep 17 00:00:00 2001
From: phillip-stephens <phillip@cs.stanford.edu>
Date: Thu, 12 Dec 2024 11:43:28 -0500
Subject: [PATCH 05/28] external lookups working

---
 src/cli/config_validation.go |  4 ---
 src/cli/modules.go           |  5 +++-
 src/zdns/lookup.go           | 47 ++++++++++++++++++++++++++++--------
 src/zdns/lookup_test.go      | 12 ++++-----
 4 files changed, 47 insertions(+), 21 deletions(-)

diff --git a/src/cli/config_validation.go b/src/cli/config_validation.go
index 459d6774..14bc41ae 100644
--- a/src/cli/config_validation.go
+++ b/src/cli/config_validation.go
@@ -128,10 +128,6 @@ func validateClientSubnetString(gc *CLIConf) error {
 }
 
 func parseNameServers(gc *CLIConf) error {
-	if gc.LookupAllNameServers && gc.NameServersString != "" {
-		log.Fatal("name servers cannot be specified in --all-nameservers mode.")
-	}
-
 	if gc.NameServersString != "" {
 		if gc.NameServerMode {
 			log.Fatal("name servers cannot be specified on command line in --name-server-mode")
diff --git a/src/cli/modules.go b/src/cli/modules.go
index e2e51885..0dd8575c 100644
--- a/src/cli/modules.go
+++ b/src/cli/modules.go
@@ -166,8 +166,11 @@ func (lm *BasicLookupModule) NewFlags() interface{} {
 }
 
 func (lm *BasicLookupModule) Lookup(resolver *zdns.Resolver, lookupName string, nameServer *zdns.NameServer) (interface{}, zdns.Trace, zdns.Status, error) {
+	if lm.LookupAllNameServers && lm.IsIterative {
+		return resolver.LookupAllNameserversIterative(&zdns.Question{Name: lookupName, Type: lm.DNSType, Class: lm.DNSClass})
+	}
 	if lm.LookupAllNameServers {
-		return resolver.LookupAllNameservers(&zdns.Question{Name: lookupName, Type: lm.DNSType, Class: lm.DNSClass})
+		return resolver.LookupAllNameserversExternal(&zdns.Question{Name: lookupName, Type: lm.DNSType, Class: lm.DNSClass}, nil)
 	}
 	if lm.IsIterative {
 		return resolver.IterativeLookup(&zdns.Question{Name: lookupName, Type: lm.DNSType, Class: lm.DNSClass})
diff --git a/src/zdns/lookup.go b/src/zdns/lookup.go
index e71b9531..e670f895 100644
--- a/src/zdns/lookup.go
+++ b/src/zdns/lookup.go
@@ -273,10 +273,41 @@ func isLookupComplete(originalName string, candidateSet map[string][]Answer, cNa
 	return false
 }
 
-// LookupAllNameservers will send a query to all name servers at each level of DNS resolution. It starts at the root,
+// LookupAllNameserversExternal will query all of a resolvers ExternalNameServers for a given question
+func (r *Resolver) LookupAllNameserversExternal(q *Question, nameServers []NameServer) ([]SingleQueryResult, Trace, Status, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), r.timeout)
+	defer cancel()
+	retv := make([]SingleQueryResult, 0)
+	var trace Trace
+	if len(nameServers) == 0 && len(r.externalNameServers) == 0 {
+		return retv, trace, StatusIllegalInput, errors.New("no external nameservers specified")
+	}
+	if len(nameServers) == 0 {
+		nameServers = r.externalNameServers
+	}
+
+	for _, ns := range nameServers {
+		if util.HasCtxExpired(ctx) {
+			return retv, trace, StatusTimeout, errors.New("context expired")
+		}
+		result, currTrace, status, err := r.ExternalLookup(q, &ns)
+		trace = append(trace, currTrace...)
+		if err != nil {
+			log.Errorf("LookupAllNameserversExternal of name %s errored for %s/%s: %v", q.Name, ns.DomainName, ns.IP.String(), err)
+			continue
+		}
+		if status == StatusNoError {
+			retv = append(retv, *result)
+			log.Debugf("LookupAllNameserversExternal of name %s succeeded for %s/%s", q.Name, ns.DomainName, ns.IP.String())
+		}
+	}
+	return retv, trace, StatusNoError, nil
+}
+
+// LookupAllNameserversIterative will send a query to all name servers at each level of DNS resolution. It starts at the root,
 // queries each NS, and then builds a de-duplicated list of the union of all responses. It repeats this process to the TLD
 // NS servers, etc.
-func (r *Resolver) LookupAllNameservers(q *Question) (*AllNameServersResult, Trace, Status, error) {
+func (r *Resolver) LookupAllNameserversIterative(q *Question) (*AllNameServersResult, Trace, Status, error) {
 	perNameServerRetriesLimit := 2
 	ctx, cancel := context.WithTimeout(context.Background(), r.timeout)
 	defer cancel()
@@ -367,8 +398,6 @@ func (r *Resolver) extractNameServersFromLayerResults(layerResults []ExtendedRes
 			}
 		}
 	}
-	// dedupe
-
 	uniqNameServersSet := make(map[string]NameServer)
 	if r.ipVersionMode != IPv6Only {
 		for _, ns := range v4NameServers {
@@ -387,12 +416,10 @@ func (r *Resolver) extractNameServersFromLayerResults(layerResults []ExtendedRes
 			}
 		}
 	}
-
 	uniqNameServers := make([]NameServer, 0, len(uniqNameServersSet))
 	for _, ns := range uniqNameServersSet {
 		uniqNameServers = append(uniqNameServers, ns)
 	}
-
 	return uniqNameServers, nil
 }
 
@@ -469,7 +496,7 @@ func (r *Resolver) queryAllNameServersInLayer(ctx context.Context, perNameServer
 			if nameServer.IP == nil {
 				nsTrace, err := r.populateNameServerIP(ctx, &nameServer)
 				if err != nil {
-					log.Debugf("LookupAllNameservers of name %s errored for %s: %v", q.Name, nameServer.DomainName, err)
+					log.Debugf("LookupAllNameserversIterative of name %s errored for %s: %v", q.Name, nameServer.DomainName, err)
 					continue
 				}
 				trace = append(trace, nsTrace...)
@@ -490,13 +517,13 @@ func (r *Resolver) queryAllNameServersInLayer(ctx context.Context, perNameServer
 				break
 			}
 			if err != nil {
-				log.Debugf("LookupAllNameservers of name %s errored for %s: %v", q.Name, nameServer.IP.String(), err)
+				log.Debugf("LookupAllNameserversIterative of name %s errored for %s: %v", q.Name, nameServer.IP.String(), err)
 			} else {
-				log.Debugf("LookupAllNameservers of name %s failed for %s: %v", q.Name, nameServer.IP.String(), status)
+				log.Debugf("LookupAllNameserversIterative of name %s failed for %s: %v", q.Name, nameServer.IP.String(), status)
 			}
 		}
 		if extResult == nil {
-			log.Debugf("LookupAllNameservers of name %s against nameserver %s ran out of retries, continueing to next nameserver", q.Name, nameServer.IP.String())
+			log.Debugf("LookupAllNameserversIterative of name %s against nameserver %s ran out of retries, continueing to next nameserver", q.Name, nameServer.IP.String())
 		} else {
 			currentLayerResults = append(currentLayerResults, *extResult)
 		}
diff --git a/src/zdns/lookup_test.go b/src/zdns/lookup_test.go
index 7f04a187..9cde0bc0 100644
--- a/src/zdns/lookup_test.go
+++ b/src/zdns/lookup_test.go
@@ -1581,7 +1581,7 @@ func TestAllNsLookupOneNs(t *testing.T) {
 		Name:  "example.com",
 	}
 
-	results, _, _, err := resolver.LookupAllNameservers(&q, ns1)
+	results, _, _, err := resolver.LookupAllNameserversIterative(&q, ns1)
 	require.NoError(t, err)
 	verifyCombinedResult(t, results.Results, expectedRes)
 }
@@ -1703,7 +1703,7 @@ func TestAllNsLookupOneNsMultipleIps(t *testing.T) {
 		Name:  "example.com",
 	}
 
-	results, _, _, err := resolver.LookupAllNameservers(&q, ns1)
+	results, _, _, err := resolver.LookupAllNameserversIterative(&q, ns1)
 	require.NoError(t, err)
 	verifyCombinedResult(t, results.Results, expectedRes)
 }
@@ -1816,7 +1816,7 @@ func TestAllNsLookupTwoNs(t *testing.T) {
 		Name:  "example.com",
 	}
 
-	results, _, _, err := resolver.LookupAllNameservers(&q, ns1)
+	results, _, _, err := resolver.LookupAllNameserversIterative(&q, ns1)
 	require.NoError(t, err)
 	verifyCombinedResult(t, results.Results, expectedRes)
 }
@@ -1916,7 +1916,7 @@ func TestAllNsLookupErrorInOne(t *testing.T) {
 		Name:  "example.com",
 	}
 
-	results, _, _, err := resolver.LookupAllNameservers(&q, ns1)
+	results, _, _, err := resolver.LookupAllNameserversIterative(&q, ns1)
 	require.NoError(t, err)
 	verifyCombinedResult(t, results.Results, expectedRes)
 }
@@ -1934,7 +1934,7 @@ func TestAllNsLookupNXDomain(t *testing.T) {
 		Name:  "example.com",
 	}
 
-	res, _, status, err := resolver.LookupAllNameservers(&q, ns1)
+	res, _, status, err := resolver.LookupAllNameserversIterative(&q, ns1)
 
 	assert.Equal(t, StatusNXDomain, status)
 	assert.Nil(t, res)
@@ -1959,7 +1959,7 @@ func TestAllNsLookupServFail(t *testing.T) {
 		Class: dns.ClassINET,
 		Name:  "example.com",
 	}
-	res, _, status, err := resolver.LookupAllNameservers(&q, ns1)
+	res, _, status, err := resolver.LookupAllNameserversIterative(&q, ns1)
 
 	assert.Equal(t, StatusServFail, status)
 	assert.Nil(t, res)

From 539a89ec66841827aba4acbae615b707889c4c2b Mon Sep 17 00:00:00 2001
From: phillip-stephens <phillip@cs.stanford.edu>
Date: Thu, 12 Dec 2024 12:29:59 -0500
Subject: [PATCH 06/28] gate --all-nameservers for all specialty CLI modules
 and add comments

---
 src/cli/modules.go                     |  8 +++++++-
 src/modules/alookup/a_lookup.go        |  3 +++
 src/modules/axfr/axfr.go               |  3 +++
 src/modules/bindversion/bindversion.go |  4 ++++
 src/modules/dmarc/dmarc.go             |  3 +++
 src/modules/mxlookup/mx_lookup.go      |  3 +++
 src/modules/nslookup/ns_lookup.go      |  3 +++
 src/modules/spf/spf.go                 |  3 +++
 src/zdns/lookup.go                     | 19 +++++++++++++------
 9 files changed, 42 insertions(+), 7 deletions(-)

diff --git a/src/cli/modules.go b/src/cli/modules.go
index 0dd8575c..53a22d08 100644
--- a/src/cli/modules.go
+++ b/src/cli/modules.go
@@ -165,9 +165,15 @@ func (lm *BasicLookupModule) NewFlags() interface{} {
 	return lm
 }
 
+// Lookup performs a DNS lookup using the given resolver and lookupName.
+// The behavior with respect to the nameServers is determined by the LookupAllNameServers and IsIterative fields.
+// non-Iterative + all-Nameservers query -> we'll send a query to each of the resolver's external nameservers
+// non-Iterative query -> we'll send a query to the nameserver provided. If none provided, a random nameserver from the resolver's external nameservers will be used
+// iterative + all-Nameservers query -> we'll send a query to each root NS and query all nameservers down the chain.
+// iterative query -> we'll send a query to a random root NS and query all nameservers down the chain.
 func (lm *BasicLookupModule) Lookup(resolver *zdns.Resolver, lookupName string, nameServer *zdns.NameServer) (interface{}, zdns.Trace, zdns.Status, error) {
 	if lm.LookupAllNameServers && lm.IsIterative {
-		return resolver.LookupAllNameserversIterative(&zdns.Question{Name: lookupName, Type: lm.DNSType, Class: lm.DNSClass})
+		return resolver.LookupAllNameserversIterative(&zdns.Question{Name: lookupName, Type: lm.DNSType, Class: lm.DNSClass}, nil)
 	}
 	if lm.LookupAllNameServers {
 		return resolver.LookupAllNameserversExternal(&zdns.Question{Name: lookupName, Type: lm.DNSType, Class: lm.DNSClass}, nil)
diff --git a/src/modules/alookup/a_lookup.go b/src/modules/alookup/a_lookup.go
index 72eeda1b..4b1cc8a8 100644
--- a/src/modules/alookup/a_lookup.go
+++ b/src/modules/alookup/a_lookup.go
@@ -34,6 +34,9 @@ func init() {
 
 // CLIInit initializes the ALookupModule with the given parameters, used to call ALOOKUP from the command line
 func (aMod *ALookupModule) CLIInit(gc *cli.CLIConf, resolverConfig *zdns.ResolverConfig) error {
+	if gc.LookupAllNameServers {
+		return errors.New("ALOOKUP module does not support --all-nameservers")
+	}
 	aMod.Init(aMod.IPv4Lookup, aMod.IPv6Lookup)
 	err := aMod.baseModule.CLIInit(gc, resolverConfig)
 	if err != nil {
diff --git a/src/modules/axfr/axfr.go b/src/modules/axfr/axfr.go
index dc6c0498..a0ef6d88 100644
--- a/src/modules/axfr/axfr.go
+++ b/src/modules/axfr/axfr.go
@@ -151,6 +151,9 @@ func (axfrMod *AxfrLookupModule) CLIInit(gc *cli.CLIConf, rc *zdns.ResolverConfi
 	if gc.IterativeResolution {
 		log.Fatal("AXFR module does not support iterative resolution")
 	}
+	if gc.LookupAllNameServers {
+		return errors.New("AXFR module does not support --all-nameservers")
+	}
 	var err error
 	if axfrMod.BlacklistPath != "" {
 		axfrMod.Blacklist = safeblacklist.New()
diff --git a/src/modules/bindversion/bindversion.go b/src/modules/bindversion/bindversion.go
index d26bb676..640dc680 100644
--- a/src/modules/bindversion/bindversion.go
+++ b/src/modules/bindversion/bindversion.go
@@ -16,6 +16,7 @@ package bindversion
 
 import (
 	"github.com/miekg/dns"
+	"github.com/pkg/errors"
 
 	"github.com/zmap/zdns/src/cli"
 	"github.com/zmap/zdns/src/zdns"
@@ -42,6 +43,9 @@ func init() {
 
 // CLIInit initializes the BindVersion lookup module
 func (bindVersionMod *BindVersionLookupModule) CLIInit(gc *cli.CLIConf, rc *zdns.ResolverConfig) error {
+	if gc.LookupAllNameServers {
+		return errors.New("AXFR module does not support --all-nameservers")
+	}
 	return bindVersionMod.BasicLookupModule.CLIInit(gc, rc)
 }
 
diff --git a/src/modules/dmarc/dmarc.go b/src/modules/dmarc/dmarc.go
index 13b749ec..6c7fd34b 100644
--- a/src/modules/dmarc/dmarc.go
+++ b/src/modules/dmarc/dmarc.go
@@ -42,6 +42,9 @@ type DmarcLookupModule struct {
 
 // CLIInit initializes the DMARC lookup module
 func (dmarcMod *DmarcLookupModule) CLIInit(gc *cli.CLIConf, rc *zdns.ResolverConfig) error {
+	if gc.LookupAllNameServers {
+		return errors.New("DMARC module does not support --all-nameservers")
+	}
 	dmarcMod.re = regexp.MustCompile(dmarcPrefixRegexp)
 	dmarcMod.BasicLookupModule.DNSType = dns.TypeTXT
 	dmarcMod.BasicLookupModule.DNSClass = dns.ClassINET
diff --git a/src/modules/mxlookup/mx_lookup.go b/src/modules/mxlookup/mx_lookup.go
index 5fdefcbc..121fcac0 100644
--- a/src/modules/mxlookup/mx_lookup.go
+++ b/src/modules/mxlookup/mx_lookup.go
@@ -56,6 +56,9 @@ type MXLookupModule struct {
 
 // CLIInit initializes the MXLookupModule with the given parameters, used to call MXLookup from the command line
 func (mxMod *MXLookupModule) CLIInit(gc *cli.CLIConf, rc *zdns.ResolverConfig) error {
+	if gc.LookupAllNameServers {
+		return errors.New("MXLOOKUP module does not support --all-nameservers")
+	}
 	if !mxMod.IPv4Lookup && !mxMod.IPv6Lookup {
 		// need to use one of the two
 		mxMod.IPv4Lookup = true
diff --git a/src/modules/nslookup/ns_lookup.go b/src/modules/nslookup/ns_lookup.go
index f25ca877..1d314743 100644
--- a/src/modules/nslookup/ns_lookup.go
+++ b/src/modules/nslookup/ns_lookup.go
@@ -37,6 +37,9 @@ type NSLookupModule struct {
 
 // CLIInit initializes the NSLookupModule with the given parameters, used to call NSLookup from the command line
 func (nsMod *NSLookupModule) CLIInit(gc *cli.CLIConf, resolverConf *zdns.ResolverConfig) error {
+	if gc.LookupAllNameServers {
+		return errors.New("NSLOOKUP module does not support --all-nameservers")
+	}
 	if !nsMod.IPv4Lookup && !nsMod.IPv6Lookup {
 		log.Debug("NSModule: neither --ipv4-lookup nor --ipv6-lookup specified, will only request A records for each NS server")
 		nsMod.IPv4Lookup = true
diff --git a/src/modules/spf/spf.go b/src/modules/spf/spf.go
index fa94cf91..702de2db 100644
--- a/src/modules/spf/spf.go
+++ b/src/modules/spf/spf.go
@@ -42,6 +42,9 @@ type SpfLookupModule struct {
 
 // CLIInit initializes the SPF lookup module
 func (spfMod *SpfLookupModule) CLIInit(gc *cli.CLIConf, rc *zdns.ResolverConfig) error {
+	if gc.LookupAllNameServers {
+		return errors.New("SPF module does not support --all-nameservers")
+	}
 	spfMod.re = regexp.MustCompile(spfPrefixRegexp)
 	spfMod.BasicLookupModule.DNSType = dns.TypeTXT
 	spfMod.BasicLookupModule.DNSClass = dns.ClassINET
diff --git a/src/zdns/lookup.go b/src/zdns/lookup.go
index e670f895..e70b0f5d 100644
--- a/src/zdns/lookup.go
+++ b/src/zdns/lookup.go
@@ -273,7 +273,8 @@ func isLookupComplete(originalName string, candidateSet map[string][]Answer, cNa
 	return false
 }
 
-// LookupAllNameserversExternal will query all of a resolvers ExternalNameServers for a given question
+// LookupAllNameserversExternal will query all nameServers with the given question and return the results
+// If nameServers is empty, it will use the externalNameServers from the resolver
 func (r *Resolver) LookupAllNameserversExternal(q *Question, nameServers []NameServer) ([]SingleQueryResult, Trace, Status, error) {
 	ctx, cancel := context.WithTimeout(context.Background(), r.timeout)
 	defer cancel()
@@ -304,10 +305,12 @@ func (r *Resolver) LookupAllNameserversExternal(q *Question, nameServers []NameS
 	return retv, trace, StatusNoError, nil
 }
 
-// LookupAllNameserversIterative will send a query to all name servers at each level of DNS resolution. It starts at the root,
-// queries each NS, and then builds a de-duplicated list of the union of all responses. It repeats this process to the TLD
-// NS servers, etc.
-func (r *Resolver) LookupAllNameserversIterative(q *Question) (*AllNameServersResult, Trace, Status, error) {
+// LookupAllNameserversIterative will send a query to all name servers at each level of DNS resolution.
+// It starts at either the provided rootNameServers or r.rootNameServers if none are provided as arguments and queries all.
+// If the responses contain an authoritative answer, the function will return the result and a trace for each queried nameserver.
+// If the responses do not contain an authoritative answer, the function will continue to the next layer of nameservers.
+// At each layer, we'll de-duplicate the referral nameservers from the previous layer and query them.
+func (r *Resolver) LookupAllNameserversIterative(q *Question, rootNameServers []NameServer) (*AllNameServersResult, Trace, Status, error) {
 	perNameServerRetriesLimit := 2
 	ctx, cancel := context.WithTimeout(context.Background(), r.timeout)
 	defer cancel()
@@ -318,7 +321,11 @@ func (r *Resolver) LookupAllNameserversIterative(q *Question) (*AllNameServersRe
 	currentLayer := "."
 	isAuthoritative := false
 	var err error
-	currentLayerNameServers := r.rootNameServers
+	currentLayerNameServers := rootNameServers
+	if len(currentLayerNameServers) == 0 {
+		// no root nameservers provided, use the resolver's root nameservers
+		currentLayerNameServers = r.rootNameServers
+	}
 
 	for isAuthoritative == false {
 		var layerResults []ExtendedResult

From 35606210c11074e03236e8d0855c8041fe67ecd1 Mon Sep 17 00:00:00 2001
From: phillip-stephens <phillip@cs.stanford.edu>
Date: Thu, 12 Dec 2024 12:44:25 -0500
Subject: [PATCH 07/28] Clarify behavior in --help for all-nameservers and
 --name-servers

---
 src/cli/cli.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/cli/cli.go b/src/cli/cli.go
index 1bf5b58e..26963bef 100644
--- a/src/cli/cli.go
+++ b/src/cli/cli.go
@@ -44,14 +44,14 @@ type StatusHandler interface {
 // GeneralOptions core options for all ZDNS modules
 // Order here is the order they'll be printed to the user, so preserve alphabetical order
 type GeneralOptions struct {
-	LookupAllNameServers bool   `long:"all-nameservers" description:"Perform the lookup via all the nameservers for the domain."`
+	LookupAllNameServers bool   `long:"all-nameservers" description:"Behavior is dependent on --iterative. In --iterative, --all-name-servers will query all root servers, then all gtld servers, etc. recording the responses at each layer. In non-iterative mode, the query will be sent to all external resolvers specified in --name-servers."`
 	CacheSize            int    `long:"cache-size" default:"10000" description:"how many items can be stored in internal recursive cache"`
 	GoMaxProcs           int    `long:"go-processes" default:"0" description:"number of OS processes (GOMAXPROCS by default)"`
 	IterationTimeout     int    `long:"iteration-timeout" default:"8" description:"timeout for a single iterative step in an iterative query, in seconds. Only applicable with --iterative"`
 	IterativeResolution  bool   `long:"iterative" description:"Perform own iteration instead of relying on recursive resolver"`
 	MaxDepth             int    `long:"max-depth" default:"10" description:"how deep should we recurse when performing iterative lookups"`
 	NameServerMode       bool   `long:"name-server-mode" description:"Treats input as nameservers to query with a static query rather than queries to send to a static name server"`
-	NameServersString    string `long:"name-servers" description:"List of DNS servers to use. Can be passed as comma-delimited string or via @/path/to/file. If no port is specified, defaults to 53."`
+	NameServersString    string `long:"name-servers" description:"List of DNS servers to use. Can be passed as comma-delimited string or via @/path/to/file. If no port is specified, defaults to 53. If not provided, defaults to either the default root servers in --iterative or the recursive resolvers specified in /etc/resolv.conf or OS equivalent."`
 	UseNanoseconds       bool   `long:"nanoseconds" description:"Use nanosecond resolution timestamps in output"`
 	NetworkTimeout       int    `long:"network-timeout" default:"2" description:"timeout for round trip network operations, in seconds"`
 	DisableFollowCNAMEs  bool   `long:"no-follow-cnames" description:"do not follow CNAMEs/DNAMEs in the lookup process"`

From 0a95c878799dab4dd86d4c472a8401bbb2199d37 Mon Sep 17 00:00:00 2001
From: phillip-stephens <phillip@cs.stanford.edu>
Date: Thu, 12 Dec 2024 15:52:37 -0500
Subject: [PATCH 08/28] handle endless loop with next layer

---
 examples/all_nameservers_lookup/main.go | 25 +++++++++++++++++++++----
 examples/single_lookup/simple.go        |  2 +-
 src/zdns/lookup.go                      | 11 +++++++----
 3 files changed, 29 insertions(+), 9 deletions(-)

diff --git a/examples/all_nameservers_lookup/main.go b/examples/all_nameservers_lookup/main.go
index c15c1cfc..ebaa342e 100644
--- a/examples/all_nameservers_lookup/main.go
+++ b/examples/all_nameservers_lookup/main.go
@@ -14,8 +14,8 @@
 package main
 
 import (
-	"context"
 	"net"
+	"time"
 
 	"github.com/miekg/dns"
 	log "github.com/sirupsen/logrus"
@@ -29,15 +29,30 @@ func main() {
 	domain := "google.com"
 	dnsQuestion := &zdns.Question{Name: domain, Type: dns.TypeA, Class: dns.ClassINET}
 	resolver := initializeResolver()
-
-	log.Warn("\n\n This lookup just used the Cloudflare recursive resolver, let's run our own recursion.")
 	// Iterative Lookups start at the root nameservers and follow the chain of referrals to the authoritative nameservers.
-	result, _, status, err := resolver.LookupAllNameservers(context.Background(), dnsQuestion, 2)
+	result, _, status, err := resolver.LookupAllNameserversIterative(dnsQuestion, nil)
+	if err != nil {
+		log.Fatal("Error looking up domain: ", err)
+	}
+	log.Warnf("Result: %v", result)
+	log.Warnf("Status: %v", status)
+	log.Info("We can also specify which root nameservers to use by setting the argument.")
+
+	result, _, status, err = resolver.LookupAllNameserversIterative(dnsQuestion, []zdns.NameServer{{IP: net.ParseIP("198.41.0.4"), Port: 53}}) // a.root-servers.net
 	if err != nil {
 		log.Fatal("Error looking up domain: ", err)
 	}
 	log.Warnf("Result: %v", result)
 	log.Warnf("Status: %v", status)
+
+	log.Info("You can query multiple recursive resolvers as well")
+
+	externalResult, _, status, err := resolver.LookupAllNameserversExternal(dnsQuestion, []zdns.NameServer{{IP: net.ParseIP("1.1.1.1"), Port: 53}, {IP: net.ParseIP("8.8.8.8"), Port: 53}}) // Cloudflare and Google recursive resolvers, respectively
+	if err != nil {
+		log.Fatal("Error looking up domain: ", err)
+	}
+	log.Warnf("Result: %v", externalResult)
+	log.Warnf("Status: %v", status)
 	resolver.Close()
 }
 
@@ -54,6 +69,8 @@ func initializeResolver() *zdns.Resolver {
 	resolverConfig.ExternalNameServersV4 = []zdns.NameServer{{IP: net.ParseIP("1.1.1.1"), Port: 53}}
 	resolverConfig.RootNameServersV4 = zdns.RootServersV4
 	resolverConfig.IPVersionMode = zdns.IPv4Only
+	resolverConfig.Timeout = time.Minute
+	resolverConfig.IterativeTimeout = time.Minute
 	// Create a new Resolver object with the ResolverConfig object, it will retain all settings set on the ResolverConfig object
 	resolver, err := zdns.InitResolver(resolverConfig)
 	if err != nil {
diff --git a/examples/single_lookup/simple.go b/examples/single_lookup/simple.go
index 74d975ea..4aa83558 100644
--- a/examples/single_lookup/simple.go
+++ b/examples/single_lookup/simple.go
@@ -30,7 +30,7 @@ func main() {
 	dnsQuestion := &zdns.Question{Name: domain, Type: dns.TypeA, Class: dns.ClassINET}
 	resolver := initializeResolver()
 
-	result, _, status, err := resolver.ExternalLookup(dnsQuestion, &zdns.NameServer{IP: net.ParseIP("1.1.1.1"), Port: 53})
+	result, _, status, err := resolver.ExternalLookup(dnsQuestion, []zdns.NameServer{{IP: net.ParseIP("1.1.1.1"), Port: 53}})
 	if err != nil {
 		log.Fatal("Error looking up domain: ", err)
 	}
diff --git a/src/zdns/lookup.go b/src/zdns/lookup.go
index e70b0f5d..ccc5ea81 100644
--- a/src/zdns/lookup.go
+++ b/src/zdns/lookup.go
@@ -337,14 +337,17 @@ func (r *Resolver) LookupAllNameserversIterative(q *Question, rootNameServers []
 		} else {
 			retv.LayeredResponses[currentLayer] = layerResults
 		}
-		if isAuthoritative {
-			return &retv, trace, StatusNoError, nil
-		}
 		// Set the next layer to query
-		currentLayer, err = nextAuthority(q.Name, currentLayer)
+		var newLayer string
+		newLayer, err = nextAuthority(q.Name, currentLayer)
 		if err != nil {
 			return &retv, trace, StatusError, errors.Wrapf(err, "error determining next authority for layer %s", currentLayer)
 		}
+		if newLayer == currentLayer {
+			// we've reached the end of the authority chain, return
+			return &retv, trace, StatusNoError, nil
+		}
+		currentLayer = newLayer
 		currentLayerNameServers, err = r.extractNameServersFromLayerResults(layerResults)
 		if err != nil {
 			return &retv, trace, StatusError, errors.Wrapf(err, "error extracting nameservers from layer %s", currentLayer)

From 04d3dec565872cc46d80a3f8811c52848e3c45e9 Mon Sep 17 00:00:00 2001
From: phillip-stephens <phillip@cs.stanford.edu>
Date: Fri, 13 Dec 2024 10:47:10 -0500
Subject: [PATCH 09/28] add integration test

---
 testing/integration_tests.py | 72 ++++++++++++++++++++++++++++++++++++
 1 file changed, 72 insertions(+)

diff --git a/testing/integration_tests.py b/testing/integration_tests.py
index 5754bf4f..3288956d 100755
--- a/testing/integration_tests.py
+++ b/testing/integration_tests.py
@@ -1419,6 +1419,78 @@ def test_external_lookup_cache(self):
         # the second query has a much smaller response time than the first to show it's being cached
         self.assertTrue(first_duration / 50 > second_duration, f"Second query {second_duration} should be faster than the first {first_duration}")
 
+    def test_lookup_all_nameservers_single_zone_iterative(self):
+        """
+        Test that --all-nameservers --iterative lookups work with domains whose nameservers are all in the same zone
+        :return:
+        """
+        # google.com's nameservers are all in the .com zone, so we should only have to query the .com nameservers
+        c = "A google.com --all-nameservers --iterative"
+        cmd,res = self.run_zdns(c, "")
+        self.assertSuccess(res, cmd, "A")
+        # Check for layers
+        self.assertIn(".", res["results"]["A"]["data"]["per_layer_responses"], "Should have the root (.) layer")
+        self.assertIn("com", res["results"]["A"]["data"]["per_layer_responses"], "Should have the .com layer")
+        self.assertIn("google.com", res["results"]["A"]["data"]["per_layer_responses"], "Should have the google.com layer")
+        # check for a.root-servers.net, b.root-servers.net, ... m.root-servers.net
+        self.check_for_existance_of_root_and_com_nses(res)
+        # check for the google.com nameservers
+        actual_google_nses = []
+        for entry in res["results"]["A"]["data"]["per_layer_responses"]["google.com"]:
+            actual_google_nses.append(entry["nameserver"])
+        expected_google_nses = ["ns1.google.com", "ns2.google.com", "ns3.google.com", "ns4.google.com"]
+        for ns in expected_google_nses:
+            self.assertIn(ns, actual_google_nses, "Should have the google.com nameservers")
+
+    def check_for_existance_of_root_and_com_nses(self, res):
+        actual_root_ns = []
+        for entry in res["results"]["A"]["data"]["per_layer_responses"]["."]:
+            actual_root_ns.append(entry["nameserver"])
+        for letter in "abcdefghijklm":
+            self.assertIn(f"{letter}.root-servers.net", actual_root_ns, "Should have the root nameservers")
+        # check for the .com nameservers
+        actual_com_nses = []
+        for entry in res["results"]["A"]["data"]["per_layer_responses"]["com"]:
+            actual_com_nses.append(entry["nameserver"])
+        for letter in "abcdefghijklm":
+            self.assertIn(f"{letter}.gtld-servers.net", actual_com_nses, "Should have the .com nameservers")
+
+    def test_lookup_all_nameservers_multi_zone_iterative(self):
+        """
+        Test that --all-nameservers lookups work with domains whose nameservers have their nameservers in different zones
+        In this case, example.com has a/b.iana-servers.net as nameservers, which are in the .com zone, but whose nameservers
+        are dig -t NS iana-servers.com -> ns.icann.org, a/b/c.iana-servers.net.
+        """
+        # example.com has nameservers in .com, .org, and .net, we'll have to iteratively figure out their IP addresses too
+        c = "A example.com --all-nameservers --iterative"
+        cmd,res = self.run_zdns(c, "")
+        self.assertSuccess(res, cmd, "A")
+        # Check for layers
+        self.assertIn(".", res["results"]["A"]["data"]["per_layer_responses"], "Should have the root (.) layer")
+        self.assertIn("com", res["results"]["A"]["data"]["per_layer_responses"], "Should have the .com layer")
+        self.assertIn("example.com", res["results"]["A"]["data"]["per_layer_responses"], "Should have the example.com layer")
+        self.check_for_existance_of_root_and_com_nses(res)
+        # check for the example.com nameservers
+        actual_example_nses = []
+        for entry in res["results"]["A"]["data"]["per_layer_responses"]["example.com"]:
+            actual_example_nses.append(entry["nameserver"])
+        expected_example_nses = ["a.iana-servers.net", "b.iana-servers.net"]
+        for ns in expected_example_nses:
+            self.assertIn(ns, actual_example_nses, "Should have the example.com nameservers")
+
+    def test_lookup_all_nameservers_external_lookup(self):
+        """
+        Test that --all-nameservers lookups work with external resolvers
+        """
+        c = "A google.com --all-nameservers --name-servers='1.1.1.1,8.8.8.8'"
+        cmd,res = self.run_zdns(c, "")
+        self.assertSuccess(res, cmd, "A")
+        actual_resolvers = []
+        for entry in res["results"]["A"]["data"]:
+            actual_resolvers.append(entry["resolver"])
+        expected_resolvers = ["1.1.1.1:53", "8.8.8.8:53"]
+        for resolver in expected_resolvers:
+            self.assertIn(resolver, actual_resolvers, "Should have the expected resolvers")
 
 
 

From 7c5bda93924ca917fc873cde611b4cb8118f96b5 Mon Sep 17 00:00:00 2001
From: phillip-stephens <phillip@cs.stanford.edu>
Date: Fri, 13 Dec 2024 10:49:09 -0500
Subject: [PATCH 10/28] fixed up comments

---
 testing/integration_tests.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/testing/integration_tests.py b/testing/integration_tests.py
index 3288956d..fe08711a 100755
--- a/testing/integration_tests.py
+++ b/testing/integration_tests.py
@@ -1422,7 +1422,8 @@ def test_external_lookup_cache(self):
     def test_lookup_all_nameservers_single_zone_iterative(self):
         """
         Test that --all-nameservers --iterative lookups work with domains whose nameservers are all in the same zone
-        :return:
+        google.com has nameservers ns1/2/3/4.google.com, which are all in the .com zone and so will have their IPs
+        provided as additionals in the .com response
         """
         # google.com's nameservers are all in the .com zone, so we should only have to query the .com nameservers
         c = "A google.com --all-nameservers --iterative"
@@ -1459,7 +1460,8 @@ def test_lookup_all_nameservers_multi_zone_iterative(self):
         """
         Test that --all-nameservers lookups work with domains whose nameservers have their nameservers in different zones
         In this case, example.com has a/b.iana-servers.net as nameservers, which are in the .com zone, but whose nameservers
-        are dig -t NS iana-servers.com -> ns.icann.org, a/b/c.iana-servers.net.
+        are dig -t NS iana-servers.com -> ns.icann.org, a/b/c.iana-servers.net. This means the .com nameservers will not
+        provide the IPs in additionals.
         """
         # example.com has nameservers in .com, .org, and .net, we'll have to iteratively figure out their IP addresses too
         c = "A example.com --all-nameservers --iterative"
@@ -1480,7 +1482,7 @@ def test_lookup_all_nameservers_multi_zone_iterative(self):
 
     def test_lookup_all_nameservers_external_lookup(self):
         """
-        Test that --all-nameservers lookups work with external resolvers
+        Test that --all-nameservers lookups work with external resolvers: cloudflare.com
         """
         c = "A google.com --all-nameservers --name-servers='1.1.1.1,8.8.8.8'"
         cmd,res = self.run_zdns(c, "")

From ff63bd6d65dc3a04d72a313b14b6c48679b1525d Mon Sep 17 00:00:00 2001
From: phillip-stephens <phillip@cs.stanford.edu>
Date: Fri, 13 Dec 2024 10:49:28 -0500
Subject: [PATCH 11/28] fixed up comments

---
 testing/integration_tests.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/testing/integration_tests.py b/testing/integration_tests.py
index fe08711a..70069347 100755
--- a/testing/integration_tests.py
+++ b/testing/integration_tests.py
@@ -1482,7 +1482,7 @@ def test_lookup_all_nameservers_multi_zone_iterative(self):
 
     def test_lookup_all_nameservers_external_lookup(self):
         """
-        Test that --all-nameservers lookups work with external resolvers: cloudflare.com
+        Test that --all-nameservers lookups work with external resolvers: cloudflare.com and google.com
         """
         c = "A google.com --all-nameservers --name-servers='1.1.1.1,8.8.8.8'"
         cmd,res = self.run_zdns(c, "")

From c17987058d5a3c6cc5766775fc1b98b3959cc06e Mon Sep 17 00:00:00 2001
From: phillip-stephens <phillip@cs.stanford.edu>
Date: Mon, 16 Dec 2024 09:43:58 -0500
Subject: [PATCH 12/28] added context to function defs

---
 .../multi_thread_lookup/multi_threaded.go     |  5 ++--
 examples/single_lookup/simple.go              |  5 ++--
 src/cli/modules.go                            |  5 ++--
 src/modules/bindversion/bindversion.go        |  5 ++--
 src/modules/bindversion/bindversion_test.go   |  3 +-
 src/modules/dmarc/dmarc_test.go               |  3 +-
 src/modules/mxlookup/mx_lookup.go             |  5 ++--
 src/modules/spf/spf_test.go                   |  3 +-
 src/zdns/alookup.go                           | 11 ++++----
 src/zdns/answers.go                           |  2 +-
 src/zdns/conf.go                              |  2 +-
 src/zdns/lookup.go                            | 28 +++++++++----------
 src/zdns/nslookup.go                          |  5 ++--
 src/zdns/qa.go                                |  6 ++--
 src/zdns/resolver.go                          | 17 +++++------
 15 files changed, 58 insertions(+), 47 deletions(-)

diff --git a/examples/multi_thread_lookup/multi_threaded.go b/examples/multi_thread_lookup/multi_threaded.go
index b46fdc09..2a823082 100644
--- a/examples/multi_thread_lookup/multi_threaded.go
+++ b/examples/multi_thread_lookup/multi_threaded.go
@@ -14,6 +14,7 @@
 package main
 
 import (
+	"context"
 	"net"
 	"sync"
 
@@ -44,7 +45,7 @@ func main() {
 	wg.Add(2)
 	go func() {
 		defer wg.Done()
-		result1, _, _, err1 := resolver1.IterativeLookup(dnsQuestion1)
+		result1, _, _, err1 := resolver1.IterativeLookup(context.Background(), dnsQuestion1)
 		if err1 != nil {
 			log.Fatal("Error looking up domain: ", err1)
 		}
@@ -52,7 +53,7 @@ func main() {
 	}()
 	go func() {
 		defer wg.Done()
-		result2, _, _, err2 := resolver2.IterativeLookup(dnsQuestion2)
+		result2, _, _, err2 := resolver2.IterativeLookup(context.Background(), dnsQuestion2)
 		if err2 != nil {
 			log.Fatal("Error looking up domain: ", err2)
 		}
diff --git a/examples/single_lookup/simple.go b/examples/single_lookup/simple.go
index 4aa83558..9d7c9176 100644
--- a/examples/single_lookup/simple.go
+++ b/examples/single_lookup/simple.go
@@ -14,6 +14,7 @@
 package main
 
 import (
+	"context"
 	"encoding/json"
 	"net"
 
@@ -30,7 +31,7 @@ func main() {
 	dnsQuestion := &zdns.Question{Name: domain, Type: dns.TypeA, Class: dns.ClassINET}
 	resolver := initializeResolver()
 
-	result, _, status, err := resolver.ExternalLookup(dnsQuestion, []zdns.NameServer{{IP: net.ParseIP("1.1.1.1"), Port: 53}})
+	result, _, status, err := resolver.ExternalLookup(context.Background(), dnsQuestion, &zdns.NameServer{IP: net.ParseIP("1.1.1.1"), Port: 53})
 	if err != nil {
 		log.Fatal("Error looking up domain: ", err)
 	}
@@ -44,7 +45,7 @@ func main() {
 
 	log.Warn("\n\n This lookup just used the Cloudflare recursive resolver, let's run our own recursion.")
 	// Iterative Lookups start at the root nameservers and follow the chain of referrals to the authoritative nameservers.
-	result, trace, status, err := resolver.IterativeLookup(&zdns.Question{Name: domain, Type: dns.TypeA, Class: dns.ClassINET})
+	result, trace, status, err := resolver.IterativeLookup(context.Background(), &zdns.Question{Name: domain, Type: dns.TypeA, Class: dns.ClassINET})
 	if err != nil {
 		log.Fatal("Error looking up domain: ", err)
 	}
diff --git a/src/cli/modules.go b/src/cli/modules.go
index 53a22d08..1b13e66b 100644
--- a/src/cli/modules.go
+++ b/src/cli/modules.go
@@ -14,6 +14,7 @@
 package cli
 
 import (
+	"context"
 	"fmt"
 
 	"github.com/miekg/dns"
@@ -179,9 +180,9 @@ func (lm *BasicLookupModule) Lookup(resolver *zdns.Resolver, lookupName string,
 		return resolver.LookupAllNameserversExternal(&zdns.Question{Name: lookupName, Type: lm.DNSType, Class: lm.DNSClass}, nil)
 	}
 	if lm.IsIterative {
-		return resolver.IterativeLookup(&zdns.Question{Name: lookupName, Type: lm.DNSType, Class: lm.DNSClass})
+		return resolver.IterativeLookup(context.Background(), &zdns.Question{Name: lookupName, Type: lm.DNSType, Class: lm.DNSClass})
 	}
-	return resolver.ExternalLookup(&zdns.Question{Type: lm.DNSType, Class: lm.DNSClass, Name: lookupName}, nameServer)
+	return resolver.ExternalLookup(context.Background(), &zdns.Question{Type: lm.DNSType, Class: lm.DNSClass, Name: lookupName}, nameServer)
 }
 
 func GetLookupModule(name string) (LookupModule, error) {
diff --git a/src/modules/bindversion/bindversion.go b/src/modules/bindversion/bindversion.go
index 640dc680..9f8d1bb2 100644
--- a/src/modules/bindversion/bindversion.go
+++ b/src/modules/bindversion/bindversion.go
@@ -15,6 +15,7 @@
 package bindversion
 
 import (
+	"context"
 	"github.com/miekg/dns"
 	"github.com/pkg/errors"
 
@@ -55,9 +56,9 @@ func (bindVersionMod *BindVersionLookupModule) Lookup(r *zdns.Resolver, lookupNa
 	var status zdns.Status
 	var err error
 	if bindVersionMod.IsIterative {
-		innerRes, trace, status, err = r.IterativeLookup(&zdns.Question{Name: BindVersionQueryName, Type: dns.TypeTXT, Class: dns.ClassCHAOS})
+		innerRes, trace, status, err = r.IterativeLookup(context.Background(), &zdns.Question{Name: BindVersionQueryName, Type: dns.TypeTXT, Class: dns.ClassCHAOS})
 	} else {
-		innerRes, trace, status, err = r.ExternalLookup(&zdns.Question{Name: BindVersionQueryName, Type: dns.TypeTXT, Class: dns.ClassCHAOS}, nameServer)
+		innerRes, trace, status, err = r.ExternalLookup(context.Background(), &zdns.Question{Name: BindVersionQueryName, Type: dns.TypeTXT, Class: dns.ClassCHAOS}, nameServer)
 	}
 	resString, resStatus, err := zdns.CheckTxtRecords(innerRes, status, nil, err)
 	res := Result{BindVersion: resString}
diff --git a/src/modules/bindversion/bindversion_test.go b/src/modules/bindversion/bindversion_test.go
index df38e4ef..8173b9c9 100644
--- a/src/modules/bindversion/bindversion_test.go
+++ b/src/modules/bindversion/bindversion_test.go
@@ -15,6 +15,7 @@
 package bindversion
 
 import (
+	"context"
 	"net"
 	"testing"
 
@@ -35,7 +36,7 @@ var queries []QueryRecord
 // DoSingleDstServerLookup(r *Resolver, q Question, nameServer string, isIterative bool) (*SingleQueryResult, Trace, Status, error)
 type MockLookup struct{}
 
-func (ml MockLookup) DoDstServersLookup(r *zdns.Resolver, question zdns.Question, nameServers []zdns.NameServer, isIterative bool) (*zdns.SingleQueryResult, zdns.Trace, zdns.Status, error) {
+func (ml MockLookup) DoDstServersLookup(ctx context.Context, r *zdns.Resolver, question zdns.Question, nameServers []zdns.NameServer, isIterative bool) (*zdns.SingleQueryResult, zdns.Trace, zdns.Status, error) {
 	queries = append(queries, QueryRecord{q: question, NameServer: &nameServers[0]})
 	if res, ok := mockResults[question.Name]; ok {
 		return res, nil, zdns.StatusNoError, nil
diff --git a/src/modules/dmarc/dmarc_test.go b/src/modules/dmarc/dmarc_test.go
index 513cb533..d6d0c2b5 100644
--- a/src/modules/dmarc/dmarc_test.go
+++ b/src/modules/dmarc/dmarc_test.go
@@ -15,6 +15,7 @@
 package dmarc
 
 import (
+	"context"
 	"net"
 	"testing"
 
@@ -35,7 +36,7 @@ var queries []QueryRecord
 
 type MockLookup struct{}
 
-func (ml MockLookup) DoDstServersLookup(r *zdns.Resolver, question zdns.Question, nameServers []zdns.NameServer, isIterative bool) (*zdns.SingleQueryResult, zdns.Trace, zdns.Status, error) {
+func (ml MockLookup) DoDstServersLookup(ctx context.Context, r *zdns.Resolver, question zdns.Question, nameServers []zdns.NameServer, isIterative bool) (*zdns.SingleQueryResult, zdns.Trace, zdns.Status, error) {
 	queries = append(queries, QueryRecord{question, &nameServers[0]})
 	if res, ok := mockResults[question.Name]; ok {
 		return res, nil, zdns.StatusNoError, nil
diff --git a/src/modules/mxlookup/mx_lookup.go b/src/modules/mxlookup/mx_lookup.go
index 121fcac0..40107e4d 100644
--- a/src/modules/mxlookup/mx_lookup.go
+++ b/src/modules/mxlookup/mx_lookup.go
@@ -14,6 +14,7 @@
 package mxlookup
 
 import (
+	"context"
 	"strings"
 
 	"github.com/miekg/dns"
@@ -95,9 +96,9 @@ func (mxMod *MXLookupModule) Lookup(r *zdns.Resolver, lookupName string, nameSer
 	var status zdns.Status
 	var err error
 	if mxMod.BasicLookupModule.IsIterative {
-		res, trace, status, err = r.IterativeLookup(&zdns.Question{Name: lookupName, Type: dns.TypeMX, Class: dns.ClassINET})
+		res, trace, status, err = r.IterativeLookup(context.Background(), &zdns.Question{Name: lookupName, Type: dns.TypeMX, Class: dns.ClassINET})
 	} else {
-		res, trace, status, err = r.ExternalLookup(&zdns.Question{Name: lookupName, Type: dns.TypeMX, Class: dns.ClassINET}, nameServer)
+		res, trace, status, err = r.ExternalLookup(context.Background(), &zdns.Question{Name: lookupName, Type: dns.TypeMX, Class: dns.ClassINET}, nameServer)
 	}
 	if status != zdns.StatusNoError || err != nil {
 		return nil, trace, status, err
diff --git a/src/modules/spf/spf_test.go b/src/modules/spf/spf_test.go
index a5a04150..d204e1ae 100644
--- a/src/modules/spf/spf_test.go
+++ b/src/modules/spf/spf_test.go
@@ -15,6 +15,7 @@
 package spf
 
 import (
+	"context"
 	"net"
 	"testing"
 
@@ -35,7 +36,7 @@ var queries []QueryRecord
 
 type MockLookup struct{}
 
-func (ml MockLookup) DoDstServersLookup(r *zdns.Resolver, question zdns.Question, nameServers []zdns.NameServer, isIterative bool) (*zdns.SingleQueryResult, zdns.Trace, zdns.Status, error) {
+func (ml MockLookup) DoDstServersLookup(ctx context.Context, r *zdns.Resolver, question zdns.Question, nameServers []zdns.NameServer, isIterative bool) (*zdns.SingleQueryResult, zdns.Trace, zdns.Status, error) {
 	queries = append(queries, QueryRecord{question, &nameServers[0]})
 	if res, ok := mockResults[question.Name]; ok {
 		return res, nil, zdns.StatusNoError, nil
diff --git a/src/zdns/alookup.go b/src/zdns/alookup.go
index 9520e1f8..3e8b5023 100644
--- a/src/zdns/alookup.go
+++ b/src/zdns/alookup.go
@@ -14,6 +14,7 @@
 package zdns
 
 import (
+	"context"
 	"strings"
 
 	"github.com/pkg/errors"
@@ -23,7 +24,7 @@ import (
 	"github.com/zmap/zdns/src/internal/util"
 )
 
-// DoTargetedLookup performs a lookup of the given domain name against the given nameserver, looking up both IPv4 and IPv6 addresses
+// DoTargetedLookup performs a lookup of the given name name against the given nameserver, looking up both IPv4 and IPv6 addresses
 // Will follow CNAME records as well as A/AAAA records to get IP addresses
 func (r *Resolver) DoTargetedLookup(name string, nameServer *NameServer, isIterative, lookupA, lookupAAAA bool) (*IPResult, Trace, Status, error) {
 	name = strings.ToLower(name)
@@ -38,9 +39,9 @@ func (r *Resolver) DoTargetedLookup(name string, nameServer *NameServer, isItera
 	var err error
 
 	if lookupA && isIterative {
-		singleQueryRes, ipv4Trace, ipv4status, err = r.IterativeLookup(&Question{Name: name, Type: dns.TypeA, Class: dns.ClassINET})
+		singleQueryRes, ipv4Trace, ipv4status, err = r.IterativeLookup(context.Background(), &Question{Name: name, Type: dns.TypeA, Class: dns.ClassINET})
 	} else if lookupA {
-		singleQueryRes, ipv4Trace, ipv4status, err = r.ExternalLookup(&Question{Name: name, Type: dns.TypeA, Class: dns.ClassINET}, nameServer)
+		singleQueryRes, ipv4Trace, ipv4status, err = r.ExternalLookup(context.Background(), &Question{Name: name, Type: dns.TypeA, Class: dns.ClassINET}, nameServer)
 	}
 	ipv4, _ = getIPAddressesFromQueryResult(singleQueryRes, "A", name)
 	if len(ipv4) > 0 {
@@ -50,9 +51,9 @@ func (r *Resolver) DoTargetedLookup(name string, nameServer *NameServer, isItera
 	}
 	singleQueryRes = &SingleQueryResult{} // reset result
 	if lookupAAAA && isIterative {
-		singleQueryRes, ipv6Trace, ipv6status, _ = r.IterativeLookup(&Question{Name: name, Type: dns.TypeAAAA, Class: dns.ClassINET})
+		singleQueryRes, ipv6Trace, ipv6status, _ = r.IterativeLookup(context.Background(), &Question{Name: name, Type: dns.TypeAAAA, Class: dns.ClassINET})
 	} else if lookupAAAA {
-		singleQueryRes, ipv6Trace, ipv6status, _ = r.ExternalLookup(&Question{Name: name, Type: dns.TypeAAAA, Class: dns.ClassINET}, nameServer)
+		singleQueryRes, ipv6Trace, ipv6status, _ = r.ExternalLookup(context.Background(), &Question{Name: name, Type: dns.TypeAAAA, Class: dns.ClassINET}, nameServer)
 	}
 	ipv6, _ = getIPAddressesFromQueryResult(singleQueryRes, "AAAA", name)
 	if len(ipv6) > 0 {
diff --git a/src/zdns/answers.go b/src/zdns/answers.go
index f85c0dd1..fef319d9 100644
--- a/src/zdns/answers.go
+++ b/src/zdns/answers.go
@@ -202,7 +202,7 @@ type SMIMEAAnswer struct {
 
 type SOAAnswer struct {
 	Answer
-	Ns      string `json:"ns" groups:"short,normal,long,trace"`
+	Ns      string `json:"IP" groups:"short,normal,long,trace"`
 	Mbox    string `json:"mbox" groups:"short,normal,long,trace"`
 	Serial  uint32 `json:"serial" groups:"short,normal,long,trace"`
 	Refresh uint32 `json:"refresh" groups:"short,normal,long,trace"`
diff --git a/src/zdns/conf.go b/src/zdns/conf.go
index d0d0f09c..97d9889b 100644
--- a/src/zdns/conf.go
+++ b/src/zdns/conf.go
@@ -23,7 +23,7 @@ const (
 )
 
 type TargetedDomain struct {
-	Domain      string   `json:"domain"`
+	Domain      string   `json:"name"`
 	Nameservers []string `json:"nameservers"`
 }
 
diff --git a/src/zdns/lookup.go b/src/zdns/lookup.go
index ccc5ea81..1db8d9a6 100644
--- a/src/zdns/lookup.go
+++ b/src/zdns/lookup.go
@@ -67,17 +67,17 @@ func GetDNSServers(path string) (ipv4, ipv6 []string, err error) {
 
 // Lookup client interface for help in mocking
 type Lookuper interface {
-	DoDstServersLookup(r *Resolver, q Question, nameServer []NameServer, isIterative bool) (*SingleQueryResult, Trace, Status, error)
+	DoDstServersLookup(ctx context.Context, r *Resolver, q Question, nameServer []NameServer, isIterative bool) (*SingleQueryResult, Trace, Status, error)
 }
 
 type LookupClient struct{}
 
 // DoDstServersLookup performs a DNS lookup for a given question against a list of interchangeable nameservers
-func (lc LookupClient) DoDstServersLookup(r *Resolver, q Question, nameServers []NameServer, isIterative bool) (*SingleQueryResult, Trace, Status, error) {
-	return r.doDstServersLookup(q, nameServers, isIterative)
+func (lc LookupClient) DoDstServersLookup(ctx context.Context, r *Resolver, q Question, nameServers []NameServer, isIterative bool) (*SingleQueryResult, Trace, Status, error) {
+	return r.doDstServersLookup(ctx, q, nameServers, isIterative)
 }
 
-func (r *Resolver) doDstServersLookup(q Question, nameServers []NameServer, isIterative bool) (*SingleQueryResult, Trace, Status, error) {
+func (r *Resolver) doDstServersLookup(ctx context.Context, q Question, nameServers []NameServer, isIterative bool) (*SingleQueryResult, Trace, Status, error) {
 	var err error
 	// nameserver is required
 	if len(nameServers) == 0 {
@@ -91,7 +91,7 @@ func (r *Resolver) doDstServersLookup(q Question, nameServers []NameServer, isIt
 		// if that looks likely, use it as is
 		if err != nil && !util.IsStringValidDomainName(q.Name) {
 			return nil, nil, StatusIllegalInput, err
-			// q.Name is a valid domain name, we can continue
+			// q.Name is a valid name name, we can continue
 		} else {
 			// remove trailing "." added by dns.ReverseAddr
 			q.Name = qname[:len(qname)-1]
@@ -291,7 +291,7 @@ func (r *Resolver) LookupAllNameserversExternal(q *Question, nameServers []NameS
 		if util.HasCtxExpired(ctx) {
 			return retv, trace, StatusTimeout, errors.New("context expired")
 		}
-		result, currTrace, status, err := r.ExternalLookup(q, &ns)
+		result, currTrace, status, err := r.ExternalLookup(ctx, q, &ns)
 		trace = append(trace, currTrace...)
 		if err != nil {
 			log.Errorf("LookupAllNameserversExternal of name %s errored for %s/%s: %v", q.Name, ns.DomainName, ns.IP.String(), err)
@@ -352,6 +352,9 @@ func (r *Resolver) LookupAllNameserversIterative(q *Question, rootNameServers []
 		if err != nil {
 			return &retv, trace, StatusError, errors.Wrapf(err, "error extracting nameservers from layer %s", currentLayer)
 		}
+		if len(currentLayerNameServers) == 0 {
+			return &retv, trace, StatusError, errors.New("no nameservers found in layer " + currentLayer)
+		}
 	}
 	return &retv, trace, StatusNoError, nil
 }
@@ -391,10 +394,7 @@ func (r *Resolver) extractNameServersFromLayerResults(layerResults []ExtendedRes
 		}
 	}
 	for _, additionals := range uniqueAdditionals {
-		if strings.HasSuffix(additionals.Name, ".") {
-			// TODO checking here to see if this is an issue
-			log.Fatalf("Name %s has a trailing dot", additionals.Name)
-		}
+		additionals.Name = strings.TrimSuffix(additionals.Name, ".")
 		if additionals.RrType == dns.TypeA {
 			if ns, ok := v4NameServers[additionals.Name]; ok {
 				ns.IP = net.ParseIP(additionals.Answer)
@@ -512,7 +512,7 @@ func (r *Resolver) queryAllNameServersInLayer(ctx context.Context, perNameServer
 				trace = append(trace, nsTrace...)
 				// we've populated NS IP, we can proceed
 			}
-			result, currTrace, status, err := r.ExternalLookup(q, &nameServer)
+			result, currTrace, status, err := r.ExternalLookup(ctx, q, &nameServer)
 			trace = append(trace, currTrace...)
 			if err == nil && status == StatusNoError {
 				extResult = &ExtendedResult{
@@ -664,7 +664,7 @@ func getRandomNonQueriedNameServer(nameServers []NameServer, queriedNameServers
 
 // cachedLookup performs a DNS lookup with caching
 // returns the result, whether it was cached, the status, and an error if one occurred
-// layer is the domain name layer we're currently querying ex: ".", "com.", "example.com."
+// layer is the name name layer we're currently querying ex: ".", "com.", "example.com."
 // depth is the current depth of the lookup, used for iterative lookups
 // requestIteration is whether to set the "recursion desired" bit in the DNS query
 // cacheBasedOnNameServer is whether to consider a cache hit based on DNS question and nameserver, or just question
@@ -1234,12 +1234,12 @@ func FindTxtRecord(res *SingleQueryResult, regex *regexp.Regexp) (string, error)
 }
 
 // populateResults is a helper function to populate the candidateSet, cnameSet, and garbage maps to follow CNAMES
-// These maps are keyed by the domain name and contain the relevant answers for that domain
+// These maps are keyed by the name name and contain the relevant answers for that name
 // candidateSet is a map of Answers that have a type matching the requested type.
 // cnameSet is a map of Answers that are CNAME records
 // dnameSet is a map of Answers that are DNAME records
 // garbage is a map of Answers that are not of the requested type or CNAME records
-// follows CNAME/DNAME and A/AAAA records to get all IPs for a given domain
+// follows CNAME/DNAME and A/AAAA records to get all IPs for a given name
 func populateResults(records []interface{}, dnsType uint16, candidateSet map[string][]Answer, cnameSet map[string][]Answer, dnameSet map[string][]Answer, garbage map[string][]Answer) {
 	var ans Answer
 	var ok bool
diff --git a/src/zdns/nslookup.go b/src/zdns/nslookup.go
index 73dec5fd..464f670c 100644
--- a/src/zdns/nslookup.go
+++ b/src/zdns/nslookup.go
@@ -14,6 +14,7 @@
 package zdns
 
 import (
+	"context"
 	"strings"
 
 	"github.com/miekg/dns"
@@ -52,9 +53,9 @@ func (r *Resolver) DoNSLookup(lookupName string, nameServer *NameServer, isItera
 	var status Status
 	var err error
 	if isIterative {
-		ns, trace, status, err = r.IterativeLookup(&Question{Name: lookupName, Type: dns.TypeNS, Class: dns.ClassINET})
+		ns, trace, status, err = r.IterativeLookup(context.Background(), &Question{Name: lookupName, Type: dns.TypeNS, Class: dns.ClassINET})
 	} else {
-		ns, trace, status, err = r.ExternalLookup(&Question{Name: lookupName, Type: dns.TypeNS, Class: dns.ClassINET}, nameServer)
+		ns, trace, status, err = r.ExternalLookup(context.Background(), &Question{Name: lookupName, Type: dns.TypeNS, Class: dns.ClassINET}, nameServer)
 
 	}
 
diff --git a/src/zdns/qa.go b/src/zdns/qa.go
index f0ded15f..0461ccf2 100644
--- a/src/zdns/qa.go
+++ b/src/zdns/qa.go
@@ -52,7 +52,7 @@ type TraceStep struct {
 	Try        int               `json:"try" groups:"trace"`
 }
 
-// Result contains all the metadata from a complete lookup(s) for a domain. Results is keyed with the ModuleName.
+// Result contains all the metadata from a complete lookup(s) for a name. Results is keyed with the ModuleName.
 type Result struct {
 	AlteredName string                        `json:"altered_name,omitempty" groups:"short,normal,long,trace"`
 	Name        string                        `json:"name,omitempty" groups:"short,normal,long,trace"`
@@ -63,7 +63,7 @@ type Result struct {
 	Results     map[string]SingleModuleResult `json:"results,omitempty" groups:"short,normal,long,trace"`
 }
 
-// SingleModuleResult contains all the metadata from a complete lookup for a domain, potentially after following many CNAMEs/etc.
+// SingleModuleResult contains all the metadata from a complete lookup for a name, potentially after following many CNAMEs/etc.
 type SingleModuleResult struct {
 	Status    string      `json:"status,omitempty" groups:"short,normal,long,trace"`
 	Error     string      `json:"error,omitempty" groups:"short,normal,long,trace"`
@@ -87,7 +87,7 @@ type SingleQueryResult struct {
 type ExtendedResult struct {
 	Res        SingleQueryResult `json:"result,omitempty" groups:"short,normal,long,trace"`
 	Status     Status            `json:"status" groups:"short,normal,long,trace"`
-	Nameserver string            `json:"nameserver" groups:"short,normal,long,trace"` // NS domain name
+	Nameserver string            `json:"nameserver" groups:"short,normal,long,trace"` // NS name name
 }
 
 type AllNameServersResult struct {
diff --git a/src/zdns/resolver.go b/src/zdns/resolver.go
index fa496fc9..51f642c5 100644
--- a/src/zdns/resolver.go
+++ b/src/zdns/resolver.go
@@ -15,6 +15,7 @@
 package zdns
 
 import (
+	"context"
 	"fmt"
 	"math/rand"
 	"net"
@@ -84,7 +85,7 @@ type ResolverConfig struct {
 	ExternalNameServersV6 []NameServer // v6 name servers used for external lookups
 	RootNameServersV4     []NameServer // v4 root servers used for iterative lookups
 	RootNameServersV6     []NameServer // v6 root servers used for iterative lookups
-	LookupAllNameServers  bool         // perform the lookup via all the nameservers for the domain
+	LookupAllNameServers  bool         // perform the lookup via all the nameservers for the name
 	FollowCNAMEs          bool         // whether iterative lookups should follow CNAMEs/DNAMEs
 	DNSConfigFilePath     string       // path to the DNS config file, ex: /etc/resolv.conf
 
@@ -279,7 +280,7 @@ type Resolver struct {
 
 	networkTimeout             time.Duration // timeout for a single on-the-wire network call
 	iterativeTimeout           time.Duration // timeout for a layer of the iterative lookup
-	timeout                    time.Duration // timeout for the entire domain lookup
+	timeout                    time.Duration // timeout for the entire name lookup
 	maxDepth                   int
 	externalNameServers        []NameServer // name servers used by external lookups (either OS or user specified)
 	rootNameServers            []NameServer // root servers used for iterative lookups
@@ -535,14 +536,14 @@ func (r *Resolver) getConnectionInfo(nameServer *NameServer) (*ConnectionInfo, e
 					}
 					var tlsConn *tls.Conn
 					if len(nameServer.DomainName) != 0 && r.verifyServerCert {
-						// domain name provided, we can verify the server's certificate
+						// name name provided, we can verify the server's certificate
 						tlsConn = tls.Client(conn, &tls.Config{
 							InsecureSkipVerify: false,
 							RootCAs:            r.rootCAs,
 							ServerName:         nameServer.DomainName,
 						})
 					} else {
-						// If no domain name is provided, we can't verify the server's certificate
+						// If no name name is provided, we can't verify the server's certificate
 						tlsConn = tls.Client(conn, &tls.Config{
 							InsecureSkipVerify: true,
 						})
@@ -596,7 +597,7 @@ func getNewTCPConn(nameServer *NameServer, connInfo *ConnectionInfo) error {
 // multiple lookups concurrently, create a new Resolver object for each concurrent lookup.
 // Returns the result of the lookup, the trace of the lookup (what each nameserver along the lookup returned), the
 // status of the lookup, and any error that occurred.
-func (r *Resolver) ExternalLookup(q *Question, dstServer *NameServer) (*SingleQueryResult, Trace, Status, error) {
+func (r *Resolver) ExternalLookup(ctx context.Context, q *Question, dstServer *NameServer) (*SingleQueryResult, Trace, Status, error) {
 	if r.isClosed {
 		log.Fatal("resolver has been closed, cannot perform lookup")
 	}
@@ -614,7 +615,7 @@ func (r *Resolver) ExternalLookup(q *Question, dstServer *NameServer) (*SingleQu
 	}
 	// dstServer has been validated and has a port, continue with lookup
 	r.lastUsedExternalNameServer = dstServer
-	lookup, trace, status, err := r.lookupClient.DoDstServersLookup(r, *q, []NameServer{*dstServer}, false)
+	lookup, trace, status, err := r.lookupClient.DoDstServersLookup(ctx, r, *q, []NameServer{*dstServer}, false)
 	return lookup, trace, status, err
 }
 
@@ -624,11 +625,11 @@ func (r *Resolver) ExternalLookup(q *Question, dstServer *NameServer) (*SingleQu
 // multiple lookups concurrently, create a new Resolver object for each concurrent lookup.
 // Returns the result of the lookup, the trace of the lookup (what each nameserver along the lookup returned), the
 // status of the lookup, and any error that occurred.
-func (r *Resolver) IterativeLookup(q *Question) (*SingleQueryResult, Trace, Status, error) {
+func (r *Resolver) IterativeLookup(ctx context.Context, q *Question) (*SingleQueryResult, Trace, Status, error) {
 	if r.isClosed {
 		log.Fatal("resolver has been closed, cannot perform lookup")
 	}
-	return r.lookupClient.DoDstServersLookup(r, *q, r.rootNameServers, true)
+	return r.lookupClient.DoDstServersLookup(ctx, r, *q, r.rootNameServers, true)
 }
 
 // Close cleans up any resources used by the resolver. This should be called when the resolver is no longer needed.

From 4adff9f2f02f79e14f379f47e5b0cbf1e3581245 Mon Sep 17 00:00:00 2001
From: phillip-stephens <phillip@cs.stanford.edu>
Date: Mon, 16 Dec 2024 10:59:59 -0500
Subject: [PATCH 13/28] First unit test passing

---
 src/zdns/lookup_test.go | 960 ++++++++++++++++++++++------------------
 1 file changed, 531 insertions(+), 429 deletions(-)

diff --git a/src/zdns/lookup_test.go b/src/zdns/lookup_test.go
index 9cde0bc0..eb3af5a2 100644
--- a/src/zdns/lookup_test.go
+++ b/src/zdns/lookup_test.go
@@ -14,7 +14,10 @@
 package zdns
 
 import (
+	"context"
 	"encoding/hex"
+	"fmt"
+	log "github.com/sirupsen/logrus"
 	"math/rand"
 	"net"
 	"reflect"
@@ -30,20 +33,20 @@ import (
 	"github.com/zmap/zdns/src/internal/util"
 )
 
-type domainNS struct {
-	domain string
-	ns     string
+type nameAndIP struct {
+	name string
+	IP   string
 }
 
-var mockResults = make(map[domainNS]SingleQueryResult)
+var mockResults = make(map[nameAndIP]SingleQueryResult)
 
-var protocolStatus = make(map[domainNS]Status)
+var protocolStatus = make(map[nameAndIP]Status)
 
 type MockLookupClient struct{}
 
-func (mc MockLookupClient) DoDstServersLookup(r *Resolver, q Question, nameServers []NameServer, isIterative bool) (*SingleQueryResult, Trace, Status, error) {
+func (mc MockLookupClient) DoDstServersLookup(ctx context.Context, r *Resolver, q Question, nameServers []NameServer, isIterative bool) (*SingleQueryResult, Trace, Status, error) {
 	ns := nameServers[rand.Intn(len(nameServers))]
-	curDomainNs := domainNS{domain: q.Name, ns: ns.String()}
+	curDomainNs := nameAndIP{name: q.Name, IP: ns.String()}
 	if res, ok := mockResults[curDomainNs]; ok {
 		var status = StatusNoError
 		if protStatus, ok := protocolStatus[curDomainNs]; ok {
@@ -56,8 +59,8 @@ func (mc MockLookupClient) DoDstServersLookup(r *Resolver, q Question, nameServe
 }
 
 func InitTest(t *testing.T) *ResolverConfig {
-	protocolStatus = make(map[domainNS]Status)
-	mockResults = make(map[domainNS]SingleQueryResult)
+	protocolStatus = make(map[nameAndIP]Status)
+	mockResults = make(map[nameAndIP]SingleQueryResult)
 
 	mc := MockLookupClient{}
 	config := NewResolverConfig()
@@ -630,7 +633,7 @@ func TestOneA(t *testing.T) {
 
 	domain1 := "example.com"
 	ns1 := &config.ExternalNameServersV4[0]
-	domainNS1 := domainNS{domain: domain1, ns: ns1.String()}
+	domainNS1 := nameAndIP{name: domain1, IP: ns1.String()}
 
 	mockResults[domainNS1] = SingleQueryResult{
 		Answers: []interface{}{Answer{
@@ -658,7 +661,7 @@ func TestTwoA(t *testing.T) {
 
 	domain1 := "example.com"
 	ns1 := &config.ExternalNameServersV4[0]
-	domainNS1 := domainNS{domain: domain1, ns: ns1.String()}
+	domainNS1 := nameAndIP{name: domain1, IP: ns1.String()}
 
 	mockResults[domainNS1] = SingleQueryResult{
 		Answers: []interface{}{Answer{
@@ -693,7 +696,7 @@ func TestQuadAWithoutFlag(t *testing.T) {
 
 	domain1 := "example.com"
 	ns1 := &config.ExternalNameServersV4[0]
-	domainNS1 := domainNS{domain: domain1, ns: ns1.String()}
+	domainNS1 := nameAndIP{name: domain1, IP: ns1.String()}
 
 	mockResults[domainNS1] = SingleQueryResult{
 		Answers: []interface{}{Answer{
@@ -729,7 +732,7 @@ func TestOnlyQuadA(t *testing.T) {
 
 	domain1 := "example.com"
 	ns1 := &config.ExternalNameServersV4[0]
-	domainNS1 := domainNS{domain: domain1, ns: ns1.String()}
+	domainNS1 := nameAndIP{name: domain1, IP: ns1.String()}
 
 	mockResults[domainNS1] = SingleQueryResult{
 		Answers: []interface{}{Answer{
@@ -759,7 +762,7 @@ func TestAandQuadA(t *testing.T) {
 
 	domain1 := "example.com"
 	ns1 := &config.ExternalNameServersV4[0]
-	domainNS1 := domainNS{domain: domain1, ns: ns1.String()}
+	domainNS1 := nameAndIP{name: domain1, IP: ns1.String()}
 
 	mockResults[domainNS1] = SingleQueryResult{
 		Answers: []interface{}{Answer{
@@ -795,7 +798,7 @@ func TestTwoQuadA(t *testing.T) {
 
 	domain1 := "example.com"
 	ns1 := &config.ExternalNameServersV4[0]
-	domainNS1 := domainNS{domain: domain1, ns: ns1.String()}
+	domainNS1 := nameAndIP{name: domain1, IP: ns1.String()}
 
 	mockResults[domainNS1] = SingleQueryResult{
 		Answers: []interface{}{Answer{
@@ -832,7 +835,7 @@ func TestNoResults(t *testing.T) {
 
 	domain1 := "example.com"
 	ns1 := &config.ExternalNameServersV4[0]
-	domainNS1 := domainNS{domain: domain1, ns: ns1.String()}
+	domainNS1 := nameAndIP{name: domain1, IP: ns1.String()}
 
 	mockResults[domainNS1] = SingleQueryResult{
 		Answers:     nil,
@@ -854,7 +857,7 @@ func TestQuadAWithCname(t *testing.T) {
 
 	domain1 := "cname.example.com"
 	ns1 := &config.ExternalNameServersV4[0]
-	domainNS1 := domainNS{domain: domain1, ns: ns1.String()}
+	domainNS1 := nameAndIP{name: domain1, IP: ns1.String()}
 
 	mockResults[domainNS1] = SingleQueryResult{
 		Answers: []interface{}{Answer{
@@ -889,7 +892,7 @@ func TestUnexpectedMxOnly(t *testing.T) {
 
 	domain1 := "example.com"
 	ns1 := &config.ExternalNameServersV4[0]
-	domainNS1 := domainNS{domain: domain1, ns: ns1.String()}
+	domainNS1 := nameAndIP{name: domain1, IP: ns1.String()}
 
 	mockResults[domainNS1] = SingleQueryResult{
 		Answers: []interface{}{Answer{
@@ -925,7 +928,7 @@ func TestMxAndAdditionals(t *testing.T) {
 
 	domain1 := "example.com"
 	ns1 := &config.ExternalNameServersV4[0]
-	domainNS1 := domainNS{domain: domain1, ns: ns1.String()}
+	domainNS1 := nameAndIP{name: domain1, IP: ns1.String()}
 
 	mockResults[domainNS1] = SingleQueryResult{
 		Answers: []interface{}{Answer{
@@ -966,7 +969,7 @@ func TestMismatchIpType(t *testing.T) {
 
 	domain1 := "example.com"
 	ns1 := &config.ExternalNameServersV4[0]
-	domainNS1 := domainNS{domain: domain1, ns: ns1.String()}
+	domainNS1 := nameAndIP{name: domain1, IP: ns1.String()}
 
 	mockResults[domainNS1] = SingleQueryResult{
 		Answers: []interface{}{Answer{
@@ -1002,7 +1005,7 @@ func TestEmptyNonTerminal(t *testing.T) {
 
 	domain1 := "leaf.intermediate.example.com"
 	ns1 := &config.ExternalNameServersV4[0]
-	domainNS1 := domainNS{domain: domain1, ns: ns1.String()}
+	domainNS1 := nameAndIP{name: domain1, IP: ns1.String()}
 
 	mockResults[domainNS1] = SingleQueryResult{
 		Answers: []interface{}{Answer{
@@ -1020,7 +1023,7 @@ func TestEmptyNonTerminal(t *testing.T) {
 
 	dom2 := "intermediate.example.com"
 
-	domainNS2 := domainNS{domain: dom2, ns: ns1.String()}
+	domainNS2 := nameAndIP{name: dom2, IP: ns1.String()}
 
 	mockResults[domainNS2] = SingleQueryResult{
 		Answers:     nil,
@@ -1038,7 +1041,7 @@ func TestEmptyNonTerminal(t *testing.T) {
 	verifyResult(t, *res, nil, nil)
 }
 
-// Test Non-existent domain in the zone returns NXDOMAIN
+// Test Non-existent name in the zone returns NXDOMAIN
 
 func TestNXDomain(t *testing.T) {
 	config := InitTest(t)
@@ -1063,9 +1066,9 @@ func TestAandQuadADedup(t *testing.T) {
 	domain2 := "cname2.example.com"
 	domain3 := "example.com"
 	ns1 := &config.ExternalNameServersV4[0]
-	domainNS1 := domainNS{domain: domain1, ns: ns1.String()}
-	domainNS2 := domainNS{domain: domain2, ns: ns1.String()}
-	domainNS3 := domainNS{domain: domain3, ns: ns1.String()}
+	domainNS1 := nameAndIP{name: domain1, IP: ns1.String()}
+	domainNS2 := nameAndIP{name: domain2, IP: ns1.String()}
+	domainNS3 := nameAndIP{name: domain3, IP: ns1.String()}
 
 	mockResults[domainNS1] = SingleQueryResult{
 		Answers: []interface{}{Answer{
@@ -1159,7 +1162,7 @@ func TestServFail(t *testing.T) {
 
 	domain1 := "example.com"
 	ns1 := &config.ExternalNameServersV4[0]
-	domainNS1 := domainNS{domain: domain1, ns: ns1.String()}
+	domainNS1 := nameAndIP{name: domain1, IP: ns1.String()}
 
 	mockResults[domainNS1] = SingleQueryResult{}
 	name := "example.com"
@@ -1196,7 +1199,7 @@ func TestNsAInAdditional(t *testing.T) {
 
 	domain1 := "example.com"
 	ns1 := &config.ExternalNameServersV4[0]
-	domainNS1 := domainNS{domain: domain1, ns: ns1.String()}
+	domainNS1 := nameAndIP{name: domain1, IP: ns1.String()}
 
 	mockResults[domainNS1] = SingleQueryResult{
 		Answers: []interface{}{
@@ -1239,7 +1242,7 @@ func TestTwoNSInAdditional(t *testing.T) {
 
 	domain1 := "example.com"
 	ns1 := &config.ExternalNameServersV4[0]
-	domainNS1 := domainNS{domain: domain1, ns: ns1.String()}
+	domainNS1 := nameAndIP{name: domain1, IP: ns1.String()}
 
 	mockResults[domainNS1] = SingleQueryResult{
 		Answers: []interface{}{
@@ -1299,7 +1302,7 @@ func TestAandQuadAInAdditional(t *testing.T) {
 
 	domain1 := "example.com"
 	ns1 := &config.ExternalNameServersV4[0]
-	domainNS1 := domainNS{domain: domain1, ns: ns1.String()}
+	domainNS1 := nameAndIP{name: domain1, IP: ns1.String()}
 
 	mockResults[domainNS1] = SingleQueryResult{
 		Answers: []interface{}{
@@ -1348,7 +1351,7 @@ func TestNsMismatchIpType(t *testing.T) {
 
 	domain1 := "example.com"
 	ns1 := &config.ExternalNameServersV4[0]
-	domainNS1 := domainNS{domain: domain1, ns: ns1.String()}
+	domainNS1 := nameAndIP{name: domain1, IP: ns1.String()}
 
 	mockResults[domainNS1] = SingleQueryResult{
 		Answers: []interface{}{
@@ -1397,7 +1400,7 @@ func TestAandQuadALookup(t *testing.T) {
 
 	domain1 := "example.com"
 	ns1 := &config.ExternalNameServersV4[0]
-	domainNS1 := domainNS{domain: domain1, ns: ns1.String()}
+	domainNS1 := nameAndIP{name: domain1, IP: ns1.String()}
 
 	mockResults[domainNS1] = SingleQueryResult{
 		Answers: []interface{}{
@@ -1417,7 +1420,7 @@ func TestAandQuadALookup(t *testing.T) {
 
 	dom2 := "ns1.example.com"
 
-	domainNS2 := domainNS{domain: dom2, ns: ns1.String()}
+	domainNS2 := nameAndIP{name: dom2, IP: ns1.String()}
 
 	mockResults[domainNS2] = SingleQueryResult{
 		Answers: []interface{}{
@@ -1470,7 +1473,7 @@ func TestNsServFail(t *testing.T) {
 
 	domain1 := "example.com"
 	ns1 := &config.ExternalNameServersV4[0]
-	domainNS1 := domainNS{domain: domain1, ns: ns1.String()}
+	domainNS1 := nameAndIP{name: domain1, IP: ns1.String()}
 
 	mockResults[domainNS1] = SingleQueryResult{}
 	protocolStatus[domainNS1] = StatusServFail
@@ -1488,7 +1491,7 @@ func TestErrorInTargetedLookup(t *testing.T) {
 
 	domain1 := "example.com"
 	ns1 := &config.ExternalNameServersV4[0]
-	domainNS1 := domainNS{domain: domain1, ns: ns1.String()}
+	domainNS1 := nameAndIP{name: domain1, IP: ns1.String()}
 
 	mockResults[domainNS1] = SingleQueryResult{
 		Answers: []interface{}{
@@ -1514,458 +1517,549 @@ func TestErrorInTargetedLookup(t *testing.T) {
 }
 
 // Test One NS with one IP with only ipv4-lookup
-func TestAllNsLookupOneNs(t *testing.T) {
+func TestAllNsLookupOneNsThreeLevels(t *testing.T) {
 	config := InitTest(t)
 	config.LocalAddrsV4 = []net.IP{net.ParseIP("127.0.0.1")}
 	resolver, err := InitResolver(config)
 	require.NoError(t, err)
+	exampleName := "example.com"
 
-	ns1 := &config.ExternalNameServersV4[0]
-	domain1 := "example.com"
-	nsDomain1 := "ns1.example.com"
-	ipv4_1 := "127.0.0.2"
-
-	domainNS1 := domainNS{domain: domain1, ns: ns1.String()}
-	mockResults[domainNS1] = SingleQueryResult{
-		Answers: []interface{}{
-			Answer{
-				TTL:    3600,
-				Type:   "NS",
-				Class:  "IN",
-				Name:   "example.com.",
-				Answer: nsDomain1 + ".",
-			},
-		},
-		Additional: []interface{}{
-			Answer{
-				TTL:    3600,
-				Type:   "A",
-				Class:  "IN",
-				Name:   nsDomain1 + ".",
-				Answer: ipv4_1,
-			},
-		},
-		Authorities: nil,
-		Protocol:    "",
-		Flags:       DNSFlags{},
-	}
-
-	domainNS2 := domainNS{domain: domain1, ns: ipv4_1 + ":53"}
-	ipv4_2 := "127.0.0.3"
-	mockResults[domainNS2] = SingleQueryResult{
-		Answers: []interface{}{
-			Answer{
-				TTL:    3600,
-				Type:   "A",
-				Class:  "IN",
-				Name:   "example.com.",
-				Answer: ipv4_2,
-			},
-		},
-		Additional:  nil,
-		Authorities: nil,
-		Protocol:    "",
-		Flags:       DNSFlags{},
-	}
-
-	expectedRes := []ExtendedResult{
-		{
-			Nameserver: nsDomain1,
-			Status:     StatusNoError,
-			Res:        mockResults[domainNS2],
-		},
-	}
-	q := Question{
-		Type:  dns.TypeNS,
-		Class: dns.ClassINET,
-		Name:  "example.com",
-	}
-
-	results, _, _, err := resolver.LookupAllNameserversIterative(&q, ns1)
-	require.NoError(t, err)
-	verifyCombinedResult(t, results.Results, expectedRes)
-}
-
-// Test One NS with two IPs with only ipv4-lookup
-
-func TestAllNsLookupOneNsMultipleIps(t *testing.T) {
-	config := InitTest(t)
-	config.IPVersionMode = IPv4Only
-	resolver, err := InitResolver(config)
-	require.NoError(t, err)
+	rootServer := "a.root-servers.net"
+	rootServerIP := "1.1.1.1"
+	comServer := "a.gtld-servers.net"
+	comServerIP := "2.2.2.2"
+	exampleNSServer := "ns1.example.com"
+	exampleNSServerIP := "3.3.3.3"
+	exampleNameAAnswer := "4.4.4.4"
 
-	ns1 := &config.ExternalNameServersV4[0]
-	domain1 := "example.com"
-	nsDomain1 := "ns1.example.com"
-	ipv4_1 := "127.0.0.2"
-	ipv4_2 := "127.0.0.3"
+	//ns1 := &config.ExternalNameServersV4[0]
+	//ipv4_1 := "127.0.0.2"
 
-	domainNS1 := domainNS{domain: domain1, ns: ns1.String()}
-	mockResults[domainNS1] = SingleQueryResult{
-		Answers: []interface{}{
+	mockResults[nameAndIP{name: exampleName, IP: rootServerIP + ":53"}] = SingleQueryResult{
+		Authorities: []interface{}{
 			Answer{
 				TTL:    3600,
 				Type:   "NS",
+				RrType: dns.TypeNS,
 				Class:  "IN",
-				Name:   "example.com.",
-				Answer: nsDomain1 + ".",
+				Name:   "com.",
+				Answer: comServer + ".",
 			},
 		},
 		Additional: []interface{}{
 			Answer{
 				TTL:    3600,
 				Type:   "A",
+				RrType: dns.TypeA,
 				Class:  "IN",
-				Name:   nsDomain1 + ".",
-				Answer: ipv4_1,
-			},
-			Answer{
-				TTL:    3600,
-				Type:   "A",
-				Class:  "IN",
-				Name:   nsDomain1 + ".",
-				Answer: ipv4_2,
-			},
-		},
-		Authorities: nil,
-		Protocol:    "",
-		Flags:       DNSFlags{},
-	}
-
-	domainNS2 := domainNS{domain: domain1, ns: ipv4_1 + ":53"}
-	ipv4_3 := "127.0.0.4"
-	ipv6_1 := "::1"
-	mockResults[domainNS2] = SingleQueryResult{
-		Answers: []interface{}{
-			Answer{
-				TTL:    3600,
-				Type:   "A",
-				Class:  "IN",
-				Name:   "example.com.",
-				Answer: ipv4_3,
-			},
-			Answer{
-				TTL:    3600,
-				Type:   "AAAA",
-				Class:  "IN",
-				Name:   "example.com.",
-				Answer: ipv6_1,
-			},
-		},
-		Additional:  nil,
-		Authorities: nil,
-		Protocol:    "",
-		Flags:       DNSFlags{},
-	}
-
-	domainNS3 := domainNS{domain: domain1, ns: ipv4_2 + ":53"}
-	ipv4_4 := "127.0.0.5"
-	ipv6_2 := "::2"
-	mockResults[domainNS3] = SingleQueryResult{
-		Answers: []interface{}{
-			Answer{
-				TTL:    3600,
-				Type:   "A",
-				Class:  "IN",
-				Name:   "example.com.",
-				Answer: ipv4_4,
-			},
-			Answer{
-				TTL:    3600,
-				Type:   "AAAA",
-				Class:  "IN",
-				Name:   "example.com.",
-				Answer: ipv6_2,
+				Name:   comServer + ".",
+				Answer: comServerIP,
 			},
 		},
-		Additional:  nil,
-		Authorities: nil,
-		Protocol:    "",
-		Flags:       DNSFlags{},
 	}
-
-	expectedRes := []ExtendedResult{
-		{
-			Nameserver: nsDomain1,
-			Status:     StatusNoError,
-			Res:        mockResults[domainNS2],
-		},
-		{
-			Nameserver: nsDomain1,
-			Status:     StatusNoError,
-			Res:        mockResults[domainNS3],
-		},
-	}
-
-	q := Question{
-		Type:  dns.TypeNS,
-		Class: dns.ClassINET,
-		Name:  "example.com",
-	}
-
-	results, _, _, err := resolver.LookupAllNameserversIterative(&q, ns1)
-	require.NoError(t, err)
-	verifyCombinedResult(t, results.Results, expectedRes)
-}
-
-// Test One NS with two IPs with only ipv4-lookup
-func TestAllNsLookupTwoNs(t *testing.T) {
-	config := InitTest(t)
-	config.IPVersionMode = IPv4Only
-	resolver, err := InitResolver(config)
-	require.NoError(t, err)
-
-	ns1 := &config.ExternalNameServersV4[0]
-	domain1 := "example.com"
-	nsDomain1 := "ns1.example.com"
-	nsDomain2 := "ns2.example.com"
-	ipv4_1 := "127.0.0.2"
-	ipv4_2 := "127.0.0.3"
-
-	domainNS1 := domainNS{domain: domain1, ns: ns1.String()}
-	mockResults[domainNS1] = SingleQueryResult{
-		Answers: []interface{}{
-			Answer{
-				TTL:    3600,
-				Type:   "NS",
-				Class:  "IN",
-				Name:   "example.com.",
-				Answer: nsDomain1 + ".",
-			},
+	mockResults[nameAndIP{name: exampleName, IP: comServerIP + ":53"}] = SingleQueryResult{
+		Authorities: []interface{}{
 			Answer{
 				TTL:    3600,
 				Type:   "NS",
+				RrType: dns.TypeNS,
 				Class:  "IN",
 				Name:   "example.com.",
-				Answer: nsDomain2 + ".",
+				Answer: exampleNSServer + ".",
 			},
 		},
 		Additional: []interface{}{
 			Answer{
 				TTL:    3600,
 				Type:   "A",
+				RrType: dns.TypeA,
 				Class:  "IN",
-				Name:   nsDomain1 + ".",
-				Answer: ipv4_1,
-			},
-			Answer{
-				TTL:    3600,
-				Type:   "A",
-				Class:  "IN",
-				Name:   nsDomain2 + ".",
-				Answer: ipv4_2,
-			},
-		},
-		Authorities: nil,
-		Protocol:    "",
-		Flags:       DNSFlags{},
-	}
-
-	domainNS2 := domainNS{domain: domain1, ns: ipv4_1 + ":53"}
-	ipv4_3 := "127.0.0.4"
-	mockResults[domainNS2] = SingleQueryResult{
-		Answers: []interface{}{
-			Answer{
-				TTL:    3600,
-				Type:   "A",
-				Class:  "IN",
-				Name:   "example.com.",
-				Answer: ipv4_3,
+				Name:   exampleNSServer + ".",
+				Answer: exampleNSServerIP,
 			},
 		},
-		Additional:  nil,
-		Authorities: nil,
-		Protocol:    "",
-		Flags:       DNSFlags{},
 	}
-
-	domainNS3 := domainNS{domain: domain1, ns: ipv4_2 + ":53"}
-	ipv4_4 := "127.0.0.5"
-	mockResults[domainNS3] = SingleQueryResult{
+	mockResults[nameAndIP{name: exampleName, IP: exampleNSServerIP + ":53"}] = SingleQueryResult{
 		Answers: []interface{}{
 			Answer{
 				TTL:    3600,
 				Type:   "A",
+				RrType: dns.TypeA,
 				Class:  "IN",
 				Name:   "example.com.",
-				Answer: ipv4_4,
+				Answer: exampleNameAAnswer,
 			},
 		},
-		Additional:  nil,
-		Authorities: nil,
-		Protocol:    "",
-		Flags:       DNSFlags{},
-	}
-
-	expectedRes := []ExtendedResult{
-		{
-			Nameserver: nsDomain1,
-			Status:     StatusNoError,
-			Res:        mockResults[domainNS2],
-		},
-		{
-			Nameserver: nsDomain2,
-			Status:     StatusNoError,
-			Res:        mockResults[domainNS3],
-		},
 	}
 
-	q := Question{
-		Type:  dns.TypeNS,
-		Class: dns.ClassINET,
-		Name:  "example.com",
-	}
-
-	results, _, _, err := resolver.LookupAllNameserversIterative(&q, ns1)
-	require.NoError(t, err)
-	verifyCombinedResult(t, results.Results, expectedRes)
-}
-
-// Test error in A lookup via targeted lookup records
-
-func TestAllNsLookupErrorInOne(t *testing.T) {
-	config := InitTest(t)
-	config.IPVersionMode = IPv4Only
-	resolver, err := InitResolver(config)
-	require.NoError(t, err)
-
-	ns1 := &config.ExternalNameServersV4[0]
-	domain1 := "example.com"
-	nsDomain1 := "ns1.example.com"
-	ipv4_1 := "127.0.0.2"
-	ipv4_2 := "127.0.0.3"
-
-	domainNS1 := domainNS{domain: domain1, ns: ns1.String()}
-	mockResults[domainNS1] = SingleQueryResult{
-		Answers: []interface{}{
-			Answer{
-				TTL:    3600,
-				Type:   "NS",
-				Class:  "IN",
-				Name:   "example.com",
-				Answer: nsDomain1 + ".",
+	expectedRes := map[string][]ExtendedResult{
+		".": {
+			{
+				Status:     StatusNoError,
+				Nameserver: rootServer,
+				Res: SingleQueryResult{
+					Authorities: []interface{}{
+						Answer{
+							TTL:    3600,
+							Type:   "NS",
+							RrType: dns.TypeNS,
+							Class:  "IN",
+							Name:   "com.",
+							Answer: "a.gtld-servers.net.",
+						},
+					},
+					Additional: []interface{}{
+						Answer{
+							TTL:    3600,
+							Type:   "A",
+							RrType: dns.TypeA,
+							Class:  "IN",
+							Name:   "a.gtld-servers.net.",
+							Answer: "2.2.2.2",
+						},
+					},
+				},
 			},
 		},
-		Additional: []interface{}{
-			Answer{
-				TTL:    3600,
-				Type:   "A",
-				Class:  "IN",
-				Name:   nsDomain1 + ".",
-				Answer: ipv4_1,
-			},
-			Answer{
-				TTL:    3600,
-				Type:   "A",
-				Class:  "IN",
-				Name:   nsDomain1 + ".",
-				Answer: ipv4_2,
+		"com": {
+			{
+				Status:     StatusNoError,
+				Nameserver: comServer,
+				Res: SingleQueryResult{
+					Authorities: []interface{}{
+						Answer{
+							TTL:    3600,
+							Type:   "NS",
+							RrType: dns.TypeNS,
+							Class:  "IN",
+							Name:   "example.com.",
+							Answer: "ns1.example.com.",
+						},
+					},
+					Additional: []interface{}{
+						Answer{
+							TTL:    3600,
+							Type:   "A",
+							RrType: dns.TypeA,
+							Class:  "IN",
+							Name:   "ns1.example.com.",
+							Answer: "3.3.3.3",
+						},
+					},
+				},
 			},
 		},
-		Authorities: nil,
-		Protocol:    "",
-		Flags:       DNSFlags{},
-	}
-
-	domainNS2 := domainNS{domain: domain1, ns: ipv4_1 + ":53"}
-	ipv4_3 := "127.0.0.4"
-	ipv6_1 := "::1"
-	mockResults[domainNS2] = SingleQueryResult{
-		Answers: []interface{}{
-			Answer{
-				TTL:    3600,
-				Type:   "A",
-				Class:  "IN",
-				Name:   "example.com.",
-				Answer: ipv4_3,
-			},
-			Answer{
-				TTL:    3600,
-				Type:   "AAAA",
-				Class:  "IN",
-				Name:   "example.com.",
-				Answer: ipv6_1,
+		"example.com": {
+			{
+				Status:     StatusNoError,
+				Nameserver: exampleNSServer,
+				Res: SingleQueryResult{
+					Answers: []interface{}{
+						Answer{
+							TTL:    3600,
+							Type:   "A",
+							RrType: dns.TypeA,
+							Class:  "IN",
+							Name:   "example.com.",
+							Answer: "4.4.4.4",
+						},
+					},
+				},
 			},
 		},
-		Additional:  nil,
-		Authorities: nil,
-		Protocol:    "",
-		Flags:       DNSFlags{},
-	}
-
-	domainNS3 := domainNS{domain: domain1, ns: ipv4_2 + ":53"}
-	protocolStatus[domainNS3] = StatusServFail
-	mockResults[domainNS3] = SingleQueryResult{}
-
-	expectedRes := []ExtendedResult{
-		{
-			Nameserver: nsDomain1,
-			Status:     StatusNoError,
-			Res:        mockResults[domainNS2],
-		},
-		{
-			Nameserver: nsDomain1,
-			Status:     StatusServFail,
-			Res:        mockResults[domainNS3],
-		},
-	}
-
-	q := Question{
-		Type:  dns.TypeNS,
-		Class: dns.ClassINET,
-		Name:  "example.com",
-	}
-
-	results, _, _, err := resolver.LookupAllNameserversIterative(&q, ns1)
-	require.NoError(t, err)
-	verifyCombinedResult(t, results.Results, expectedRes)
-}
-
-func TestAllNsLookupNXDomain(t *testing.T) {
-	config := InitTest(t)
-	config.IPVersionMode = IPv4Only
-	resolver, err := InitResolver(config)
-	require.NoError(t, err)
-
-	ns1 := &config.ExternalNameServersV4[0]
-	q := Question{
-		Type:  dns.TypeNS,
-		Class: dns.ClassINET,
-		Name:  "example.com",
 	}
-
-	res, _, status, err := resolver.LookupAllNameserversIterative(&q, ns1)
-
-	assert.Equal(t, StatusNXDomain, status)
-	assert.Nil(t, res)
-	require.NoError(t, err)
-}
-
-func TestAllNsLookupServFail(t *testing.T) {
-	config := InitTest(t)
-	config.IPVersionMode = IPv4Only
-	resolver, err := InitResolver(config)
-	require.NoError(t, err)
-
-	ns1 := &config.ExternalNameServersV4[0]
-	domain1 := "example.com"
-	domainNS1 := domainNS{domain: domain1, ns: ns1.String()}
-
-	protocolStatus[domainNS1] = StatusServFail
-	mockResults[domainNS1] = SingleQueryResult{}
-
 	q := Question{
-		Type:  dns.TypeNS,
+		Type:  dns.TypeA,
 		Class: dns.ClassINET,
 		Name:  "example.com",
 	}
-	res, _, status, err := resolver.LookupAllNameserversIterative(&q, ns1)
 
-	assert.Equal(t, StatusServFail, status)
-	assert.Nil(t, res)
+	results, _, _, err := resolver.LookupAllNameserversIterative(&q, []NameServer{{DomainName: rootServer, IP: net.ParseIP(rootServerIP)}})
 	require.NoError(t, err)
+	verifyCombinedResult(t, results.LayeredResponses, expectedRes)
+	log.Warn(results, expectedRes)
 }
 
+// // Test One NS with two IPs with only ipv4-lookup
+//
+//	func TestAllNsLookupOneNsMultipleIps(t *testing.T) {
+//		config := InitTest(t)
+//		config.IPVersionMode = IPv4Only
+//		resolver, err := InitResolver(config)
+//		require.NoError(t, err)
+//
+//		ns1 := &config.ExternalNameServersV4[0]
+//		domain1 := "example.com"
+//		nsDomain1 := "ns1.example.com"
+//		ipv4_1 := "127.0.0.2"
+//		ipv4_2 := "127.0.0.3"
+//
+//		domainNS1 := nameAndIP{name: domain1, IP: ns1.String()}
+//		mockResults[domainNS1] = SingleQueryResult{
+//			Answers: []interface{}{
+//				Answer{
+//					TTL:    3600,
+//					Type:   "NS",
+//					Class:  "IN",
+//					Name:   "example.com.",
+//					Answer: nsDomain1 + ".",
+//				},
+//			},
+//			Additional: []interface{}{
+//				Answer{
+//					TTL:    3600,
+//					Type:   "A",
+//					Class:  "IN",
+//					Name:   nsDomain1 + ".",
+//					Answer: ipv4_1,
+//				},
+//				Answer{
+//					TTL:    3600,
+//					Type:   "A",
+//					Class:  "IN",
+//					Name:   nsDomain1 + ".",
+//					Answer: ipv4_2,
+//				},
+//			},
+//			Authorities: nil,
+//			Protocol:    "",
+//			Flags:       DNSFlags{},
+//		}
+//
+//		domainNS2 := nameAndIP{name: domain1, IP: ipv4_1 + ":53"}
+//		ipv4_3 := "127.0.0.4"
+//		ipv6_1 := "::1"
+//		mockResults[domainNS2] = SingleQueryResult{
+//			Answers: []interface{}{
+//				Answer{
+//					TTL:    3600,
+//					Type:   "A",
+//					Class:  "IN",
+//					Name:   "example.com.",
+//					Answer: ipv4_3,
+//				},
+//				Answer{
+//					TTL:    3600,
+//					Type:   "AAAA",
+//					Class:  "IN",
+//					Name:   "example.com.",
+//					Answer: ipv6_1,
+//				},
+//			},
+//			Additional:  nil,
+//			Authorities: nil,
+//			Protocol:    "",
+//			Flags:       DNSFlags{},
+//		}
+//
+//		domainNS3 := nameAndIP{name: domain1, IP: ipv4_2 + ":53"}
+//		ipv4_4 := "127.0.0.5"
+//		ipv6_2 := "::2"
+//		mockResults[domainNS3] = SingleQueryResult{
+//			Answers: []interface{}{
+//				Answer{
+//					TTL:    3600,
+//					Type:   "A",
+//					Class:  "IN",
+//					Name:   "example.com.",
+//					Answer: ipv4_4,
+//				},
+//				Answer{
+//					TTL:    3600,
+//					Type:   "AAAA",
+//					Class:  "IN",
+//					Name:   "example.com.",
+//					Answer: ipv6_2,
+//				},
+//			},
+//			Additional:  nil,
+//			Authorities: nil,
+//			Protocol:    "",
+//			Flags:       DNSFlags{},
+//		}
+//
+//		expectedRes := []ExtendedResult{
+//			{
+//				Nameserver: nsDomain1,
+//				Status:     StatusNoError,
+//				Res:        mockResults[domainNS2],
+//			},
+//			{
+//				Nameserver: nsDomain1,
+//				Status:     StatusNoError,
+//				Res:        mockResults[domainNS3],
+//			},
+//		}
+//
+//		q := Question{
+//			Type:  dns.TypeNS,
+//			Class: dns.ClassINET,
+//			Name:  "example.com",
+//		}
+//
+//		results, _, _, err := resolver.LookupAllNameserversIterative(&q, ns1)
+//		require.NoError(t, err)
+//		verifyCombinedResult(t, results.Results, expectedRes)
+//	}
+//
+// // Test One NS with two IPs with only ipv4-lookup
+//
+//	func TestAllNsLookupTwoNs(t *testing.T) {
+//		config := InitTest(t)
+//		config.IPVersionMode = IPv4Only
+//		resolver, err := InitResolver(config)
+//		require.NoError(t, err)
+//
+//		ns1 := &config.ExternalNameServersV4[0]
+//		domain1 := "example.com"
+//		nsDomain1 := "ns1.example.com"
+//		nsDomain2 := "ns2.example.com"
+//		ipv4_1 := "127.0.0.2"
+//		ipv4_2 := "127.0.0.3"
+//
+//		domainNS1 := nameAndIP{name: domain1, IP: ns1.String()}
+//		mockResults[domainNS1] = SingleQueryResult{
+//			Answers: []interface{}{
+//				Answer{
+//					TTL:    3600,
+//					Type:   "NS",
+//					Class:  "IN",
+//					Name:   "example.com.",
+//					Answer: nsDomain1 + ".",
+//				},
+//				Answer{
+//					TTL:    3600,
+//					Type:   "NS",
+//					Class:  "IN",
+//					Name:   "example.com.",
+//					Answer: nsDomain2 + ".",
+//				},
+//			},
+//			Additional: []interface{}{
+//				Answer{
+//					TTL:    3600,
+//					Type:   "A",
+//					Class:  "IN",
+//					Name:   nsDomain1 + ".",
+//					Answer: ipv4_1,
+//				},
+//				Answer{
+//					TTL:    3600,
+//					Type:   "A",
+//					Class:  "IN",
+//					Name:   nsDomain2 + ".",
+//					Answer: ipv4_2,
+//				},
+//			},
+//			Authorities: nil,
+//			Protocol:    "",
+//			Flags:       DNSFlags{},
+//		}
+//
+//		domainNS2 := nameAndIP{name: domain1, IP: ipv4_1 + ":53"}
+//		ipv4_3 := "127.0.0.4"
+//		mockResults[domainNS2] = SingleQueryResult{
+//			Answers: []interface{}{
+//				Answer{
+//					TTL:    3600,
+//					Type:   "A",
+//					Class:  "IN",
+//					Name:   "example.com.",
+//					Answer: ipv4_3,
+//				},
+//			},
+//			Additional:  nil,
+//			Authorities: nil,
+//			Protocol:    "",
+//			Flags:       DNSFlags{},
+//		}
+//
+//		domainNS3 := nameAndIP{name: domain1, IP: ipv4_2 + ":53"}
+//		ipv4_4 := "127.0.0.5"
+//		mockResults[domainNS3] = SingleQueryResult{
+//			Answers: []interface{}{
+//				Answer{
+//					TTL:    3600,
+//					Type:   "A",
+//					Class:  "IN",
+//					Name:   "example.com.",
+//					Answer: ipv4_4,
+//				},
+//			},
+//			Additional:  nil,
+//			Authorities: nil,
+//			Protocol:    "",
+//			Flags:       DNSFlags{},
+//		}
+//
+//		expectedRes := []ExtendedResult{
+//			{
+//				Nameserver: nsDomain1,
+//				Status:     StatusNoError,
+//				Res:        mockResults[domainNS2],
+//			},
+//			{
+//				Nameserver: nsDomain2,
+//				Status:     StatusNoError,
+//				Res:        mockResults[domainNS3],
+//			},
+//		}
+//
+//		q := Question{
+//			Type:  dns.TypeNS,
+//			Class: dns.ClassINET,
+//			Name:  "example.com",
+//		}
+//
+//		results, _, _, err := resolver.LookupAllNameserversIterative(&q, ns1)
+//		require.NoError(t, err)
+//		verifyCombinedResult(t, results.Results, expectedRes)
+//	}
+//
+// // Test error in A lookup via targeted lookup records
+//
+//	func TestAllNsLookupErrorInOne(t *testing.T) {
+//		config := InitTest(t)
+//		config.IPVersionMode = IPv4Only
+//		resolver, err := InitResolver(config)
+//		require.NoError(t, err)
+//
+//		ns1 := &config.ExternalNameServersV4[0]
+//		domain1 := "example.com"
+//		nsDomain1 := "ns1.example.com"
+//		ipv4_1 := "127.0.0.2"
+//		ipv4_2 := "127.0.0.3"
+//
+//		domainNS1 := nameAndIP{name: domain1, IP: ns1.String()}
+//		mockResults[domainNS1] = SingleQueryResult{
+//			Answers: []interface{}{
+//				Answer{
+//					TTL:    3600,
+//					Type:   "NS",
+//					Class:  "IN",
+//					Name:   "example.com",
+//					Answer: nsDomain1 + ".",
+//				},
+//			},
+//			Additional: []interface{}{
+//				Answer{
+//					TTL:    3600,
+//					Type:   "A",
+//					Class:  "IN",
+//					Name:   nsDomain1 + ".",
+//					Answer: ipv4_1,
+//				},
+//				Answer{
+//					TTL:    3600,
+//					Type:   "A",
+//					Class:  "IN",
+//					Name:   nsDomain1 + ".",
+//					Answer: ipv4_2,
+//				},
+//			},
+//			Authorities: nil,
+//			Protocol:    "",
+//			Flags:       DNSFlags{},
+//		}
+//
+//		domainNS2 := nameAndIP{name: domain1, IP: ipv4_1 + ":53"}
+//		ipv4_3 := "127.0.0.4"
+//		ipv6_1 := "::1"
+//		mockResults[domainNS2] = SingleQueryResult{
+//			Answers: []interface{}{
+//				Answer{
+//					TTL:    3600,
+//					Type:   "A",
+//					Class:  "IN",
+//					Name:   "example.com.",
+//					Answer: ipv4_3,
+//				},
+//				Answer{
+//					TTL:    3600,
+//					Type:   "AAAA",
+//					Class:  "IN",
+//					Name:   "example.com.",
+//					Answer: ipv6_1,
+//				},
+//			},
+//			Additional:  nil,
+//			Authorities: nil,
+//			Protocol:    "",
+//			Flags:       DNSFlags{},
+//		}
+//
+//		domainNS3 := nameAndIP{name: domain1, IP: ipv4_2 + ":53"}
+//		protocolStatus[domainNS3] = StatusServFail
+//		mockResults[domainNS3] = SingleQueryResult{}
+//
+//		expectedRes := []ExtendedResult{
+//			{
+//				Nameserver: nsDomain1,
+//				Status:     StatusNoError,
+//				Res:        mockResults[domainNS2],
+//			},
+//			{
+//				Nameserver: nsDomain1,
+//				Status:     StatusServFail,
+//				Res:        mockResults[domainNS3],
+//			},
+//		}
+//
+//		q := Question{
+//			Type:  dns.TypeNS,
+//			Class: dns.ClassINET,
+//			Name:  "example.com",
+//		}
+//
+//		results, _, _, err := resolver.LookupAllNameserversIterative(&q, ns1)
+//		require.NoError(t, err)
+//		verifyCombinedResult(t, results.Results, expectedRes)
+//	}
+//
+//	func TestAllNsLookupNXDomain(t *testing.T) {
+//		config := InitTest(t)
+//		config.IPVersionMode = IPv4Only
+//		resolver, err := InitResolver(config)
+//		require.NoError(t, err)
+//
+//		ns1 := &config.ExternalNameServersV4[0]
+//		q := Question{
+//			Type:  dns.TypeNS,
+//			Class: dns.ClassINET,
+//			Name:  "example.com",
+//		}
+//
+//		res, _, status, err := resolver.LookupAllNameserversIterative(&q, ns1)
+//
+//		assert.Equal(t, StatusNXDomain, status)
+//		assert.Nil(t, res)
+//		require.NoError(t, err)
+//	}
+//
+//	func TestAllNsLookupServFail(t *testing.T) {
+//		config := InitTest(t)
+//		config.IPVersionMode = IPv4Only
+//		resolver, err := InitResolver(config)
+//		require.NoError(t, err)
+//
+//		ns1 := &config.ExternalNameServersV4[0]
+//		domain1 := "example.com"
+//		domainNS1 := nameAndIP{name: domain1, IP: ns1.String()}
+//
+//		protocolStatus[domainNS1] = StatusServFail
+//		mockResults[domainNS1] = SingleQueryResult{}
+//
+//		q := Question{
+//			Type:  dns.TypeNS,
+//			Class: dns.ClassINET,
+//			Name:  "example.com",
+//		}
+//		res, _, status, err := resolver.LookupAllNameserversIterative(&q, ns1)
+//
+//		assert.Equal(t, StatusServFail, status)
+//		assert.Nil(t, res)
+//		require.NoError(t, err)
+//	}
 func TestInvalidInputsLookup(t *testing.T) {
 	config := InitTest(t)
 	config.LocalAddrsV4 = []net.IP{net.ParseIP("127.0.0.1")}
@@ -1979,11 +2073,11 @@ func TestInvalidInputsLookup(t *testing.T) {
 	}
 
 	t.Run("no port attached to nameserver", func(t *testing.T) {
-		_, _, _, err := resolver.ExternalLookup(&q, &NameServer{IP: net.ParseIP("127.0.0.53")})
+		_, _, _, err := resolver.ExternalLookup(context.Background(), &q, &NameServer{IP: net.ParseIP("127.0.0.53")})
 		assert.Nil(t, err)
 	})
 	t.Run("invalid nameserver address", func(t *testing.T) {
-		result, trace, status, err := resolver.ExternalLookup(&q, &NameServer{IP: net.ParseIP("987.987.987.987"), Port: 53})
+		result, trace, status, err := resolver.ExternalLookup(context.Background(), &q, &NameServer{IP: net.ParseIP("987.987.987.987"), Port: 53})
 		assert.Nil(t, result)
 		assert.Nil(t, trace)
 		assert.Equal(t, StatusIllegalInput, status)
@@ -2014,7 +2108,15 @@ func verifyNsResult(t *testing.T, servers []NSRecord, expectedServersMap map[str
 	}
 }
 
-func verifyCombinedResult(t *testing.T, records []ExtendedResult, expectedRecords []ExtendedResult) {
+func verifyCombinedResult(t *testing.T, records map[string][]ExtendedResult, expectedRecords map[string][]ExtendedResult) {
+	for layer, _ := range expectedRecords {
+		assert.Contains(t, records, layer, fmt.Sprintf("Layer %s not found in combined result", layer))
+	}
+	for layer, expectedLayerResults := range expectedRecords {
+		if !reflect.DeepEqual(records[layer], expectedLayerResults) {
+			t.Errorf("Combined result not matching for layer %s, expected %v, found %v", layer, expectedLayerResults, records[layer])
+		}
+	}
 	if !reflect.DeepEqual(records, expectedRecords) {
 		t.Errorf("Combined result not matching, expected %v, found %v", expectedRecords, records)
 	}

From 501dafcbceb1ff5d62b812fc1e07fb66c16e3869 Mon Sep 17 00:00:00 2001
From: phillip-stephens <phillip@cs.stanford.edu>
Date: Mon, 16 Dec 2024 12:08:17 -0500
Subject: [PATCH 14/28] added another unit test, 2 .com NSes and one fails

---
 src/zdns/lookup.go      |  11 +-
 src/zdns/lookup_test.go | 544 +++++++++++++++-------------------------
 2 files changed, 211 insertions(+), 344 deletions(-)

diff --git a/src/zdns/lookup.go b/src/zdns/lookup.go
index 1db8d9a6..1acafbc5 100644
--- a/src/zdns/lookup.go
+++ b/src/zdns/lookup.go
@@ -514,12 +514,11 @@ func (r *Resolver) queryAllNameServersInLayer(ctx context.Context, perNameServer
 			}
 			result, currTrace, status, err := r.ExternalLookup(ctx, q, &nameServer)
 			trace = append(trace, currTrace...)
-			if err == nil && status == StatusNoError {
-				extResult = &ExtendedResult{
-					Res:        *result,
-					Status:     status,
-					Nameserver: nameServer.DomainName,
-				}
+			extResult = &ExtendedResult{Status: status, Nameserver: nameServer.DomainName}
+			if result != nil {
+				extResult.Res = *result
+			}
+			if err == nil && status == StatusNoError && result != nil {
 				if result.Flags.Authoritative {
 					isAuthoritative = true
 				}
diff --git a/src/zdns/lookup_test.go b/src/zdns/lookup_test.go
index eb3af5a2..00e936a3 100644
--- a/src/zdns/lookup_test.go
+++ b/src/zdns/lookup_test.go
@@ -17,12 +17,12 @@ import (
 	"context"
 	"encoding/hex"
 	"fmt"
-	log "github.com/sirupsen/logrus"
 	"math/rand"
 	"net"
 	"reflect"
 	"regexp"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/require"
 
@@ -1677,345 +1677,213 @@ func TestAllNsLookupOneNsThreeLevels(t *testing.T) {
 	results, _, _, err := resolver.LookupAllNameserversIterative(&q, []NameServer{{DomainName: rootServer, IP: net.ParseIP(rootServerIP)}})
 	require.NoError(t, err)
 	verifyCombinedResult(t, results.LayeredResponses, expectedRes)
-	log.Warn(results, expectedRes)
 }
 
-// // Test One NS with two IPs with only ipv4-lookup
-//
-//	func TestAllNsLookupOneNsMultipleIps(t *testing.T) {
-//		config := InitTest(t)
-//		config.IPVersionMode = IPv4Only
-//		resolver, err := InitResolver(config)
-//		require.NoError(t, err)
-//
-//		ns1 := &config.ExternalNameServersV4[0]
-//		domain1 := "example.com"
-//		nsDomain1 := "ns1.example.com"
-//		ipv4_1 := "127.0.0.2"
-//		ipv4_2 := "127.0.0.3"
-//
-//		domainNS1 := nameAndIP{name: domain1, IP: ns1.String()}
-//		mockResults[domainNS1] = SingleQueryResult{
-//			Answers: []interface{}{
-//				Answer{
-//					TTL:    3600,
-//					Type:   "NS",
-//					Class:  "IN",
-//					Name:   "example.com.",
-//					Answer: nsDomain1 + ".",
-//				},
-//			},
-//			Additional: []interface{}{
-//				Answer{
-//					TTL:    3600,
-//					Type:   "A",
-//					Class:  "IN",
-//					Name:   nsDomain1 + ".",
-//					Answer: ipv4_1,
-//				},
-//				Answer{
-//					TTL:    3600,
-//					Type:   "A",
-//					Class:  "IN",
-//					Name:   nsDomain1 + ".",
-//					Answer: ipv4_2,
-//				},
-//			},
-//			Authorities: nil,
-//			Protocol:    "",
-//			Flags:       DNSFlags{},
-//		}
-//
-//		domainNS2 := nameAndIP{name: domain1, IP: ipv4_1 + ":53"}
-//		ipv4_3 := "127.0.0.4"
-//		ipv6_1 := "::1"
-//		mockResults[domainNS2] = SingleQueryResult{
-//			Answers: []interface{}{
-//				Answer{
-//					TTL:    3600,
-//					Type:   "A",
-//					Class:  "IN",
-//					Name:   "example.com.",
-//					Answer: ipv4_3,
-//				},
-//				Answer{
-//					TTL:    3600,
-//					Type:   "AAAA",
-//					Class:  "IN",
-//					Name:   "example.com.",
-//					Answer: ipv6_1,
-//				},
-//			},
-//			Additional:  nil,
-//			Authorities: nil,
-//			Protocol:    "",
-//			Flags:       DNSFlags{},
-//		}
-//
-//		domainNS3 := nameAndIP{name: domain1, IP: ipv4_2 + ":53"}
-//		ipv4_4 := "127.0.0.5"
-//		ipv6_2 := "::2"
-//		mockResults[domainNS3] = SingleQueryResult{
-//			Answers: []interface{}{
-//				Answer{
-//					TTL:    3600,
-//					Type:   "A",
-//					Class:  "IN",
-//					Name:   "example.com.",
-//					Answer: ipv4_4,
-//				},
-//				Answer{
-//					TTL:    3600,
-//					Type:   "AAAA",
-//					Class:  "IN",
-//					Name:   "example.com.",
-//					Answer: ipv6_2,
-//				},
-//			},
-//			Additional:  nil,
-//			Authorities: nil,
-//			Protocol:    "",
-//			Flags:       DNSFlags{},
-//		}
-//
-//		expectedRes := []ExtendedResult{
-//			{
-//				Nameserver: nsDomain1,
-//				Status:     StatusNoError,
-//				Res:        mockResults[domainNS2],
-//			},
-//			{
-//				Nameserver: nsDomain1,
-//				Status:     StatusNoError,
-//				Res:        mockResults[domainNS3],
-//			},
-//		}
-//
-//		q := Question{
-//			Type:  dns.TypeNS,
-//			Class: dns.ClassINET,
-//			Name:  "example.com",
-//		}
-//
-//		results, _, _, err := resolver.LookupAllNameserversIterative(&q, ns1)
-//		require.NoError(t, err)
-//		verifyCombinedResult(t, results.Results, expectedRes)
-//	}
-//
-// // Test One NS with two IPs with only ipv4-lookup
-//
-//	func TestAllNsLookupTwoNs(t *testing.T) {
-//		config := InitTest(t)
-//		config.IPVersionMode = IPv4Only
-//		resolver, err := InitResolver(config)
-//		require.NoError(t, err)
-//
-//		ns1 := &config.ExternalNameServersV4[0]
-//		domain1 := "example.com"
-//		nsDomain1 := "ns1.example.com"
-//		nsDomain2 := "ns2.example.com"
-//		ipv4_1 := "127.0.0.2"
-//		ipv4_2 := "127.0.0.3"
-//
-//		domainNS1 := nameAndIP{name: domain1, IP: ns1.String()}
-//		mockResults[domainNS1] = SingleQueryResult{
-//			Answers: []interface{}{
-//				Answer{
-//					TTL:    3600,
-//					Type:   "NS",
-//					Class:  "IN",
-//					Name:   "example.com.",
-//					Answer: nsDomain1 + ".",
-//				},
-//				Answer{
-//					TTL:    3600,
-//					Type:   "NS",
-//					Class:  "IN",
-//					Name:   "example.com.",
-//					Answer: nsDomain2 + ".",
-//				},
-//			},
-//			Additional: []interface{}{
-//				Answer{
-//					TTL:    3600,
-//					Type:   "A",
-//					Class:  "IN",
-//					Name:   nsDomain1 + ".",
-//					Answer: ipv4_1,
-//				},
-//				Answer{
-//					TTL:    3600,
-//					Type:   "A",
-//					Class:  "IN",
-//					Name:   nsDomain2 + ".",
-//					Answer: ipv4_2,
-//				},
-//			},
-//			Authorities: nil,
-//			Protocol:    "",
-//			Flags:       DNSFlags{},
-//		}
-//
-//		domainNS2 := nameAndIP{name: domain1, IP: ipv4_1 + ":53"}
-//		ipv4_3 := "127.0.0.4"
-//		mockResults[domainNS2] = SingleQueryResult{
-//			Answers: []interface{}{
-//				Answer{
-//					TTL:    3600,
-//					Type:   "A",
-//					Class:  "IN",
-//					Name:   "example.com.",
-//					Answer: ipv4_3,
-//				},
-//			},
-//			Additional:  nil,
-//			Authorities: nil,
-//			Protocol:    "",
-//			Flags:       DNSFlags{},
-//		}
-//
-//		domainNS3 := nameAndIP{name: domain1, IP: ipv4_2 + ":53"}
-//		ipv4_4 := "127.0.0.5"
-//		mockResults[domainNS3] = SingleQueryResult{
-//			Answers: []interface{}{
-//				Answer{
-//					TTL:    3600,
-//					Type:   "A",
-//					Class:  "IN",
-//					Name:   "example.com.",
-//					Answer: ipv4_4,
-//				},
-//			},
-//			Additional:  nil,
-//			Authorities: nil,
-//			Protocol:    "",
-//			Flags:       DNSFlags{},
-//		}
-//
-//		expectedRes := []ExtendedResult{
-//			{
-//				Nameserver: nsDomain1,
-//				Status:     StatusNoError,
-//				Res:        mockResults[domainNS2],
-//			},
-//			{
-//				Nameserver: nsDomain2,
-//				Status:     StatusNoError,
-//				Res:        mockResults[domainNS3],
-//			},
-//		}
-//
-//		q := Question{
-//			Type:  dns.TypeNS,
-//			Class: dns.ClassINET,
-//			Name:  "example.com",
-//		}
-//
-//		results, _, _, err := resolver.LookupAllNameserversIterative(&q, ns1)
-//		require.NoError(t, err)
-//		verifyCombinedResult(t, results.Results, expectedRes)
-//	}
-//
-// // Test error in A lookup via targeted lookup records
-//
-//	func TestAllNsLookupErrorInOne(t *testing.T) {
-//		config := InitTest(t)
-//		config.IPVersionMode = IPv4Only
-//		resolver, err := InitResolver(config)
-//		require.NoError(t, err)
-//
-//		ns1 := &config.ExternalNameServersV4[0]
-//		domain1 := "example.com"
-//		nsDomain1 := "ns1.example.com"
-//		ipv4_1 := "127.0.0.2"
-//		ipv4_2 := "127.0.0.3"
-//
-//		domainNS1 := nameAndIP{name: domain1, IP: ns1.String()}
-//		mockResults[domainNS1] = SingleQueryResult{
-//			Answers: []interface{}{
-//				Answer{
-//					TTL:    3600,
-//					Type:   "NS",
-//					Class:  "IN",
-//					Name:   "example.com",
-//					Answer: nsDomain1 + ".",
-//				},
-//			},
-//			Additional: []interface{}{
-//				Answer{
-//					TTL:    3600,
-//					Type:   "A",
-//					Class:  "IN",
-//					Name:   nsDomain1 + ".",
-//					Answer: ipv4_1,
-//				},
-//				Answer{
-//					TTL:    3600,
-//					Type:   "A",
-//					Class:  "IN",
-//					Name:   nsDomain1 + ".",
-//					Answer: ipv4_2,
-//				},
-//			},
-//			Authorities: nil,
-//			Protocol:    "",
-//			Flags:       DNSFlags{},
-//		}
-//
-//		domainNS2 := nameAndIP{name: domain1, IP: ipv4_1 + ":53"}
-//		ipv4_3 := "127.0.0.4"
-//		ipv6_1 := "::1"
-//		mockResults[domainNS2] = SingleQueryResult{
-//			Answers: []interface{}{
-//				Answer{
-//					TTL:    3600,
-//					Type:   "A",
-//					Class:  "IN",
-//					Name:   "example.com.",
-//					Answer: ipv4_3,
-//				},
-//				Answer{
-//					TTL:    3600,
-//					Type:   "AAAA",
-//					Class:  "IN",
-//					Name:   "example.com.",
-//					Answer: ipv6_1,
-//				},
-//			},
-//			Additional:  nil,
-//			Authorities: nil,
-//			Protocol:    "",
-//			Flags:       DNSFlags{},
-//		}
-//
-//		domainNS3 := nameAndIP{name: domain1, IP: ipv4_2 + ":53"}
-//		protocolStatus[domainNS3] = StatusServFail
-//		mockResults[domainNS3] = SingleQueryResult{}
-//
-//		expectedRes := []ExtendedResult{
-//			{
-//				Nameserver: nsDomain1,
-//				Status:     StatusNoError,
-//				Res:        mockResults[domainNS2],
-//			},
-//			{
-//				Nameserver: nsDomain1,
-//				Status:     StatusServFail,
-//				Res:        mockResults[domainNS3],
-//			},
-//		}
-//
-//		q := Question{
-//			Type:  dns.TypeNS,
-//			Class: dns.ClassINET,
-//			Name:  "example.com",
-//		}
-//
-//		results, _, _, err := resolver.LookupAllNameserversIterative(&q, ns1)
-//		require.NoError(t, err)
-//		verifyCombinedResult(t, results.Results, expectedRes)
-//	}
-//
+// Test AllNameservers with a ".", ".com", and "example.com". We'll have two .com servers and one will error. Should still be able to resolve the query.
+func TestAllNsLookupErrorInOne(t *testing.T) {
+	config := InitTest(t)
+	config.LocalAddrsV4 = []net.IP{net.ParseIP("127.0.0.1")}
+	config.Timeout = time.Hour
+	config.IterativeTimeout = time.Hour
+	resolver, err := InitResolver(config)
+	require.NoError(t, err)
+	exampleName := "example.com"
+
+	rootServer := "a.root-servers.net"
+	rootServerIP := "1.1.1.1"
+	comServerA := "a.gtld-servers.net"
+	comServerB := "b.gtld-servers.net"
+	comServerAIP := "2.2.2.2"
+	comServerBIP := "3.3.3.3"
+	exampleNSServer := "ns1.example.com"
+	exampleNSServerIP := "4.4.4.4"
+	exampleNameAAnswer := "5.5.5.5"
+
+	mockResults[nameAndIP{name: exampleName, IP: rootServerIP + ":53"}] = SingleQueryResult{
+		Authorities: []interface{}{
+			Answer{
+				TTL:    3600,
+				Type:   "NS",
+				RrType: dns.TypeNS,
+				Class:  "IN",
+				Name:   "com.",
+				Answer: comServerA + ".",
+			},
+			Answer{
+				TTL:    3600,
+				Type:   "NS",
+				RrType: dns.TypeNS,
+				Class:  "IN",
+				Name:   "com.",
+				Answer: comServerB + ".",
+			},
+		},
+		Additional: []interface{}{
+			Answer{
+				TTL:    3600,
+				Type:   "A",
+				RrType: dns.TypeA,
+				Class:  "IN",
+				Name:   comServerA + ".",
+				Answer: comServerAIP,
+			},
+			Answer{
+				TTL:    3600,
+				Type:   "A",
+				RrType: dns.TypeA,
+				Class:  "IN",
+				Name:   comServerB + ".",
+				Answer: comServerBIP,
+			},
+		},
+	}
+	// Error in comServerB
+	mockResults[nameAndIP{name: exampleName, IP: comServerAIP + ":53"}] = SingleQueryResult{}
+	protocolStatus[nameAndIP{name: exampleName, IP: comServerAIP + ":53"}] = StatusServFail
+	// Success in comServerA
+	mockResults[nameAndIP{name: exampleName, IP: comServerBIP + ":53"}] = SingleQueryResult{
+		Authorities: []interface{}{
+			Answer{
+				TTL:    3600,
+				Type:   "NS",
+				RrType: dns.TypeNS,
+				Class:  "IN",
+				Name:   "example.com.",
+				Answer: exampleNSServer + ".",
+			},
+		},
+		Additional: []interface{}{
+			Answer{
+				TTL:    3600,
+				Type:   "A",
+				RrType: dns.TypeA,
+				Class:  "IN",
+				Name:   exampleNSServer + ".",
+				Answer: exampleNSServerIP,
+			},
+		},
+	}
+	mockResults[nameAndIP{name: exampleName, IP: exampleNSServerIP + ":53"}] = SingleQueryResult{
+		Answers: []interface{}{
+			Answer{
+				TTL:    3600,
+				Type:   "A",
+				RrType: dns.TypeA,
+				Class:  "IN",
+				Name:   "example.com.",
+				Answer: exampleNameAAnswer,
+			},
+		},
+	}
+
+	expectedRes := map[string][]ExtendedResult{
+		".": {
+			{
+				Status:     StatusNoError,
+				Nameserver: rootServer,
+				Res: SingleQueryResult{
+					Authorities: []interface{}{
+						Answer{
+							TTL:    3600,
+							Type:   "NS",
+							RrType: dns.TypeNS,
+							Class:  "IN",
+							Name:   "com.",
+							Answer: "a.gtld-servers.net.",
+						},
+						Answer{
+							TTL:    3600,
+							Type:   "NS",
+							RrType: dns.TypeNS,
+							Class:  "IN",
+							Name:   "com.",
+							Answer: "b.gtld-servers.net.",
+						},
+					},
+					Additional: []interface{}{
+						Answer{
+							TTL:    3600,
+							Type:   "A",
+							RrType: dns.TypeA,
+							Class:  "IN",
+							Name:   "a.gtld-servers.net.",
+							Answer: comServerAIP,
+						},
+						Answer{
+							TTL:    3600,
+							Type:   "A",
+							RrType: dns.TypeA,
+							Class:  "IN",
+							Name:   "b.gtld-servers.net.",
+							Answer: comServerBIP,
+						},
+					},
+				},
+			},
+		},
+		"com": {
+			{
+				Status:     StatusServFail,
+				Nameserver: comServerA,
+				Res:        SingleQueryResult{},
+			},
+			{
+				Status:     StatusNoError,
+				Nameserver: comServerB,
+				Res: SingleQueryResult{
+					Authorities: []interface{}{
+						Answer{
+							TTL:    3600,
+							Type:   "NS",
+							RrType: dns.TypeNS,
+							Class:  "IN",
+							Name:   "example.com.",
+							Answer: exampleNSServer + ".",
+						},
+					},
+					Additional: []interface{}{
+						Answer{
+							TTL:    3600,
+							Type:   "A",
+							RrType: dns.TypeA,
+							Class:  "IN",
+							Name:   "ns1.example.com.",
+							Answer: exampleNSServerIP,
+						},
+					},
+				},
+			},
+		},
+		"example.com": {
+			{
+				Status:     StatusNoError,
+				Nameserver: exampleNSServer,
+				Res: SingleQueryResult{
+					Answers: []interface{}{
+						Answer{
+							TTL:    3600,
+							Type:   "A",
+							RrType: dns.TypeA,
+							Class:  "IN",
+							Name:   "example.com.",
+							Answer: exampleNameAAnswer,
+						},
+					},
+				},
+			},
+		},
+	}
+	q := Question{
+		Type:  dns.TypeA,
+		Class: dns.ClassINET,
+		Name:  "example.com",
+	}
+
+	results, _, _, err := resolver.LookupAllNameserversIterative(&q, []NameServer{{DomainName: rootServer, IP: net.ParseIP(rootServerIP)}})
+	require.NoError(t, err)
+	verifyCombinedResult(t, results.LayeredResponses, expectedRes)
+}
+
 //	func TestAllNsLookupNXDomain(t *testing.T) {
 //		config := InitTest(t)
 //		config.IPVersionMode = IPv4Only

From a7f37981580343ce1fec6bbd2e5c9f37170a15b5 Mon Sep 17 00:00:00 2001
From: phillip-stephens <phillip@cs.stanford.edu>
Date: Mon, 16 Dec 2024 12:21:30 -0500
Subject: [PATCH 15/28] fixed up another test

---
 src/zdns/lookup_test.go | 69 +++++++++++++++--------------------------
 1 file changed, 25 insertions(+), 44 deletions(-)

diff --git a/src/zdns/lookup_test.go b/src/zdns/lookup_test.go
index 00e936a3..92ffb2af 100644
--- a/src/zdns/lookup_test.go
+++ b/src/zdns/lookup_test.go
@@ -1884,50 +1884,31 @@ func TestAllNsLookupErrorInOne(t *testing.T) {
 	verifyCombinedResult(t, results.LayeredResponses, expectedRes)
 }
 
-//	func TestAllNsLookupNXDomain(t *testing.T) {
-//		config := InitTest(t)
-//		config.IPVersionMode = IPv4Only
-//		resolver, err := InitResolver(config)
-//		require.NoError(t, err)
-//
-//		ns1 := &config.ExternalNameServersV4[0]
-//		q := Question{
-//			Type:  dns.TypeNS,
-//			Class: dns.ClassINET,
-//			Name:  "example.com",
-//		}
-//
-//		res, _, status, err := resolver.LookupAllNameserversIterative(&q, ns1)
-//
-//		assert.Equal(t, StatusNXDomain, status)
-//		assert.Nil(t, res)
-//		require.NoError(t, err)
-//	}
-//
-//	func TestAllNsLookupServFail(t *testing.T) {
-//		config := InitTest(t)
-//		config.IPVersionMode = IPv4Only
-//		resolver, err := InitResolver(config)
-//		require.NoError(t, err)
-//
-//		ns1 := &config.ExternalNameServersV4[0]
-//		domain1 := "example.com"
-//		domainNS1 := nameAndIP{name: domain1, IP: ns1.String()}
-//
-//		protocolStatus[domainNS1] = StatusServFail
-//		mockResults[domainNS1] = SingleQueryResult{}
-//
-//		q := Question{
-//			Type:  dns.TypeNS,
-//			Class: dns.ClassINET,
-//			Name:  "example.com",
-//		}
-//		res, _, status, err := resolver.LookupAllNameserversIterative(&q, ns1)
-//
-//		assert.Equal(t, StatusServFail, status)
-//		assert.Nil(t, res)
-//		require.NoError(t, err)
-//	}
+func TestAllNsLookupNXDomain(t *testing.T) {
+	config := InitTest(t)
+	config.IPVersionMode = IPv4Only
+	resolver, err := InitResolver(config)
+	require.NoError(t, err)
+
+	ns1 := &config.ExternalNameServersV4[0]
+	q := Question{
+		Type:  dns.TypeNS,
+		Class: dns.ClassINET,
+		Name:  "example.com",
+	}
+
+	res, _, status, err := resolver.LookupAllNameserversIterative(&q, []NameServer{*ns1})
+
+	expectedResponse := map[string][]ExtendedResult{
+		".": {{Status: StatusNXDomain}},
+	}
+	if !reflect.DeepEqual(res.LayeredResponses, expectedResponse) {
+		t.Errorf("Expected %v, Received %v", expectedResponse, res.LayeredResponses)
+	}
+	assert.Equal(t, StatusError, status)
+	require.Error(t, err) // could not successfully complete lookup, so this should error
+}
+
 func TestInvalidInputsLookup(t *testing.T) {
 	config := InitTest(t)
 	config.LocalAddrsV4 = []net.IP{net.ParseIP("127.0.0.1")}

From 45a7842435d024b11dea5d4625fba24d5bd5506b Mon Sep 17 00:00:00 2001
From: phillip-stephens <phillip@cs.stanford.edu>
Date: Mon, 16 Dec 2024 14:07:44 -0500
Subject: [PATCH 16/28] handle the extra sibling NS case that issue 352 brings
 up

---
 src/zdns/lookup.go | 61 +++++++++++++++++++++++++++++++++++++---------
 1 file changed, 50 insertions(+), 11 deletions(-)

diff --git a/src/zdns/lookup.go b/src/zdns/lookup.go
index 1acafbc5..186e013d 100644
--- a/src/zdns/lookup.go
+++ b/src/zdns/lookup.go
@@ -19,6 +19,7 @@ import (
 	"io"
 	"math/rand"
 	"net"
+	"reflect"
 	"regexp"
 	"strings"
 
@@ -319,23 +320,32 @@ func (r *Resolver) LookupAllNameserversIterative(q *Question, rootNameServers []
 	}
 	var trace Trace
 	currentLayer := "."
-	isAuthoritative := false
+	queryingForFinalResult := false
 	var err error
 	currentLayerNameServers := rootNameServers
 	if len(currentLayerNameServers) == 0 {
 		// no root nameservers provided, use the resolver's root nameservers
 		currentLayerNameServers = r.rootNameServers
 	}
+	originalQuestionType := q.Type
 
-	for isAuthoritative == false {
+	q.Type = dns.TypeNS
+	for {
+		// Getting the NameServers
 		var layerResults []ExtendedResult
 		currTrace := make(Trace, 0)
-		layerResults, currTrace, isAuthoritative, err = r.queryAllNameServersInLayer(ctx, perNameServerRetriesLimit, q, currentLayerNameServers)
+		layerResults, currTrace, _, err = r.queryAllNameServersInLayer(ctx, perNameServerRetriesLimit, q, currentLayerNameServers)
 		trace = append(trace, currTrace...)
 		if err != nil {
 			return &retv, trace, StatusError, errors.Wrapf(err, "error encountered on layer %s", currentLayer)
-		} else {
+		} else if len(retv.LayeredResponses[currentLayer]) == 0 {
 			retv.LayeredResponses[currentLayer] = layerResults
+		} else {
+			retv.LayeredResponses[currentLayer] = append(retv.LayeredResponses[currentLayer], layerResults...)
+		}
+		if queryingForFinalResult {
+			// we've found an authoritative answer, we can return
+			break
 		}
 		// Set the next layer to query
 		var newLayer string
@@ -343,18 +353,29 @@ func (r *Resolver) LookupAllNameserversIterative(q *Question, rootNameServers []
 		if err != nil {
 			return &retv, trace, StatusError, errors.Wrapf(err, "error determining next authority for layer %s", currentLayer)
 		}
-		if newLayer == currentLayer {
-			// we've reached the end of the authority chain, return
-			return &retv, trace, StatusNoError, nil
-		}
+		//if newLayer == currentLayer {
+		//	// TODO need a better way to know when to break out
+		//	// we've reached the end of the authority chain, return
+		//	//return &retv, trace, StatusNoError, nil
+		//}
 		currentLayer = newLayer
-		currentLayerNameServers, err = r.extractNameServersFromLayerResults(layerResults)
+		var newNameServers []NameServer
+		newNameServers, err = r.extractNameServersFromLayerResults(layerResults)
 		if err != nil {
 			return &retv, trace, StatusError, errors.Wrapf(err, "error extracting nameservers from layer %s", currentLayer)
 		}
 		if len(currentLayerNameServers) == 0 {
 			return &retv, trace, StatusError, errors.New("no nameservers found in layer " + currentLayer)
 		}
+		sameNameServers := reflect.DeepEqual(currentLayerNameServers, newNameServers)
+		if sameNameServers {
+			// we've found all authoritative nameservers, now we can query them
+			q.Type = originalQuestionType
+			queryingForFinalResult = true
+		} else {
+			// we've found either new nameservers or the next layer. Continuing
+			currentLayerNameServers = newNameServers
+		}
 	}
 	return &retv, trace, StatusNoError, nil
 }
@@ -369,6 +390,7 @@ func (r *Resolver) extractNameServersFromLayerResults(layerResults []ExtendedRes
 	}
 	uniqueAdditionals := make(map[mapKey]Answer)
 	uniqueAuthorities := make(map[mapKey]Answer)
+	uniqueAnswers := make(map[mapKey]Answer)
 	for _, res := range layerResults {
 		if res.Status != StatusNoError {
 			continue
@@ -383,6 +405,13 @@ func (r *Resolver) extractNameServersFromLayerResults(layerResults []ExtendedRes
 				uniqueAuthorities[mapKey{Type: a.RrType, Name: a.Name, Answer: a.Answer}] = a
 			}
 		}
+		for _, ans := range res.Res.Answers {
+			if a, ok := ans.(Answer); ok {
+				if a.RrType == dns.TypeNS {
+					uniqueAnswers[mapKey{Type: a.RrType, Name: a.Name, Answer: a.Answer}] = a
+				}
+			}
+		}
 	}
 	// We have a map of unique additional and authority records. Now we need to extract the nameservers from them.
 	v4NameServers := make(map[string]NameServer)
@@ -426,6 +455,16 @@ func (r *Resolver) extractNameServersFromLayerResults(layerResults []ExtendedRes
 			}
 		}
 	}
+	// append any NS answers too
+	for _, answer := range uniqueAnswers {
+		ns := NameServer{
+			DomainName: strings.TrimSuffix(answer.Answer, "."),
+		}
+		key := ns.DomainName
+		if _, ok := uniqNameServersSet[key]; !ok {
+			uniqNameServersSet[key] = ns
+		}
+	}
 	uniqNameServers := make([]NameServer, 0, len(uniqNameServersSet))
 	for _, ns := range uniqNameServersSet {
 		uniqNameServers = append(uniqNameServers, ns)
@@ -607,8 +646,8 @@ func (r *Resolver) iterativeLookup(ctx context.Context, qWithMeta *QuestionWithM
 func (r *Resolver) cyclingLookup(ctx context.Context, qWithMeta *QuestionWithMetadata, nameServers []NameServer, layer string, depth int, recursionDesired bool) (*SingleQueryResult, IsCached, Status, error) {
 	var cacheBasedOnNameServer bool
 	var cacheNonAuthoritative bool
-	if recursionDesired {
-		// we're doing an external lookup and need to set the recursionDesired bit
+	if recursionDesired || r.lookupAllNameServers {
+		// we're doing an external or all-nameservers lookup and need to set the recursionDesired bit
 		// Additionally, in external mode we may perform the same lookup against multiple nameservers, so the cache should be based on the nameserver as well
 		cacheBasedOnNameServer = true
 		cacheNonAuthoritative = true

From 6cff29f5e1b765f4d88a84a520ea3153a0e4a5ec Mon Sep 17 00:00:00 2001
From: phillip-stephens <phillip@cs.stanford.edu>
Date: Mon, 16 Dec 2024 14:25:22 -0500
Subject: [PATCH 17/28] remove comment

---
 src/zdns/lookup.go | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/src/zdns/lookup.go b/src/zdns/lookup.go
index 186e013d..77bfea5b 100644
--- a/src/zdns/lookup.go
+++ b/src/zdns/lookup.go
@@ -353,11 +353,6 @@ func (r *Resolver) LookupAllNameserversIterative(q *Question, rootNameServers []
 		if err != nil {
 			return &retv, trace, StatusError, errors.Wrapf(err, "error determining next authority for layer %s", currentLayer)
 		}
-		//if newLayer == currentLayer {
-		//	// TODO need a better way to know when to break out
-		//	// we've reached the end of the authority chain, return
-		//	//return &retv, trace, StatusNoError, nil
-		//}
 		currentLayer = newLayer
 		var newNameServers []NameServer
 		newNameServers, err = r.extractNameServersFromLayerResults(layerResults)

From 2882d832151da50658339dcad2492edb64fe9211 Mon Sep 17 00:00:00 2001
From: phillip-stephens <phillip@cs.stanford.edu>
Date: Mon, 16 Dec 2024 16:37:16 -0500
Subject: [PATCH 18/28] bug fixes, query for all leaf NSes first, then followup
 with original query type

---
 src/modules/bindversion/bindversion.go |  1 +
 src/zdns/lookup.go                     | 62 ++++++++++++++------------
 src/zdns/lookup_test.go                |  2 +-
 src/zdns/qa.go                         |  1 +
 4 files changed, 37 insertions(+), 29 deletions(-)

diff --git a/src/modules/bindversion/bindversion.go b/src/modules/bindversion/bindversion.go
index 9f8d1bb2..729b73e1 100644
--- a/src/modules/bindversion/bindversion.go
+++ b/src/modules/bindversion/bindversion.go
@@ -16,6 +16,7 @@ package bindversion
 
 import (
 	"context"
+
 	"github.com/miekg/dns"
 	"github.com/pkg/errors"
 
diff --git a/src/zdns/lookup.go b/src/zdns/lookup.go
index 77bfea5b..5df7eeec 100644
--- a/src/zdns/lookup.go
+++ b/src/zdns/lookup.go
@@ -19,7 +19,6 @@ import (
 	"io"
 	"math/rand"
 	"net"
-	"reflect"
 	"regexp"
 	"strings"
 
@@ -98,8 +97,6 @@ func (r *Resolver) doDstServersLookup(ctx context.Context, q Question, nameServe
 			q.Name = qname[:len(qname)-1]
 		}
 	}
-	ctx, cancel := context.WithTimeout(context.Background(), r.timeout)
-	defer cancel()
 	retries := r.retries
 	questionWithMeta := QuestionWithMetadata{
 		Q:                q,
@@ -320,7 +317,6 @@ func (r *Resolver) LookupAllNameserversIterative(q *Question, rootNameServers []
 	}
 	var trace Trace
 	currentLayer := "."
-	queryingForFinalResult := false
 	var err error
 	currentLayerNameServers := rootNameServers
 	if len(currentLayerNameServers) == 0 {
@@ -328,12 +324,11 @@ func (r *Resolver) LookupAllNameserversIterative(q *Question, rootNameServers []
 		currentLayerNameServers = r.rootNameServers
 	}
 	originalQuestionType := q.Type
-
 	q.Type = dns.TypeNS
+	var layerResults []ExtendedResult
+	var currTrace Trace
 	for {
 		// Getting the NameServers
-		var layerResults []ExtendedResult
-		currTrace := make(Trace, 0)
 		layerResults, currTrace, _, err = r.queryAllNameServersInLayer(ctx, perNameServerRetriesLimit, q, currentLayerNameServers)
 		trace = append(trace, currTrace...)
 		if err != nil {
@@ -343,9 +338,10 @@ func (r *Resolver) LookupAllNameserversIterative(q *Question, rootNameServers []
 		} else {
 			retv.LayeredResponses[currentLayer] = append(retv.LayeredResponses[currentLayer], layerResults...)
 		}
-		if queryingForFinalResult {
-			// we've found an authoritative answer, we can return
-			break
+		var newNameServers []NameServer
+		newNameServers, err = r.extractNameServersFromLayerResults(layerResults)
+		if err != nil {
+			return &retv, trace, StatusError, errors.Wrapf(err, "error extracting nameservers from layer %s", currentLayer)
 		}
 		// Set the next layer to query
 		var newLayer string
@@ -353,25 +349,35 @@ func (r *Resolver) LookupAllNameserversIterative(q *Question, rootNameServers []
 		if err != nil {
 			return &retv, trace, StatusError, errors.Wrapf(err, "error determining next authority for layer %s", currentLayer)
 		}
-		currentLayer = newLayer
-		var newNameServers []NameServer
-		newNameServers, err = r.extractNameServersFromLayerResults(layerResults)
-		if err != nil {
-			return &retv, trace, StatusError, errors.Wrapf(err, "error extracting nameservers from layer %s", currentLayer)
-		}
-		if len(currentLayerNameServers) == 0 {
-			return &retv, trace, StatusError, errors.New("no nameservers found in layer " + currentLayer)
-		}
-		sameNameServers := reflect.DeepEqual(currentLayerNameServers, newNameServers)
-		if sameNameServers {
-			// we've found all authoritative nameservers, now we can query them
-			q.Type = originalQuestionType
-			queryingForFinalResult = true
-		} else {
-			// we've found either new nameservers or the next layer. Continuing
-			currentLayerNameServers = newNameServers
+		if newLayer == currentLayer {
+			// we've reached the final layer
+			currentLayerNameServers = append(currentLayerNameServers, newNameServers...)
+			break
 		}
+		currentLayerNameServers = newNameServers
+		currentLayer = newLayer
+	}
+	// de-dupe nameservers
+	uniqNameServersSet := make(map[string]NameServer)
+	for _, ns := range currentLayerNameServers {
+		uniqNameServersSet[ns.DomainName] = ns
+	}
+	uniqNameServers := make([]NameServer, 0, len(uniqNameServersSet))
+	for _, ns := range uniqNameServersSet {
+		uniqNameServers = append(uniqNameServers, ns)
 	}
+	// Now that we have an exhaustive list of leaf NSes, we'll query the original NSes
+	q.Type = originalQuestionType
+	layerResults, currTrace, _, err = r.queryAllNameServersInLayer(ctx, perNameServerRetriesLimit, q, uniqNameServers)
+	trace = append(trace, currTrace...)
+	if err != nil {
+		return &retv, trace, StatusError, errors.Wrapf(err, "error encountered on layer %s", currentLayer)
+	} else if len(retv.LayeredResponses[currentLayer]) == 0 {
+		retv.LayeredResponses[currentLayer] = layerResults
+	} else {
+		retv.LayeredResponses[currentLayer] = append(retv.LayeredResponses[currentLayer], layerResults...)
+	}
+
 	return &retv, trace, StatusNoError, nil
 }
 
@@ -548,7 +554,7 @@ func (r *Resolver) queryAllNameServersInLayer(ctx context.Context, perNameServer
 			}
 			result, currTrace, status, err := r.ExternalLookup(ctx, q, &nameServer)
 			trace = append(trace, currTrace...)
-			extResult = &ExtendedResult{Status: status, Nameserver: nameServer.DomainName}
+			extResult = &ExtendedResult{Status: status, Nameserver: nameServer.DomainName, Type: dns.TypeToString[q.Type]}
 			if result != nil {
 				extResult.Res = *result
 			}
diff --git a/src/zdns/lookup_test.go b/src/zdns/lookup_test.go
index 92ffb2af..b03314d5 100644
--- a/src/zdns/lookup_test.go
+++ b/src/zdns/lookup_test.go
@@ -1958,7 +1958,7 @@ func verifyNsResult(t *testing.T, servers []NSRecord, expectedServersMap map[str
 }
 
 func verifyCombinedResult(t *testing.T, records map[string][]ExtendedResult, expectedRecords map[string][]ExtendedResult) {
-	for layer, _ := range expectedRecords {
+	for layer := range expectedRecords {
 		assert.Contains(t, records, layer, fmt.Sprintf("Layer %s not found in combined result", layer))
 	}
 	for layer, expectedLayerResults := range expectedRecords {
diff --git a/src/zdns/qa.go b/src/zdns/qa.go
index 0461ccf2..a64a2568 100644
--- a/src/zdns/qa.go
+++ b/src/zdns/qa.go
@@ -85,6 +85,7 @@ type SingleQueryResult struct {
 }
 
 type ExtendedResult struct {
+	Type       string            `json:"type" groups:"short,normal,long,trace"`
 	Res        SingleQueryResult `json:"result,omitempty" groups:"short,normal,long,trace"`
 	Status     Status            `json:"status" groups:"short,normal,long,trace"`
 	Nameserver string            `json:"nameserver" groups:"short,normal,long,trace"` // NS name name

From 6b6e93ddcaf2a97057be41d72c1fa3a4f0967af6 Mon Sep 17 00:00:00 2001
From: phillip-stephens <phillip@cs.stanford.edu>
Date: Mon, 16 Dec 2024 16:49:33 -0500
Subject: [PATCH 19/28] lint/tests

---
 src/zdns/lookup.go      |  4 ++++
 src/zdns/lookup_test.go | 46 +++++++++++++++++++++++++++++++++++++----
 2 files changed, 46 insertions(+), 4 deletions(-)

diff --git a/src/zdns/lookup.go b/src/zdns/lookup.go
index 5df7eeec..20588595 100644
--- a/src/zdns/lookup.go
+++ b/src/zdns/lookup.go
@@ -354,6 +354,10 @@ func (r *Resolver) LookupAllNameserversIterative(q *Question, rootNameServers []
 			currentLayerNameServers = append(currentLayerNameServers, newNameServers...)
 			break
 		}
+		if len(newNameServers) == 0 {
+			// we have no more nameservers to query, error
+			return &retv, trace, StatusError, errors.Errorf("no nameservers found for layer %s", currentLayer)
+		}
 		currentLayerNameServers = newNameServers
 		currentLayer = newLayer
 	}
diff --git a/src/zdns/lookup_test.go b/src/zdns/lookup_test.go
index b03314d5..401ab478 100644
--- a/src/zdns/lookup_test.go
+++ b/src/zdns/lookup_test.go
@@ -1532,9 +1532,6 @@ func TestAllNsLookupOneNsThreeLevels(t *testing.T) {
 	exampleNSServerIP := "3.3.3.3"
 	exampleNameAAnswer := "4.4.4.4"
 
-	//ns1 := &config.ExternalNameServersV4[0]
-	//ipv4_1 := "127.0.0.2"
-
 	mockResults[nameAndIP{name: exampleName, IP: rootServerIP + ":53"}] = SingleQueryResult{
 		Authorities: []interface{}{
 			Answer{
@@ -1597,6 +1594,7 @@ func TestAllNsLookupOneNsThreeLevels(t *testing.T) {
 			{
 				Status:     StatusNoError,
 				Nameserver: rootServer,
+				Type:       "NS",
 				Res: SingleQueryResult{
 					Authorities: []interface{}{
 						Answer{
@@ -1624,6 +1622,7 @@ func TestAllNsLookupOneNsThreeLevels(t *testing.T) {
 		"com": {
 			{
 				Status:     StatusNoError,
+				Type:       "NS",
 				Nameserver: comServer,
 				Res: SingleQueryResult{
 					Authorities: []interface{}{
@@ -1652,6 +1651,24 @@ func TestAllNsLookupOneNsThreeLevels(t *testing.T) {
 		"example.com": {
 			{
 				Status:     StatusNoError,
+				Type:       "NS",
+				Nameserver: exampleNSServer,
+				Res: SingleQueryResult{
+					Answers: []interface{}{
+						Answer{
+							TTL:    3600,
+							Type:   "A",
+							RrType: dns.TypeA,
+							Class:  "IN",
+							Name:   "example.com.",
+							Answer: "4.4.4.4",
+						},
+					},
+				},
+			},
+			{
+				Status:     StatusNoError,
+				Type:       "A",
 				Nameserver: exampleNSServer,
 				Res: SingleQueryResult{
 					Answers: []interface{}{
@@ -1780,6 +1797,7 @@ func TestAllNsLookupErrorInOne(t *testing.T) {
 		".": {
 			{
 				Status:     StatusNoError,
+				Type:       "NS",
 				Nameserver: rootServer,
 				Res: SingleQueryResult{
 					Authorities: []interface{}{
@@ -1824,11 +1842,13 @@ func TestAllNsLookupErrorInOne(t *testing.T) {
 		"com": {
 			{
 				Status:     StatusServFail,
+				Type:       "NS",
 				Nameserver: comServerA,
 				Res:        SingleQueryResult{},
 			},
 			{
 				Status:     StatusNoError,
+				Type:       "NS",
 				Nameserver: comServerB,
 				Res: SingleQueryResult{
 					Authorities: []interface{}{
@@ -1857,6 +1877,24 @@ func TestAllNsLookupErrorInOne(t *testing.T) {
 		"example.com": {
 			{
 				Status:     StatusNoError,
+				Type:       "NS",
+				Nameserver: exampleNSServer,
+				Res: SingleQueryResult{
+					Answers: []interface{}{
+						Answer{
+							TTL:    3600,
+							Type:   "A",
+							RrType: dns.TypeA,
+							Class:  "IN",
+							Name:   "example.com.",
+							Answer: exampleNameAAnswer,
+						},
+					},
+				},
+			},
+			{
+				Status:     StatusNoError,
+				Type:       "A",
 				Nameserver: exampleNSServer,
 				Res: SingleQueryResult{
 					Answers: []interface{}{
@@ -1900,7 +1938,7 @@ func TestAllNsLookupNXDomain(t *testing.T) {
 	res, _, status, err := resolver.LookupAllNameserversIterative(&q, []NameServer{*ns1})
 
 	expectedResponse := map[string][]ExtendedResult{
-		".": {{Status: StatusNXDomain}},
+		".": {{Status: StatusNXDomain, Type: "NS"}},
 	}
 	if !reflect.DeepEqual(res.LayeredResponses, expectedResponse) {
 		t.Errorf("Expected %v, Received %v", expectedResponse, res.LayeredResponses)

From 12ba0ae0ad6587abacb8fcd3e5f9174e17c99181 Mon Sep 17 00:00:00 2001
From: phillip-stephens <phillip@cs.stanford.edu>
Date: Mon, 16 Dec 2024 17:21:15 -0500
Subject: [PATCH 20/28] integration tests

---
 testing/integration_tests.py | 38 ++++++++++++++++++++++++++----------
 1 file changed, 28 insertions(+), 10 deletions(-)

diff --git a/testing/integration_tests.py b/testing/integration_tests.py
index 70069347..875fb898 100755
--- a/testing/integration_tests.py
+++ b/testing/integration_tests.py
@@ -1422,26 +1422,44 @@ def test_external_lookup_cache(self):
     def test_lookup_all_nameservers_single_zone_iterative(self):
         """
         Test that --all-nameservers --iterative lookups work with domains whose nameservers are all in the same zone
-        google.com has nameservers ns1/2/3/4.google.com, which are all in the .com zone and so will have their IPs
+        zdns-testing.com has nameservers ns-cloud-c1/2/3/4.googledomains.com, which are all in the .com zone and so will have their IPs
         provided as additionals in the .com response
         """
-        # google.com's nameservers are all in the .com zone, so we should only have to query the .com nameservers
-        c = "A google.com --all-nameservers --iterative"
+        # zdns-testing.com's nameservers are all in the .com zone, so we should only have to query the .com nameservers
+        c = "A zdns-testing.com --all-nameservers --iterative"
         cmd,res = self.run_zdns(c, "")
         self.assertSuccess(res, cmd, "A")
         # Check for layers
         self.assertIn(".", res["results"]["A"]["data"]["per_layer_responses"], "Should have the root (.) layer")
         self.assertIn("com", res["results"]["A"]["data"]["per_layer_responses"], "Should have the .com layer")
-        self.assertIn("google.com", res["results"]["A"]["data"]["per_layer_responses"], "Should have the google.com layer")
+        self.assertIn("zdns-testing.com", res["results"]["A"]["data"]["per_layer_responses"], "Should have the google.com layer")
         # check for a.root-servers.net, b.root-servers.net, ... m.root-servers.net
         self.check_for_existance_of_root_and_com_nses(res)
         # check for the google.com nameservers
-        actual_google_nses = []
-        for entry in res["results"]["A"]["data"]["per_layer_responses"]["google.com"]:
-            actual_google_nses.append(entry["nameserver"])
-        expected_google_nses = ["ns1.google.com", "ns2.google.com", "ns3.google.com", "ns4.google.com"]
-        for ns in expected_google_nses:
-            self.assertIn(ns, actual_google_nses, "Should have the google.com nameservers")
+        actual_zdns_testing_leaf_NS_answers = []
+        actual_zdns_testing_leaf_A_answers = []
+        for entry in res["results"]["A"]["data"]["per_layer_responses"]["zdns-testing.com"]:
+            if entry["type"] == "NS":
+                actual_zdns_testing_leaf_NS_answers.append(entry)
+            elif entry["type"] == "A":
+                actual_zdns_testing_leaf_A_answers.append(entry)
+            else:
+                self.fail(f"Unexpected record type {entry['type']}")
+
+
+
+        # Check that we have "1.2.3.4", "2.3.4.5", and "3.4.5.6" as the A records and valid NS records for all expected Leaf NSes
+        if len(actual_zdns_testing_leaf_A_answers) != 4 or len(actual_zdns_testing_leaf_NS_answers) != 4:
+            self.fail("Should have 4 A  and 4 NS record sets")
+        expectedAnswers = ["1.2.3.4", "2.3.4.5", "3.4.5.6"]
+        for entry in actual_zdns_testing_leaf_A_answers:
+            actualAnswers = []
+            for answer in entry["result"]["answers"]:
+                actualAnswers.append(answer["answer"])
+            # sort
+            actualAnswers.sort()
+            expectedAnswers.sort()
+            self.assertEqual(actualAnswers, expectedAnswers, "Should have the expected A records")
 
     def check_for_existance_of_root_and_com_nses(self, res):
         actual_root_ns = []

From e065f93567f77119e94619616fea8d7a0175acb1 Mon Sep 17 00:00:00 2001
From: phillip-stephens <phillip@cs.stanford.edu>
Date: Mon, 16 Dec 2024 17:40:19 -0500
Subject: [PATCH 21/28] fix over-zealous replace operation

---
 src/zdns/alookup.go  | 2 +-
 src/zdns/answers.go  | 2 +-
 src/zdns/lookup.go   | 6 +++---
 src/zdns/qa.go       | 2 +-
 src/zdns/resolver.go | 4 ++--
 5 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/src/zdns/alookup.go b/src/zdns/alookup.go
index 3e8b5023..9e4c1d1b 100644
--- a/src/zdns/alookup.go
+++ b/src/zdns/alookup.go
@@ -24,7 +24,7 @@ import (
 	"github.com/zmap/zdns/src/internal/util"
 )
 
-// DoTargetedLookup performs a lookup of the given name name against the given nameserver, looking up both IPv4 and IPv6 addresses
+// DoTargetedLookup performs a lookup of the given name against the given nameserver, looking up both IPv4 and IPv6 addresses
 // Will follow CNAME records as well as A/AAAA records to get IP addresses
 func (r *Resolver) DoTargetedLookup(name string, nameServer *NameServer, isIterative, lookupA, lookupAAAA bool) (*IPResult, Trace, Status, error) {
 	name = strings.ToLower(name)
diff --git a/src/zdns/answers.go b/src/zdns/answers.go
index fef319d9..f85c0dd1 100644
--- a/src/zdns/answers.go
+++ b/src/zdns/answers.go
@@ -202,7 +202,7 @@ type SMIMEAAnswer struct {
 
 type SOAAnswer struct {
 	Answer
-	Ns      string `json:"IP" groups:"short,normal,long,trace"`
+	Ns      string `json:"ns" groups:"short,normal,long,trace"`
 	Mbox    string `json:"mbox" groups:"short,normal,long,trace"`
 	Serial  uint32 `json:"serial" groups:"short,normal,long,trace"`
 	Refresh uint32 `json:"refresh" groups:"short,normal,long,trace"`
diff --git a/src/zdns/lookup.go b/src/zdns/lookup.go
index 20588595..f9255196 100644
--- a/src/zdns/lookup.go
+++ b/src/zdns/lookup.go
@@ -91,7 +91,7 @@ func (r *Resolver) doDstServersLookup(ctx context.Context, q Question, nameServe
 		// if that looks likely, use it as is
 		if err != nil && !util.IsStringValidDomainName(q.Name) {
 			return nil, nil, StatusIllegalInput, err
-			// q.Name is a valid name name, we can continue
+			// q.Name is a valid name, we can continue
 		} else {
 			// remove trailing "." added by dns.ReverseAddr
 			q.Name = qname[:len(qname)-1]
@@ -707,7 +707,7 @@ func getRandomNonQueriedNameServer(nameServers []NameServer, queriedNameServers
 
 // cachedLookup performs a DNS lookup with caching
 // returns the result, whether it was cached, the status, and an error if one occurred
-// layer is the name name layer we're currently querying ex: ".", "com.", "example.com."
+// layer is the name layer we're currently querying ex: ".", "com.", "example.com."
 // depth is the current depth of the lookup, used for iterative lookups
 // requestIteration is whether to set the "recursion desired" bit in the DNS query
 // cacheBasedOnNameServer is whether to consider a cache hit based on DNS question and nameserver, or just question
@@ -1277,7 +1277,7 @@ func FindTxtRecord(res *SingleQueryResult, regex *regexp.Regexp) (string, error)
 }
 
 // populateResults is a helper function to populate the candidateSet, cnameSet, and garbage maps to follow CNAMES
-// These maps are keyed by the name name and contain the relevant answers for that name
+// These maps are keyed by the name and contain the relevant answers for that name
 // candidateSet is a map of Answers that have a type matching the requested type.
 // cnameSet is a map of Answers that are CNAME records
 // dnameSet is a map of Answers that are DNAME records
diff --git a/src/zdns/qa.go b/src/zdns/qa.go
index a64a2568..c28712e7 100644
--- a/src/zdns/qa.go
+++ b/src/zdns/qa.go
@@ -88,7 +88,7 @@ type ExtendedResult struct {
 	Type       string            `json:"type" groups:"short,normal,long,trace"`
 	Res        SingleQueryResult `json:"result,omitempty" groups:"short,normal,long,trace"`
 	Status     Status            `json:"status" groups:"short,normal,long,trace"`
-	Nameserver string            `json:"nameserver" groups:"short,normal,long,trace"` // NS name name
+	Nameserver string            `json:"nameserver" groups:"short,normal,long,trace"` // NS domain name
 }
 
 type AllNameServersResult struct {
diff --git a/src/zdns/resolver.go b/src/zdns/resolver.go
index 51f642c5..c7f74dac 100644
--- a/src/zdns/resolver.go
+++ b/src/zdns/resolver.go
@@ -536,14 +536,14 @@ func (r *Resolver) getConnectionInfo(nameServer *NameServer) (*ConnectionInfo, e
 					}
 					var tlsConn *tls.Conn
 					if len(nameServer.DomainName) != 0 && r.verifyServerCert {
-						// name name provided, we can verify the server's certificate
+						// domain name provided, we can verify the server's certificate
 						tlsConn = tls.Client(conn, &tls.Config{
 							InsecureSkipVerify: false,
 							RootCAs:            r.rootCAs,
 							ServerName:         nameServer.DomainName,
 						})
 					} else {
-						// If no name name is provided, we can't verify the server's certificate
+						// If no name is provided, we can't verify the server's certificate
 						tlsConn = tls.Client(conn, &tls.Config{
 							InsecureSkipVerify: true,
 						})

From 524dc0637eff44bfd9cc10b81eac97d4b4030fa3 Mon Sep 17 00:00:00 2001
From: phillip-stephens <phillip@cs.stanford.edu>
Date: Mon, 16 Dec 2024 17:48:52 -0500
Subject: [PATCH 22/28] sort for consistent unit test

---
 src/zdns/lookup_test.go | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/src/zdns/lookup_test.go b/src/zdns/lookup_test.go
index 401ab478..8c2d4cd3 100644
--- a/src/zdns/lookup_test.go
+++ b/src/zdns/lookup_test.go
@@ -21,6 +21,7 @@ import (
 	"net"
 	"reflect"
 	"regexp"
+	"sort"
 	"testing"
 	"time"
 
@@ -2000,6 +2001,12 @@ func verifyCombinedResult(t *testing.T, records map[string][]ExtendedResult, exp
 		assert.Contains(t, records, layer, fmt.Sprintf("Layer %s not found in combined result", layer))
 	}
 	for layer, expectedLayerResults := range expectedRecords {
+		sort.Slice(records[layer], func(i, j int) bool {
+			return records[layer][i].Nameserver < records[layer][j].Nameserver
+		})
+		sort.Slice(records[layer], func(i, j int) bool {
+			return expectedLayerResults[i].Nameserver < expectedLayerResults[j].Nameserver
+		})
 		if !reflect.DeepEqual(records[layer], expectedLayerResults) {
 			t.Errorf("Combined result not matching for layer %s, expected %v, found %v", layer, expectedLayerResults, records[layer])
 		}

From d2d8625f949662f35079e6f4f81acccb8dabb292 Mon Sep 17 00:00:00 2001
From: phillip-stephens <phillip@cs.stanford.edu>
Date: Mon, 16 Dec 2024 18:01:15 -0500
Subject: [PATCH 23/28] add timeout error to all-nameservers

---
 src/zdns/lookup.go | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/src/zdns/lookup.go b/src/zdns/lookup.go
index f9255196..fc9a83eb 100644
--- a/src/zdns/lookup.go
+++ b/src/zdns/lookup.go
@@ -33,6 +33,8 @@ import (
 	"github.com/zmap/zdns/src/internal/util"
 )
 
+var ErrorContextExpired = errors.New("context expired")
+
 // GetDNSServers returns a list of IPv4, IPv6 DNS servers from a file, or an error if one occurs
 func GetDNSServers(path string) (ipv4, ipv6 []string, err error) {
 	c, err := dns.ClientConfigFromFile(path)
@@ -287,7 +289,7 @@ func (r *Resolver) LookupAllNameserversExternal(q *Question, nameServers []NameS
 
 	for _, ns := range nameServers {
 		if util.HasCtxExpired(ctx) {
-			return retv, trace, StatusTimeout, errors.New("context expired")
+			return retv, trace, StatusTimeout, ErrorContextExpired
 		}
 		result, currTrace, status, err := r.ExternalLookup(ctx, q, &ns)
 		trace = append(trace, currTrace...)
@@ -331,7 +333,9 @@ func (r *Resolver) LookupAllNameserversIterative(q *Question, rootNameServers []
 		// Getting the NameServers
 		layerResults, currTrace, _, err = r.queryAllNameServersInLayer(ctx, perNameServerRetriesLimit, q, currentLayerNameServers)
 		trace = append(trace, currTrace...)
-		if err != nil {
+		if err != nil && errors.Is(err, ErrorContextExpired) {
+			return &retv, trace, StatusTimeout, err
+		} else if err != nil {
 			return &retv, trace, StatusError, errors.Wrapf(err, "error encountered on layer %s", currentLayer)
 		} else if len(retv.LayeredResponses[currentLayer]) == 0 {
 			retv.LayeredResponses[currentLayer] = layerResults
@@ -545,7 +549,7 @@ func (r *Resolver) queryAllNameServersInLayer(ctx context.Context, perNameServer
 		var extResult *ExtendedResult
 		for retry := 0; retry < perNameServerRetriesLimit; retry++ {
 			if util.HasCtxExpired(ctx) {
-				return currentLayerResults, trace, false, errors.New("context expired")
+				return currentLayerResults, trace, false, ErrorContextExpired
 			}
 			if nameServer.IP == nil {
 				nsTrace, err := r.populateNameServerIP(ctx, &nameServer)

From 6583c6f29f270529a1fa1c701b17c31acabaaf2c Mon Sep 17 00:00:00 2001
From: phillip-stephens <phillip@cs.stanford.edu>
Date: Wed, 18 Dec 2024 11:41:07 -0500
Subject: [PATCH 24/28] handle de-duplicating nameservers by name, ensureing we
 only query one IPv4/v6 addr for a given nameserver

---
 src/zdns/lookup.go           | 64 +++++++++++++++++++++++++++++++-----
 testing/integration_tests.py |  4 +--
 2 files changed, 58 insertions(+), 10 deletions(-)

diff --git a/src/zdns/lookup.go b/src/zdns/lookup.go
index fc9a83eb..344e9663 100644
--- a/src/zdns/lookup.go
+++ b/src/zdns/lookup.go
@@ -305,6 +305,59 @@ func (r *Resolver) LookupAllNameserversExternal(q *Question, nameServers []NameS
 	return retv, trace, StatusNoError, nil
 }
 
+// filterNameServersForUniqueNames will filter out duplicate nameservers based on the name.
+// Usually we'll have duplicates if a nameserver has both an IPv4 and IPv6 address. We'll use r.ipVersionMode and r.iterationIPPreference to determine which to keep.
+func (r *Resolver) filterNameServersForUniqueNames(nameServers []NameServer) []NameServer {
+	uniqNameServersSet := make(map[string][]NameServer)
+	for _, ns := range nameServers {
+		if _, ok := uniqNameServersSet[ns.DomainName]; !ok {
+			// no slice, add one
+			uniqNameServersSet[ns.DomainName] = make([]NameServer, 0, 1)
+		}
+		uniqNameServersSet[ns.DomainName] = append(uniqNameServersSet[ns.DomainName], ns)
+	}
+	// nameservers not grouped by name
+	filteredNameServersSet := make([]NameServer, 0, len(uniqNameServersSet))
+	for _, nsSlice := range uniqNameServersSet {
+		var ipv4NS, ipv6NS *NameServer
+		for _, ns := range nsSlice {
+			if ns.IP.To4() != nil {
+				ipv4NS = &ns
+			} else if util.IsIPv6(&ns.IP) {
+				ipv6NS = &ns
+			}
+		}
+		if ipv4NS == nil && ipv6NS == nil {
+			// can be the case that nameservers don't have IPs (like if we have an authority but no additionals)
+			// use the first NS if so
+			if len(nsSlice) > 0 {
+				filteredNameServersSet = append(filteredNameServersSet, nsSlice[0])
+				continue
+			}
+		}
+		// If we only have one IP version, we'll keep that
+		if ipv4NS == nil {
+			filteredNameServersSet = append(filteredNameServersSet, *ipv6NS)
+			continue
+		}
+		if ipv6NS == nil {
+			filteredNameServersSet = append(filteredNameServersSet, *ipv4NS)
+			continue
+		}
+		// If we have both, we'll use the resolver's settings to determine which to keep
+		if r.ipVersionMode == IPv4Only {
+			filteredNameServersSet = append(filteredNameServersSet, *ipv4NS)
+		} else if r.ipVersionMode == IPv6Only {
+			filteredNameServersSet = append(filteredNameServersSet, *ipv6NS)
+		} else if r.iterationIPPreference == PreferIPv4 {
+			filteredNameServersSet = append(filteredNameServersSet, *ipv4NS)
+		} else if r.iterationIPPreference == PreferIPv6 {
+			filteredNameServersSet = append(filteredNameServersSet, *ipv6NS)
+		}
+	}
+	return filteredNameServersSet
+}
+
 // LookupAllNameserversIterative will send a query to all name servers at each level of DNS resolution.
 // It starts at either the provided rootNameServers or r.rootNameServers if none are provided as arguments and queries all.
 // If the responses contain an authoritative answer, the function will return the result and a trace for each queried nameserver.
@@ -330,6 +383,8 @@ func (r *Resolver) LookupAllNameserversIterative(q *Question, rootNameServers []
 	var layerResults []ExtendedResult
 	var currTrace Trace
 	for {
+		// Filter out duplicate nameservers by name, we'll treat IPv4 and IPv6 addresses as the same nameserver
+		currentLayerNameServers = r.filterNameServersForUniqueNames(currentLayerNameServers)
 		// Getting the NameServers
 		layerResults, currTrace, _, err = r.queryAllNameServersInLayer(ctx, perNameServerRetriesLimit, q, currentLayerNameServers)
 		trace = append(trace, currTrace...)
@@ -366,14 +421,7 @@ func (r *Resolver) LookupAllNameserversIterative(q *Question, rootNameServers []
 		currentLayer = newLayer
 	}
 	// de-dupe nameservers
-	uniqNameServersSet := make(map[string]NameServer)
-	for _, ns := range currentLayerNameServers {
-		uniqNameServersSet[ns.DomainName] = ns
-	}
-	uniqNameServers := make([]NameServer, 0, len(uniqNameServersSet))
-	for _, ns := range uniqNameServersSet {
-		uniqNameServers = append(uniqNameServers, ns)
-	}
+	uniqNameServers := r.filterNameServersForUniqueNames(currentLayerNameServers)
 	// Now that we have an exhaustive list of leaf NSes, we'll query the original NSes
 	q.Type = originalQuestionType
 	layerResults, currTrace, _, err = r.queryAllNameServersInLayer(ctx, perNameServerRetriesLimit, q, uniqNameServers)
diff --git a/testing/integration_tests.py b/testing/integration_tests.py
index 875fb898..bd79993f 100755
--- a/testing/integration_tests.py
+++ b/testing/integration_tests.py
@@ -1426,7 +1426,7 @@ def test_lookup_all_nameservers_single_zone_iterative(self):
         provided as additionals in the .com response
         """
         # zdns-testing.com's nameservers are all in the .com zone, so we should only have to query the .com nameservers
-        c = "A zdns-testing.com --all-nameservers --iterative"
+        c = "A zdns-testing.com --all-nameservers --iterative --timeout=30"
         cmd,res = self.run_zdns(c, "")
         self.assertSuccess(res, cmd, "A")
         # Check for layers
@@ -1482,7 +1482,7 @@ def test_lookup_all_nameservers_multi_zone_iterative(self):
         provide the IPs in additionals.
         """
         # example.com has nameservers in .com, .org, and .net, we'll have to iteratively figure out their IP addresses too
-        c = "A example.com --all-nameservers --iterative"
+        c = "A example.com --all-nameservers --iterative --timeout=30"
         cmd,res = self.run_zdns(c, "")
         self.assertSuccess(res, cmd, "A")
         # Check for layers

From a1c4e8b8759dbb6142751640736fa1c0c4dc8e00 Mon Sep 17 00:00:00 2001
From: phillip-stephens <phillip@cs.stanford.edu>
Date: Wed, 18 Dec 2024 11:49:25 -0500
Subject: [PATCH 25/28] PR cleanup

---
 examples/all_nameservers_lookup/main.go | 2 +-
 src/zdns/lookup.go                      | 6 +++++-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/examples/all_nameservers_lookup/main.go b/examples/all_nameservers_lookup/main.go
index ebaa342e..d1c55123 100644
--- a/examples/all_nameservers_lookup/main.go
+++ b/examples/all_nameservers_lookup/main.go
@@ -29,7 +29,7 @@ func main() {
 	domain := "google.com"
 	dnsQuestion := &zdns.Question{Name: domain, Type: dns.TypeA, Class: dns.ClassINET}
 	resolver := initializeResolver()
-	// Iterative Lookups start at the root nameservers and follow the chain of referrals to the authoritative nameservers.
+	// LookupAllNameserversIterative will query all root nameservers, and then all TLD nameservers, and then all authoritative nameservers for the domain.
 	result, _, status, err := resolver.LookupAllNameserversIterative(dnsQuestion, nil)
 	if err != nil {
 		log.Fatal("Error looking up domain: ", err)
diff --git a/src/zdns/lookup.go b/src/zdns/lookup.go
index 344e9663..e6a71b79 100644
--- a/src/zdns/lookup.go
+++ b/src/zdns/lookup.go
@@ -362,7 +362,11 @@ func (r *Resolver) filterNameServersForUniqueNames(nameServers []NameServer) []N
 // It starts at either the provided rootNameServers or r.rootNameServers if none are provided as arguments and queries all.
 // If the responses contain an authoritative answer, the function will return the result and a trace for each queried nameserver.
 // If the responses do not contain an authoritative answer, the function will continue to the next layer of nameservers.
-// At each layer, we'll de-duplicate the referral nameservers from the previous layer and query them.
+// At each layer, we'll de-duplicate the referral nameservers from the previous layer and query them. For example, if all
+// root nameservers return a-m.gtld-servers.net, we'll only query each gtld-server once.
+//
+// Additionally, we'll query each layer for NS records, and once we have the set of authoritative nameservers, we'll query with
+// the original question type. This helps find sibling nameservers that aren't listed with the TLD.
 func (r *Resolver) LookupAllNameserversIterative(q *Question, rootNameServers []NameServer) (*AllNameServersResult, Trace, Status, error) {
 	perNameServerRetriesLimit := 2
 	ctx, cancel := context.WithTimeout(context.Background(), r.timeout)

From d4885d2ac9dfbb1144bf602038a0868d8af52ce5 Mon Sep 17 00:00:00 2001
From: phillip-stephens <phillip@cs.stanford.edu>
Date: Wed, 18 Dec 2024 13:12:25 -0500
Subject: [PATCH 26/28] don't error for CNAME referrals, we won't implement
 this

---
 src/zdns/lookup.go | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/src/zdns/lookup.go b/src/zdns/lookup.go
index e6a71b79..7e031361 100644
--- a/src/zdns/lookup.go
+++ b/src/zdns/lookup.go
@@ -418,6 +418,23 @@ func (r *Resolver) LookupAllNameserversIterative(q *Question, rootNameServers []
 			break
 		}
 		if len(newNameServers) == 0 {
+			// check if we have no referral nameservers because we've hit a CNAME or DNAME
+			foundReferral := false
+			for _, res := range layerResults {
+				for _, ans := range res.Res.Answers {
+					if a, ok := ans.(Answer); ok {
+						if a.RrType == dns.TypeCNAME || a.RrType == dns.TypeDNAME {
+							foundReferral = true
+							break
+						}
+					}
+				}
+			}
+			if foundReferral {
+				// we don't handle iterative all-nameservers lookups with C/DNAMEs, returning the results we've collected
+				// thus far
+				return &retv, trace, StatusNoError, nil
+			}
 			// we have no more nameservers to query, error
 			return &retv, trace, StatusError, errors.Errorf("no nameservers found for layer %s", currentLayer)
 		}

From 657ddf70b3a4ec130556081ad3156c87777975b3 Mon Sep 17 00:00:00 2001
From: phillip-stephens <phillip@cs.stanford.edu>
Date: Wed, 18 Dec 2024 13:21:16 -0500
Subject: [PATCH 27/28] cleanup

---
 src/zdns/qa.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/zdns/qa.go b/src/zdns/qa.go
index c28712e7..91d1072a 100644
--- a/src/zdns/qa.go
+++ b/src/zdns/qa.go
@@ -88,11 +88,11 @@ type ExtendedResult struct {
 	Type       string            `json:"type" groups:"short,normal,long,trace"`
 	Res        SingleQueryResult `json:"result,omitempty" groups:"short,normal,long,trace"`
 	Status     Status            `json:"status" groups:"short,normal,long,trace"`
-	Nameserver string            `json:"nameserver" groups:"short,normal,long,trace"` // NS domain name
+	Nameserver string            `json:"nameserver" groups:"short,normal,long,trace"` // NS name queried for this result
 }
 
 type AllNameServersResult struct {
-	LayeredResponses map[string][]ExtendedResult `json:"per_layer_responses" groups:"short,normal,long,trace"` //
+	LayeredResponses map[string][]ExtendedResult `json:"per_layer_responses" groups:"short,normal,long,trace"`
 }
 
 type IPResult struct {

From 82f146e8db0f585cf3f50b8c1beeb547ef75e302 Mon Sep 17 00:00:00 2001
From: phillip-stephens <phillip@cs.stanford.edu>
Date: Wed, 18 Dec 2024 13:27:22 -0500
Subject: [PATCH 28/28] up timeouts on integration tests

---
 testing/integration_tests.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/testing/integration_tests.py b/testing/integration_tests.py
index bd79993f..9eabbe85 100755
--- a/testing/integration_tests.py
+++ b/testing/integration_tests.py
@@ -1426,7 +1426,7 @@ def test_lookup_all_nameservers_single_zone_iterative(self):
         provided as additionals in the .com response
         """
         # zdns-testing.com's nameservers are all in the .com zone, so we should only have to query the .com nameservers
-        c = "A zdns-testing.com --all-nameservers --iterative --timeout=30"
+        c = "A zdns-testing.com --all-nameservers --iterative --timeout=60"
         cmd,res = self.run_zdns(c, "")
         self.assertSuccess(res, cmd, "A")
         # Check for layers
@@ -1482,7 +1482,7 @@ def test_lookup_all_nameservers_multi_zone_iterative(self):
         provide the IPs in additionals.
         """
         # example.com has nameservers in .com, .org, and .net, we'll have to iteratively figure out their IP addresses too
-        c = "A example.com --all-nameservers --iterative --timeout=30"
+        c = "A example.com --all-nameservers --iterative --timeout=60"
         cmd,res = self.run_zdns(c, "")
         self.assertSuccess(res, cmd, "A")
         # Check for layers