diff --git a/README.md b/README.md index 3d2dd735..adc32d8a 100644 --- a/README.md +++ b/README.md @@ -31,19 +31,13 @@ Usage ===== ZDNS was originally built as a CLI tool only. Work has been done to convert -this into a library with a CLI that calls this library. Currently, the library -has been separated out and a new, separate CLI has been added. Work is ongoing -to clean up the interface between the CLI (or any other client program of the -ZDNS library) and the ZDNS library itself. +this [library](github.com/zmap/zdns/src/zdns) into a standalone library and let a separate [CLI](github.com/zmap/zdns/src/cli) wrap the library. -The ZDNS library lives in `github.com/zmap/zdns/pkg/zdns`. A function there, -`zdns.Run()`, is used to start the ZDNS tool and do the requested lookups. -Currently, this tool is intended to accept a `zdns.GlobalConf` object, `plfag` -flags, and other information, but this interface is undergoing revisions to be -more generally usable and continue to decouple the CLI from the library. - -The CLI for this library lives in `github.com/zmap/zdns` under the main -package. Its functionality is described below. +The library consists of a `ResolverConfig` struct which will contain all config options for all lookups made. +The `ResolverConfig` is used to create 1+ `Resolver` struct(s) which will make all lookups. A `Resolver` +should only make a single lookup at a time (it is not thread-safe) and multiple `Resolver` structs should be +used for parallelism. See our [examples](github.com/zmap/zdns/examples) for how to use the +library. [Modules](github.com/zmap/zdns/src/modules) are used to define the behavior of the lookups. ZDNS provides several types of modules: @@ -51,7 +45,8 @@ ZDNS provides several types of modules: but in JSON. There is a module for (nearly) every type of DNS record - *Lookup modules* provide more helpful responses when multiple queries are - required (e.g., completing additional `A` lookup if a `CNAME` is received) + required (e.g., completing additional `A` lookup for IP addresses if a `NS` is + received in `NSLOOKUP`) - *Misc modules* provide other additional means of querying servers (e.g., `bind.version`) @@ -76,40 +71,42 @@ For example, the command: returns: ```json { - "name": "censys.io", - "class": "IN", - "status": "NOERROR", - "data": { - "answers": [ - { - "ttl": 300, - "type": "A", - "class": "IN", - "name": "censys.io", - "data": "216.239.38.21" + "name": "censys.io", + "results": { + "A": { + "data": { + "additionals": [ + { + "flags": "", + "type": "EDNS0", + "udpsize": 512, + "version": 0 + } + ], + "answers": [ + { + "answer": "104.18.10.85", + "class": "IN", + "name": "censys.io", + "ttl": 300, + "type": "A" + }, + { + "answer": "104.18.11.85", + "class": "IN", + "name": "censys.io", + "ttl": 300, + "type": "A" + } + ], + "protocol": "udp", + "resolver": "[2603:6013:9d00:3302::1]:53" + }, + "duration": 0.285295416, + "status": "NOERROR", + "timestamp": "2024-08-23T13:12:43-04:00" } - ], - "additionals": [ - { - "ttl": 34563, - "type": "A", - "class": "IN", - "name": "ns-cloud-e1.googledomains.com", - "data": "216.239.32.110" - }, - ], - "authorities": [ - { - "ttl": 53110, - "type": "NS", - "class": "IN", - "name": "censys.io", - "data": "ns-cloud-e1.googledomains.com." - }, - ], - "protocol": "udp", - "resolver": "30.128.52.190:53" - } + } } ``` @@ -119,13 +116,12 @@ Lookup Modules Raw DNS responses frequently do not provide the data you _want_. For example, an MX response may not include the associated A records in the additionals section requiring an additional lookup. To address this gap and provide a -friendlier interface, we also provide several _lookup_ modules: `alookup` and -`mxlookup`. +friendlier interface, we also provide several _lookup_ modules: `alookup`, +`mxlookup`, and `nslookup`. -`mxlookup` will additionally do an A lookup for the IP addresses that -correspond with an exchange record. `alookup` acts similar to nslookup and will -follow CNAME records. `nslookup` will additionally do an A/AAAA lookup for IP -addresses that correspond with an NS record +`alookup` acts similar to nslookup and will follow CNAME records. +`mxlookup` will additionally do an A lookup for the IP addresses that correspond with an exchange record. +`nslookup` will additionally do an A/AAAA lookup for IP addresses that correspond with an NS record For example, @@ -134,32 +130,38 @@ For example, returns: ```json { - "name": "censys.io", - "status": "NOERROR", - "data": { - "exchanges": [ - { - "name": "aspmx.l.google.com", - "type": "MX", - "class": "IN", - "preference": 1, - "ipv4_addresses": [ - "74.125.28.26" - ], - "ttl": 288 - }, - { - "name": "alt1.aspmx.l.google.com", - "type": "MX", - "class": "IN", - "preference": 5, - "ipv4_addresses": [ - "64.233.182.26" - ], - "ttl": 288 + "name": "censys.io", + "results": { + "MXLOOKUP": { + "data": { + "exchanges": [ + { + "class": "IN", + "ipv4_addresses": [ + "209.85.202.27" + ], + "name": "alt1.aspmx.l.google.com", + "preference": 5, + "ttl": 300, + "type": "MX" + }, + { + "class": "IN", + "ipv4_addresses": [ + "142.250.31.26" + ], + "name": "aspmx.l.google.com", + "preference": 1, + "ttl": 300, + "type": "MX" + } + ] + }, + "duration": 0.154786958, + "status": "NOERROR", + "timestamp": "2024-08-23T13:10:11-04:00" } - ] - } + } } ``` @@ -283,6 +285,44 @@ There is a feature available to perform a certain DNS query against all nameserv ```echo "google.com" | ./zdns A --all-nameservers``` +Dig-style Domain Input +---------------------- +Similiar to dig, zdns can take a domain as input and perform a lookup on it. The only requirement is that the module is +the first argument and domains follow. +For example: + +``` +./zdns A google.com +``` + +Multiple Lookup Modules +----------------------- +ZDNS supports using multiple lookup modules in a single invocation. For example, let's say you want to perform an A, +AAAA, and MXLOOKUP for a set of domains and you want to perform them with iterative resolution. You will need to use the +`MULTIPLE` module and provide a config file with the modules and module-specific flags you want to use. + +Please see `./zdns --help` and `./zdns --help` for Global and Module-specific options that can be used in the config file. + +For example: + +``` +cat 1000k_domains.txt | ./zdns MULTIPLE --multi-config-file="./multiple.ini" +``` +Where `multiple.ini` is a file that looks like: +``` +; Specify Global Options here +[Application Options] +iterative=true +; List out modules and their respective module-specific options here. A module can only be listed once +[MXLOOKUP] +ipv4-lookup = true +; You can use default values and just list modules if you don't need to specify any options +[A] +[AAAA] +``` + +A sample `multiple.ini` file is provided in [github.com/zmap/zdns/cli/multiple.ini](github.com/zmap/zdns/cli/multiple.ini) + Running ZDNS ------------ diff --git a/benchmark/stats.go b/benchmark/stats.go index e5225d6a..d9cdaa71 100644 --- a/benchmark/stats.go +++ b/benchmark/stats.go @@ -135,7 +135,14 @@ func updateStats(line string, s *Stats) { log.Panicf("failed to unmarshal JSON (%s): %v", line, err) } domainName := res.Name - resolveTime := time.Duration(res.Duration * float64(time.Second)) + // TODO - this will only work for a single module benchmark, we'll need to adjust this if we want to benchmark multi-module + var duration float64 + var status zdns.Status + for _, moduleResult := range res.Results { + duration = moduleResult.Duration + status = zdns.Status(moduleResult.Status) + } + resolveTime := time.Duration(duration * float64(time.Second)) if resolveTime < s.MinResolveTime || s.MinResolveTime == 0 { s.MinResolveTime = resolveTime @@ -173,7 +180,6 @@ func updateStats(line string, s *Stats) { } s.numberOfResolutions++ - status := zdns.Status(res.Status) if status == zdns.StatusNoError { s.SuccessfulResolutions++ } else if status == zdns.StatusTimeout || status == zdns.StatusIterTimeout { diff --git a/src/cli/cli.go b/src/cli/cli.go index 68e3b581..e9c2c39d 100644 --- a/src/cli/cli.go +++ b/src/cli/cli.go @@ -83,19 +83,20 @@ type NetworkOptions struct { // InputOutputOptions options for controlling the input and output behavior of zdns. Applicable to all modules. type InputOutputOptions struct { - AlexaFormat bool `long:"alexa" description:"is input file from Alexa Top Million download"` - BlacklistFilePath string `long:"blacklist-file" description:"blacklist file for servers to exclude from lookups"` - ConfigFilePath string `long:"conf-file" default:"/etc/resolv.conf" description:"config file for DNS servers"` - IncludeInOutput string `long:"include-fields" description:"Comma separated list of fields to additionally output beyond result verbosity. Options: class, protocol, ttl, resolver, flags"` - InputFilePath string `short:"f" long:"input-file" default:"-" description:"names to read, defaults to stdin"` - LogFilePath string `long:"log-file" default:"-" description:"where should JSON logs be saved, defaults to stderr"` - MetadataFilePath string `long:"metadata-file" description:"where should JSON metadata be saved, defaults to no metadata output. Use '-' for stderr."` - MetadataFormat bool `long:"metadata-passthrough" description:"if input records have the form 'name,METADATA', METADATA will be propagated to the output"` - OutputFilePath string `short:"o" long:"output-file" default:"-" description:"where should JSON output be saved, defaults to stdout"` - NameOverride string `long:"override-name" description:"name overrides all passed in names. Commonly used with --name-server-mode."` - NamePrefix string `long:"prefix" description:"name to be prepended to what's passed in (e.g., www.)"` - ResultVerbosity string `long:"result-verbosity" default:"normal" description:"Sets verbosity of each output record. Options: short, normal, long, trace"` - Verbosity int `long:"verbosity" default:"3" description:"log verbosity: 1 (lowest)--5 (highest)"` + AlexaFormat bool `long:"alexa" description:"is input file from Alexa Top Million download"` + BlacklistFilePath string `long:"blacklist-file" description:"blacklist file for servers to exclude from lookups"` + DNSConfigFilePath string `long:"conf-file" default:"/etc/resolv.conf" description:"config file for DNS servers"` + MultipleModuleConfigFilePath string `short:"c" long:"multi-config-file" description:"config file path for multiple module"` + IncludeInOutput string `long:"include-fields" description:"Comma separated list of fields to additionally output beyond result verbosity. Options: class, protocol, ttl, resolver, flags"` + InputFilePath string `short:"f" long:"input-file" default:"-" description:"names to read, defaults to stdin"` + LogFilePath string `long:"log-file" default:"-" description:"where should JSON logs be saved, defaults to stderr"` + MetadataFilePath string `long:"metadata-file" description:"where should JSON metadata be saved, defaults to no metadata output. Use '-' for stderr."` + MetadataFormat bool `long:"metadata-passthrough" description:"if input records have the form 'name,METADATA', METADATA will be propagated to the output"` + OutputFilePath string `short:"o" long:"output-file" default:"-" description:"where should JSON output be saved, defaults to stdout"` + NameOverride string `long:"override-name" description:"name overrides all passed in names. Commonly used with --name-server-mode."` + NamePrefix string `long:"prefix" description:"name to be prepended to what's passed in (e.g., www.)"` + ResultVerbosity string `long:"result-verbosity" default:"normal" description:"Sets verbosity of each output record. Options: short, normal, long, trace"` + Verbosity int `long:"verbosity" default:"3" description:"log verbosity: 1 (lowest)--5 (highest)"` } type CLIConf struct { @@ -106,12 +107,15 @@ type CLIConf struct { OutputGroups []string TimeFormat string NameServers []string // recursive resolvers if not in iterative mode, root servers/servers to start iteration if in iterative mode + Domains []string // if user provides domain names as arguments, dig-style LocalAddrSpecified bool LocalAddrs []net.IP ClientSubnet *dns.EDNS0_SUBNET InputHandler InputHandler OutputHandler OutputHandler - Module string + CLIModule string // the module name as passed in by the user + ActiveModuleNames []string // names of modules that are active in this invocation of zdns. Mostly used with MULTIPLE + ActiveModules map[string]LookupModule // map of module names to modules Class uint16 } @@ -121,9 +125,51 @@ var GC CLIConf // This is called by main.main(). It only needs to happen once to the rootCmd. func Execute() { parseArgs() + if strings.EqualFold(GC.CLIModule, "MULTIPLE") { + err := handleMultipleModule(&GC) + if err != nil { + log.Fatalf("error in handling multiple modules: %v", err) + } + } else { + lookupModule, err := GetLookupModule(GC.CLIModule) + if err != nil { + log.Fatal("could not get lookup module: ", err) + } + GC.ActiveModules = make(map[string]LookupModule) + GC.ActiveModules[GC.CLIModule] = lookupModule + GC.ActiveModuleNames = []string{GC.CLIModule} + } Run(GC) } +func handleMultipleModule(GC *CLIConf) error { + // need to parse the multiple module config file first + if GC.MultipleModuleConfigFilePath == "" { + return errors.New("must specify a config file for the multiple module, see -c") + } + ini := flags.NewIniParser(parser) + moduleStrings, modules, err := ini.ParseFile(GC.MultipleModuleConfigFilePath) + if err != nil { + return fmt.Errorf("could not parse multi-module file: %v", err) + } + if len(moduleStrings) != len(modules) { + return errors.New("number of module names does not match number of modules retrieved from file") + } + GC.ActiveModuleNames = moduleStrings + GC.ActiveModules = make(map[string]LookupModule, len(moduleStrings)) + for i, name := range moduleStrings { + lm, ok := modules[i].(LookupModule) + if !ok { + return fmt.Errorf("module %s is not a LookupModule", name) + } + if _, ok = GC.ActiveModules[name]; ok { + return fmt.Errorf("module %s is defined multiple times in the config file", name) + } + GC.ActiveModules[name] = lm + } + return nil +} + // parseArgs parses the command line arguments and sets the global configuration // One limitation of the zflags library is you can't have "command-less" flags like ./zdns --version without turning // SubCommandsOptional = true. But then you don't get ZFlag's great command suggestion if you barely mistype a cmd. @@ -153,7 +199,7 @@ func parseArgs() { } parser.SubcommandsOptional = false parser.Options = flags.Default - _, moduleType, _, err := parser.ParseCommandLine(os.Args[1:]) + args, moduleType, _, err := parser.ParseCommandLine(os.Args[1:]) if err != nil { var flagErr *flags.Error if errors.As(err, &flagErr) { @@ -163,18 +209,30 @@ func parseArgs() { // exit and print log.Fatal(err) } - GC.Module = strings.ToUpper(moduleType) + if len(args) != 0 { + GC.Domains = args + } + GC.CLIModule = strings.ToUpper(moduleType) } func init() { parser = flags.NewParser(nil, flags.None) // options set in Execute() parser.Command.SubcommandsOptional = true // without this, the user must use a command, makes ./zdns --version impossible, we'll enforce specifying modules ourselves parser.Name = "zdns" + // ZFlags will pre-pend the parser.Name and append "" to the Usage string. So this is a work-around to indicate + // to users that [DOMAINS] must come after the command. ex: "./zdns A google.com yahoo.com + parser.Usage = "[OPTIONS] [DOMAINS]\n zdns [OPTIONS]" parser.ShortDescription = "High-speed, low-drag DNS lookups" parser.LongDescription = `ZDNS is a library and CLI tool for making very fast DNS requests. It's built upon https://github.com/zmap/dns (and in turn https://github.com/miekg/dns) for constructing and parsing raw DNS packets. +ZDNS also includes its own recursive resolution and a cache to further optimize performance. + +Domains can optionally passed into ZDNS similiar to dig, ex: zdns A google.com yahoo.com +If no domains are passed, ZDNS will read from stdin or the --input-file flag, if specified. + ZDNS also includes its own recursive resolution and a cache to further optimize performance.` + _, err := parser.AddGroup("General Options", "General options for controlling the behavior of zdns", &GC.GeneralOptions) if err != nil { log.Fatalf("could not add ZDNS Options group: %v", err) @@ -191,5 +249,10 @@ ZDNS also includes its own recursive resolution and a cache to further optimize if err != nil { log.Fatalf("could not add Input/Output Options group: %v", err) } - + // this is necessary since the .ini file parser expects that all global options are a part of Application Options + appOptions, err := parser.AddGroup("Application Options", "Hidden group including all global options", &GC) + if err != nil { + log.Fatalf("could not add Application Options group: %v", err) + } + appOptions.Hidden = true } diff --git a/src/cli/iohandlers/string_slice_input_handler.go b/src/cli/iohandlers/string_slice_input_handler.go new file mode 100644 index 00000000..6d1622a1 --- /dev/null +++ b/src/cli/iohandlers/string_slice_input_handler.go @@ -0,0 +1,42 @@ +/* + * ZDNS Copyright 2024 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package iohandlers + +import ( + "sync" + + log "github.com/sirupsen/logrus" +) + +// StringSliceInputHandler Feeds a channel with the strings in the slice. +type StringSliceInputHandler struct { + Names []string +} + +func NewStringSliceInputHandler(domains []string) *StringSliceInputHandler { + if len(domains) == 0 { + log.Fatal("No domains provided, cannot create a string slice input handler") + } + return &StringSliceInputHandler{Names: domains} +} + +func (h *StringSliceInputHandler) FeedChannel(in chan<- string, wg *sync.WaitGroup) error { + defer close(in) + defer wg.Done() + for _, name := range h.Names { + in <- name + } + return nil +} diff --git a/src/cli/modules.go b/src/cli/modules.go index 0db1771a..c3281a77 100644 --- a/src/cli/modules.go +++ b/src/cli/modules.go @@ -16,9 +16,8 @@ package cli import ( "fmt" - log "github.com/sirupsen/logrus" - "github.com/pkg/errors" + log "github.com/sirupsen/logrus" "github.com/zmap/dns" "github.com/zmap/zdns/src/zdns" @@ -27,8 +26,10 @@ import ( type LookupModule interface { CLIInit(gc *CLIConf, rc *zdns.ResolverConfig) error Lookup(resolver *zdns.Resolver, lookupName, nameServer string) (interface{}, zdns.Trace, zdns.Status, error) - Help() string - Description() string + Help() string // needed to satisfy the ZCommander interface in ZFlags. + GetDescription() string // needed to add a command to the parser, printed to the user. Printed to the user when they run the help command for a given module + Validate(args []string) error // needed to satisfy the ZCommander interface in ZFlags + NewFlags() interface{} // needed to satisfy the ZModule interface in ZFlags } const ( @@ -108,11 +109,17 @@ func init() { RegisterLookupModule("UNSPEC", &BasicLookupModule{DNSType: dns.TypeUNSPEC, DNSClass: dns.ClassINET}) RegisterLookupModule("URI", &BasicLookupModule{DNSType: dns.TypeURI, DNSClass: dns.ClassINET}) RegisterLookupModule("ANY", &BasicLookupModule{DNSType: dns.TypeANY, DNSClass: dns.ClassINET}) + RegisterLookupModule("MULTIPLE", &BasicLookupModule{ + DNSType: dns.TypeANY, + DNSClass: dns.ClassINET, + Description: "MULTIPLE is a lookup module used from the CLI to use multiple lookup modules at once with the " + + "help of a configuration file provided with --multi-config-file/-c. See README.md/Multiple Lookup Modules " + + "for more information."}) } func RegisterLookupModule(name string, lm LookupModule) { moduleToLookupModule[name] = lm - _, err := parser.AddCommand(name, "", lm.Description(), lm) + _, err := parser.AddCommand(name, "", lm.GetDescription(), lm) if err != nil { log.Fatalf("could not add command: %v", err) } @@ -123,6 +130,7 @@ type BasicLookupModule struct { LookupAllNameServers bool DNSType uint16 DNSClass uint16 + Description string } func (lm *BasicLookupModule) CLIInit(gc *CLIConf, rc *zdns.ResolverConfig) error { @@ -141,8 +149,16 @@ func (lm *BasicLookupModule) Help() string { return "" } -func (lm *BasicLookupModule) Description() string { - return "" +func (lm *BasicLookupModule) GetDescription() string { + return lm.Description +} + +func (lm *BasicLookupModule) Validate(args []string) error { + return nil +} + +func (lm *BasicLookupModule) NewFlags() interface{} { + return lm } func (lm *BasicLookupModule) Lookup(resolver *zdns.Resolver, lookupName, nameServer string) (interface{}, zdns.Trace, zdns.Status, error) { diff --git a/src/cli/multiple.ini b/src/cli/multiple.ini new file mode 100644 index 00000000..0038a984 --- /dev/null +++ b/src/cli/multiple.ini @@ -0,0 +1,11 @@ +; Specify Global Options here +[Application Options] +iterative=true +prefer-ipv6-iteration="true" +; List out modules and their respective module-specific options here. A module can only be listed once +[ALOOKUP] +ipv4-lookup = true +; You can use default values and just list modules if you don't need to specify any options +[A] +[AAAA] +[CNAME] \ No newline at end of file diff --git a/src/cli/worker_manager.go b/src/cli/worker_manager.go index 59ecc89c..5d44f88f 100644 --- a/src/cli/worker_manager.go +++ b/src/cli/worker_manager.go @@ -44,12 +44,14 @@ const ( ) type routineMetadata struct { - Names int - Status map[zdns.Status]int + Names int // number of domain names processed + Lookups int // number of lookups performed + Status map[zdns.Status]int } type Metadata struct { Names int `json:"names"` + Lookups int `json:"lookups"` Status map[string]int `json:"statuses"` StartTime string `json:"start_time"` EndTime string `json:"end_time"` @@ -139,7 +141,7 @@ func populateCLIConfig(gc *CLIConf) *CLIConf { if gc.NameServerMode && gc.MetadataFormat { log.Fatal("Metadata mode is incompatible with name server mode") } - if gc.NameServerMode && gc.NameOverride == "" && gc.Module != BINDVERSION { + if gc.NameServerMode && gc.NameOverride == "" && gc.CLIModule != BINDVERSION { log.Fatal("Static Name must be defined with --override-name in --name-server-mode unless DNS module does not expect names (e.g., BINDVERSION).") } // Output Groups are defined by a base + any additional fields that the user wants @@ -152,7 +154,10 @@ func populateCLIConfig(gc *CLIConf) *CLIConf { gc.OutputGroups = append(gc.OutputGroups, groups...) // setup i/o if not specified - if gc.InputHandler == nil { + if len(GC.Domains) > 0 { + // using domains from command line + gc.InputHandler = iohandlers.NewStringSliceInputHandler(GC.Domains) + } else if gc.InputHandler == nil { gc.InputHandler = iohandlers.NewFileInputHandler(gc.InputFilePath) } if gc.OutputHandler == nil { @@ -184,7 +189,7 @@ func populateResolverConfig(gc *CLIConf) *zdns.ResolverConfig { config.CheckingDisabledBit = gc.CheckingDisabled config.ShouldRecycleSockets = !gc.DisableRecycleSockets config.DNSSecEnabled = gc.Dnssec - config.DNSConfigFilePath = gc.ConfigFilePath + config.DNSConfigFilePath = gc.DNSConfigFilePath config.LogLevel = log.Level(gc.Verbosity) @@ -460,13 +465,12 @@ func Run(gc CLIConf) { if err != nil { log.Fatalf("resolver config did not pass validation: %v", err) } - lookupModule, err := GetLookupModule(gc.Module) - if err != nil { - log.Fatal("could not get lookup module: ", err) - } - err = lookupModule.CLIInit(&gc, resolverConfig) - if err != nil { - log.Fatalf("could not initialize lookup module (type: %s): %v", gc.Module, err) + for _, module := range gc.ActiveModules { + // init all modules + err = module.CLIInit(&gc, resolverConfig) + if err != nil { + log.Fatalf("could not initialize lookup module (type: %s): %v", gc.CLIModule, err) + } } // DoLookup: // - n threads that do processing from in and place results in out @@ -511,7 +515,7 @@ func Run(gc CLIConf) { for i := 0; i < gc.Threads; i++ { i := i go func(threadID int) { - initWorkerErr := doLookupWorker(&gc, lookupModule, resolverConfig, inChan, outChan, metaChan, &lookupWG) + initWorkerErr := doLookupWorker(&gc, resolverConfig, inChan, outChan, metaChan, &lookupWG) if initWorkerErr != nil { log.Fatalf("could not start lookup worker #%d: %v", i, initWorkerErr) } @@ -562,7 +566,7 @@ func Run(gc CLIConf) { } // doLookupWorker is a single worker thread that processes lookups from the input channel. It calls wg.Done when it is finished. -func doLookupWorker(gc *CLIConf, lookup LookupModule, rc *zdns.ResolverConfig, input <-chan string, output chan<- string, metaChan chan<- routineMetadata, wg *sync.WaitGroup) error { +func doLookupWorker(gc *CLIConf, rc *zdns.ResolverConfig, input <-chan string, output chan<- string, metaChan chan<- routineMetadata, wg *sync.WaitGroup) error { defer wg.Done() resolver, err := zdns.InitResolver(rc) if err != nil { @@ -571,13 +575,9 @@ func doLookupWorker(gc *CLIConf, lookup LookupModule, rc *zdns.ResolverConfig, i var metadata routineMetadata metadata.Status = make(map[zdns.Status]int) for line := range input { - var res zdns.Result - var innerRes interface{} - var trace zdns.Trace - var status zdns.Status - var err error - var changed bool - var lookupName string + // we'll process each module sequentially, parallelism is per-domain + res := zdns.Result{Results: make(map[string]zdns.SingleModuleResult, len(gc.ActiveModules))} + // get the fields that won't change for each lookup module rawName := "" nameServer := "" var rank int @@ -596,25 +596,41 @@ func doLookupWorker(gc *CLIConf, lookup LookupModule, rc *zdns.ResolverConfig, i } else { rawName, nameServer = parseNormalInputLine(line) } - lookupName, changed = makeName(rawName, gc.NamePrefix, gc.NameOverride) - if changed { - res.AlteredName = lookupName - } res.Name = rawName - res.Class = dns.Class(gc.Class).String() + // handle per-module lookups + for moduleName, module := range gc.ActiveModules { + var innerRes interface{} + var trace zdns.Trace + var status zdns.Status + var err error + var changed bool + var lookupName string + lookupName, changed = makeName(rawName, gc.NamePrefix, gc.NameOverride) + if changed { + res.AlteredName = lookupName + } + res.Class = dns.Class(gc.Class).String() - startTime := time.Now() - innerRes, trace, status, err = lookup.Lookup(resolver, lookupName, nameServer) + startTime := time.Now() + innerRes, trace, status, err = module.Lookup(resolver, lookupName, nameServer) - res.Timestamp = time.Now().Format(gc.TimeFormat) - res.Duration = time.Since(startTime).Seconds() - if status != zdns.StatusNoOutput { - res.Status = string(status) - res.Data = innerRes - res.Trace = trace - if err != nil { - res.Error = err.Error() + lookupRes := zdns.SingleModuleResult{ + Timestamp: time.Now().Format(gc.TimeFormat), + Duration: time.Since(startTime).Seconds(), + } + if status != zdns.StatusNoOutput { + lookupRes.Status = string(status) + lookupRes.Data = innerRes + lookupRes.Trace = trace + if err != nil { + lookupRes.Error = err.Error() + } + res.Results[moduleName] = lookupRes } + metadata.Status[status]++ + metadata.Lookups++ + } + if len(res.Results) > 0 { v, _ := version.NewVersion("0.0.0") o := &sheriff.Options{ Groups: gc.OutputGroups, @@ -631,7 +647,6 @@ func doLookupWorker(gc *CLIConf, lookup LookupModule, rc *zdns.ResolverConfig, i output <- string(jsonRes) } metadata.Names++ - metadata.Status[status]++ } metaChan <- metadata return nil @@ -689,6 +704,7 @@ func aggregateMetadata(c <-chan routineMetadata) Metadata { meta.Status = make(map[string]int) for m := range c { meta.Names += m.Names + meta.Lookups += m.Lookups for k, v := range m.Status { meta.Status[string(k)] += v } diff --git a/src/modules/alookup/a_lookup.go b/src/modules/alookup/a_lookup.go index b0ba2db6..3affc024 100644 --- a/src/modules/alookup/a_lookup.go +++ b/src/modules/alookup/a_lookup.go @@ -53,12 +53,18 @@ func (aMod *ALookupModule) Lookup(r *zdns.Resolver, lookupName, nameServer strin return ipResult, trace, status, err } -// Help returns the module's help string func (aMod *ALookupModule) Help() string { return "" } -// Description returns the module's description -func (aMod *ALookupModule) Description() string { +func (aMod *ALookupModule) Validate(args []string) error { + return nil +} + +func (aMod *ALookupModule) NewFlags() interface{} { + return aMod +} + +func (aMod *ALookupModule) GetDescription() string { return "alookup will get the information that is typically desired, instead of just the information that exists in a single record. Specifically, alookup acts similar to nslookup and will follow CNAME records." } diff --git a/src/modules/axfr/axfr.go b/src/modules/axfr/axfr.go index d2f03e56..88a893f8 100644 --- a/src/modules/axfr/axfr.go +++ b/src/modules/axfr/axfr.go @@ -123,12 +123,22 @@ func (axfrMod *AxfrLookupModule) Lookup(resolver *zdns.Resolver, name, nameServe return retv, nil, zdns.StatusNoError, nil } -// Command-line Help Documentation. This is the descriptive text what is -// returned when you run zdns module --help func (axfrMod *AxfrLookupModule) Help() string { return "" } +func (axfrMod *AxfrLookupModule) Validate(args []string) error { + return nil +} + +func (axfrMod *AxfrLookupModule) NewFlags() interface{} { + return axfrMod +} + +func (axfrMod *AxfrLookupModule) GetDescription() string { + return "" +} + // CLIInit initializes the AxfrLookupModule with the given parameters, used to call AXFR from the command line func (axfrMod *AxfrLookupModule) CLIInit(gc *cli.CLIConf, rc *zdns.ResolverConfig) error { if gc == nil { diff --git a/src/modules/bindversion/bindversion.go b/src/modules/bindversion/bindversion.go index f68c0bcd..d84ac9e1 100644 --- a/src/modules/bindversion/bindversion.go +++ b/src/modules/bindversion/bindversion.go @@ -59,3 +59,19 @@ func (bindVersionMod *BindVersionLookupModule) Lookup(r *zdns.Resolver, lookupNa res := Result{BindVersion: resString} return res, trace, resStatus, err } + +func (bindVersionMod *BindVersionLookupModule) Help() string { + return "" +} + +func (bindVersionMod *BindVersionLookupModule) GetDescription() string { + return "" +} + +func (bindVersionMod *BindVersionLookupModule) Validate(args []string) error { + return nil +} + +func (bindVersionMod *BindVersionLookupModule) NewFlags() interface{} { + return bindVersionMod +} diff --git a/src/modules/dmarc/dmarc.go b/src/modules/dmarc/dmarc.go index 7d91f50d..1618567b 100644 --- a/src/modules/dmarc/dmarc.go +++ b/src/modules/dmarc/dmarc.go @@ -59,7 +59,18 @@ func (dmarcMod *DmarcLookupModule) Lookup(r *zdns.Resolver, lookupName, nameServ return res, trace, resStatus, err } -// Help func (dmarcMod *DmarcLookupModule) Help() string { return "" } + +func (dmarcMod *DmarcLookupModule) Validate(args []string) error { + return nil +} + +func (dmarcMod *DmarcLookupModule) GetDescription() string { + return "" +} + +func (dmarcMod *DmarcLookupModule) NewFlags() interface{} { + return dmarcMod +} diff --git a/src/modules/mxlookup/mx_lookup.go b/src/modules/mxlookup/mx_lookup.go index a116a257..a9d8c39b 100644 --- a/src/modules/mxlookup/mx_lookup.go +++ b/src/modules/mxlookup/mx_lookup.go @@ -135,11 +135,18 @@ func (mxMod *MXLookupModule) Lookup(r *zdns.Resolver, lookupName, nameServer str return &retv, trace, zdns.StatusNoError, nil } -// Help returns the module's help string func (mxMod *MXLookupModule) Help() string { return "" } -func (mxMod *MXLookupModule) Description() string { +func (mxMod *MXLookupModule) Validate(args []string) error { + return nil +} + +func (mxMod *MXLookupModule) GetDescription() string { return "MXLOOKUP will additionally do an A lookup for the IP addresses that correspond with an exchange record." } + +func (mxMod *MXLookupModule) NewFlags() interface{} { + return mxMod +} diff --git a/src/modules/nslookup/ns_lookup.go b/src/modules/nslookup/ns_lookup.go index 784458d2..f1d560e6 100644 --- a/src/modules/nslookup/ns_lookup.go +++ b/src/modules/nslookup/ns_lookup.go @@ -76,10 +76,18 @@ func (nsMod *NSLookupModule) Help() string { return "" } +func (nsMod *NSLookupModule) Validate(args []string) error { + return nil +} + func (nsMod *NSLookupModule) WithTestingLookup(f func(r *zdns.Resolver, lookupName string, nameServer string) (interface{}, zdns.Trace, zdns.Status, error)) { nsMod.testingLookup = f } -func (nsMod *NSLookupModule) Description() string { +func (nsMod *NSLookupModule) GetDescription() string { return "Run a more exhaustive ns lookup, will additionally do an A/AAAA lookup for the IP addresses that correspond with name server records." } + +func (nsMod *NSLookupModule) NewFlags() interface{} { + return nsMod +} diff --git a/src/modules/spf/spf.go b/src/modules/spf/spf.go index 6d97c337..ad820cb4 100644 --- a/src/modules/spf/spf.go +++ b/src/modules/spf/spf.go @@ -63,3 +63,17 @@ func (spfMod *SpfLookupModule) Lookup(r *zdns.Resolver, name, resolver string) ( func (spfMod *SpfLookupModule) Help() string { return "" } + +// Validate +func (spfMod *SpfLookupModule) Validate(args []string) error { + return nil +} + +// Description +func (spfMod *SpfLookupModule) GetDescription() string { + return "" +} + +func (spfMod *SpfLookupModule) NewFlags() interface{} { + return spfMod +} diff --git a/src/zdns/qa.go b/src/zdns/qa.go index f961dd37..1523c65b 100644 --- a/src/zdns/qa.go +++ b/src/zdns/qa.go @@ -46,20 +46,25 @@ type TraceStep struct { Try int `json:"try" groups:"trace"` } -// Result contains all the metadata from a complete lookup, potentailly after following many CNAMEs/etc. +// Result contains all the metadata from a complete lookup(s) for a domain. Results is keyed with the ModuleName. type Result struct { - AlteredName string `json:"altered_name,omitempty" groups:"short,normal,long,trace"` - Name string `json:"name,omitempty" groups:"short,normal,long,trace"` - Nameserver string `json:"nameserver,omitempty" groups:"normal,long,trace"` - Class string `json:"class,omitempty" groups:"long,trace"` - AlexaRank int `json:"alexa_rank,omitempty" groups:"short,normal,long,trace"` - Metadata string `json:"metadata,omitempty" groups:"short,normal,long,trace"` - Status string `json:"status,omitempty" groups:"short,normal,long,trace"` - Error string `json:"error,omitempty" groups:"short,normal,long,trace"` - Timestamp string `json:"timestamp,omitempty" groups:"short,normal,long,trace"` - Duration float64 `json:"duration,omitempty" groups:"short,normal,long,trace"` // in seconds - Data interface{} `json:"data,omitempty" groups:"short,normal,long,trace"` - Trace Trace `json:"trace,omitempty" groups:"trace"` + AlteredName string `json:"altered_name,omitempty" groups:"short,normal,long,trace"` + Name string `json:"name,omitempty" groups:"short,normal,long,trace"` + Nameserver string `json:"nameserver,omitempty" groups:"normal,long,trace"` + Class string `json:"class,omitempty" groups:"long,trace"` + AlexaRank int `json:"alexa_rank,omitempty" groups:"short,normal,long,trace"` + Metadata string `json:"metadata,omitempty" groups:"short,normal,long,trace"` + Results map[string]SingleModuleResult `json:"results,omitempty" groups:"short,normal,long,trace"` +} + +// SingleModuleResult contains all the metadata from a complete lookup for a domain, potentially after following many CNAMEs/etc. +type SingleModuleResult struct { + Status string `json:"status,omitempty" groups:"short,normal,long,trace"` + Error string `json:"error,omitempty" groups:"short,normal,long,trace"` + Timestamp string `json:"timestamp,omitempty" groups:"short,normal,long,trace"` + Duration float64 `json:"duration,omitempty" groups:"short,normal,long,trace"` // in seconds + Data interface{} `json:"data,omitempty" groups:"short,normal,long,trace"` + Trace Trace `json:"trace,omitempty" groups:"trace"` } // SingleQueryResult contains the results of a single DNS query diff --git a/testing/integration_tests.py b/testing/integration_tests.py index 646473ad..c6ee4f40 100755 --- a/testing/integration_tests.py +++ b/testing/integration_tests.py @@ -40,35 +40,39 @@ def dictSort(d): class Tests(unittest.TestCase): maxDiff = None ZDNS_EXECUTABLE = "./zdns" + ADDITIONAL_FLAGS = " --threads=10" # flags used with every test def run_zdns_check_failure(self, flags, name, expected_err, executable=ZDNS_EXECUTABLE): - flags = flags + " --threads=10" + flags = flags + self.ADDITIONAL_FLAGS c = f"echo '{name}' | {executable} {flags}; exit 0" o = subprocess.check_output(c, shell=True, stderr=subprocess.STDOUT) self.assertEqual(expected_err in o.decode(), True) def run_zdns(self, flags, name, executable=ZDNS_EXECUTABLE): - flags = flags + " --threads=10" + flags = flags + self.ADDITIONAL_FLAGS c = f"echo '{name}' | {executable} {flags}" o = subprocess.check_output(c, shell=True) return c, json.loads(o.rstrip()) - def run_zdns_multiline(self, flags, names, executable=ZDNS_EXECUTABLE): - d = tempfile.mkdtemp - f = "/".join([d, "temp"]) - with open(f) as fd: - for name in names: - fd.writeline(name) - flags = flags + " --threads=10" - c = f"cat '{f}' | {executable} {flags}" + # Runs zdns with a given name(s) input and flags, returns the command and JSON objects from the piped JSON-Lines output + # Used when running a ZDNS command that should return multiple lines of output, and you want those in a list + def run_zdns_multiline_output(self, flags, name, executable=ZDNS_EXECUTABLE): + flags = flags + self.ADDITIONAL_FLAGS + c = f"echo '{name}' | {executable} {flags}" o = subprocess.check_output(c, shell=True) - os.rm(f) - return c, [json.loads(l.rstrip()) for l in o] + output_lines = o.decode('utf-8').strip().splitlines() + json_objects = [json.loads(line.rstrip()) for line in output_lines] + return c, json_objects + + ROOT_A_ZDNS_TESTING_COM = {"1.2.3.4", "2.3.4.5", "3.4.5.6"} # zdns-testing.com - ROOT_A = {"1.2.3.4", "2.3.4.5", "3.4.5.6"} + ROOT_A_A_ZDNS_TESTING_COM = {"21.9.87.65"} # a.zdns-testing.com ROOT_A_ANSWERS = [{"type": "A", "class": "IN", "answer": x, - "name": "zdns-testing.com"} for x in ROOT_A] + "name": "zdns-testing.com"} for x in ROOT_A_ZDNS_TESTING_COM] + + ROOT_A_A_ZDNS_TESTING_COM_ANSWERS = [{"type": "A", "class": "IN", "answer": x, + "name": "a.zdns-testing.com"} for x in ROOT_A_A_ZDNS_TESTING_COM] ROOT_AAAA = {"fd5a:3bce:8713::1", "fde6:9bb3:dbd6::2", "fdb3:ac76:a577::3"} @@ -81,6 +85,15 @@ def run_zdns_multiline(self, flags, names, executable=ZDNS_EXECUTABLE): {"answer": "mx1.censys.io.", "preference": 10, "type": "MX", "class": "IN", 'name': 'zdns-testing.com'}, ] + A_MX1_ZDNS_TESTING_COM = {"1.2.3.4", "2.3.4.5"} + + AAAA_MX1_ZDNS_TESTING_COM = {"fdb3:ac76:a577::4", "fdb3:ac76:a577::5"} + + A_MX1_ZDNS_TESTING_COM_ANSWERS = [{"type": "A", "class": "IN", "answer": x, "name": "mx1.zdns-testing.com"} + for x in A_MX1_ZDNS_TESTING_COM] + AAAA_MX1_ZDNS_TESTING_COM_ANSWERS = [{"type": "AAAA", "class": "IN", "answer": x, "name": "mx1.zdns-testing.com"} + for x in AAAA_MX1_ZDNS_TESTING_COM] + NS_SERVERS = [ {"type": "NS", "class": "IN", "name": "zdns-testing.com", "answer": "ns-cloud-c2.googledomains.com."}, @@ -100,138 +113,171 @@ def run_zdns_multiline(self, flags, names, executable=ZDNS_EXECUTABLE): MX_LOOKUP_ANSWER = { "name": "zdns-testing.com", - "class": "IN", - "status": "NOERROR", - "data": { - "exchanges": [ - { - "name": "mx1.zdns-testing.com", - "type": "MX", - "class": "IN", - "preference": 1, - "ipv4_addresses": [ - "1.2.3.4", - "2.3.4.5" - ], - "ipv6_addresses": [ - "fdb3:ac76:a577::4", - "fdb3:ac76:a577::5" - ], - - }, - { - "name": "mx2.zdns-testing.com", - "type": "MX", - "class": "IN", - "preference": 5, - "ipv4_addresses": [ - "5.6.7.8" - ], - }, - { - "name": "mx1.censys.io", - "type": "MX", - "class": "IN", - "preference": 10, + "results": { + "MXLOOKUP": { + "class": "IN", + "status": "NOERROR", + "data": { + "exchanges": [ + { + "name": "mx1.zdns-testing.com", + "type": "MX", + "class": "IN", + "preference": 1, + "ipv4_addresses": [ + "1.2.3.4", + "2.3.4.5" + ], + "ipv6_addresses": [ + "fdb3:ac76:a577::4", + "fdb3:ac76:a577::5" + ], + + }, + { + "name": "mx2.zdns-testing.com", + "type": "MX", + "class": "IN", + "preference": 5, + "ipv4_addresses": [ + "5.6.7.8" + ], + }, + { + "name": "mx1.censys.io", + "type": "MX", + "class": "IN", + "preference": 10, + } + ] } - ] + } } } MX_LOOKUP_ANSWER_IPV4 = copy.deepcopy(MX_LOOKUP_ANSWER) - del MX_LOOKUP_ANSWER_IPV4["data"]["exchanges"][0]["ipv6_addresses"] + del MX_LOOKUP_ANSWER_IPV4["results"]["MXLOOKUP"]["data"]["exchanges"][0]["ipv6_addresses"] MX_LOOKUP_ANSWER_IPV6 = copy.deepcopy(MX_LOOKUP_ANSWER) - del MX_LOOKUP_ANSWER_IPV6["data"]["exchanges"][0]["ipv4_addresses"] - del MX_LOOKUP_ANSWER_IPV6["data"]["exchanges"][1]["ipv4_addresses"] + del MX_LOOKUP_ANSWER_IPV6["results"]["MXLOOKUP"]["data"]["exchanges"][0]["ipv4_addresses"] + del MX_LOOKUP_ANSWER_IPV6["results"]["MXLOOKUP"]["data"]["exchanges"][1]["ipv4_addresses"] A_LOOKUP_WWW_ZDNS_TESTING = { "name": "www.zdns-testing.com", - "class": "IN", - "status": "NOERROR", - "data": { - "ipv4_addresses": [ - "1.2.3.4", - "2.3.4.5", - "3.4.5.6" - ], - "ipv6_addresses": [ - "fde6:9bb3:dbd6::2", - "fd5a:3bce:8713::1", - "fdb3:ac76:a577::3" - ] + "results": { + "ALOOKUP": { + "class": "IN", + "status": "NOERROR", + "data": { + "ipv4_addresses": [ + "1.2.3.4", + "2.3.4.5", + "3.4.5.6" + ], + "ipv6_addresses": [ + "fde6:9bb3:dbd6::2", + "fd5a:3bce:8713::1", + "fdb3:ac76:a577::3" + ] + } + } + } + } + + A_LOOKUP_WWW_ZDNS_TESTING_IPv6 = { + "name": "www.zdns-testing.com", + "results": { + "ALOOKUP": { + "status": "NOERROR", + "class": "IN", + "data": { + "ipv6_addresses": [ + "fde6:9bb3:dbd6::2", + "fd5a:3bce:8713::1", + "fdb3:ac76:a577::3" + ] + } + } } } A_LOOKUP_CNAME_CHAIN_03 = { "name": "cname-chain-03.esrg.stanford.edu", - "class": "IN", - "status": "NOERROR", - "data": { - "ipv4_addresses": [ - "1.2.3.4", - ], + "results": { + "ALOOKUP": { + "status": "NOERROR", + "class": "IN", + "data": { + "ipv4_addresses": [ + "1.2.3.4", + ] + } + } } } A_LOOKUP_IPV4_WWW_ZDNS_TESTING = copy.deepcopy(A_LOOKUP_WWW_ZDNS_TESTING) - del A_LOOKUP_IPV4_WWW_ZDNS_TESTING["data"]["ipv6_addresses"] + del A_LOOKUP_IPV4_WWW_ZDNS_TESTING["results"]["ALOOKUP"]["data"]["ipv6_addresses"] A_LOOKUP_IPV6_WWW_ZDNS_TESTING = copy.deepcopy(A_LOOKUP_WWW_ZDNS_TESTING) - del A_LOOKUP_IPV6_WWW_ZDNS_TESTING["data"]["ipv4_addresses"] + del A_LOOKUP_IPV6_WWW_ZDNS_TESTING["results"]["ALOOKUP"]["data"]["ipv4_addresses"] NS_LOOKUP_WWW_ZDNS_TESTING = { "name": "www.zdns-testing.com", - "status": "NOERROR", - "data": { - "servers": [ - { - "ipv4_addresses": [ - "216.239.34.108" - ], - "ipv6_addresses": [ - "2001:4860:4802:34::6c" - ], - "name": "ns-cloud-c2.googledomains.com", - "type": "NS" - }, - { - "ipv4_addresses": [ - "216.239.32.108" - ], - "ipv6_addresses": [ - "2001:4860:4802:32::6c" - ], - "name": "ns-cloud-c1.googledomains.com", - "type": "NS" - }, - { - "ipv4_addresses": [ - "216.239.38.108" - ], - "ipv6_addresses": [ - "2001:4860:4802:38::6c" - ], - "name": "ns-cloud-c4.googledomains.com", - "type": "NS" - }, - { - "ipv4_addresses": [ - "216.239.36.108" - ], - "ipv6_addresses": [ - "2001:4860:4802:36::6c" - ], - "name": "ns-cloud-c3.googledomains.com", - "type": "NS" + "results": { + "NSLOOKUP": { + "status": "NOERROR", + "data": { + "servers": [ + { + "ipv4_addresses": [ + "216.239.34.108" + ], + "ipv6_addresses": [ + "2001:4860:4802:34::6c" + ], + "name": "ns-cloud-c2.googledomains.com", + "type": "NS" + }, + { + "ipv4_addresses": [ + "216.239.32.108" + ], + "ipv6_addresses": [ + "2001:4860:4802:32::6c" + ], + "name": "ns-cloud-c1.googledomains.com", + "type": "NS" + }, + { + "ipv4_addresses": [ + "216.239.38.108" + ], + "ipv6_addresses": [ + "2001:4860:4802:38::6c" + ], + "name": "ns-cloud-c4.googledomains.com", + "type": "NS" + }, + { + "ipv4_addresses": [ + "216.239.36.108" + ], + "ipv6_addresses": [ + "2001:4860:4802:36::6c" + ], + "name": "ns-cloud-c3.googledomains.com", + "type": "NS" + } + ] } - ] + } } } NS_LOOKUP_IPV4_WWW_ZDNS_TESTING = copy.deepcopy(NS_LOOKUP_WWW_ZDNS_TESTING) - for server in NS_LOOKUP_IPV4_WWW_ZDNS_TESTING["data"]["servers"]: + for server in NS_LOOKUP_IPV4_WWW_ZDNS_TESTING["results"]["NSLOOKUP"]["data"]["servers"]: del server["ipv6_addresses"] NS_LOOKUP_IPV6_WWW_ZDNS_TESTING = copy.deepcopy(NS_LOOKUP_WWW_ZDNS_TESTING) - for server in NS_LOOKUP_IPV6_WWW_ZDNS_TESTING["data"]["servers"]: + for server in NS_LOOKUP_IPV6_WWW_ZDNS_TESTING["results"]["NSLOOKUP"]["data"]["servers"]: del server["ipv4_addresses"] PTR_LOOKUP_GOOGLE_PUB = [ @@ -533,17 +579,17 @@ def run_zdns_multiline(self, flags, names, executable=ZDNS_EXECUTABLE): "129.127.149.0/24": "1.2.3.4" } - def assertSuccess(self, res, cmd): - self.assertEqual(res["status"], "NOERROR", cmd) + def assertSuccess(self, res, cmd, query_type): + self.assertEqual(res["results"][query_type]["status"], "NOERROR", cmd) - def assertServFail(self, res, cmd): - self.assertEqual(res["status"], "SERVFAIL", cmd) + def assertServFail(self, res, cmd, query_type): + self.assertEqual(res["results"][query_type]["status"], "SERVFAIL", cmd) - def assertEqualAnswers(self, res, correct, cmd, key="answer"): - self.assertIn("answers", res["data"]) - for answer in res["data"]["answers"]: + def assertEqualAnswers(self, res, correct, cmd, query_type, key="answer"): + self.assertIn("answers", res["results"][query_type]["data"]) + for answer in res["results"][query_type]["data"]["answers"]: del answer["ttl"] - a = sorted(res["data"]["answers"], key=lambda x: x[key]) + a = sorted(res["results"][query_type]["data"]["answers"], key=lambda x: x[key]) b = sorted(correct, key=lambda x: x[key]) helptext = "%s\nExpected:\n%s\n\nActual:\n%s" % (cmd, json.dumps(b, indent=4), json.dumps(a, indent=4)) @@ -561,37 +607,39 @@ def _lowercase(obj): _lowercase(b) self.assertEqual(a, b, helptext) - def assertEqualNXDOMAIN(self, res, correct): + def assertEqualNXDOMAIN(self, res, correct, query_type): self.assertEqual(res["name"], correct["name"]) - self.assertEqual(res["status"], correct["status"]) + self.assertEqual(res["results"][query_type]["status"], correct["status"]) def assertEqualMXLookup(self, res, correct): self.assertEqual(res["name"], correct["name"]) - self.assertEqual(res["status"], correct["status"]) - for exchange in res["data"]["exchanges"]: + self.assertEqual(res["results"]["MXLOOKUP"]["status"], correct["results"]["MXLOOKUP"]["status"]) + for exchange in res["results"]["MXLOOKUP"]["data"]["exchanges"]: del exchange["ttl"] - self.assertEqual(recursiveSort(res["data"]["exchanges"]), recursiveSort(correct["data"]["exchanges"])) + self.assertEqual(recursiveSort(res["results"]["MXLOOKUP"]["data"]["exchanges"]), recursiveSort(correct["results"]["MXLOOKUP"]["data"]["exchanges"])) - def assertEqualALookup(self, res, correct): + def assertEqualALookup(self, res, correct, query_type): self.assertEqual(res["name"], correct["name"]) - self.assertEqual(res["status"], correct["status"]) - if "ipv4_addresses" in correct["data"]: + res = res["results"][query_type] + correct_A_lookup = correct["results"][query_type] + self.assertEqual(res["status"], correct_A_lookup["status"]) + if "ipv4_addresses" in correct_A_lookup["data"]: self.assertIn("ipv4_addresses", res["data"]) - self.assertEqual(sorted(res["data"]["ipv4_addresses"]), sorted(correct["data"]["ipv4_addresses"])) + self.assertEqual(sorted(res["data"]["ipv4_addresses"]), sorted(correct_A_lookup["data"]["ipv4_addresses"])) else: self.assertNotIn("ipv4_addresses", res["data"]) - if "ipv6_addresses" in correct["data"]: + if "ipv6_addresses" in correct_A_lookup["data"]: self.assertIn("ipv6_addresses", res["data"]) - self.assertEqual(sorted(res["data"]["ipv6_addresses"]), sorted(correct["data"]["ipv6_addresses"])) + self.assertEqual(sorted(res["data"]["ipv6_addresses"]), sorted(correct_A_lookup["data"]["ipv6_addresses"])) else: self.assertNotIn("ipv6_addresses", res["data"]) def assertEqualNSLookup(self, res, correct): self.assertEqual(res["name"], correct["name"]) - self.assertEqual(res["status"], correct["status"]) - for server in res["data"]["servers"]: + self.assertEqual(res["results"]["NSLOOKUP"]["status"], correct["results"]["NSLOOKUP"]["status"]) + for server in res["results"]["NSLOOKUP"]["data"]["servers"]: del server["ttl"] - self.assertEqual(recursiveSort(res["data"]["servers"]), recursiveSort(correct["data"]["servers"])) + self.assertEqual(recursiveSort(res["results"]["NSLOOKUP"]["data"]["servers"]), recursiveSort(correct["results"]["NSLOOKUP"]["data"]["servers"])) def assertEqualTypes(self, res, list): res_types = set() @@ -637,182 +685,280 @@ def test_a(self): c = "A" name = "zdns-testing.com" cmd, res = self.run_zdns(c, name) - self.assertSuccess(res, cmd) - self.assertEqualAnswers(res, self.ROOT_A_ANSWERS, cmd) + self.assertSuccess(res, cmd, "A") + self.assertEqualAnswers(res, self.ROOT_A_ANSWERS, cmd, "A") + + def test_a_dig_style_args(self): + c = "A zdns-testing.com" + name = "" + cmd, res = self.run_zdns(c, name) + self.assertSuccess(res, cmd, "A") + self.assertEqualAnswers(res, self.ROOT_A_ANSWERS, cmd, "A") + + def test_a_multiple_domains_dig_style(self): + c = "A zdns-testing.com a.zdns-testing.com --iterative" + name = "" + cmd, res = self.run_zdns_multiline_output(c, name) + self.assertSuccess(res[0], cmd, "A") + self.assertSuccess(res[1], cmd, "A") + if res[0]["name"] == "zdns-testing.com": + self.assertEqualAnswers(res[0], self.ROOT_A_ANSWERS, cmd, "A") + self.assertEqualAnswers(res[1], self.ROOT_A_A_ZDNS_TESTING_COM_ANSWERS, cmd, "A") + else: + self.assertEqualAnswers(res[0], self.ROOT_A_A_ZDNS_TESTING_COM_ANSWERS, cmd, "A") + self.assertEqualAnswers(res[1], self.ROOT_A_ANSWERS, cmd, "A") + + def test_multiple_modules(self): + iniFileContents = """ + [Application Options] + name-servers = "1.1.1.1" + [A] + [AAAA] + """ + file_name = "./test_multiple_modules.ini" + with open(file_name, "w") as f: + f.write(iniFileContents) + c = "MULTIPLE -c " + file_name + name = "zdns-testing.com" + cmd, res = self.run_zdns(c, name) + self.assertSuccess(res, cmd, "A") + self.assertSuccess(res, cmd, "AAAA") + self.assertEqualAnswers(res, self.ROOT_A_ANSWERS, cmd, "A") + self.assertEqualAnswers(res, self.ROOT_AAAA_ANSWERS, cmd, "AAAA") + # delete file + cmd = f"rm {file_name}" + subprocess.check_output(cmd, shell=True) + + def test_multiple_modules_multiple_domains(self): + iniFileContents = """ + [Application Options] + name-servers = "1.1.1.1" + [A] + [AAAA] + """ + file_name = "./test_multiple_modules_multiple_domains.ini" + with open(file_name, "w") as f: + f.write(iniFileContents) + c = "MULTIPLE -c " + file_name + " zdns-testing.com mx1.zdns-testing.com" + name = "" + + cmd, res = self.run_zdns_multiline_output(c, name) + self.assertSuccess(res[0], cmd, "A") + self.assertSuccess(res[0], cmd, "AAAA") + self.assertSuccess(res[1], cmd, "A") + self.assertSuccess(res[1], cmd, "AAAA") + for r in res: + for query_type, query_res in r["results"].items(): + if query_res["data"]["resolver"] != "1.1.1.1:53": + self.fail("Unexpected resolver") + if r["name"] == "zdns-testing.com" and query_type == "A": + self.assertEqualAnswers(r, self.ROOT_A_ANSWERS, cmd, "A") + elif r["name"] == "zdns-testing.com" and query_type == "AAAA": + self.assertEqualAnswers(r, self.ROOT_AAAA_ANSWERS, cmd, "AAAA") + elif r["name"] == "mx1.zdns-testing.com" and query_type == "A": + self.assertEqualAnswers(r, self.A_MX1_ZDNS_TESTING_COM_ANSWERS, cmd, "A") + elif r["name"] == "mx1.zdns-testing.com" and query_type == "AAAA": + self.assertEqualAnswers(r, self.AAAA_MX1_ZDNS_TESTING_COM_ANSWERS, cmd, "AAAA") + else: + self.fail("Unexpected response") + # delete file + cmd = f"rm {file_name}" + subprocess.check_output(cmd, shell=True) + + def test_multiple_modules_with_special_modules(self): + iniFileContents = """ + [Application Options] + name-servers = "1.1.1.1" + [ALOOKUP] + ipv4-lookup=false + ipv6-lookup = true + """ + file_name = "./test_multiple_modules.ini" + with open(file_name, "w") as f: + f.write(iniFileContents) + c = "MULTIPLE -c " + file_name + name = "www.zdns-testing.com" + cmd, res = self.run_zdns_multiline_output(c, name) + self.assertSuccess(res[0], cmd, "ALOOKUP") + self.assertEqualALookup(res[0], self.A_LOOKUP_WWW_ZDNS_TESTING_IPv6, "ALOOKUP") + # delete file + cmd = f"rm {file_name}" + subprocess.check_output(cmd, shell=True) + def test_cname(self): c = "CNAME" name = "www.zdns-testing.com" cmd, res = self.run_zdns(c, name) - self.assertSuccess(res, cmd) - self.assertEqualAnswers(res, self.WWW_CNAME_ANSWERS, cmd) + self.assertSuccess(res, cmd, "CNAME") + self.assertEqualAnswers(res, self.WWW_CNAME_ANSWERS, cmd, "CNAME") def test_cname_loop_iterative(self): c = "A --iterative" name = "cname-loop.zdns-testing.com" cmd, res = self.run_zdns(c, name) - self.assertSuccess(res, cmd) - self.assertEqualAnswers(res, self.CNAME_LOOP_ANSWERS, cmd) + self.assertSuccess(res, cmd, "A") + self.assertEqualAnswers(res, self.CNAME_LOOP_ANSWERS, cmd, "A") def test_a_behind_cname(self): c = "A" name = "www.zdns-testing.com" cmd, res = self.run_zdns(c, name) - self.assertSuccess(res, cmd) - self.assertEqualAnswers(res, self.WWW_CNAME_AND_A_ANSWERS, cmd) + self.assertSuccess(res, cmd, "A") + self.assertEqualAnswers(res, self.WWW_CNAME_AND_A_ANSWERS, cmd, "A") def test_aaaa_behind_cname(self): c = "AAAA" name = "www.zdns-testing.com" cmd, res = self.run_zdns(c, name) - self.assertSuccess(res, cmd) - self.assertEqualAnswers(res, self.WWW_CNAME_AND_AAAA_ANSWERS, cmd) + self.assertSuccess(res, cmd, "AAAA") + self.assertEqualAnswers(res, self.WWW_CNAME_AND_AAAA_ANSWERS, cmd, "AAAA") def test_caa(self): c = "CAA" name = "zdns-testing.com" cmd, res = self.run_zdns(c, name) - self.assertSuccess(res, cmd) - self.assertEqualAnswers(res, self.CAA_RECORD, cmd, key="name") + self.assertSuccess(res, cmd, "CAA") + self.assertEqualAnswers(res, self.CAA_RECORD, cmd, key="name", query_type="CAA") def test_txt(self): c = "TXT" name = "test_txt.zdns-testing.com" cmd, res = self.run_zdns(c, name) - self.assertSuccess(res, cmd) - self.assertEqualAnswers(res, self.TXT_RECORD, cmd) + self.assertSuccess(res, cmd, "TXT") + self.assertEqualAnswers(res, self.TXT_RECORD, cmd, "TXT") def test_a_iterative(self): c = "A --iterative" name = "zdns-testing.com" cmd, res = self.run_zdns(c, name) - self.assertSuccess(res, cmd) - self.assertEqualAnswers(res, self.ROOT_A_ANSWERS, cmd) + self.assertSuccess(res, cmd, "A") + self.assertEqualAnswers(res, self.ROOT_A_ANSWERS, cmd, "A") def test_a_iterative_nxdomain(self): c = "A --iterative" name = "zdns-testing-nxdomain.com" cmd, res = self.run_zdns(c, name) - self.assertEqualNXDOMAIN(res, self.NXDOMAIN_ANSWER) + self.assertEqualNXDOMAIN(res, self.NXDOMAIN_ANSWER, "A") def test_aaaa(self): c = "AAAA" name = "zdns-testing.com" cmd, res = self.run_zdns(c, name) - self.assertSuccess(res, cmd) - self.assertEqualAnswers(res, self.ROOT_AAAA_ANSWERS, cmd) + self.assertSuccess(res, cmd, "AAAA") + self.assertEqualAnswers(res, self.ROOT_AAAA_ANSWERS, cmd, "AAAA") def test_aaaa_iterative(self): c = "AAAA --iterative" name = "zdns-testing.com" cmd, res = self.run_zdns(c, name) - self.assertSuccess(res, cmd) - self.assertEqualAnswers(res, self.ROOT_AAAA_ANSWERS, cmd) + self.assertSuccess(res, cmd, "AAAA") + self.assertEqualAnswers(res, self.ROOT_AAAA_ANSWERS, cmd, "AAAA") def test_mx(self): c = "MX" name = "zdns-testing.com" cmd, res = self.run_zdns(c, name) - self.assertSuccess(res, cmd) - self.assertEqualAnswers(res, self.MX_SERVERS, cmd) + self.assertSuccess(res, cmd, "MX") + self.assertEqualAnswers(res, self.MX_SERVERS, cmd, "MX") def test_mx_iterative(self): c = "MX --iterative" name = "zdns-testing.com" cmd, res = self.run_zdns(c, name) - self.assertSuccess(res, cmd) - self.assertEqualAnswers(res, self.MX_SERVERS, cmd) + self.assertSuccess(res, cmd, "MX") + self.assertEqualAnswers(res, self.MX_SERVERS, cmd, "MX") def test_ns(self): c = "NS" name = "zdns-testing.com" cmd, res = self.run_zdns(c, name) - self.assertSuccess(res, cmd) - self.assertEqualAnswers(res, self.NS_SERVERS, cmd) + self.assertSuccess(res, cmd, "NS") + self.assertEqualAnswers(res, self.NS_SERVERS, cmd, "NS") def test_ns_iterative(self): c = "NS --iterative" name = "zdns-testing.com" cmd, res = self.run_zdns(c, name) - self.assertSuccess(res, cmd) - self.assertEqualAnswers(res, self.NS_SERVERS, cmd) + self.assertSuccess(res, cmd, "NS") + self.assertEqualAnswers(res, self.NS_SERVERS, cmd, "NS") def test_mx_lookup(self): c = "mxlookup --ipv4-lookup --ipv6-lookup" name = "zdns-testing.com" cmd, res = self.run_zdns(c, name) - self.assertSuccess(res, cmd) + self.assertSuccess(res, cmd, "MXLOOKUP") self.assertEqualMXLookup(res, self.MX_LOOKUP_ANSWER) def test_mx_lookup_iterative(self): c = "mxlookup --ipv4-lookup --ipv6-lookup --iterative" name = "zdns-testing.com" cmd, res = self.run_zdns(c, name) - self.assertSuccess(res, cmd) + self.assertSuccess(res, cmd, "MXLOOKUP") self.assertEqualMXLookup(res, self.MX_LOOKUP_ANSWER) def test_mx_lookup_ipv4(self): c = "mxlookup --ipv4-lookup" name = "zdns-testing.com" cmd, res = self.run_zdns(c, name) - self.assertSuccess(res, cmd) + self.assertSuccess(res, cmd, "MXLOOKUP") self.assertEqualMXLookup(res, self.MX_LOOKUP_ANSWER_IPV4) def test_mx_lookup_ipv6(self): c = "mxlookup --ipv6-lookup" name = "zdns-testing.com" cmd, res = self.run_zdns(c, name) - self.assertSuccess(res, cmd) + self.assertSuccess(res, cmd, "MXLOOKUP") self.assertEqualMXLookup(res, self.MX_LOOKUP_ANSWER_IPV6) def test_mx_lookup_default(self): c = "mxlookup" name = "zdns-testing.com" cmd, res = self.run_zdns(c, name) - self.assertSuccess(res, cmd) + self.assertSuccess(res, cmd, "MXLOOKUP") self.assertEqualMXLookup(res, self.MX_LOOKUP_ANSWER_IPV4) def test_a_lookup(self): c = "alookup --ipv4-lookup --ipv6-lookup" name = "www.zdns-testing.com" cmd, res = self.run_zdns(c, name) - self.assertSuccess(res, cmd) - self.assertEqualALookup(res, self.A_LOOKUP_WWW_ZDNS_TESTING) + self.assertSuccess(res, cmd, "ALOOKUP") + self.assertEqualALookup(res, self.A_LOOKUP_WWW_ZDNS_TESTING, "ALOOKUP") def test_a_lookup_iterative(self): c = "alookup --ipv4-lookup --ipv6-lookup --iterative" name = "www.zdns-testing.com" cmd, res = self.run_zdns(c, name) - self.assertSuccess(res, cmd) - self.assertEqualALookup(res, self.A_LOOKUP_WWW_ZDNS_TESTING) + self.assertSuccess(res, cmd, "ALOOKUP") + self.assertEqualALookup(res, self.A_LOOKUP_WWW_ZDNS_TESTING, "ALOOKUP") def test_a_lookup_ipv4(self): c = "alookup --ipv4-lookup" name = "www.zdns-testing.com" cmd, res = self.run_zdns(c, name) - self.assertSuccess(res, cmd) - self.assertEqualALookup(res, self.A_LOOKUP_IPV4_WWW_ZDNS_TESTING) + self.assertSuccess(res, cmd, "ALOOKUP") + self.assertEqualALookup(res, self.A_LOOKUP_IPV4_WWW_ZDNS_TESTING, "ALOOKUP") def test_a_lookup_ipv6(self): c = "alookup --ipv6-lookup" name = "www.zdns-testing.com" cmd, res = self.run_zdns(c, name) - self.assertSuccess(res, cmd) - self.assertEqualALookup(res, self.A_LOOKUP_IPV6_WWW_ZDNS_TESTING) + self.assertSuccess(res, cmd, "ALOOKUP") + self.assertEqualALookup(res, self.A_LOOKUP_IPV6_WWW_ZDNS_TESTING, "ALOOKUP") def test_a_lookup_default(self): c = "alookup" name = "www.zdns-testing.com" cmd, res = self.run_zdns(c, name) - self.assertSuccess(res, cmd) - self.assertEqualALookup(res, self.A_LOOKUP_IPV4_WWW_ZDNS_TESTING) + self.assertSuccess(res, cmd, "ALOOKUP") + self.assertEqualALookup(res, self.A_LOOKUP_IPV4_WWW_ZDNS_TESTING, "ALOOKUP") def test_a_lookup_iterative_cname_loop(self): c = "alookup --iterative" name = "cname-loop.zdns-testing.com" cmd, res = self.run_zdns(c, name) - self.assertSuccess(res, cmd) - assert len(res["data"]) == 0 + self.assertSuccess(res, cmd, "ALOOKUP") + assert len(res["results"]["ALOOKUP"]["data"]) == 0 # There exists DNS records in esrg.stanford.edu and zdns-testing.com of the form: # cname-chain-01.esrg.stanford.edu CNAME cname-chain-02.zdns-testing.com. @@ -825,120 +971,105 @@ def test_a_lookup_cname_chain_too_long(self): c = "alookup --iterative --ipv4-lookup" name = "cname-chain-01.esrg.stanford.edu" cmd, res = self.run_zdns(c, name) - self.assertServFail(res, cmd) + self.assertServFail(res, cmd, "ALOOKUP") def test_a_lookup_cname_chain(self): c = "alookup --iterative --ipv4-lookup" name = "cname-chain-03.esrg.stanford.edu" cmd, res = self.run_zdns(c, name) - self.assertSuccess(res, cmd) - self.assertEqualALookup(res, self.A_LOOKUP_CNAME_CHAIN_03) + self.assertSuccess(res, cmd, "ALOOKUP") + self.assertEqualALookup(res, self.A_LOOKUP_CNAME_CHAIN_03, "ALOOKUP") + + def test_type_option_server_mode_a_lookup_ipv4(self): + c = "A --override-name=www.zdns-testing.com --name-server-mode" + name = "8.8.8.8" + cmd, res = self.run_zdns(c, name) + self.assertEqual(res["results"]["A"]["data"]["resolver"], "8.8.8.8:53") + self.assertEqualAnswers(res, self.WWW_CNAME_AND_A_ANSWERS, cmd, "A") + def test_dig_style_type_option_server_mode_a_lookup_ipv4(self): + c = "A 8.8.8.8 --override-name=www.zdns-testing.com --name-server-mode" + name = "" + cmd, res = self.run_zdns(c, name) + self.assertEqual(res["results"]["A"]["data"]["resolver"], "8.8.8.8:53") + self.assertEqualAnswers(res, self.WWW_CNAME_AND_A_ANSWERS, cmd, "A") def test_ns_lookup(self): c = "nslookup --ipv4-lookup --ipv6-lookup" name = "www.zdns-testing.com" cmd, res = self.run_zdns(c, name) - self.assertSuccess(res, cmd) + self.assertSuccess(res, cmd, "NSLOOKUP") self.assertEqualNSLookup(res, self.NS_LOOKUP_WWW_ZDNS_TESTING) def test_ns_lookup_iterative(self): c = "nslookup --ipv4-lookup --ipv6-lookup --iterative" name = "www.zdns-testing.com" cmd, res = self.run_zdns(c, name) - self.assertSuccess(res, cmd) - self.assertEqualALookup(res, self.NS_LOOKUP_WWW_ZDNS_TESTING) + self.assertSuccess(res, cmd, "NSLOOKUP") + self.assertEqualALookup(res, self.NS_LOOKUP_WWW_ZDNS_TESTING, "NSLOOKUP") def test_ns_lookup_default(self): c = "nslookup" name = "www.zdns-testing.com" cmd, res = self.run_zdns(c, name) - self.assertSuccess(res, cmd) + self.assertSuccess(res, cmd, "NSLOOKUP") self.assertEqualNSLookup(res, self.NS_LOOKUP_IPV4_WWW_ZDNS_TESTING) def test_ns_lookup_ipv4(self): c = "nslookup --ipv4-lookup" name = "www.zdns-testing.com" cmd, res = self.run_zdns(c, name) - self.assertSuccess(res, cmd) + self.assertSuccess(res, cmd, "NSLOOKUP") self.assertEqualNSLookup(res, self.NS_LOOKUP_IPV4_WWW_ZDNS_TESTING) def test_ns_lookup_ipv6(self): c = "nslookup --ipv6-lookup" name = "www.zdns-testing.com" cmd, res = self.run_zdns(c, name) - self.assertSuccess(res, cmd) + self.assertSuccess(res, cmd, "NSLOOKUP") self.assertEqualNSLookup(res, self.NS_LOOKUP_IPV6_WWW_ZDNS_TESTING) def test_spf_lookup(self): c = "spf" name = "zdns-testing.com" cmd, res = self.run_zdns(c, name) - self.assertSuccess(res, cmd) - self.assertEqual(res["data"], self.SPF_ANSWER["data"]) + self.assertSuccess(res, cmd, "SPF") + self.assertEqual(res["results"]["SPF"]["data"], self.SPF_ANSWER["data"]) def test_spf_lookup_iterative(self): c = "spf --iterative" name = "zdns-testing.com" cmd, res = self.run_zdns(c, name) - self.assertSuccess(res, cmd) - self.assertEqual(res["data"], self.SPF_ANSWER["data"]) + self.assertSuccess(res, cmd, "SPF") + self.assertEqual(res["results"]["SPF"]["data"], self.SPF_ANSWER["data"]) def test_dmarc_lookup(self): c = "dmarc" name = "_dmarc.zdns-testing.com" cmd, res = self.run_zdns(c, name) - self.assertSuccess(res, cmd) - self.assertEqual(res["data"], self.DMARC_ANSWER["data"]) + self.assertSuccess(res, cmd, "DMARC") + self.assertEqual(res["results"]["DMARC"]["data"], self.DMARC_ANSWER["data"]) def test_dmarc_lookup_iterative(self): c = "dmarc --iterative" name = "_dmarc.zdns-testing.com" cmd, res = self.run_zdns(c, name) - self.assertSuccess(res, cmd) - self.assertEqual(res["data"], self.DMARC_ANSWER["data"]) + self.assertSuccess(res, cmd, "DMARC") + self.assertEqual(res["results"]["DMARC"]["data"], self.DMARC_ANSWER["data"]) def test_ptr(self): c = "PTR" name = "8.8.8.8" cmd, res = self.run_zdns(c, name) - self.assertSuccess(res, cmd) - self.assertEqualAnswers(res, self.PTR_LOOKUP_GOOGLE_PUB, cmd) + self.assertSuccess(res, cmd, "PTR") + self.assertEqualAnswers(res, self.PTR_LOOKUP_GOOGLE_PUB, cmd, "PTR") def test_ptr_iterative(self): c = "PTR --iterative" name = "8.8.8.8" cmd, res = self.run_zdns(c, name) - self.assertSuccess(res, cmd) - self.assertEqualAnswers(res, self.PTR_LOOKUP_GOOGLE_PUB, cmd) - - def test_spf(self): - c = "SPF" - name = "zdns-testing.com" - cmd, res = self.run_zdns(c, name) - self.assertSuccess(res, cmd) - self.assertEqual(res["data"], self.SPF_ANSWER["data"]) - - def test_spf_iterative(self): - c = "SPF --iterative" - name = "zdns-testing.com" - cmd, res = self.run_zdns(c, name) - self.assertSuccess(res, cmd) - self.assertEqual(res["data"], self.SPF_ANSWER["data"]) - - def test_dmarc(self): - c = "DMARC" - name = "_dmarc.zdns-testing.com" - cmd, res = self.run_zdns(c, name) - self.assertSuccess(res, cmd) - self.assertEqual(res["data"], self.DMARC_ANSWER["data"]) - - def test_dmarc_iterative(self): - c = "DMARC --iterative" - name = "_dmarc.zdns-testing.com" - cmd, res = self.run_zdns(c, name) - self.assertSuccess(res, cmd) - self.assertEqual(res["data"], self.DMARC_ANSWER["data"]) + self.assertSuccess(res, cmd, "PTR") + self.assertEqualAnswers(res, self.PTR_LOOKUP_GOOGLE_PUB, cmd, "PTR") def test_axfr(self): # In this test, we just check for few specific records @@ -948,72 +1079,72 @@ def test_axfr(self): c = "axfr" name = "zonetransfer.me" cmd, res = self.run_zdns(c, name) - self.assertSuccess(res, cmd) + self.assertSuccess(res, cmd, "AXFR") f = open("testing/axfr.json") axfr_answer = json.load(f) - self.assertEqualAxfrLookup(res["data"]["servers"][0]["records"], axfr_answer["data"]["servers"][0]["records"]) + self.assertEqualAxfrLookup(res["results"]["AXFR"]["data"]["servers"][0]["records"], axfr_answer["data"]["servers"][0]["records"]) f.close() def test_soa(self): c = "SOA" name = "zdns-testing.com" cmd, res = self.run_zdns(c, name) - self.assertSuccess(res, cmd) - self.assertEqualAnswers(res, self.SOA_ANSWERS, cmd, key="serial") + self.assertSuccess(res, cmd, c) + self.assertEqualAnswers(res, self.SOA_ANSWERS, cmd, c, key="serial") def test_srv(self): c = "SRV" name = "_sip._udp.sip.voice.google.com" cmd, res = self.run_zdns(c, name) - self.assertSuccess(res, cmd) - self.assertEqualAnswers(res, self.SRV_ANSWERS, cmd, key="target") + self.assertSuccess(res, cmd, c) + self.assertEqualAnswers(res, self.SRV_ANSWERS, cmd,c, key="target") def test_tlsa(self): c = "TLSA" name = "_25._tcp.mail.ietf.org" cmd, res = self.run_zdns(c, name) - self.assertSuccess(res, cmd) - self.assertEqualAnswers(res, self.TLSA_ANSWERS, cmd, key="certificate") + self.assertSuccess(res, cmd, c) + self.assertEqualAnswers(res, self.TLSA_ANSWERS, cmd, c, key="certificate") def test_too_big_txt_udp(self): c = "TXT --udp-only --name-servers=8.8.8.8:53" name = "large-text.zdns-testing.com" cmd, res = self.run_zdns(c, name) - self.assertEqual(res["status"], "TRUNCATED") - self.assertEqual(res["data"]["protocol"], "udp") + self.assertEqual(res["results"]["TXT"]["status"], "TRUNCATED") + self.assertEqual(res["results"]["TXT"]["data"]["protocol"], "udp") def test_too_big_txt_tcp(self): c = "TXT --tcp-only --name-servers=8.8.8.8:53" # Azure DNS does not provide results. name = "large-text.zdns-testing.com" cmd, res = self.run_zdns(c, name) - self.assertEqualAnswers(res, self.TCP_LARGE_TXT_ANSWERS, cmd, key="answer") + self.assertEqualAnswers(res, self.TCP_LARGE_TXT_ANSWERS, cmd, "TXT", key="answer") def test_too_big_txt_all(self): c = "TXT --name-servers=8.8.8.8:53" name = "large-text.zdns-testing.com" cmd, res = self.run_zdns(c, name) - self.assertEqual(res["data"]["protocol"], "tcp") - self.assertEqualAnswers(res, self.TCP_LARGE_TXT_ANSWERS, cmd, key="answer") + self.assertEqual(res["results"]["TXT"]["data"]["protocol"], "tcp") + self.assertEqualAnswers(res, self.TCP_LARGE_TXT_ANSWERS, cmd, "TXT", key="answer") def test_override_name(self): c = "A --override-name=zdns-testing.com" name = "notrealname.com" cmd, res = self.run_zdns(c, name) - self.assertEqualAnswers(res, self.ROOT_A_ANSWERS, cmd) + self.assertEqualAnswers(res, self.ROOT_A_ANSWERS, cmd, "A") def test_server_mode_a_lookup_ipv4(self): c = "A --override-name=zdns-testing.com --name-server-mode" name = "8.8.8.8:53" cmd, res = self.run_zdns(c, name) - self.assertEqual(res["data"]["resolver"], "8.8.8.8:53") - self.assertEqualAnswers(res, self.ROOT_A_ANSWERS, cmd) + self.assertEqual(res["results"]["A"]["data"]["resolver"], "8.8.8.8:53") + self.assertEqualAnswers(res, self.ROOT_A_ANSWERS, cmd, "A") def test_mixed_mode_a_lookup_ipv4(self): c = "A --name-servers=0.0.0.0" name = "zdns-testing.com,8.8.8.8:53" cmd, res = self.run_zdns(c, name) - self.assertEqual(res["data"]["resolver"], "8.8.8.8:53") - self.assertEqualAnswers(res, self.ROOT_A_ANSWERS, cmd) + self.assertEqual(res["results"]["A"]["data"]["resolver"], "8.8.8.8:53") + self.assertEqualAnswers(res, self.ROOT_A_ANSWERS, cmd, "A") def test_local_addr_interface_warning(self): c = "A --local-addr 192.168.1.5 --local-interface en0" @@ -1026,22 +1157,25 @@ def test_edns0_client_subnet(self): # Hardcoding a name server that supports ECS; Github's default recursive does not. c = f"A --client-subnet {subnet} --name-servers=8.8.8.8:53" cmd, res = self.run_zdns(c, name) - self.assertSuccess(res, cmd) + self.assertSuccess(res, cmd, "A") address, netmask = tuple(subnet.split("/")) family = 1 if ip_address(address).version == 4 else 2 + original_res = res + res = res["results"]["A"] self.assertEqual(address, res["data"]['additionals'][0]['csubnet']['address']) self.assertEqual(int(netmask), res["data"]['additionals'][0]['csubnet']["source_netmask"]) self.assertEqual(family, res["data"]['additionals'][0]['csubnet']['family']) self.assertTrue("source_scope" in res["data"]['additionals'][0]['csubnet']) correct = [{"type": "A", "class": "IN", "answer": ip_addr, "name": "ecs-geo.zdns-testing.com"}] - self.assertEqualAnswers(res, correct, cmd) + self.assertEqualAnswers(original_res, correct, cmd, "A") def test_edns0_nsid(self): name = "google.com" # using Google Public DNS for testing as its NSID always follows format 'gpdns-' c = f"A --nsid --name-servers=8.8.8.8:53" cmd, res = self.run_zdns(c, name) - self.assertSuccess(res, cmd) + self.assertSuccess(res, cmd, "A") + res = res["results"]["A"] self.assertTrue("nsid" in res["data"]['additionals'][0]) self.assertTrue(res["data"]['additionals'][0]['nsid']['nsid'].startswith("gpdns-")) @@ -1050,7 +1184,8 @@ def test_edns0_ede_1(self): # using Cloudflare Public DNS (1.1.1.1) that implements EDE c = f"A --name-servers=1.1.1.1:53" cmd, res = self.run_zdns(c, name) - self.assertServFail(res, cmd) + self.assertServFail(res, cmd, 'A') + res = res["results"]["A"] self.assertTrue("ede" in res["data"]['additionals'][0]) ede_obj = res["data"]['additionals'][0]["ede"][0] self.assertEqual("DNSKEY Missing", ede_obj["error_text"]) @@ -1062,7 +1197,8 @@ def test_edns0_ede_2_cd(self): # using Cloudflare Public DNS (1.1.1.1) that implements EDE, checking disabled resulting in NOERROR c = f"A --name-servers=1.1.1.1:53 --checking-disabled" cmd, res = self.run_zdns(c, name) - self.assertSuccess(res, cmd) + self.assertSuccess(res, cmd, "A") + res = res["results"]["A"] self.assertTrue("ede" in res["data"]['additionals'][0]) ede_obj = res["data"]['additionals'][0]["ede"][0] self.assertEqual("DNSKEY Missing", ede_obj["error_text"]) @@ -1074,7 +1210,8 @@ def test_dnssec_response(self): c = f"SOA --dnssec --name-servers=8.8.8.8:53" name = "." cmd, res = self.run_zdns(c, name) - self.assertSuccess(res, cmd) + self.assertSuccess(res, cmd, "SOA") + res = res["results"]["SOA"] self.assertEqual('do', res["data"]['additionals'][0]['flags']) self.assertEqualTypes(res, ["SOA", "RRSIG"]) @@ -1082,19 +1219,20 @@ def test_cd_bit_not_set(self): c = "A --name-servers=8.8.8.8:53" name = "dnssec-failed.org" cmd, res = self.run_zdns(c, name) - self.assertServFail(res, cmd) + self.assertServFail(res, cmd, "A") def test_cd_bit_set(self): c = "A --name-servers=8.8.8.8:53 --checking-disabled" name = "dnssec-failed.org" cmd, res = self.run_zdns(c, name) - self.assertSuccess(res, cmd) + self.assertSuccess(res, cmd, "A") def test_timetamps(self): c = "A" name = "zdns-testing.com" cmd, res = self.run_zdns(c, name) - self.assertSuccess(res, cmd) + self.assertSuccess(res, cmd, "A") + res = res["results"]["A"] assert "timestamp" in res date = datetime.datetime.strptime(res["timestamp"], "%Y-%m-%dT%H:%M:%S%z") self.assertTrue(date.microsecond == 0) # microseconds should be 0 since we didn't call with --nanoseconds @@ -1103,7 +1241,8 @@ def test_timetamps_nanoseconds(self): c = "A --nanoseconds" name = "zdns-testing.com" cmd, res = self.run_zdns(c, name) - self.assertSuccess(res, cmd) + self.assertSuccess(res, cmd, "A") + res = res["results"]["A"] assert "timestamp" in res date = parser.parse(res["timestamp"]) self.assertTrue(date.microsecond != 0)