Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Output output from multiple modules in the same output line, similar to Grab #429

Merged
merged 6 commits into from
Aug 23, 2024
Merged

Output output from multiple modules in the same output line, similar to Grab #429

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
dee2ee9
only add 'lookup_module' if we're using multiple modules to save stor…
phillip-stephens Aug 23, 2024
b665ad9
collect output
phillip-stephens Aug 23, 2024
2e5eae4
sort imports
phillip-stephens Aug 23, 2024
92a2bc7
fixed up integration tests to match new schema
phillip-stephens Aug 23, 2024
7febad6
edit todo
phillip-stephens Aug 23, 2024
30ca109
update README.md
phillip-stephens Aug 23, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
124 changes: 66 additions & 58 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,40 +71,42 @@ For example, the command:
returns:
```json
{
"name": "censys.io",
"class": "IN",
"status": "NOERROR",
"data": {
"answers": [
{
"ttl": 300,
"type": "A",
"class": "IN",
"name": "censys.io",
"data": "216.239.38.21"
"name": "censys.io",
"results": {
"A": {
"data": {
"additionals": [
{
"flags": "",
"type": "EDNS0",
"udpsize": 512,
"version": 0
}
],
"answers": [
{
"answer": "104.18.10.85",
"class": "IN",
"name": "censys.io",
"ttl": 300,
"type": "A"
},
{
"answer": "104.18.11.85",
"class": "IN",
"name": "censys.io",
"ttl": 300,
"type": "A"
}
],
"protocol": "udp",
"resolver": "[2603:6013:9d00:3302::1]:53"
},
"duration": 0.285295416,
"status": "NOERROR",
"timestamp": "2024-08-23T13:12:43-04:00"
}
],
"additionals": [
{
"ttl": 34563,
"type": "A",
"class": "IN",
"name": "ns-cloud-e1.googledomains.com",
"data": "216.239.32.110"
},
],
"authorities": [
{
"ttl": 53110,
"type": "NS",
"class": "IN",
"name": "censys.io",
"data": "ns-cloud-e1.googledomains.com."
},
],
"protocol": "udp",
"resolver": "30.128.52.190:53"
}
}
}
```

Expand All @@ -128,32 +130,38 @@ For example,
returns:
```json
{
"name": "censys.io",
"status": "NOERROR",
"data": {
"exchanges": [
{
"name": "aspmx.l.google.com",
"type": "MX",
"class": "IN",
"preference": 1,
"ipv4_addresses": [
"74.125.28.26"
],
"ttl": 288
},
{
"name": "alt1.aspmx.l.google.com",
"type": "MX",
"class": "IN",
"preference": 5,
"ipv4_addresses": [
"64.233.182.26"
],
"ttl": 288
"name": "censys.io",
"results": {
"MXLOOKUP": {
"data": {
"exchanges": [
{
"class": "IN",
"ipv4_addresses": [
"209.85.202.27"
],
"name": "alt1.aspmx.l.google.com",
"preference": 5,
"ttl": 300,
"type": "MX"
},
{
"class": "IN",
"ipv4_addresses": [
"142.250.31.26"
],
"name": "aspmx.l.google.com",
"preference": 1,
"ttl": 300,
"type": "MX"
}
]
},
"duration": 0.154786958,
"status": "NOERROR",
"timestamp": "2024-08-23T13:10:11-04:00"
}
]
}
}
}
```

Expand Down
10 changes: 8 additions & 2 deletions benchmark/stats.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -135,7 +135,14 @@ func updateStats(line string, s *Stats) {
log.Panicf("failed to unmarshal JSON (%s): %v", line, err)
}
domainName := res.Name
resolveTime := time.Duration(res.Duration * float64(time.Second))
// TODO - this will only work for a single module benchmark, we'll need to adjust this if we want to benchmark multi-module
var duration float64
var status zdns.Status
for _, moduleResult := range res.Results {
duration = moduleResult.Duration
status = zdns.Status(moduleResult.Status)
}
resolveTime := time.Duration(duration * float64(time.Second))

if resolveTime < s.MinResolveTime || s.MinResolveTime == 0 {
s.MinResolveTime = resolveTime
Expand Down Expand Up @@ -173,7 +180,6 @@ func updateStats(line string, s *Stats) {
}
s.numberOfResolutions++

status := zdns.Status(res.Status)
if status == zdns.StatusNoError {
s.SuccessfulResolutions++
} else if status == zdns.StatusTimeout || status == zdns.StatusIterTimeout {
Expand Down
87 changes: 47 additions & 40 deletions src/cli/worker_manager.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -576,69 +576,76 @@ func doLookupWorker(gc *CLIConf, rc *zdns.ResolverConfig, input <-chan string, o
metadata.Status = make(map[zdns.Status]int)
for line := range input {
// we'll process each module sequentially, parallelism is per-domain
res := zdns.Result{Results: make(map[string]zdns.SingleModuleResult, len(gc.ActiveModules))}
// get the fields that won't change for each lookup module
rawName := ""
nameServer := ""
var rank int
var entryMetadata string
if gc.AlexaFormat {
rawName, rank = parseAlexa(line)
res.AlexaRank = rank
} else if gc.MetadataFormat {
rawName, entryMetadata = parseMetadataInputLine(line)
res.Metadata = entryMetadata
} else if gc.NameServerMode {
nameServer, err = util.AddDefaultPortToDNSServerName(line)
if err != nil {
log.Fatal("unable to parse name server: ", line)
}
} else {
rawName, nameServer = parseNormalInputLine(line)
}
res.Name = rawName
// handle per-module lookups
for moduleName, module := range gc.ActiveModules {
res := zdns.Result{Module: moduleName}
var innerRes interface{}
var trace zdns.Trace
var status zdns.Status
var err error
var changed bool
var lookupName string
rawName := ""
nameServer := ""
var rank int
var entryMetadata string
if gc.AlexaFormat {
rawName, rank = parseAlexa(line)
res.AlexaRank = rank
} else if gc.MetadataFormat {
rawName, entryMetadata = parseMetadataInputLine(line)
res.Metadata = entryMetadata
} else if gc.NameServerMode {
nameServer, err = util.AddDefaultPortToDNSServerName(line)
if err != nil {
log.Fatal("unable to parse name server: ", line)
}
} else {
rawName, nameServer = parseNormalInputLine(line)
}
lookupName, changed = makeName(rawName, gc.NamePrefix, gc.NameOverride)
if changed {
res.AlteredName = lookupName
}
res.Name = rawName
res.Class = dns.Class(gc.Class).String()

startTime := time.Now()
innerRes, trace, status, err = module.Lookup(resolver, lookupName, nameServer)

res.Timestamp = time.Now().Format(gc.TimeFormat)
res.Duration = time.Since(startTime).Seconds()
lookupRes := zdns.SingleModuleResult{
Timestamp: time.Now().Format(gc.TimeFormat),
Duration: time.Since(startTime).Seconds(),
}
if status != zdns.StatusNoOutput {
res.Status = string(status)
res.Data = innerRes
res.Trace = trace
if err != nil {
res.Error = err.Error()
}
v, _ := version.NewVersion("0.0.0")
o := &sheriff.Options{
Groups: gc.OutputGroups,
ApiVersion: v,
}
data, err := sheriff.Marshal(o, res)
lookupRes.Status = string(status)
lookupRes.Data = innerRes
lookupRes.Trace = trace
if err != nil {
log.Fatalf("unable to marshal result to JSON: %v", err)
lookupRes.Error = err.Error()
}
jsonRes, err := json.Marshal(data)
if err != nil {
log.Fatalf("unable to marshal JSON result: %v", err)
}
output <- string(jsonRes)
res.Results[moduleName] = lookupRes
}
metadata.Status[status]++
metadata.Lookups++
}
if len(res.Results) > 0 {
v, _ := version.NewVersion("0.0.0")
o := &sheriff.Options{
Groups: gc.OutputGroups,
ApiVersion: v,
}
data, err := sheriff.Marshal(o, res)
if err != nil {
log.Fatalf("unable to marshal result to JSON: %v", err)
}
jsonRes, err := json.Marshal(data)
if err != nil {
log.Fatalf("unable to marshal JSON result: %v", err)
}
output <- string(jsonRes)
}
metadata.Names++
}
metaChan <- metadata
Expand Down
32 changes: 18 additions & 14 deletions src/zdns/qa.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,21 +46,25 @@ type TraceStep struct {
Try int `json:"try" groups:"trace"`
}

// Result contains all the metadata from a complete lookup, potentailly after following many CNAMEs/etc.
// Result contains all the metadata from a complete lookup(s) for a domain. Results is keyed with the ModuleName.
type Result struct {
AlteredName string `json:"altered_name,omitempty" groups:"short,normal,long,trace"`
Name string `json:"name,omitempty" groups:"short,normal,long,trace"`
Nameserver string `json:"nameserver,omitempty" groups:"normal,long,trace"`
Class string `json:"class,omitempty" groups:"long,trace"`
AlexaRank int `json:"alexa_rank,omitempty" groups:"short,normal,long,trace"`
Metadata string `json:"metadata,omitempty" groups:"short,normal,long,trace"`
Module string `json:"lookup_module,omitempty" groups:"short,normal,long,trace"`
Status string `json:"status,omitempty" groups:"short,normal,long,trace"`
Error string `json:"error,omitempty" groups:"short,normal,long,trace"`
Timestamp string `json:"timestamp,omitempty" groups:"short,normal,long,trace"`
Duration float64 `json:"duration,omitempty" groups:"short,normal,long,trace"` // in seconds
Data interface{} `json:"data,omitempty" groups:"short,normal,long,trace"`
Trace Trace `json:"trace,omitempty" groups:"trace"`
AlteredName string `json:"altered_name,omitempty" groups:"short,normal,long,trace"`
Name string `json:"name,omitempty" groups:"short,normal,long,trace"`
Nameserver string `json:"nameserver,omitempty" groups:"normal,long,trace"`
Class string `json:"class,omitempty" groups:"long,trace"`
AlexaRank int `json:"alexa_rank,omitempty" groups:"short,normal,long,trace"`
Metadata string `json:"metadata,omitempty" groups:"short,normal,long,trace"`
Results map[string]SingleModuleResult `json:"results,omitempty" groups:"short,normal,long,trace"`
}

// SingleModuleResult contains all the metadata from a complete lookup for a domain, potentially after following many CNAMEs/etc.
type SingleModuleResult struct {
Status string `json:"status,omitempty" groups:"short,normal,long,trace"`
Error string `json:"error,omitempty" groups:"short,normal,long,trace"`
Timestamp string `json:"timestamp,omitempty" groups:"short,normal,long,trace"`
Duration float64 `json:"duration,omitempty" groups:"short,normal,long,trace"` // in seconds
Data interface{} `json:"data,omitempty" groups:"short,normal,long,trace"`
Trace Trace `json:"trace,omitempty" groups:"trace"`
}

// SingleQueryResult contains the results of a single DNS query
Expand Down
Loading