From e22629cbd2f852820db967e425727efdb28efb36 Mon Sep 17 00:00:00 2001 From: Bastien Faure Date: Tue, 24 Nov 2020 17:46:26 -0800 Subject: [PATCH 1/2] Fixed the OTX provider that did not return proper data when a hostname (not a top domain) is submitted as target --- providers/otx.go | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/providers/otx.go b/providers/otx.go index 59ae749..249dce9 100644 --- a/providers/otx.go +++ b/providers/otx.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "strings" + "github.com/bobesa/go-domain-util/domainutil" ) type OTXProvider struct { @@ -31,9 +32,19 @@ func NewOTXProvider(config *Config) Provider { } func (o *OTXProvider) formatURL(domain string, page int) string { - return fmt.Sprintf("https://otx.alienvault.com/api/v1/indicators/domain/%s/url_list?limit=%d&page=%d", - domain, otxResultsLimit, page, - ) + if !domainutil.HasSubdomain(domain) { + return fmt.Sprintf("https://otx.alienvault.com/api/v1/indicators/domain/%s/url_list?limit=%d&page=%d", + domain, otxResultsLimit, page, + ) + } else if domainutil.HasSubdomain(domain) && o.IncludeSubdomains { + return fmt.Sprintf("https://otx.alienvault.com/api/v1/indicators/domain/%s/url_list?limit=%d&page=%d", + domain, otxResultsLimit, page, + ) + } else { + return fmt.Sprintf("https://otx.alienvault.com/api/v1/indicators/hostname/%s/url_list?limit=%d&page=%d", + domain, otxResultsLimit, page, + ) + } } func (o *OTXProvider) Fetch(domain string, results chan<- string) error { @@ -53,7 +64,13 @@ func (o *OTXProvider) Fetch(domain string, results chan<- string) error { for _, entry := range result.URLList { if o.IncludeSubdomains { - results <- entry.URL + if !domainutil.HasSubdomain(domain) { + results <- entry.URL + } else { + if strings.Contains(strings.ToLower(entry.Hostname), strings.ToLower(domain)) { + results <- entry.URL + } + } } else { if strings.EqualFold(domain, entry.Hostname) { results <- entry.URL From 4faab620ea7346ff2f929bfc9874d265a1d21b05 Mon Sep 17 00:00:00 2001 From: Bastien Faure Date: Tue, 24 Nov 2020 18:20:03 -0800 Subject: [PATCH 2/2] When non top-domain hostname is submitted but subdomains are to included, query the OTX API with the extracted domain --- providers/otx.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/providers/otx.go b/providers/otx.go index 249dce9..8c0958e 100644 --- a/providers/otx.go +++ b/providers/otx.go @@ -38,7 +38,7 @@ func (o *OTXProvider) formatURL(domain string, page int) string { ) } else if domainutil.HasSubdomain(domain) && o.IncludeSubdomains { return fmt.Sprintf("https://otx.alienvault.com/api/v1/indicators/domain/%s/url_list?limit=%d&page=%d", - domain, otxResultsLimit, page, + domainutil.Domain(domain), otxResultsLimit, page, ) } else { return fmt.Sprintf("https://otx.alienvault.com/api/v1/indicators/hostname/%s/url_list?limit=%d&page=%d",