refactor: Revise intelx workings

hueristiq · Jul 26, 2023 · 8e840b7 · 8e840b7
1 parent 11134a8
commit 8e840b7
Show file tree

Hide file tree

Showing 4 changed files with 128 additions and 125 deletions.
diff --git a/pkg/xurlfind3r/sources/github/github.go b/pkg/xurlfind3r/sources/github/github.go
@@ -115,7 +115,7 @@ func (source *Source) Enumerate(searchReqURL, domain string, tokens *Tokens, URL
 		URLs := mdExtractor.FindAllString(string(getRawContentRes.Body()), -1)
 
 		for _, URL := range URLs {
-			URL = fixURL(URL)
+			URL = sources.FixURL(URL)
 
 			parsedURL, err := hqgourl.Parse(URL)
 			if err != nil {
@@ -135,7 +135,7 @@ func (source *Source) Enumerate(searchReqURL, domain string, tokens *Tokens, URL
 			URLs := mdExtractor.FindAllString(textMatch.Fragment, -1)
 
 			for _, URL := range URLs {
-				URL = fixURL(URL)
+				URL = sources.FixURL(URL)
 
 				parsedURL, err := hqgourl.Parse(URL)
 				if err != nil {

diff --git a/pkg/xurlfind3r/sources/github/utils.go b/pkg/xurlfind3r/sources/github/utils.go
@@ -11,100 +11,3 @@ func getRawContentURL(URL string) (rawContentURL string) {
 
 	return
 }
-
-func fixURL(URL string) (fixedURL string) {
-	fixedURL = URL
-
-	// ',",`,
-	quotes := []rune{'\'', '"', '`'}
-
-	for i := range quotes {
-		quote := quotes[i]
-
-		indexOfQuote := findUnbalancedQuote(URL, quote)
-		if indexOfQuote <= len(fixedURL) && indexOfQuote >= 0 {
-			fixedURL = fixedURL[:indexOfQuote]
-		}
-	}
-
-	// (),[],{}
-	parentheses := []struct {
-		Opening, Closing rune
-	}{
-		{'[', ']'},
-		{'(', ')'},
-		{'{', '}'},
-	}
-
-	for i := range parentheses {
-		parenthesis := parentheses[i]
-
-		indexOfParenthesis := findUnbalancedBracket(URL, parenthesis.Opening, parenthesis.Closing)
-		if indexOfParenthesis <= len(fixedURL) && indexOfParenthesis >= 0 {
-			fixedURL = fixedURL[:indexOfParenthesis]
-		}
-	}
-
-	// ;
-	indexOfComma := strings.Index(fixedURL, ";")
-	if indexOfComma <= len(fixedURL) && indexOfComma >= 0 {
-		fixedURL = fixedURL[:indexOfComma]
-	}
-
-	return
-}
-
-func findUnbalancedQuote(s string, quoteChar rune) int {
-	insideQuotes := false
-
-	for _, ch := range s {
-		if ch == quoteChar {
-			if insideQuotes {
-				insideQuotes = false
-			} else {
-				insideQuotes = true
-			}
-		}
-	}
-
-	// If still inside quotes at the end of the string,
-	// find the index of the opening quote
-	if insideQuotes {
-		for i, ch := range s {
-			if ch == quoteChar {
-				return i
-			}
-		}
-	}
-
-	return -1 // return -1 if all quotes are balanced
-}
-
-func findUnbalancedBracket(s string, openChar, closeChar rune) int {
-	openCount := 0
-
-	var firstOpenIndex int
-
-	for i, ch := range s {
-		if ch == openChar {
-			if openCount == 0 {
-				firstOpenIndex = i
-			}
-
-			openCount++
-		} else if ch == closeChar {
-			openCount--
-
-			if openCount < 0 {
-				return i // Found an unbalanced closing bracket
-			}
-		}
-	}
-
-	// If there are unmatched opening brackets
-	if openCount > 0 {
-		return firstOpenIndex
-	}
-
-	return -1 // All brackets are balanced
-}
diff --git a/pkg/xurlfind3r/sources/intelx/intelx.go b/pkg/xurlfind3r/sources/intelx/intelx.go
@@ -1,13 +1,12 @@
-// Package intelx implements functions to search URLs from intelx.
 package intelx
 
 import (
 	"encoding/json"
 	"fmt"
-	"net/mail"
 	"strings"
 	"time"
 
+	"github.com/hueristiq/hqgourl"
 	"github.com/hueristiq/xurlfind3r/pkg/xurlfind3r/httpclient"
 	"github.com/hueristiq/xurlfind3r/pkg/xurlfind3r/sources"
 	"github.com/valyala/fasthttp"
@@ -16,6 +15,7 @@ import (
 type searchRequest struct {
 	Term       string        `json:"term"`
 	Timeout    time.Duration `json:"timeout"`
+	Target     int           `json:"target"`
 	MaxResults int           `json:"maxresults"`
 	Media      int           `json:"media"`
 }
@@ -24,7 +24,7 @@ type searchResponse struct {
 	Status int    `json:"status"`
 }
 
-type resultsResponse struct {
+type getResultsResponse struct {
 	Selectors []struct {
 		Selectvalue string `json:"selectorvalue"`
 	} `json:"selectors"`
@@ -60,58 +60,67 @@ func (source *Source) Run(config *sources.Configuration, domain string) (URLsCha
 			return
 		}
 
-		searchURL := fmt.Sprintf("https://%s/phonebook/search?k=%s", intelXHost, intelXKey)
+		searchReqURL := fmt.Sprintf("https://%s/phonebook/search?k=%s", intelXHost, intelXKey)
 		searchReqBody := searchRequest{
-			Term:       domain,
+			Term:       "*" + domain,
 			MaxResults: 100000,
 			Media:      0,
+			Target:     3, // 1 = Domains | 2 = Emails | 3 = URLs
 			Timeout:    20,
 		}
 
-		var body []byte
+		var searchReqBodyBytes []byte
 
-		body, err = json.Marshal(searchReqBody)
+		searchReqBodyBytes, err = json.Marshal(searchReqBody)
 		if err != nil {
 			return
 		}
 
-		var res *fasthttp.Response
+		var searchRes *fasthttp.Response
 
-		res, err = httpclient.SimplePost(searchURL, "application/json", body)
+		searchRes, err = httpclient.SimplePost(searchReqURL, "application/json", searchReqBodyBytes)
 		if err != nil {
 			return
 		}
 
-		var searchResponseData searchResponse
+		var searchResData searchResponse
 
-		if err = json.Unmarshal(res.Body(), &searchResponseData); err != nil {
+		if err = json.Unmarshal(searchRes.Body(), &searchResData); err != nil {
 			return
 		}
 
-		resultsURL := fmt.Sprintf("https://%s/phonebook/search/result?k=%s&id=%s&limit=10000", intelXHost, intelXKey, searchResponseData.ID)
+		getResultsReqURL := fmt.Sprintf("https://%s/phonebook/search/result?k=%s&id=%s&limit=10000", intelXHost, intelXKey, searchResData.ID)
 		status := 0
 
 		for status == 0 || status == 3 {
-			res, err = httpclient.Get(resultsURL, "", nil)
+			var getResultsRes *fasthttp.Response
+
+			getResultsRes, err = httpclient.Get(getResultsReqURL, "", nil)
 			if err != nil {
 				return
 			}
 
-			var resultsResponseData resultsResponse
+			var getResultsResData getResultsResponse
 
-			if err = json.Unmarshal(res.Body(), &resultsResponseData); err != nil {
+			if err = json.Unmarshal(getResultsRes.Body(), &getResultsResData); err != nil {
 				return
 			}
 
-			status = resultsResponseData.Status
+			status = getResultsResData.Status
 
-			for _, hostname := range resultsResponseData.Selectors {
+			for _, hostname := range getResultsResData.Selectors {
 				URL := hostname.Selectvalue
+				URL = sources.FixURL(URL)
 
-				if isEmail(URL) {
-					continue
+				parsedURL, err := hqgourl.Parse(URL)
+				if err != nil {
+					return
 				}
 
+				parsedURL.Path = strings.Split(parsedURL.Path, ":")[0]
+
+				URL = parsedURL.String()
+
 				if !sources.IsInScope(URL, domain, config.IncludeSubdomains) {
 					continue
 				}
@@ -124,13 +133,6 @@ func (source *Source) Run(config *sources.Configuration, domain string) (URLsCha
 	return
 }
 
-func isEmail(URL string) (isEmail bool) {
-	_, err := mail.ParseAddress(URL)
-	isEmail = err == nil
-
-	return
-}
-
 func (source *Source) Name() string {
 	return "intelx"
 }
diff --git a/pkg/xurlfind3r/sources/utils.go b/pkg/xurlfind3r/sources/utils.go
@@ -4,6 +4,7 @@ import (
 	"crypto/rand"
 	"fmt"
 	"math/big"
+	"strings"
 
 	"github.com/hueristiq/hqgourl"
 )
@@ -58,3 +59,100 @@ func IsInScope(URL, domain string, includeSubdomains bool) (isInScope bool) {
 
 	return
 }
+
+func FixURL(URL string) (fixedURL string) {
+	fixedURL = URL
+
+	// ',",`,
+	quotes := []rune{'\'', '"', '`'}
+
+	for i := range quotes {
+		quote := quotes[i]
+
+		indexOfQuote := findUnbalancedQuote(URL, quote)
+		if indexOfQuote <= len(fixedURL) && indexOfQuote >= 0 {
+			fixedURL = fixedURL[:indexOfQuote]
+		}
+	}
+
+	// (),[],{}
+	parentheses := []struct {
+		Opening, Closing rune
+	}{
+		{'[', ']'},
+		{'(', ')'},
+		{'{', '}'},
+	}
+
+	for i := range parentheses {
+		parenthesis := parentheses[i]
+
+		indexOfParenthesis := findUnbalancedBracket(URL, parenthesis.Opening, parenthesis.Closing)
+		if indexOfParenthesis <= len(fixedURL) && indexOfParenthesis >= 0 {
+			fixedURL = fixedURL[:indexOfParenthesis]
+		}
+	}
+
+	// ;
+	indexOfComma := strings.Index(fixedURL, ";")
+	if indexOfComma <= len(fixedURL) && indexOfComma >= 0 {
+		fixedURL = fixedURL[:indexOfComma]
+	}
+
+	return
+}
+
+func findUnbalancedQuote(s string, quoteChar rune) int {
+	insideQuotes := false
+
+	for _, ch := range s {
+		if ch == quoteChar {
+			if insideQuotes {
+				insideQuotes = false
+			} else {
+				insideQuotes = true
+			}
+		}
+	}
+
+	// If still inside quotes at the end of the string,
+	// find the index of the opening quote
+	if insideQuotes {
+		for i, ch := range s {
+			if ch == quoteChar {
+				return i
+			}
+		}
+	}
+
+	return -1 // return -1 if all quotes are balanced
+}
+
+func findUnbalancedBracket(s string, openChar, closeChar rune) int {
+	openCount := 0
+
+	var firstOpenIndex int
+
+	for i, ch := range s {
+		if ch == openChar {
+			if openCount == 0 {
+				firstOpenIndex = i
+			}
+
+			openCount++
+		} else if ch == closeChar {
+			openCount--
+
+			if openCount < 0 {
+				return i // Found an unbalanced closing bracket
+			}
+		}
+	}
+
+	// If there are unmatched opening brackets
+	if openCount > 0 {
+		return firstOpenIndex
+	}
+
+	return -1 // All brackets are balanced
+}