chore: -

.golangci.yaml

@@ -73,7 +73,7 @@ linters:
         # - ireturn # Accept Interfaces, Return Concrete Types [fast: false, auto-fix: false]
         # - lll # Reports long lines [fast: true, auto-fix: false]
         - loggercheck # (logrlint) # Checks key value pairs for common logger libraries (kitlog,klog,logr,zap). [fast: false, auto-fix: false]
-        - maintidx # maintidx measures the maintainability index of each function. [fast: true, auto-fix: false]
+        # - maintidx # maintidx measures the maintainability index of each function. [fast: true, auto-fix: false]
         - makezero # Finds slice declarations with non-zero initial length [fast: false, auto-fix: false]
         # - maligned # [deprecated] # Tool to detect Go structs that would take less memory if their fields were sorted [fast: false, auto-fix: false]
         - mirror # reports wrong mirror patterns of bytes/strings usage [fast: false, auto-fix: false]

Makefile

@@ -17,6 +17,8 @@ ifneq ($(shell go env GOOS),darwin)
 LDFLAGS := -extldflags "-static"
 endif
 
+all: build
+
 .PHONY: tidy
 tidy:
 	$(GOMOD) tidy
@@ -26,13 +28,16 @@ update-deps:
 	$(GOGET) -f -t -u ./...
 	$(GOGET) -f -u ./...
 
-.PHONY: format
-format:
+.PHONY: _gofmt
+_gofmt:
 	$(GOFMT) ./...
 
+.PHONY: _golangci-lint
+_golangci-lint:
+	$(GOLANGCILINTRUN) $(GOLANGCILINT) ./...
+
 .PHONY: lint
-lint:
-	$(GOLANGCILINTRUN) ./...
+lint: _gofmt _golangci-lint
 
 .PHONY: test
 test:
@@ -44,4 +49,4 @@ build:
 
 .PHONY: install
 install:
-	$(GOINSTALL) $(GOFLAGS) ./...
+	$(GOINSTALL) $(GOFLAGS) ./...

cmd/xurlfind3r/main.go

@@ -64,11 +64,11 @@ func init() {
 		fmt.Fprintln(os.Stderr, configuration.BANNER)
 
 		h := "\nUSAGE:\n"
-		h += fmt.Sprintf("  %s [OPTIONS]\n", configuration.NAME)
+		h += fmt.Sprintf(" %s [OPTIONS]\n", configuration.NAME)
 
 		h += "\nCONFIGURATION:\n"
 		defaultConfigurationFilePath := strings.ReplaceAll(configuration.ConfigurationFilePath, configuration.UserDotConfigDirectoryPath, "$HOME/.config")
-		h += fmt.Sprintf(" -c, --configuration string          configuration file path (default: %s)\n", defaultConfigurationFilePath)
+		h += fmt.Sprintf(" -c, --configuration string          configuration file (default: %s)\n", defaultConfigurationFilePath)
 
 		h += "\nINPUT:\n"
 		h += " -d, --domain string[]               target domain\n"

go.mod

@@ -4,22 +4,22 @@ go 1.20
 
 require (
 	dario.cat/mergo v1.0.0
+	github.com/hueristiq/hqgohttp v0.0.0-20231024010818-fdb48fa4aead
 	github.com/hueristiq/hqgolimit v0.0.0-20230623113203-3e14552a97f8
 	github.com/hueristiq/hqgolog v0.0.0-20230623113334-a6018965a34f
 	github.com/hueristiq/hqgourl v0.0.0-20230821112831-e12f907b5a53
 	github.com/logrusorgru/aurora/v3 v3.0.0
 	github.com/spf13/cast v1.5.1
 	github.com/spf13/pflag v1.0.5
 	github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80
-	github.com/valyala/fasthttp v1.50.0
 	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
-	github.com/andybalholm/brotli v1.0.5 // indirect
-	github.com/klauspost/compress v1.17.0 // indirect
-	github.com/valyala/bytebufferpool v1.0.0 // indirect
-	golang.org/x/net v0.17.0 // indirect
-	golang.org/x/sys v0.13.0 // indirect
-	golang.org/x/term v0.13.0 // indirect
+	github.com/Mzack9999/go-http-digest-auth-client v0.6.0 // indirect
+	github.com/hueristiq/hqgoutils v0.0.0-20231024005153-bd2c47932440 // indirect
+	golang.org/x/net v0.18.0 // indirect
+	golang.org/x/sys v0.14.0 // indirect
+	golang.org/x/term v0.14.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
 )

go.sum

@@ -1,38 +1,41 @@
 dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk=
 dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
-github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs=
-github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
+github.com/Mzack9999/go-http-digest-auth-client v0.6.0 h1:LXVNMsj7qiNVmlZByFbjJmXf6SOm/uoo04XmnNcWPms=
+github.com/Mzack9999/go-http-digest-auth-client v0.6.0/go.mod h1:gbwaYYXwA15ZfIxMyY5QU1acATDyNKEuG5TylBCL7AM=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY=
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/hueristiq/hqgohttp v0.0.0-20231024010818-fdb48fa4aead h1:Iep2G2h3hSwc7w0qr1iVVAptgXqjn7DRXVQ33luPmhk=
+github.com/hueristiq/hqgohttp v0.0.0-20231024010818-fdb48fa4aead/go.mod h1:Faf/mOhyfNnLIfhoYj2vfPrjt0nKBr4WaU+OQ0C7r6U=
 github.com/hueristiq/hqgolimit v0.0.0-20230623113203-3e14552a97f8 h1:r4ze6pX8H//X4SJEIcn8wHPgAhaGKEaa44lyHh1epXY=
 github.com/hueristiq/hqgolimit v0.0.0-20230623113203-3e14552a97f8/go.mod h1:CzhJzxz2rv/NMKNz5b4eKFh1epdcED05YTHT32NFyrI=
 github.com/hueristiq/hqgolog v0.0.0-20230623113334-a6018965a34f h1:JAgZOIJ+UbkENpRiOTlfg51CW0UNrUkgwLjUGiH+x9g=
 github.com/hueristiq/hqgolog v0.0.0-20230623113334-a6018965a34f/go.mod h1:S5J3E3Azva5+JKv67uc+Hh3XwLDvkVYDGjEaMTFrIqg=
 github.com/hueristiq/hqgourl v0.0.0-20230821112831-e12f907b5a53 h1:6pwdpEJoB1woSToh0cxLh5QirNOAp2z7DzvMKiaqdro=
 github.com/hueristiq/hqgourl v0.0.0-20230821112831-e12f907b5a53/go.mod h1:Fc2vfWpIVFWUmCv1S0xVsz3mIPYwdgsa6f2vCgL4CrA=
-github.com/klauspost/compress v1.17.0 h1:Rnbp4K9EjcDuVuHtd0dgA4qNuv9yKDYKK1ulpJwgrqM=
-github.com/klauspost/compress v1.17.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
+github.com/hueristiq/hqgoutils v0.0.0-20231024005153-bd2c47932440 h1:dpHAa9c74HgAXkZ2WPd84q2cCiF76eluuSGRw7bk7To=
+github.com/hueristiq/hqgoutils v0.0.0-20231024005153-bd2c47932440/go.mod h1:NlZ117o///yWDbRAbgYD7/Y44qce8z1Dj4caUsjunSY=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/logrusorgru/aurora/v3 v3.0.0 h1:R6zcoZZbvVcGMvDCKo45A9U/lzYyzl5NfYIvznmDfE4=
 github.com/logrusorgru/aurora/v3 v3.0.0/go.mod h1:vsR12bk5grlLvLXAYrBsb5Oc/N+LxAlxggSjiwMnCUc=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
 github.com/spf13/cast v1.5.1 h1:R+kOtfhWQE6TVQzY+4D7wJLBgkdVasCEFxSUBYBYIlA=
 github.com/spf13/cast v1.5.1/go.mod h1:b9PdjNptOpzXr7Rq1q9gJML/2cdGQAo69NKzQ10KN48=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 h1:nrZ3ySNYwJbSpD6ce9duiP+QkD3JuLCcWkdaehUS/3Y=
 github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80/go.mod h1:iFyPdL66DjUD96XmzVL3ZntbzcflLnznH0fr99w5VqE=
-github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
-github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
-github.com/valyala/fasthttp v1.50.0 h1:H7fweIlBm0rXLs2q0XbalvJ6r0CUPFWK3/bB4N13e9M=
-github.com/valyala/fasthttp v1.50.0/go.mod h1:k2zXd82h/7UZc3VOdJ2WaUqt1uZ/XpXAfE9i+HBC3lA=
-golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
-golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
-golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
-golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek=
-golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
+golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg=
+golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ=
+golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q=
+golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/term v0.14.0 h1:LGK9IlZ8T9jvdy6cTdfKUCltatMFOehAQo9SRC46UQ8=
+golang.org/x/term v0.14.0/go.mod h1:TySc+nGkYR6qt8km8wUhuFRTVSMIX3XPR58y2lC8vww=
+golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
+golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

pkg/httpclient/client.go

@@ -1,39 +1,52 @@
 package httpclient
 
 import (
-	"github.com/valyala/fasthttp"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+
+	"github.com/hueristiq/hqgohttp"
+	"github.com/hueristiq/hqgohttp/methods"
+	"github.com/hueristiq/hqgohttp/status"
+	"github.com/hueristiq/xurlfind3r/internal/configuration"
 )
 
-var client = &fasthttp.Client{}
+var client *hqgohttp.Client
 
-func httpRequestWrapper(req *fasthttp.Request) (res *fasthttp.Response, err error) {
-	res = fasthttp.AcquireResponse()
+func init() {
+	options := hqgohttp.DefaultOptionsSpraying
 
-	if err = client.Do(req, res); err != nil {
+	client, _ = hqgohttp.New(options)
+}
+
+func httpRequestWrapper(req *hqgohttp.Request) (res *http.Response, err error) {
+	res, err = client.Do(req)
+	if err != nil {
 		return
 	}
 
-	return
-}
+	if res.StatusCode != status.OK {
+		requestURL, _ := url.QueryUnescape(req.URL.String())
 
-func Request(method, URL, cookies string, headers map[string]string, body []byte) (res *fasthttp.Response, err error) {
-	req := fasthttp.AcquireRequest()
+		err = fmt.Errorf("unexpected status code %d received from %s", res.StatusCode, requestURL)
 
-	req.SetRequestURI(URL)
-	req.SetBody(body)
-	req.Header.SetMethod(method)
+		return
+	}
 
-	var agent string
+	return
+}
 
-	agent, err = UserAgent()
+// HTTPRequest makes any HTTP request to a URL with extended parameters
+func HTTPRequest(method, requestURL, cookies string, headers map[string]string, body io.Reader) (*http.Response, error) {
+	req, err := hqgohttp.NewRequest(method, requestURL, body)
 	if err != nil {
-		return
+		return nil, err
 	}
 
-	req.Header.Set("User-Agent", agent)
 	req.Header.Set("Accept", "*/*")
 	req.Header.Set("Accept-Language", "en")
-	req.Header.Set("Connection", "close")
+	req.Header.Set("User-Agent", fmt.Sprintf("%s v%s (https://github.com/hueristiq/%s)", configuration.NAME, configuration.VERSION, configuration.NAME))
 
 	if cookies != "" {
 		req.Header.Set("Cookie", cookies)
@@ -46,18 +59,28 @@ func Request(method, URL, cookies string, headers map[string]string, body []byte
 	return httpRequestWrapper(req)
 }
 
-func SimpleGet(URL string) (*fasthttp.Response, error) {
-	return Request(fasthttp.MethodGet, URL, "", map[string]string{}, nil)
+// Get makes a GET request to a URL with extended parameters
+func Get(URL, cookies string, headers map[string]string) (*http.Response, error) {
+	return HTTPRequest(methods.Get, URL, cookies, headers, nil)
 }
 
-func Get(URL, cookies string, headers map[string]string) (*fasthttp.Response, error) {
-	return Request(fasthttp.MethodGet, URL, cookies, headers, nil)
+// SimpleGet makes a simple GET request to a URL
+func SimpleGet(URL string) (*http.Response, error) {
+	return HTTPRequest(methods.Get, URL, "", map[string]string{}, nil)
 }
 
-func SimplePost(URL, contentType string, body []byte) (*fasthttp.Response, error) {
-	return Request(fasthttp.MethodPost, URL, "", map[string]string{"Content-Type": contentType}, body)
+// Post makes a POST request to a URL with extended parameters
+func Post(URL, cookies string, headers map[string]string, body io.Reader) (*http.Response, error) {
+	return HTTPRequest(methods.Post, URL, cookies, headers, body)
 }
 
-func Post(URL, cookies string, headers map[string]string, body []byte) (*fasthttp.Response, error) {
-	return Request(fasthttp.MethodPost, URL, cookies, headers, body)
+func DiscardResponse(response *http.Response) {
+	if response != nil {
+		_, err := io.Copy(io.Discard, response.Body)
+		if err != nil {
+			return
+		}
+
+		response.Body.Close()
+	}
 }

pkg/scraper/scraper.go

@@ -57,14 +57,8 @@ func (finder *Finder) Scrape(domain string) (results chan sources.Result) {
 							continue
 						}
 
-						if finder.MatchRegex != nil {
-							if !finder.MatchRegex.MatchString(sResult.Value) {
-								continue
-							}
-						} else if finder.FilterRegex != nil && finder.MatchRegex == nil {
-							if finder.FilterRegex.MatchString(sResult.Value) {
-								continue
-							}
+						if (finder.MatchRegex != nil && !finder.MatchRegex.MatchString(sResult.Value)) || (finder.FilterRegex != nil && finder.MatchRegex == nil && finder.FilterRegex.MatchString(sResult.Value)) {
+							continue
 						}
 					}
 

pkg/scraper/sources/bevigil/bevigil.go

@@ -3,10 +3,10 @@ package bevigil
 import (
 	"encoding/json"
 	"fmt"
+	"net/http"
 
 	"github.com/hueristiq/xurlfind3r/pkg/httpclient"
 	"github.com/hueristiq/xurlfind3r/pkg/scraper/sources"
-	"github.com/valyala/fasthttp"
 )
 
 type getURLsResponse struct {
@@ -47,7 +47,7 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s
 
 		getURLsReqURL := fmt.Sprintf("https://osint.bevigil.com/api/%s/urls/", domain)
 
-		var getURLsRes *fasthttp.Response
+		var getURLsRes *http.Response
 
 		getURLsRes, err = httpclient.Get(getURLsReqURL, "", getURLsReqHeaders)
 		if err != nil {
@@ -59,13 +59,14 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s
 
 			results <- result
 
+			httpclient.DiscardResponse(getURLsRes)
+
 			return
 		}
 
 		var getURLsResData getURLsResponse
 
-		err = json.Unmarshal(getURLsRes.Body(), &getURLsResData)
-		if err != nil {
+		if err = json.NewDecoder(getURLsRes.Body).Decode(&getURLsResData); err != nil {
 			result := sources.Result{
 				Type:   sources.Error,
 				Source: source.Name(),
@@ -74,9 +75,13 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s
 
 			results <- result
 
+			getURLsRes.Body.Close()
+
 			return
 		}
 
+		getURLsRes.Body.Close()
+
 		for _, URL := range getURLsResData.URLs {
 			if !sources.IsInScope(URL, domain, config.IncludeSubdomains) {
 				continue