From 02313f840076ef05c929f7779f92f4b54ea23e96 Mon Sep 17 00:00:00 2001
From: lc <corben@lynxsecurity.io>
Date: Wed, 9 Dec 2020 19:13:12 -0600
Subject: [PATCH] add blacklist flag

---
 .gitignore            |  1 +
 go.mod                |  5 ++++-
 go.sum                |  6 ++++++
 main.go               | 11 +++++++++--
 output/output.go      | 35 ++++++++++++++++++++++++++++++++---
 providers/provider.go |  1 +
 6 files changed, 53 insertions(+), 6 deletions(-)

diff --git a/.gitignore b/.gitignore
index 37e93dd..5a49f69 100644
--- a/.gitignore
+++ b/.gitignore
@@ -16,5 +16,6 @@
 # Dependency directories (remove the comment below to include it)
 # vendor/
 .DS_Store
+.idea
 dist
 gau
diff --git a/go.mod b/go.mod
index 0f88e43..f634e2b 100644
--- a/go.mod
+++ b/go.mod
@@ -2,4 +2,7 @@ module github.com/lc/gau
 
 go 1.14
 
-require github.com/json-iterator/go v1.1.10
+require (
+	github.com/bobesa/go-domain-util v0.0.0-20190911083921-4033b5f7dd89
+	github.com/json-iterator/go v1.1.10
+)
diff --git a/go.sum b/go.sum
index 9d969b9..891a408 100644
--- a/go.sum
+++ b/go.sum
@@ -1,3 +1,5 @@
+github.com/bobesa/go-domain-util v0.0.0-20190911083921-4033b5f7dd89 h1:2pkAuIM8OF1fy4ToFpMnI4oE+VeUNRbGrpSLKshK0oQ=
+github.com/bobesa/go-domain-util v0.0.0-20190911083921-4033b5f7dd89/go.mod h1:/09nEjna1UMoasyyQDhOrIn8hi2v2kiJglPWed1idck=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
@@ -10,3 +12,7 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+golang.org/x/net v0.0.0-20180811021610-c39426892332 h1:efGso+ep0DjyCBJPjvoz0HI6UldX4Md2F1rZFe1ir0E=
+golang.org/x/net v0.0.0-20180811021610-c39426892332/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
diff --git a/main.go b/main.go
index 38de92f..3277656 100644
--- a/main.go
+++ b/main.go
@@ -57,12 +57,12 @@ func run(config *providers.Config, domains []string) {
 	writewg.Add(1)
 	if config.JSON {
 		go func() {
-			output.WriteURLsJSON(results, out)
+			output.WriteURLsJSON(results, out, config.Blacklist)
 			writewg.Done()
 		}()
 	} else {
 		go func() {
-			output.WriteURLs(results, out)
+			output.WriteURLs(results, out, config.Blacklist)
 			writewg.Done()
 		}()
 	}
@@ -102,6 +102,7 @@ func main() {
 	output := flag.String("o", "", "filename to write results to")
 	jsonOut := flag.Bool("json", false, "write output as json")
 	randomAgent := flag.Bool("random-agent", false, "use random user-agent")
+	blacklist := flag.String("b","","extensions to skip, ex: ttf,woff,svg,png,jpg")
 	flag.Parse()
 
 	if *version {
@@ -132,6 +133,11 @@ func main() {
 		}
 	}
 
+	extensions := strings.Split(*blacklist,",")
+	extMap := make(map[string]struct{})
+	for _, ext := range extensions {
+		extMap[ext] = struct{}{}
+	}
 	config := providers.Config{
 		Verbose:           *verbose,
 		RandomAgent:       *randomAgent,
@@ -139,6 +145,7 @@ func main() {
 		IncludeSubdomains: *includeSubs,
 		Output:            *output,
 		JSON:              *jsonOut,
+		Blacklist: 			extMap,
 		Client: &http.Client{
 			Timeout:   time.Second * 15,
 			Transport: tr,
diff --git a/output/output.go b/output/output.go
index 2b2f38e..561255e 100644
--- a/output/output.go
+++ b/output/output.go
@@ -3,6 +3,8 @@ package output
 import (
 	"bufio"
 	"io"
+	"net/url"
+	"path"
 	"strings"
 
 	jsoniter "github.com/json-iterator/go"
@@ -11,11 +13,24 @@ import (
 type JSONResult struct {
 	Url string `json:"url"`
 }
-
-func WriteURLs(results <-chan string, writer io.Writer) error {
+func WriteURLs(results <-chan string, writer io.Writer, blacklistMap map[string]struct{}) error {
 	wr := bufio.NewWriter(writer)
 	str := &strings.Builder{}
 	for result := range results {
+		if len(blacklistMap) != 0 {
+			u, err := url.Parse(result)
+			if err != nil {
+				continue
+			}
+			base := strings.Split(path.Base(u.Path),".")
+			ext := base[len(base)-1]
+			if ext != "" {
+				_, ok := blacklistMap[ext]
+				if ok {
+					continue
+				}
+			}
+		}
 		str.WriteString(result)
 		str.WriteRune('\n')
 		_, err := wr.WriteString(str.String())
@@ -27,10 +42,24 @@ func WriteURLs(results <-chan string, writer io.Writer) error {
 	}
 	return wr.Flush()
 }
-func WriteURLsJSON(results <-chan string, writer io.Writer) {
+func WriteURLsJSON(results <-chan string, writer io.Writer, blacklistMap map[string]struct{}) {
 	var jr JSONResult
 	enc := jsoniter.NewEncoder(writer)
 	for result := range results {
+		if len(blacklistMap) != 0 {
+			u, err := url.Parse(result)
+			if err != nil {
+				continue
+			}
+			base := strings.Split(path.Base(u.Path),".")
+			ext := base[len(base)-1]
+			if ext != "" {
+				_, ok := blacklistMap[ext]
+				if ok {
+					continue
+				}
+			}
+		}
 		jr.Url = result
 		enc.Encode(jr)
 	}
diff --git a/providers/provider.go b/providers/provider.go
index 2ac0f76..6705581 100644
--- a/providers/provider.go
+++ b/providers/provider.go
@@ -24,6 +24,7 @@ type Config struct {
 	IncludeSubdomains bool
 	Client            *http.Client
 	Providers         []string
+	Blacklist 		  map[string]struct{}
 	Output            string
 	JSON              bool
 }