chore: Remove default filtering

A default filtering had been added during the introduction of the
filtering feature (commit: 3adfac7).
The default filtering is removed in this commit.

README.md

@@ -52,10 +52,8 @@ sigurlfind3r -d tesla.com -is
 
 #### Regex filter out URLs
 
-> sigurlfind3r filters out `.(jpg|jpeg|gif|png|ico|css|eot|tif|tiff|ttf|woff|woff2)` by default
-
 ```bash
-sigurlfind3r -d tesla.com -f ".(png|ico|jpg|gif|css|woff|svg|ttf|eot|pdf)"
+sigurlfind3r -d tesla.com -f ".(jpg|jpeg|gif|png|ico|css|eot|tif|tiff|ttf|woff|woff2)"
 ```
 
 ## Installation
@@ -75,10 +73,10 @@ GO111MODULE=on go get -u -v github.com/signedsecurity/sigurlfind3r/cmd/sigurlfin
 #### From Github
 
 ```bash
-▶ git clone https://github.com/signedsecurity/sigurlfind3r.git
-▶ cd sigurlfind3r/cmd/sigurlfind3r/
-▶ go build; mv sigurlfind3r /usr/local/bin/
-▶ sigurlfind3r -h
+git clone https://github.com/signedsecurity/sigurlfind3r.git && \
+cd sigurlfind3r/cmd/sigurlfind3r/ && \
+go build; mv sigurlfind3r /usr/local/bin/ && \
+sigurlfind3r -h
 ```
 
 ## Post Installation

cmd/sigurlfind3r/main.go

@@ -35,19 +35,19 @@ func banner() {
 func init() {
 	flag.StringVar(&o.Domain, "d", "", "")
 	flag.StringVar(&o.Domain, "domain", "", "")
-	flag.StringVar(&o.SourcesToExclude, "es", "", "")
+	flag.StringVar(&o.SourcesToExclude, "eS", "", "")
 	flag.StringVar(&o.SourcesToExclude, "exclude-sources", "", "")
-	flag.StringVar(&o.FilterRegex, "f", ".(jpg|jpeg|gif|png|ico|css|eot|tif|tiff|ttf|woff|woff2)", "")
-	flag.StringVar(&o.FilterRegex, "filter", ".(jpg|jpeg|gif|png|ico|css|eot|tif|tiff|ttf|woff|woff2)", "")
-	flag.BoolVar(&o.IncludeSubdomains, "is", false, "")
+	flag.StringVar(&o.FilterRegex, "f", "", "")
+	flag.StringVar(&o.FilterRegex, "filter", "", "")
+	flag.BoolVar(&o.IncludeSubdomains, "iS", false, "")
 	flag.BoolVar(&o.IncludeSubdomains, "include-subs", false, "")
-	flag.BoolVar(&o.ListSources, "ls", false, "")
+	flag.BoolVar(&o.ListSources, "lS", false, "")
 	flag.BoolVar(&o.ListSources, "list-sources", false, "")
-	flag.BoolVar(&o.NoColor, "ns", false, "")
+	flag.BoolVar(&o.NoColor, "nC", false, "")
 	flag.BoolVar(&o.NoColor, "no-color", false, "")
 	flag.BoolVar(&o.Silent, "s", false, "")
 	flag.BoolVar(&o.Silent, "silent", false, "")
-	flag.StringVar(&o.SourcesToUse, "us", "", "")
+	flag.StringVar(&o.SourcesToUse, "uS", "", "")
 	flag.StringVar(&o.SourcesToUse, "use-sources", "", "")
 
 	flag.Usage = func() {
@@ -57,14 +57,14 @@ func init() {
 		h += "  sigurlfind3r [OPTIONS]\n"
 
 		h += "\nOPTIONS:\n"
-		h += "  -d,  --domain            domain to fetch urls for\n"
-		h += "  -es, --exclude-sources   comma(,) separated list of sources to exclude\n"
-		h += "  -f,  --filter            URL filtering regex\n"
-		h += "  -is, --include-subs      include subdomains' urls\n"
-		h += "  -ls, --list-sources      list all the available sources\n"
-		h += "  -nc, --no-color          no color mode\n"
-		h += "  -s,  --silent            silent mode: output urls only\n"
-		h += "  -us, --use-sources       comma(,) separated list of sources to use\n"
+		h += "   -d, --domain            domain to fetch urls for\n"
+		h += "  -eS, --exclude-sources   comma(,) separated list of sources to exclude\n"
+		h += "   -f, --filter            URL filtering regex\n"
+		h += "  -iS, --include-subs      include subdomains' urls\n"
+		h += "  -lS, --list-sources      list all the available sources\n"
+		h += "  -nC, --no-color          no color mode\n"
+		h += "   -s  --silent            silent mode: output urls only\n"
+		h += "  -uS, --use-sources       comma(,) separated list of sources to use\n"
 
 		fmt.Println(h)
 	}

go.mod

@@ -7,6 +7,7 @@ require (
 	github.com/kr/text v0.2.0 // indirect
 	github.com/logrusorgru/aurora/v3 v3.0.0
 	github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80
+	golang.org/x/net v0.0.0-20210614182718-04defd469f4e // indirect
 	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
 )

go.sum

@@ -13,6 +13,8 @@ github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 h1:nrZ3ySNYwJ
 github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80/go.mod h1:iFyPdL66DjUD96XmzVL3ZntbzcflLnznH0fr99w5VqE=
 golang.org/x/net v0.0.0-20210510120150-4163338589ed h1:p9UgmWI9wKpfYmgaV/IZKGdXc5qEK45tDwwwDyjS26I=
 golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20210614182718-04defd469f4e h1:XpT3nA5TvE525Ne3hInMh6+GETgn27Zfm9dxsThnX2Q=
+golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=

pkg/sigurlfind3r/scraping/utils.go

@@ -18,8 +18,10 @@ func NormalizeURL(URL string, scope session.Scope) (string, bool) {
 		return URL, false
 	}
 
-	if scope.FilterRegex.MatchString(parsedURL.Path) {
-		return URL, false
+	if scope.FilterRegex != nil {
+		if scope.FilterRegex.MatchString(parsedURL.Path) {
+			return URL, false
+		}
 	}
 
 	if parsedURL.ETLDPlus1 == "" || parsedURL.ETLDPlus1 != scope.Domain {

pkg/sigurlfind3r/sigurlfind3r.go

@@ -43,11 +43,15 @@ func New(options ...*Options) (runner *Runner) {
 	}
 
 	runner = &Runner{
-		FilterRegex: regexp.MustCompile(opts.FilterRegex),
-		Passive:     passive.New(opts.SourcesToUse, opts.SourcesToExclude),
-		Options:     opts,
+		Options: opts,
 	}
 
+	if opts.FilterRegex != "" {
+		runner.FilterRegex = regexp.MustCompile(opts.FilterRegex)
+	}
+
+	runner.Passive = passive.New(opts.SourcesToUse, opts.SourcesToExclude)
+
 	return
 }