refactor: set a log level based on verbose flag

lc · Jul 14, 2023 · 7097804 · 7097804
1 parent 04528bb
commit 7097804
Show file tree

Hide file tree

Showing 7 changed files with 19 additions and 26 deletions.
diff --git a/pkg/providers/commoncrawl/commoncrawl.go b/pkg/providers/commoncrawl/commoncrawl.go
@@ -60,9 +60,7 @@ func (c *Client) Fetch(ctx context.Context, domain string, results chan string)
 	}
 	// 0 pages means no results
 	if p.Pages == 0 {
-		if c.config.Verbose {
-			logrus.WithFields(logrus.Fields{"provider": Name}).Infof("no results for %s", domain)
-		}
+		logrus.WithFields(logrus.Fields{"provider": Name}).Infof("no results for %s", domain)
 		return nil
 	}
 
@@ -72,9 +70,7 @@ paginate:
 		case <-ctx.Done():
 			break paginate
 		default:
-			if c.config.Verbose {
-				logrus.WithFields(logrus.Fields{"provider": Name, "page": page}).Infof("fetching %s", domain)
-			}
+			logrus.WithFields(logrus.Fields{"provider": Name, "page": page}).Infof("fetching %s", domain)
 			apiURL := c.formatURL(domain, page)
 			resp, err := httpclient.MakeRequest(c.config.Client, apiURL, c.config.MaxRetries, c.config.Timeout)
 			if err != nil {

diff --git a/pkg/providers/otx/otx.go b/pkg/providers/otx/otx.go
@@ -52,9 +52,7 @@ paginate:
 		case <-ctx.Done():
 			break paginate
 		default:
-			if c.config.Verbose {
-				logrus.WithFields(logrus.Fields{"provider": Name, "page": page - 1}).Infof("fetching %s", domain)
-			}
+			logrus.WithFields(logrus.Fields{"provider": Name, "page": page - 1}).Infof("fetching %s", domain)
 			apiURL := c.formatURL(domain, page)
 			resp, err := httpclient.MakeRequest(c.config.Client, apiURL, c.config.MaxRetries, c.config.Timeout)
 			if err != nil {

diff --git a/pkg/providers/providers.go b/pkg/providers/providers.go
@@ -22,7 +22,6 @@ type URLScan struct {
 type Config struct {
 	Threads           uint
 	Timeout           uint
-	Verbose           bool
 	MaxRetries        uint
 	IncludeSubdomains bool
 	RemoveParameters  bool

diff --git a/pkg/providers/urlscan/urlscan.go b/pkg/providers/urlscan/urlscan.go
@@ -47,9 +47,7 @@ paginate:
 		case <-ctx.Done():
 			break paginate
 		default:
-			if c.config.Verbose {
-				logrus.WithFields(logrus.Fields{"provider": Name, "page": page}).Infof("fetching %s", domain)
-			}
+			logrus.WithFields(logrus.Fields{"provider": Name, "page": page}).Infof("fetching %s", domain)
 			apiURL := c.formatURL(domain, searchAfter)
 			resp, err := httpclient.MakeRequest(c.config.Client, apiURL, c.config.MaxRetries, c.config.Timeout, header)
 			if err != nil {
@@ -63,9 +61,7 @@ paginate:
 			}
 			// rate limited
 			if result.Status == 429 {
-				if c.config.Verbose {
-					logrus.WithField("provider", "urlscan").Warnf("urlscan responded with 429, probably being rate limited")
-				}
+				logrus.WithField("provider", "urlscan").Warnf("urlscan responded with 429, probably being rate limited")
 				break paginate
 			}
 

diff --git a/pkg/providers/wayback/wayback.go b/pkg/providers/wayback/wayback.go
@@ -43,14 +43,13 @@ func (c *Client) Fetch(ctx context.Context, domain string, results chan string)
 	if err != nil {
 		return fmt.Errorf("failed to fetch wayback pagination: %s", err)
 	}
+
 	for page := uint(0); page < pages; page++ {
 		select {
 		case <-ctx.Done():
 			return nil
 		default:
-			if c.config.Verbose {
-				logrus.WithFields(logrus.Fields{"provider": Name, "page": page}).Infof("fetching %s", domain)
-			}
+			logrus.WithFields(logrus.Fields{"provider": Name, "page": page}).Infof("fetching %s", domain)
 			apiURL := c.formatURL(domain, page)
 			// make HTTP request
 			resp, err := httpclient.MakeRequest(c.config.Client, apiURL, c.config.MaxRetries, c.config.Timeout)

diff --git a/runner/flags/flags.go b/runner/flags/flags.go
@@ -4,18 +4,20 @@ import (
 	"crypto/tls"
 	"flag"
 	"fmt"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
 	mapset "github.com/deckarep/golang-set/v2"
 	"github.com/lc/gau/v2/pkg/providers"
 	"github.com/lynxsecurity/pflag"
 	"github.com/lynxsecurity/viper"
+	"github.com/sirupsen/logrus"
 	log "github.com/sirupsen/logrus"
 	"github.com/valyala/fasthttp"
 	"github.com/valyala/fasthttp/fasthttpproxy"
-	"net/url"
-	"os"
-	"path/filepath"
-	"strings"
-	"time"
 )
 
 type URLScanConfig struct {
@@ -61,7 +63,6 @@ func (c *Config) ProviderConfig() (*providers.Config, error) {
 	pc := &providers.Config{
 		Threads:           c.Threads,
 		Timeout:           c.Timeout,
-		Verbose:           c.Verbose,
 		MaxRetries:        c.MaxRetries,
 		IncludeSubdomains: c.IncludeSubdomains,
 		RemoveParameters:  c.RemoveParameters,
@@ -81,6 +82,10 @@ func (c *Config) ProviderConfig() (*providers.Config, error) {
 		OTX: c.OTX,
 	}
 
+	logrus.SetLevel(log.ErrorLevel)
+	if c.Verbose {
+		logrus.SetLevel(log.InfoLevel)
+	}
 	pc.Blacklist = mapset.NewThreadUnsafeSet(c.Blacklist...)
 	pc.Blacklist.Add("")
 	return pc, nil

diff --git a/runner/runner.go b/runner/runner.go
@@ -79,7 +79,7 @@ work:
 					wg.Add(1)
 					go func(p providers.Provider) {
 						defer wg.Done()
-						if err := p.Fetch(ctx, domain, results); err != nil && r.config.Verbose {
+						if err := p.Fetch(ctx, domain, results); err != nil {
 							logrus.WithField("provider", p.Name()).Warnf("%s - %v", domain, err)
 						}
 					}(p)