Merge pull request #5 from signedsecurity/save-output-to-file

Save output to file
hueristiq · Aug 11, 2021 · d58ccfd · d58ccfd
2 parents 4bc7931 + 5a5e33e
commit d58ccfd
Show file tree

Hide file tree

Showing 3 changed files with 145 additions and 81 deletions.
diff --git a/README.md b/README.md
@@ -4,10 +4,74 @@
 
 sigurlfind3r is a passive reconnaissance tool, it fetches known URLs from **[AlienVault's OTX](https://otx.alienvault.com/)**, **[Common Crawl](https://commoncrawl.org/)**, **[URLScan](https://urlscan.io/)**, **[Github](https://github.com)** and the **[Wayback Machine](https://archive.org/web/)**.
 
-**DiSCLAIMER:** fetching urls from github is a bit slow.
+## Resource
+
+* [Features](#features)
+* [Installation](#installation)
+	* [From Binary](#from-binary)
+	* [From source](#from-source)
+	* [From github](#from-github)
+* [Post Installation](#post-installation)
+* [Usage](#usage)
+	* [Examples](#examples)
+		* [Basic](#basic)
+		* [Include Subdomains' URLs](#include-subdomains-urls)
+		* [Regex filter URLs](#regex-filter-urls)
+* [Contribution](#contribution)
+
+## Features
+
+* Fetches known URLs from AlienVault's OTX, Common Crawl, URLScan, Github & the Wayback Machine.
+* Save output to file.
+* Regex filter URLs.
+
+## Installation
+
+### From Binary
+
+You can download the pre-built binary for your platform from this repository's [releases](https://github.com/signedsecurity/sigurlfind3r/releases/) page, extract, then move it to your `$PATH`and you're ready to go.
+
+### From Source
+
+sigurlfind3r requires **go1.14+** to install successfully. Run the following command to get the repo
+
+```bash
+GO111MODULE=on go get -u -v github.com/signedsecurity/sigurlfind3r/cmd/sigurlfind3r
+```
+
+### From Github
+
+```bash
+git clone https://github.com/signedsecurity/sigurlfind3r.git && \
+cd sigurlfind3r/cmd/sigurlfind3r/ && \
+go build; mv sigurlfind3r /usr/local/bin/ && \
+sigurlfind3r -h
+```
+
+## Post Installation
+
+sigurlfind3r will work after [installation](#installation). However, to configure sigurlfind3r to work with certain services - currently github - you will need to have setup API keys. The API keys are stored in the `$HOME/.config/sigurlfind3r/conf.yaml` file - created upon first run - and uses the YAML format. Multiple API keys can be specified for each of these services.
+
+Example:
+
+```yaml
+version: 1.1.0
+sources:
+    - commoncrawl
+    - github
+    - otx
+    - urlscan
+    - wayback
+keys:
+    github:
+        - d23a554bbc1aabb208c9acfbd2dd41ce7fc9db39
+        - asdsd54bbc1aabb208c9acfbd2dd41ce7fc9db39
+```
 
 ## Usage
 
+**DiSCLAIMER:** fetching urls from github is a bit slow.
+
 ```bash
 sigurlfind3r -h
 ```
@@ -19,7 +83,7 @@ This will display help for the tool.
  ___(_) __ _ _   _ _ __| |/ _(_)_ __   __| |___ / _ __
 / __| |/ _` | | | | '__| | |_| | '_ \ / _` | |_ \| '__|
 \__ \ | (_| | |_| | |  | |  _| | | | | (_| |___) | |
-|___/_|\__, |\__,_|_|  |_|_| |_|_| |_|\__,_|____/|_| v1.2.0
+|___/_|\__, |\__,_|_|  |_|_| |_|_| |_|\__,_|____/|_| 1.2.0
        |___/
 
 USAGE:
@@ -34,6 +98,7 @@ OPTIONS:
   -nC, --no-color          no color mode
    -s  --silent            silent mode: output urls only
   -uS, --use-sources       comma(,) separated list of sources to use
+   -o, --output            output file
 ```
 
 ### Examples
@@ -44,61 +109,18 @@ OPTIONS:
 sigurlfind3r -d tesla.com
 ```
 
-#### Include Subdomains
+#### Include Subdomains' URLs
 
 ```bash
-sigurlfind3r -d tesla.com -is
+sigurlfind3r -d tesla.com -iS
 ```
 
-#### Regex filter out URLs
+#### Regex filter URLs
 
 ```bash
 sigurlfind3r -d tesla.com -f ".(jpg|jpeg|gif|png|ico|css|eot|tif|tiff|ttf|woff|woff2)"
 ```
 
-## Installation
-
-#### From Binary
-
-You can download the pre-built binary for your platform from this repository's [releases](https://github.com/signedsecurity/sigurlfind3r/releases/) page, extract, then move it to your `$PATH`and you're ready to go.
-
-#### From Source
-
-sigurlfind3r requires **go1.14+** to install successfully. Run the following command to get the repo
-
-```bash
-GO111MODULE=on go get -u -v github.com/signedsecurity/sigurlfind3r/cmd/sigurlfind3r
-```
-
-#### From Github
-
-```bash
-git clone https://github.com/signedsecurity/sigurlfind3r.git && \
-cd sigurlfind3r/cmd/sigurlfind3r/ && \
-go build; mv sigurlfind3r /usr/local/bin/ && \
-sigurlfind3r -h
-```
-
-## Post Installation
-
-sigurlfind3r will work after [installation](#installation). However, to configure sigurlfind3r to work with certain services - currently github - you will need to have setup API keys. The API keys are stored in the `$HOME/.config/sigurlfind3r/conf.yaml` file - created upon first run - and uses the YAML format. Multiple API keys can be specified for each of these services.
-
-Example:
-
-```yaml
-version: 1.1.0
-sources:
-    - commoncrawl
-    - github
-    - otx
-    - urlscan
-    - wayback
-keys:
-    github:
-        - d23a554bbc1aabb208c9acfbd2dd41ce7fc9db39
-        - asdsd54bbc1aabb208c9acfbd2dd41ce7fc9db39
-```
-
 ## Contribution
 
 [Issues](https://github.com/signedsecurity/sigurlfind3r/issues) and [Pull Requests](https://github.com/signedsecurity/sigurlfind3r/pulls) are welcome!
diff --git a/cmd/sigurlfind3r/main.go b/cmd/sigurlfind3r/main.go
@@ -1,11 +1,13 @@
 package main
 
 import (
+	"bufio"
 	"context"
 	"flag"
 	"fmt"
 	"log"
 	"os"
+	"path/filepath"
 	"reflect"
 	"strconv"
 	"strings"
@@ -17,19 +19,15 @@ import (
 )
 
 var (
-	au aurora.Aurora
-	o  configuration.CLIOptions
+	au      aurora.Aurora
+	o       configuration.CLIOptions
+	output  string
+	silent  bool
+	noColor bool
 )
 
 func banner() {
-	fmt.Fprintln(os.Stderr, aurora.BrightBlue(`
-     _                  _  __ _           _ _____
- ___(_) __ _ _   _ _ __| |/ _(_)_ __   __| |___ / _ __
-/ __| |/ _`+"`"+` | | | | '__| | |_| | '_ \ / _`+"`"+` | |_ \| '__|
-\__ \ | (_| | |_| | |  | |  _| | | | | (_| |___) | |
-|___/_|\__, |\__,_|_|  |_|_| |_|_| |_|\__,_|____/|_| v1.2.0
-       |___/
-`).Bold())
+	fmt.Fprintln(os.Stderr, configuration.BANNER)
 }
 
 func init() {
@@ -43,12 +41,14 @@ func init() {
 	flag.BoolVar(&o.IncludeSubdomains, "include-subs", false, "")
 	flag.BoolVar(&o.ListSources, "lS", false, "")
 	flag.BoolVar(&o.ListSources, "list-sources", false, "")
-	flag.BoolVar(&o.NoColor, "nC", false, "")
-	flag.BoolVar(&o.NoColor, "no-color", false, "")
-	flag.BoolVar(&o.Silent, "s", false, "")
-	flag.BoolVar(&o.Silent, "silent", false, "")
+	flag.BoolVar(&noColor, "nC", false, "")
+	flag.BoolVar(&noColor, "no-color", false, "")
+	flag.BoolVar(&silent, "s", false, "")
+	flag.BoolVar(&silent, "silent", false, "")
 	flag.StringVar(&o.SourcesToUse, "uS", "", "")
 	flag.StringVar(&o.SourcesToUse, "use-sources", "", "")
+	flag.StringVar(&output, "o", "", "")
+	flag.StringVar(&output, "output", "", "")
 
 	flag.Usage = func() {
 		banner()
@@ -65,13 +65,14 @@ func init() {
 		h += "  -nC, --no-color          no color mode\n"
 		h += "   -s  --silent            silent mode: output urls only\n"
 		h += "  -uS, --use-sources       comma(,) separated list of sources to use\n"
+		h += "   -o, --output            output file\n"
 
 		fmt.Println(h)
 	}
 
 	flag.Parse()
 
-	au = aurora.NewAurora(!o.NoColor)
+	au = aurora.NewAurora(!noColor)
 }
 
 func main() {
@@ -80,7 +81,7 @@ func main() {
 		log.Fatalln(err)
 	}
 
-	if !o.Silent {
+	if !silent {
 		banner()
 	}
 
@@ -109,7 +110,7 @@ func main() {
 		os.Exit(0)
 	}
 
-	if !o.Silent {
+	if !silent {
 		fmt.Println("[", au.BrightBlue("INF"), "] fetching urls for", au.Underline(options.Domain).Bold())
 
 		if options.IncludeSubdomains {
@@ -134,11 +135,44 @@ func main() {
 		log.Fatalln(err)
 	}
 
-	for URL := range URLs {
-		if o.Silent {
-			fmt.Println(URL.Value)
-		} else {
-			fmt.Printf("[%s] %s\n", au.BrightBlue(URL.Source), URL.Value)
+	if output != "" {
+		directory := filepath.Dir(output)
+
+		if _, err := os.Stat(directory); os.IsNotExist(err) {
+			if err = os.MkdirAll(directory, os.ModePerm); err != nil {
+				log.Fatalln(err)
+			}
+		}
+
+		file, err := os.OpenFile(output, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+		if err != nil {
+			log.Fatalln(err)
+		}
+
+		defer file.Close()
+
+		writer := bufio.NewWriter(file)
+
+		for URL := range URLs {
+			if silent {
+				fmt.Println(URL.Value)
+			} else {
+				fmt.Printf("[%s] %s\n", au.BrightBlue(URL.Source), URL.Value)
+			}
+
+			fmt.Fprintln(writer, URL.Value)
+		}
+
+		if err = writer.Flush(); err != nil {
+			log.Fatalln(err)
+		}
+	} else {
+		for URL := range URLs {
+			if silent {
+				fmt.Println(URL.Value)
+			} else {
+				fmt.Printf("[%s] %s\n", au.BrightBlue(URL.Source), URL.Value)
+			}
 		}
 	}
 }
diff --git a/internal/configuration/configuration.go b/internal/configuration/configuration.go
@@ -1,6 +1,7 @@
 package configuration
 
 import (
+	"fmt"
 	"os"
 	"path"
 	"strings"
@@ -23,8 +24,6 @@ type CLIOptions struct {
 	FilterRegex       string
 	IncludeSubdomains bool
 	ListSources       bool
-	NoColor           bool
-	Silent            bool
 	SourcesToExclude  string
 	SourcesToUse      string
 }
@@ -34,17 +33,28 @@ type Options struct {
 	FilterRegex       string
 	IncludeSubdomains bool
 	ListSources       bool
-	NoColor           bool
-	Silent            bool
 	SourcesToExclude  []string
 	SourcesToUse      []string
 	YAML              YAMLConfiguration
 }
 
+const (
+	VERSION = "1.2.0"
+)
+
+var (
+	BANNER string = fmt.Sprintf(`
+     _                  _  __ _           _ _____
+ ___(_) __ _ _   _ _ __| |/ _(_)_ __   __| |___ / _ __
+/ __| |/ _`+"`"+` | | | | '__| | |_| | '_ \ / _`+"`"+` | |_ \| '__|
+\__ \ | (_| | |_| | |  | |  _| | | | | (_| |___) | |
+|___/_|\__, |\__,_|_|  |_|_| |_|_| |_|\__,_|____/|_| %s
+       |___/
+`, VERSION)
+)
+
 // ParseCLIOptions parse the command line flags and read config file
 func ParseCLIOptions(options *CLIOptions) (parsedOptions *Options, err error) {
-	version := "1.2.0"
-
 	directory, err := os.UserHomeDir()
 	if err != nil {
 		return
@@ -57,8 +67,6 @@ func ParseCLIOptions(options *CLIOptions) (parsedOptions *Options, err error) {
 		FilterRegex:       options.FilterRegex,
 		IncludeSubdomains: options.IncludeSubdomains,
 		ListSources:       options.ListSources,
-		NoColor:           options.NoColor,
-		Silent:            options.Silent,
 	}
 
 	if options.SourcesToUse != "" {
@@ -73,7 +81,7 @@ func ParseCLIOptions(options *CLIOptions) (parsedOptions *Options, err error) {
 
 	if _, err = os.Stat(configPath); os.IsNotExist(err) {
 		configuration := YAMLConfiguration{
-			Version: version,
+			Version: VERSION,
 			Sources: scraping.SourcesList,
 		}
 
@@ -94,9 +102,9 @@ func ParseCLIOptions(options *CLIOptions) (parsedOptions *Options, err error) {
 			return nil, err
 		}
 
-		if configuration.Version != version {
+		if configuration.Version != VERSION {
 			configuration.Sources = scraping.SourcesList
-			configuration.Version = version
+			configuration.Version = VERSION
 
 			if err = configuration.MarshalWrite(configPath); err != nil {
 				return nil, err