Skip to content

Commit

Permalink
refactor: Reorganize, rename and revise components
Browse files Browse the repository at this point in the history 
Reorganize and rename core packages, revise the components as well
in terms of interconnection and also minor enhancements
  • Loading branch information
@enenumxela
enenumxela committed Feb 2, 2023
1 parent f308acc commit 3b9e348
Show file tree
Hide file tree
Showing 23 changed files with 705 additions and 704 deletions.
99 changes: 61 additions & 38 deletions cmd/hqurlfind3r/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ package main

import (
"bufio"
"context"
"regexp"

"fmt"
"log"
Expand All @@ -13,35 +13,44 @@ import (
"strings"

"github.com/hueristiq/hqurlfind3r/internal/configuration"
"github.com/hueristiq/hqurlfind3r/pkg/hqurlfind3r"
"github.com/hueristiq/hqurlfind3r/pkg/runner"
"github.com/hueristiq/hqurlfind3r/pkg/runner/collector"
"github.com/hueristiq/hqurlfind3r/pkg/runner/collector/filter"
"github.com/hueristiq/hqurlfind3r/pkg/runner/collector/sources"
"github.com/logrusorgru/aurora/v3"
flag "github.com/spf13/pflag"
"github.com/spf13/pflag"
)

var (
au aurora.Aurora
o configuration.CLIOptions
output string
monochrome, silent bool
listSources bool

domain string
sourcesToUse, sourcesToExclude []string
includeSubdomains bool
filterRegex string
monochrome, silent bool
output string

au aurora.Aurora
)

func printBanner() {
fmt.Fprintln(os.Stderr, configuration.BANNER)
}

func init() {
flag.StringVarP(&o.Domain, "domain", "d", "", "target domain")
flag.BoolVar(&o.IncludeSubdomains, "include-subdomains", false, "include subdomains")
flag.StringVarP(&o.FilterRegex, "filter", "f", "", "URL filtering regex")
flag.StringSliceVar(&o.SourcesToUse, "use-sources", []string{}, "comma(,) separated sources to use")
flag.StringSliceVar(&o.SourcesToExclude, "exclude-sources", []string{}, "comma(,) separated sources to exclude")
flag.BoolVar(&o.ListSources, "list-sources", false, "list all the available sources")
flag.BoolVarP(&monochrome, "monochrome", "m", false, "no colored output mode")
flag.BoolVarP(&silent, "silent", "s", false, "silent output mode")
flag.StringVarP(&output, "output", "o", "", "output file")

flag.CommandLine.SortFlags = false
flag.Usage = func() {
pflag.StringVarP(&domain, "domain", "d", "", "target domain")
pflag.BoolVar(&includeSubdomains, "include-subdomains", false, "include subdomains")
pflag.StringVarP(&filterRegex, "filter", "f", "", "URL filtering regex")
pflag.StringSliceVar(&sourcesToUse, "use-sources", []string{}, "comma(,) separated sources to use")
pflag.StringSliceVar(&sourcesToExclude, "exclude-sources", []string{}, "comma(,) separated sources to exclude")
pflag.BoolVar(&listSources, "list-sources", false, "list all the available sources")
pflag.BoolVarP(&monochrome, "monochrome", "m", false, "no colored output mode")
pflag.BoolVarP(&silent, "silent", "s", false, "silent output mode")
pflag.StringVarP(&output, "output", "o", "", "output file")

pflag.CommandLine.SortFlags = false
pflag.Usage = func() {
printBanner()

h := "USAGE:\n"
Expand All @@ -61,35 +70,44 @@ func init() {
fmt.Fprintln(os.Stderr, h)
}

flag.Parse()
pflag.Parse()

au = aurora.NewAurora(!monochrome)
}

func main() {
options, err := configuration.ParseCLIOptions(&o)
if err != nil {
log.Fatalln(err)
}
var (
keys sources.Keys
regex *regexp.Regexp
ftr filter.Filter
clr *collector.Collector
rnr *runner.Runner
)

if !silent {
printBanner()
}

if o.ListSources {
fmt.Println("[", au.BrightBlue("INF"), "] current list of the available", au.Underline(strconv.Itoa(len(options.YAML.Sources))+" sources").Bold())
config, err := configuration.Read()
if err != nil {
log.Fatalln(err)
}

keys = config.GetKeys()

if listSources {
fmt.Println("[", au.BrightBlue("INF"), "] current list of the available", au.Underline(strconv.Itoa(len(config.Sources))+" sources").Bold())
fmt.Println("[", au.BrightBlue("INF"), "] sources marked with an * needs key or token")
fmt.Println("")

keys := options.YAML.GetKeys()
needsKey := make(map[string]interface{})
keysElem := reflect.ValueOf(&keys).Elem()

for i := 0; i < keysElem.NumField(); i++ {
needsKey[strings.ToLower(keysElem.Type().Field(i).Name)] = keysElem.Field(i).Interface()
}

for _, source := range options.YAML.Sources {
for _, source := range config.Sources {
if _, ok := needsKey[source]; ok {
fmt.Println(">", source, "*")
} else {
Expand All @@ -102,24 +120,29 @@ func main() {
}

if !silent {
fmt.Println("[", au.BrightBlue("INF"), "] fetching urls for", au.Underline(options.Domain).Bold())
fmt.Println("[", au.BrightBlue("INF"), "] fetching urls for", au.Underline(domain).Bold())

if options.IncludeSubdomains {
if includeSubdomains {
fmt.Println("[", au.BrightBlue("INF"), "] `--include-subdomains` used: includes subdomains' urls")
}

fmt.Println("")
}

runner := hqurlfind3r.New(&hqurlfind3r.Options{
FilterRegex: options.FilterRegex,
SourcesToUse: options.SourcesToUse,
SourcesToExclude: options.SourcesToExclude,
IncludeSubdomains: options.IncludeSubdomains,
Keys: options.YAML.GetKeys(),
})
if filterRegex != "" {
regex = regexp.MustCompile(filterRegex)
}

ftr = filter.Filter{
Domain: domain,
IncludeSubdomains: includeSubdomains,
ExcludeRegex: regex,
}

clr = collector.New(sourcesToUse, sourcesToExclude, keys, ftr)
rnr = runner.New(clr)

URLs, err := runner.Run(context.Background(), options.Domain)
URLs, err := rnr.Run()
if err != nil {
log.Fatalln(err)
}
Expand Down
144 changes: 28 additions & 116 deletions internal/configuration/configuration.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,17 +2,17 @@ package configuration

import (
"fmt"
"log"
"math/rand"
"os"
"path"
"path/filepath"
"strings"

"github.com/hueristiq/hqurlfind3r/pkg/hqurlfind3r/scraping"
"github.com/hueristiq/hqurlfind3r/pkg/hqurlfind3r/session"
"github.com/hueristiq/hqurlfind3r/pkg/runner/collector/sources"
"gopkg.in/yaml.v3"
)

type YAMLConfiguration struct {
type Configuration struct {
Version string `yaml:"version"`
Sources []string `yaml:"sources"`
Keys struct {
Expand All @@ -21,25 +21,6 @@ type YAMLConfiguration struct {
}
}

type CLIOptions struct {
Domain string
FilterRegex string
IncludeSubdomains bool
ListSources bool
SourcesToExclude []string
SourcesToUse []string
}

type Options struct {
Domain string
FilterRegex string
IncludeSubdomains bool
ListSources bool
SourcesToExclude []string
SourcesToUse []string
YAML YAMLConfiguration
}

const (
VERSION = "1.9.0"
)
Expand All @@ -53,123 +34,54 @@ var (
|_| |_|\__, |\__,_|_| |_|_| |_|_| |_|\__,_|____/|_| v%s
|_|
`, VERSION)
)

// ParseCLIOptions parse the command line flags and read config file
func ParseCLIOptions(options *CLIOptions) (parsedOptions *Options, err error) {
directory, err := os.UserHomeDir()
if err != nil {
return
}

configPath := directory + "/.config/hqurlfind3r/conf.yaml"

parsedOptions = &Options{
Domain: options.Domain,
FilterRegex: options.FilterRegex,
IncludeSubdomains: options.IncludeSubdomains,
ListSources: options.ListSources,
}

if len(options.SourcesToUse) > 0 {
parsedOptions.SourcesToUse = append(parsedOptions.SourcesToUse, options.SourcesToUse...)
} else {
parsedOptions.SourcesToUse = append(parsedOptions.SourcesToUse, scraping.SourcesList...)
}

if len(options.SourcesToExclude) > 0 {
parsedOptions.SourcesToExclude = append(parsedOptions.SourcesToExclude, options.SourcesToExclude...)
}

if _, err = os.Stat(configPath); os.IsNotExist(err) {
configuration := YAMLConfiguration{
Version: VERSION,
Sources: scraping.SourcesList,
}

directory, _ := path.Split(configPath)

if err = makeDirectory(directory); err != nil {
return
}

if err = configuration.MarshalWrite(configPath); err != nil {
return
}

parsedOptions.YAML = configuration
} else {
configuration, err := UnmarshalRead(configPath)
ConfigurationFilePath = func() string {
home, err := os.UserHomeDir()
if err != nil {
return nil, err
log.Fatalln(err)
}

if configuration.Version != VERSION {
configuration.Sources = scraping.SourcesList
configuration.Version = VERSION

if err = configuration.MarshalWrite(configPath); err != nil {
return nil, err
}
}

parsedOptions.YAML = configuration
}

return
}

func makeDirectory(directory string) error {
if _, err := os.Stat(directory); os.IsNotExist(err) {
if directory != "" {
err = os.MkdirAll(directory, os.ModePerm)
if err != nil {
return err
}
}
}

return nil
}
return filepath.Join(home, "/.config/hqurlfind3r/conf.yaml")
}()
)

func (config *YAMLConfiguration) MarshalWrite(file string) error {
f, err := os.OpenFile(file, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0755)
func (configuration *Configuration) Write() error {
file, err := os.OpenFile(ConfigurationFilePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0755)
if err != nil {
return err
}

enc := yaml.NewEncoder(f)
enc := yaml.NewEncoder(file)
enc.SetIndent(4)
err = enc.Encode(&config)
f.Close()
err = enc.Encode(&configuration)
file.Close()
return err
}

func UnmarshalRead(file string) (YAMLConfiguration, error) {
config := YAMLConfiguration{}
func Read() (Configuration, error) {
configuration := Configuration{}

f, err := os.Open(file)
file, err := os.Open(ConfigurationFilePath)
if err != nil {
return config, err
return configuration, err
}

err = yaml.NewDecoder(f).Decode(&config)
err = yaml.NewDecoder(file).Decode(&configuration)

f.Close()
file.Close()

return config, err
return configuration, err
}

func (config *YAMLConfiguration) GetKeys() session.Keys {
keys := session.Keys{}
func (configuration *Configuration) GetKeys() sources.Keys {
keys := sources.Keys{}

if len(config.Keys.GitHub) > 0 {
keys.GitHub = config.Keys.GitHub
if len(configuration.Keys.GitHub) > 0 {
keys.GitHub = configuration.Keys.GitHub
}

intelxKeysCount := len(config.Keys.Intelx)
intelxKeysCount := len(configuration.Keys.Intelx)
if intelxKeysCount > 0 {
intelxKeys := config.Keys.Intelx[rand.Intn(intelxKeysCount)]
intelxKeys := configuration.Keys.Intelx[rand.Intn(intelxKeysCount)]
parts := strings.Split(intelxKeys, ":")
if len(parts) == 2 {
keys.IntelXHost = parts[0]
Expand Down
Loading

0 comments on commit 3b9e348

Please sign in to comment.