Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Cleanup migration code #941

Merged
merged 4 commits into from
Aug 18, 2023
Merged

Cleanup migration code #941

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
048c92f
cleanup migration code
dogancanbakir Aug 16, 2023
3c907cf
create if not exists
dogancanbakir Aug 17, 2023
f565351
fixing syntax error
Mzack9999 Aug 18, 2023
2916dff
fixing variable name
Mzack9999 Aug 18, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,4 +7,5 @@ v2/subfinder
vendor/
.idea
.devcontainer
.vscode
dist
12 changes: 10 additions & 2 deletions v2/pkg/runner/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,14 +30,22 @@ func GetConfigDirectory() (string, error) {
return config, nil
}

// CreateProviderConfigYAML marshals the input map to the given location on the disk
func CreateProviderConfigYAML(configFilePath string, sourcesRequiringApiKeysMap map[string][]string) error {
// createProviderConfigYAML marshals the input map to the given location on the disk
func createProviderConfigYAML(configFilePath string) error {
configFile, err := os.Create(configFilePath)
if err != nil {
return err
}
defer configFile.Close()

sourcesRequiringApiKeysMap := make(map[string][]string)
for _, source := range passive.AllSources {
if source.NeedsKey() {
sourceName := strings.ToLower(source.Name())
sourcesRequiringApiKeysMap[sourceName] = []string{}
}
}

return yaml.NewEncoder(configFile).Encode(sourcesRequiringApiKeysMap)
}

Expand Down
63 changes: 6 additions & 57 deletions v2/pkg/runner/options.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,6 @@ import (
"regexp"
"strings"

"gopkg.in/yaml.v3"

"github.com/projectdiscovery/goflags"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/subfinder/v2/pkg/passive"
Expand Down Expand Up @@ -76,18 +74,6 @@ type OnResultCallback func(result *resolve.HostEntry)
func ParseOptions() *Options {
logutil.DisableDefaultLogger()

// Migrate config to provider config
if fileutil.FileExists(defaultConfigLocation) && !fileutil.FileExists(defaultProviderConfigLocation) {
gologger.Info().Msgf("Detected old %s config file, trying to migrate providers to %s\n", defaultConfigLocation, defaultProviderConfigLocation)
if err := migrateToProviderConfig(defaultConfigLocation, defaultProviderConfigLocation); err != nil {
gologger.Warning().Msgf("Could not migrate providers from existing config %s to provider config %s: %s\n", defaultConfigLocation, defaultProviderConfigLocation, err)
} else {
// cleanup the existing config file post migration
_ = os.Remove(defaultConfigLocation)
gologger.Info().Msgf("Migration successful from %s to %s.\n", defaultConfigLocation, defaultProviderConfigLocation)
}
}

options := &Options{}

var err error
Expand Down Expand Up @@ -159,6 +145,12 @@ func ParseOptions() *Options {
os.Exit(1)
}

if exists := fileutil.FileExists(defaultProviderConfigLocation); !exists {
if err := createProviderConfigYAML(defaultProviderConfigLocation); err != nil {
gologger.Error().Msgf("Could not create provider config file: %s\n", err)
}
}

if options.Config != defaultConfigLocation {
// An empty source file is not a fatal error
if err := flagSet.MergeConfigFile(options.Config); err != nil && !errors.Is(err, io.EOF) {
Expand Down Expand Up @@ -226,49 +218,6 @@ func (options *Options) loadProvidersFrom(location string) {
}
}

func migrateToProviderConfig(defaultConfigLocation, defaultProviderLocation string) error {
configs, err := unMarshalToLowerCaseMap(defaultConfigLocation)
if err != nil {
return err
}

sourcesRequiringApiKeysMap := make(map[string][]string)
for _, source := range passive.AllSources {
if source.NeedsKey() {
sourceName := strings.ToLower(source.Name())
if sourceKeys, ok := configs[sourceName]; ok {
sourcesRequiringApiKeysMap[sourceName] = sourceKeys
} else {
sourcesRequiringApiKeysMap[sourceName] = []string{}
}
}
}

return CreateProviderConfigYAML(defaultProviderLocation, sourcesRequiringApiKeysMap)
}

func unMarshalToLowerCaseMap(defaultConfigLocation string) (map[string][]string, error) {
defaultConfigFile, err := os.Open(defaultConfigLocation)
if err != nil {
return nil, err
}
defer defaultConfigFile.Close()

configs := map[string][]string{}
if err := yaml.NewDecoder(defaultConfigFile).Decode(configs); isFatalErr(err) {
return nil, err
}

for k, v := range configs {
configs[strings.ToLower(k)] = v
}
return configs, nil
}

func isFatalErr(err error) bool {
return err != nil && !errors.Is(err, io.EOF)
}

func listSources(options *Options) {
gologger.Info().Msgf("Current list of available sources. [%d]\n", len(passive.AllSources))
gologger.Info().Msgf("Sources marked with an * need key(s) or token(s) to work.\n")
Expand Down