Skip to content

Commit

Permalink
chore: Misc cleanup
Browse files Browse the repository at this point in the history
  • Loading branch information
@enenumxela
enenumxela committed Jun 28, 2023
1 parent 0ba90ae commit 6ef3338
Show file tree
Hide file tree
Showing 7 changed files with 131 additions and 106 deletions.
28 changes: 15 additions & 13 deletions pkg/xurlfind3r/sources/bevigil/bevigil.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import (
"github.com/valyala/fasthttp"
)

type response struct {
type Response struct {
Domain string `json:"domain"`
URLs []string `json:"urls"`
}
Expand All @@ -23,38 +23,40 @@ func (source *Source) Run(config *sources.Configuration) (URLsChannel chan sourc
defer close(URLsChannel)

var (
key string
err error
res *fasthttp.Response
headers = map[string]string{}
err error
key string
)

key, err = sources.PickRandom(config.Keys.Bevigil)
if key == "" || err != nil {
return
}

reqHeaders := map[string]string{}

if len(config.Keys.Bevigil) > 0 {
headers["X-Access-Token"] = key
reqHeaders["X-Access-Token"] = key
}

reqURL := fmt.Sprintf("https://osint.bevigil.com/api/%s/urls/", config.Domain)

res, err = httpclient.Request(fasthttp.MethodGet, reqURL, "", headers, nil)
var res *fasthttp.Response

res, err = httpclient.Request(fasthttp.MethodGet, reqURL, "", reqHeaders, nil)
if err != nil {
return
}

body := res.Body()
var data Response

var results response

if err = json.Unmarshal(body, &results); err != nil {
if err = json.Unmarshal(res.Body(), &data); err != nil {
return
}

for _, i := range results.URLs {
URLsChannel <- sources.URL{Source: source.Name(), Value: i}
for index := range data.URLs {
URL := data.URLs[index]

URLsChannel <- sources.URL{Source: source.Name(), Value: URL}
}
}()

Expand Down
57 changes: 31 additions & 26 deletions pkg/xurlfind3r/sources/commoncrawl/commoncrawl.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,17 +13,17 @@ import (
"github.com/valyala/fasthttp"
)

type Source struct{}
type API struct {
ID string `json:"id"`
API string `json:"cdx-API"`
}

type CDXAPIResult struct {
type Response struct {
URL string `json:"url"`
Error string `json:"error"`
}

type Index struct {
ID string `json:"id"`
CDX_API string `json:"cdx-API"` //nolint:revive,stylecheck // Is as is
}
type Source struct{}

func (source *Source) Run(config *sources.Configuration) (URLsChannel chan sources.URL) {
URLsChannel = make(chan sources.URL)
Expand All @@ -33,71 +33,76 @@ func (source *Source) Run(config *sources.Configuration) (URLsChannel chan sourc

var (
err error
res *fasthttp.Response
)

var res *fasthttp.Response

res, err = httpclient.SimpleGet("https://index.commoncrawl.org/collinfo.json")
if err != nil {
return
}

var commonCrawlIndexes []Index
var APIs []API

if err = json.Unmarshal(res.Body(), &commonCrawlIndexes); err != nil {
if err = json.Unmarshal(res.Body(), &APIs); err != nil {
return
}

wg := new(sync.WaitGroup)

for index := range commonCrawlIndexes {
for index := range APIs {
wg.Add(1)

commonCrawlIndex := commonCrawlIndexes[index]
API := APIs[index]

go func(API string) {
defer wg.Done()

var (
err error
headers = map[string]string{"Host": "index.commoncrawl.org"}
res *fasthttp.Response
err error
// headers = map[string]string{"Host": "index.commoncrawl.org"}
// res *fasthttp.Response
)

res, err = httpclient.Get(fmt.Sprintf("%s?url=*.%s/*&output=json&fl=url", API, config.Domain), "", headers)
reqHeaders := map[string]string{"Host": "index.commoncrawl.org"}

var res *fasthttp.Response

res, err = httpclient.Get(fmt.Sprintf("%s?url=*.%s/*&output=json&fl=url", API, config.Domain), "", reqHeaders)
if err != nil {
return
}

scanner := bufio.NewScanner(bytes.NewReader(res.Body()))

for scanner.Scan() {
var result CDXAPIResult
var data Response

if err = json.Unmarshal(scanner.Bytes(), &result); err != nil {
if err = json.Unmarshal(scanner.Bytes(), &data); err != nil {
return
}

if result.Error != "" {
if data.Error != "" {
return
}

URL := result.URL
URL := data.URL

if !sources.IsValid(URL) {
return
}
// if !sources.IsValid(URL) {
// return
// }

if !sources.IsInScope(URL, config.Domain, config.IncludeSubdomains) {
return
}
// if !sources.IsInScope(URL, config.Domain, config.IncludeSubdomains) {
// return
// }

URLsChannel <- sources.URL{Source: source.Name(), Value: URL}
}

if scanner.Err() != nil {
return
}
}(commonCrawlIndex.CDX_API)
}(API.API)
}

wg.Wait()
Expand Down
18 changes: 9 additions & 9 deletions pkg/xurlfind3r/sources/github/github.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -137,13 +137,13 @@ func proccesItems(items []item, domainRegexp *regexp.Regexp, name string, URLsCh
}

for _, URL = range domainRegexp.FindAllString(normalizeContent(line), -1) {
if !sources.IsValid(URL) {
continue
}
// if !sources.IsValid(URL) {
// continue
// }

if !sources.IsInScope(URL, config.Domain, config.IncludeSubdomains) {
return
}
// if !sources.IsInScope(URL, config.Domain, config.IncludeSubdomains) {
// return
// }

URLsChannel <- sources.URL{Source: name, Value: URL}
}
Expand All @@ -152,9 +152,9 @@ func proccesItems(items []item, domainRegexp *regexp.Regexp, name string, URLsCh

for _, textMatch := range item.TextMatches {
for _, URL = range domainRegexp.FindAllString(normalizeContent(textMatch.Fragment), -1) {
if !sources.IsValid(URL) {
continue
}
// if !sources.IsValid(URL) {
// continue
// }

URLsChannel <- sources.URL{Source: name, Value: URL}
}
Expand Down
28 changes: 15 additions & 13 deletions pkg/xurlfind3r/sources/intelx/intelx.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ import (
"github.com/valyala/fasthttp"
)

type searchResponseType struct {
type SearchResponse struct {
ID string `json:"id"`
Status int `json:"status"`
}
Expand Down Expand Up @@ -42,10 +42,12 @@ func (source *Source) Run(config *sources.Configuration) (URLsChannel chan sourc
defer close(URLsChannel)

var (
key string
err error
res *fasthttp.Response
err error
key string

body []byte

res *fasthttp.Response
)

key, err = sources.PickRandom(config.Keys.Intelx)
Expand All @@ -62,14 +64,14 @@ func (source *Source) Run(config *sources.Configuration) (URLsChannel chan sourc
}

searchURL := fmt.Sprintf("https://%s/phonebook/search?k=%s", intelXHost, intelXKey)
reqBody := requestBody{
searchReqBody := requestBody{
Term: config.Domain,
MaxResults: 100000,
Media: 0,
Timeout: 20,
}

body, err = json.Marshal(reqBody)
body, err = json.Marshal(searchReqBody)
if err != nil {
return
}
Expand All @@ -79,7 +81,7 @@ func (source *Source) Run(config *sources.Configuration) (URLsChannel chan sourc
return
}

var response searchResponseType
var response SearchResponse

if err = json.Unmarshal(res.Body(), &response); err != nil {
return
Expand All @@ -105,13 +107,13 @@ func (source *Source) Run(config *sources.Configuration) (URLsChannel chan sourc
for _, hostname := range response.Selectors {
URL := hostname.Selectvalue

if !sources.IsValid(URL) {
continue
}
// if !sources.IsValid(URL) {
// continue
// }

if !sources.IsInScope(URL, config.Domain, config.IncludeSubdomains) {
return
}
// if !sources.IsInScope(URL, config.Domain, config.IncludeSubdomains) {
// return
// }

URLsChannel <- sources.URL{Source: source.Name(), Value: URL}
}
Expand Down
44 changes: 24 additions & 20 deletions pkg/xurlfind3r/sources/otx/otx.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,7 @@ import (
"github.com/valyala/fasthttp"
)

type Source struct{}

type response struct {
type Response struct {
URLList []struct {
Domain string `json:"domain"`
URL string `json:"url"`
Expand All @@ -29,44 +27,50 @@ type response struct {
ActualSize int `json:"actual_size"`
}

type Source struct{}

func (source *Source) Run(config *sources.Configuration) (URLsChannel chan sources.URL) {
URLsChannel = make(chan sources.URL)

go func() {
defer close(URLsChannel)

var (
err error
res *fasthttp.Response
)

for page := 1; ; page++ {
res, err = httpclient.SimpleGet(fmt.Sprintf("https://otx.alienvault.com/api/v1/indicators/domain/%s/url_list?limit=%d&page=%d", config.Domain, 200, page))
var (
err error
)

reqURL := fmt.Sprintf("https://otx.alienvault.com/api/v1/indicators/domain/%s/url_list?limit=%d&page=%d", config.Domain, 200, page)

var res *fasthttp.Response

res, err = httpclient.SimpleGet(reqURL)
if err != nil {
return
}

var results response
var data Response

if err = json.Unmarshal(res.Body(), &results); err != nil {
if err = json.Unmarshal(res.Body(), &data); err != nil {
return
}

for _, i := range results.URLList {
URL := i.URL
for index := range data.URLList {
URL := data.URLList[index].URL
// URL := i.URL

if !sources.IsValid(URL) {
continue
}
// if !sources.IsValid(URL) {
// continue
// }

if !sources.IsInScope(URL, config.Domain, config.IncludeSubdomains) {
return
}
// if !sources.IsInScope(URL, config.Domain, config.IncludeSubdomains) {
// return
// }

URLsChannel <- sources.URL{Source: source.Name(), Value: URL}
}

if !results.HasNext {
if !data.HasNext {
break
}
}
Expand Down
Loading

0 comments on commit 6ef3338

Please sign in to comment.