Skip to content

Commit

Permalink
added intelx source which also supports https://phonebook.cz/
Browse files Browse the repository at this point in the history
  • Loading branch information
akhil-reni committed Jun 7, 2020
1 parent 2c6ff84 commit fdb25a9
Show file tree
Hide file tree
Showing 7 changed files with 151 additions and 8 deletions.
3 changes: 2 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
.DS_Store
cmd/subfinder/subfinder
vendor/
vendor/
.idea
15 changes: 9 additions & 6 deletions config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ sources:
- entrust
- googleter
- hackertarget
- intelx
- ipv4info
- passivetotal
- rapiddns
Expand All @@ -41,19 +42,21 @@ sources:
- zoomeye
censys:
- <key-here>
binaryedge:
binaryedge:
- <key-here>
certspotter:
certspotter:
- <key-here>
facebook: []
passivetotal:
intelx:
- <hostname:key-here>
passivetotal:
- <email:key-here>
securitytrails:
securitytrails:
- <key-here>
urlscan: []
virustotal:
virustotal:
- <key-here>
chaos:
chaos:
- <key-here>
spyse:
- <key-here>
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
github.com/alexbrainman/sspi v0.0.0-20180613141037-e580b900e9f5 h1:P5U+E4x5OkVEKQDklVPmzs71WM56RTTRqV4OrDC//Y4=
github.com/alexbrainman/sspi v0.0.0-20180613141037-e580b900e9f5/go.mod h1:976q2ETgjT2snVCf2ZaBnyBbVoPERGjUz+0sofzEfro=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
Expand All @@ -22,6 +23,7 @@ github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGn
github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k=
github.com/k0kubun/pp v2.3.0+incompatible/go.mod h1:GWse8YhT0p8pT4ir3ZgBbfZild3tgzSScAn6HmfYukg=
github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/lib/pq v1.6.0 h1:I5DPxhYJChW9KYc66se+oKFFQX6VuQrKiprsX6ivRZc=
github.com/lib/pq v1.6.0/go.mod h1:4vXEAYvW1fRQ2/FhZ78H73A60MHw1geSm145z2mdY1g=
Expand Down
4 changes: 4 additions & 0 deletions pkg/passive/sources.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import (
"github.com/projectdiscovery/subfinder/pkg/subscraping/sources/dnsdumpster"
"github.com/projectdiscovery/subfinder/pkg/subscraping/sources/entrust"
"github.com/projectdiscovery/subfinder/pkg/subscraping/sources/hackertarget"
"github.com/projectdiscovery/subfinder/pkg/subscraping/sources/intelx"
"github.com/projectdiscovery/subfinder/pkg/subscraping/sources/ipv4info"
"github.com/projectdiscovery/subfinder/pkg/subscraping/sources/passivetotal"
"github.com/projectdiscovery/subfinder/pkg/subscraping/sources/rapiddns"
Expand Down Expand Up @@ -49,6 +50,7 @@ var DefaultSources = []string{
"entrust",
"hackertarget",
"ipv4info",
"intelx",
"passivetotal",
"rapiddns",
"securitytrails",
Expand Down Expand Up @@ -116,6 +118,8 @@ func (a *Agent) addSources(sources []string) {
a.sources[source] = &hackertarget.Source{}
case "ipv4info":
a.sources[source] = &ipv4info.Source{}
case "intelx":
a.sources[source] = &intelx.Source{}
case "passivetotal":
a.sources[source] = &passivetotal.Source{}
case "rapiddns":
Expand Down
12 changes: 11 additions & 1 deletion pkg/runner/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,11 @@ type ConfigFile struct {
Certspotter []string `yaml:"certspotter"`
Chaos []string `yaml:"chaos"`
DNSDB []string `yaml:"dnsdb"`
IntelX []string `yaml:"intelx"`
PassiveTotal []string `yaml:"passivetotal"`
SecurityTrails []string `yaml:"securitytrails"`
Shodan []string `yaml:"shodan"`
Spyse []string `yaml:"spyse"`
Spyse []string `yaml:"spyse"`
URLScan []string `yaml:"urlscan"`
Virustotal []string `yaml:"virustotal"`
ZoomEye []string `yaml:"zoomeye"`
Expand Down Expand Up @@ -118,6 +119,15 @@ func (c ConfigFile) GetKeys() subscraping.Keys {
keys.DNSDB = c.DNSDB[rand.Intn(len(c.DNSDB))]
}

if len(c.IntelX) > 0 {
intelxKeys := c.IntelX[rand.Intn(len(c.IntelX))]
parts := strings.Split(intelxKeys, ":")
if len(parts) == 2 {
keys.IntelXHost = parts[0]
keys.IntelXKey = parts[1]
}
}

if len(c.PassiveTotal) > 0 {
passiveTotalKeys := c.PassiveTotal[rand.Intn(len(c.PassiveTotal))]
parts := strings.Split(passiveTotalKeys, ":")
Expand Down
121 changes: 121 additions & 0 deletions pkg/subscraping/sources/intelx/intelx.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
package intelx

import (
"bytes"
"context"
"encoding/json"
"fmt"
jsoniter "github.com/json-iterator/go"
"github.com/projectdiscovery/subfinder/pkg/subscraping"
"io/ioutil"
"net/http"
)

type searchResponseType struct {
Id string `json:"id"`
Status int `json:"status"`
}

type selectorType struct {
Selectvalue string `json:"selectorvalue"`
}

type searchResultType struct {
Selectors []selectorType `json:"selectors"`
Status int `json:"status"`
}

type requestBody struct {
Term string
Maxresults int
Media int
Target int
Terminate []int
Timeout int
}

type Source struct{}

func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {

results := make(chan subscraping.Result)

go func() {
if session.Keys.IntelXKey == "" || session.Keys.IntelXHost == "" {
fmt.Println(session.Keys)
close(results)
return
}

search_url := fmt.Sprintf("https://%s/phonebook/search?k=%s", session.Keys.IntelXHost, session.Keys.IntelXKey)

reqBody := requestBody{
Term: domain,
Maxresults: 100000,
Media: 0,
Target: 1,
Timeout: 20,
}

body, err := json.Marshal(reqBody)

if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
close(results)
return
}

resp, err := http.Post(search_url, "application/json", bytes.NewBuffer(body))

var response searchResponseType

err = jsoniter.NewDecoder(resp.Body).Decode(&response)

if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
close(results)
return
}

results_url := fmt.Sprintf("https://%s/phonebook/search/result?k=%s&id=%s&limit=10000", session.Keys.IntelXHost, session.Keys.IntelXKey, response.Id)

var status = 0

for status == 0 {

resp, err = session.Get(ctx, results_url, "", map[string]string{})
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
resp.Body.Close()
close(results)
return
}
var response searchResultType
err = jsoniter.NewDecoder(resp.Body).Decode(&response)

if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
close(results)
return
}

body, err = ioutil.ReadAll(resp.Body)

status = response.Status

for _, hostname := range response.Selectors {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: hostname.Selectvalue}

}

}

close(results)
}()

return results
}

func (s *Source) Name() string {
return "intelx"
}
2 changes: 2 additions & 0 deletions pkg/subscraping/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,8 @@ type Keys struct {
Certspotter string `json:"certspotter"`
Chaos string `json:"chaos"`
DNSDB string `json:"dnsdb"`
IntelXHost string `json:"intelXHost"`
IntelXKey string `json:"intelXKey"`
PassiveTotalUsername string `json:"passivetotal_username"`
PassiveTotalPassword string `json:"passivetotal_password"`
Securitytrails string `json:"securitytrails"`
Expand Down

0 comments on commit fdb25a9

Please sign in to comment.