diff --git a/cmd/amass/enum.go b/cmd/amass/enum.go index 6a85ae2ed..daf235fd2 100644 --- a/cmd/amass/enum.go +++ b/cmd/amass/enum.go @@ -13,6 +13,7 @@ import ( "io/ioutil" "log" "math/rand" + "net" "os" "os/signal" "path/filepath" @@ -49,6 +50,7 @@ type enumArgs struct { Domains stringset.Set Excluded stringset.Set Included stringset.Set + Interface string MaxDNSQueries int MinForRecursive int Names stringset.Set @@ -101,6 +103,7 @@ func defineEnumArgumentFlags(enumFlags *flag.FlagSet, args *enumArgs) { enumFlags.Var(&args.Domains, "d", "Domain names separated by commas (can be used multiple times)") enumFlags.Var(&args.Excluded, "exclude", "Data source names separated by commas to be excluded") enumFlags.Var(&args.Included, "include", "Data source names separated by commas to be included") + enumFlags.StringVar(&args.Interface, "i", "", "Provide the network interface to send the traffic through") enumFlags.IntVar(&args.MaxDNSQueries, "max-dns-queries", 0, "Maximum number of concurrent DNS queries") enumFlags.IntVar(&args.MinForRecursive, "min-for-recursive", 1, "Subdomain labels seen before recursive brute forcing") enumFlags.Var(&args.Ports, "p", "Ports separated by commas (default: 443)") @@ -173,7 +176,7 @@ func runEnumCommand(clArgs []string) { os.Exit(1) } defer sys.Shutdown() - sys.SetDataSources(datasrcs.GetAllSources(sys)) + sys.SetDataSources(datasrcs.GetAllSources(sys, true)) // Expand data source category names into the associated source names cfg.SourceFilter.Sources = expandCategoryNames(cfg.SourceFilter.Sources, generateCategoryMap(sys)) @@ -280,6 +283,17 @@ func argsAndConfig(clArgs []string) (*config.Config, *enumArgs) { return nil, &args } + if args.Interface != "" { + iface, err := net.InterfaceByName(args.Interface) + if err != nil || iface == nil { + fmt.Fprint(color.Output, format.InterfaceInfo()) + os.Exit(1) + } + if err := assignNetInterface(iface); err != nil { + r.Fprintf(color.Error, "%v\n", err) + os.Exit(1) + } + } if args.Options.NoColor { color.NoColor = true } @@ -334,8 +348,8 @@ func argsAndConfig(clArgs []string) (*config.Config, *enumArgs) { // Check if the user has requested the data source names if args.Options.ListSources { - for _, info := range GetAllSourceInfo(cfg) { - g.Println(info) + for _, line := range GetAllSourceInfo(cfg) { + fmt.Fprintln(color.Output, line) } return nil, &args } diff --git a/cmd/amass/main.go b/cmd/amass/main.go index b548da9e0..c0ae098c1 100644 --- a/cmd/amass/main.go +++ b/cmd/amass/main.go @@ -27,6 +27,7 @@ import ( "errors" "flag" "fmt" + "net" "os" "path" "sort" @@ -38,7 +39,7 @@ import ( "github.com/OWASP/Amass/v3/eventbus" "github.com/OWASP/Amass/v3/format" "github.com/OWASP/Amass/v3/graph" - "github.com/OWASP/Amass/v3/net" + amassnet "github.com/OWASP/Amass/v3/net" "github.com/OWASP/Amass/v3/requests" "github.com/OWASP/Amass/v3/stringfilter" "github.com/OWASP/Amass/v3/systems" @@ -134,7 +135,7 @@ func main() { } } -// GetAllSourceInfo returns the names of all Amass data sources. +// GetAllSourceInfo returns the output for the 'list' flag. func GetAllSourceInfo(cfg *config.Config) []string { var names []string @@ -146,10 +147,21 @@ func GetAllSourceInfo(cfg *config.Config) []string { if err != nil { return names } - sys.SetDataSources(datasrcs.GetAllSources(sys)) + sys.SetDataSources(datasrcs.GetAllSources(sys, false)) + + names = append(names, fmt.Sprintf("%-35s%-35s%s", blue("Data Source"), blue("| Type"), blue("| Available"))) + var line string + for i := 0; i < 8; i++ { + line += blue("----------") + } + names = append(names, line) for _, src := range sys.DataSources() { - names = append(names, fmt.Sprintf("%-20s\t%s", src.String(), src.Type())) + var avail string + if src.CheckConfig() == nil { + avail = "*" + } + names = append(names, fmt.Sprintf("%-35s %-35s %s", green(src.String()), yellow(src.Type()), yellow(avail))) } sys.Shutdown() @@ -238,9 +250,10 @@ func orderedEvents(events []string, db *graph.Graph) ([]string, []time.Time, []t e1, l1 := db.EventDateRange(events[i]) e2, l2 := db.EventDateRange(events[j]) - if l1.After(l2) || e2.Before(e1) { + if l2.After(l1) || e1.Before(e2) { less = true } + return less }) @@ -287,8 +300,14 @@ func memGraphForScope(domains []string, from *graph.Graph) (*graph.Graph, error) return nil, errors.New("Failed to create the in-memory graph database") } + var err error // Migrate the event data into the in-memory graph database - if err := from.MigrateEventsInScope(db, domains); err != nil { + if len(domains) == 0 { + err = from.MigrateEvents(db) + } else { + err = from.MigrateEventsInScope(db, domains) + } + if err != nil { return nil, fmt.Errorf("Failed to move the data into the in-memory graph database: %v", err) } @@ -297,7 +316,7 @@ func memGraphForScope(domains []string, from *graph.Graph) (*graph.Graph, error) func getEventOutput(uuids []string, db *graph.Graph) []*requests.Output { var output []*requests.Output - cache := net.NewASNCache() + cache := amassnet.NewASNCache() filter := stringfilter.NewStringFilter() for i := len(uuids) - 1; i >= 0; i-- { @@ -324,7 +343,7 @@ func domainNameInScope(name string, scope []string) bool { } func healASInfo(uuids []string, db *graph.Graph) bool { - cache := net.NewASNCache() + cache := amassnet.NewASNCache() db.ASNCacheFill(cache) cfg := config.NewConfig() @@ -333,7 +352,7 @@ func healASInfo(uuids []string, db *graph.Graph) bool { if err != nil { return false } - sys.SetDataSources(datasrcs.GetAllSources(sys)) + sys.SetDataSources(datasrcs.GetAllSources(sys, true)) defer sys.Shutdown() bus := eventbus.NewEventBus() @@ -370,3 +389,30 @@ func healASInfo(uuids []string, db *graph.Graph) bool { return updated } + +func assignNetInterface(iface *net.Interface) error { + addrs, err := iface.Addrs() + if err != nil { + return fmt.Errorf("Network interface '%s' has no assigned addresses", iface.Name) + } + + var best net.Addr + for _, addr := range addrs { + if a, ok := addr.(*net.IPNet); ok { + if best == nil { + best = a + } + if amassnet.IsIPv4(a.IP) { + best = a + break + } + } + } + + if best == nil { + return fmt.Errorf("Network interface '%s' does not have assigned IP addresses", iface.Name) + } + + amassnet.LocalAddr = best + return nil +} diff --git a/datasrcs/dnsdumpster.go b/datasrcs/dnsdumpster.go index ce2fc366c..1c55355d7 100644 --- a/datasrcs/dnsdumpster.go +++ b/datasrcs/dnsdumpster.go @@ -7,7 +7,6 @@ import ( "context" "fmt" "io/ioutil" - "net" "net/http" "net/url" "regexp" @@ -114,13 +113,6 @@ func (d *DNSDumpster) postForm(ctx context.Context, token, domain string) (strin return "", fmt.Errorf("%s failed to obtain the EventBus from Context", d.String()) } - dial := net.Dialer{} - client := &http.Client{ - Transport: &http.Transport{ - DialContext: dial.DialContext, - TLSHandshakeTimeout: 10 * time.Second, - }, - } params := url.Values{ "csrfmiddlewaretoken": {token}, "targetip": {domain}, @@ -147,7 +139,7 @@ func (d *DNSDumpster) postForm(ctx context.Context, token, domain string) (strin req.Header.Set("Referer", "https://dnsdumpster.com") req.Header.Set("X-CSRF-Token", token) - resp, err := client.Do(req) + resp, err := amasshttp.DefaultClient.Do(req) if err != nil { bus.Publish(requests.LogTopic, eventbus.PriorityHigh, fmt.Sprintf("%s: The POST request failed: %v", d.String(), err)) diff --git a/datasrcs/radb.go b/datasrcs/radb.go index 380b21a69..74f77efcb 100644 --- a/datasrcs/radb.go +++ b/datasrcs/radb.go @@ -8,12 +8,12 @@ import ( "context" "encoding/json" "fmt" - "net" "strconv" "strings" "time" "github.com/OWASP/Amass/v3/eventbus" + amassnet "github.com/OWASP/Amass/v3/net" "github.com/OWASP/Amass/v3/net/http" "github.com/OWASP/Amass/v3/requests" "github.com/OWASP/Amass/v3/resolvers" @@ -343,8 +343,7 @@ func (r *RADb) ipToASN(ctx context.Context, cidr string) int { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() - d := net.Dialer{} - conn, err := d.DialContext(ctx, "tcp", r.addr+":43") + conn, err := amassnet.DialContext(ctx, "tcp", r.addr+":43") if err != nil { bus.Publish(requests.LogTopic, eventbus.PriorityHigh, fmt.Sprintf("%s: %v", r.String(), err)) return 0 diff --git a/datasrcs/shadowserver.go b/datasrcs/shadowserver.go index 83f53bd30..2cf1772f5 100644 --- a/datasrcs/shadowserver.go +++ b/datasrcs/shadowserver.go @@ -205,8 +205,7 @@ func (s *ShadowServer) netblocks(ctx context.Context, asn int) stringset.Set { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() - d := net.Dialer{} - conn, err := d.DialContext(ctx, "tcp", s.addr+":43") + conn, err := amassnet.DialContext(ctx, "tcp", s.addr+":43") if err != nil { bus.Publish(requests.LogTopic, eventbus.PriorityHigh, fmt.Sprintf("%s: %v", s.String(), err)) return netblocks diff --git a/format/print.go b/format/print.go index 932fd7e17..d329280e8 100644 --- a/format/print.go +++ b/format/print.go @@ -6,10 +6,11 @@ package format import ( "fmt" "io" + "net" "strconv" "strings" - "github.com/OWASP/Amass/v3/net" + amassnet "github.com/OWASP/Amass/v3/net" "github.com/OWASP/Amass/v3/requests" "github.com/fatih/color" ) @@ -198,7 +199,7 @@ func censorString(input string, start, end int) string { // OutputLineParts returns the parts of a line to be printed for a requests.Output. func OutputLineParts(out *requests.Output, src, addrs, demo bool) (source, name, ips string) { if src { - source = fmt.Sprintf("%-18s", "["+out.Source+"] ") + source = fmt.Sprintf("%-18s", "["+out.Sources[0]+"] ") } if addrs { for i, a := range out.Addresses { @@ -230,12 +231,40 @@ func DesiredAddrTypes(addrs []requests.AddressInfo, ipv4, ipv6 bool) []requests. var keep []requests.AddressInfo for _, addr := range addrs { - if net.IsIPv4(addr.Address) && !ipv4 { + if amassnet.IsIPv4(addr.Address) && !ipv4 { continue - } else if net.IsIPv6(addr.Address) && !ipv6 { + } else if amassnet.IsIPv6(addr.Address) && !ipv6 { continue } keep = append(keep, addr) } return keep } + +// InterfaceInfo returns network interface information specific to the current host. +func InterfaceInfo() string { + var output string + + if ifaces, err := net.Interfaces(); err == nil { + for _, i := range ifaces { + addrs, err := i.Addrs() + if err != nil { + continue + } + output += fmt.Sprintf("%s%s%s\n", blue(i.Name+": "), green("flags="), yellow("<"+strings.ToUpper(i.Flags.String()+">"))) + if i.HardwareAddr.String() != "" { + output += fmt.Sprintf("\t%s%s\n", green("ether: "), yellow(i.HardwareAddr.String())) + } + for _, addr := range addrs { + inet := "inet" + if a, ok := addr.(*net.IPNet); ok && amassnet.IsIPv6(a.IP) { + inet += "6" + } + inet += ": " + output += fmt.Sprintf("\t%s%s\n", green(inet), yellow(addr.String())) + } + } + } + + return output +} diff --git a/net/http/http.go b/net/http/http.go index abb270116..c0426d5d8 100644 --- a/net/http/http.go +++ b/net/http/http.go @@ -12,7 +12,6 @@ import ( "fmt" "io" "io/ioutil" - "net" "net/http" "net/http/cookiejar" "net/url" @@ -20,6 +19,7 @@ import ( "strings" "time" + amassnet "github.com/OWASP/Amass/v3/net" "github.com/OWASP/Amass/v3/net/dns" "github.com/OWASP/Amass/v3/stringset" ) @@ -38,20 +38,15 @@ const ( defaultHandshakeDeadline = 5 * time.Second ) -var ( - defaultClient *http.Client -) +// DefaultClient is the same HTTP client used by the package methods. +var DefaultClient *http.Client func init() { jar, _ := cookiejar.New(nil) - defaultClient = &http.Client{ + DefaultClient = &http.Client{ Timeout: 30 * time.Second, Transport: &http.Transport{ - DialContext: (&net.Dialer{ - Timeout: 30 * time.Second, - KeepAlive: 30 * time.Second, - DualStack: true, - }).DialContext, + DialContext: amassnet.DialContext, MaxIdleConns: 200, IdleConnTimeout: 90 * time.Second, TLSHandshakeTimeout: 20 * time.Second, @@ -68,14 +63,14 @@ func init() { func CopyCookies(src string, dest string) { srcURL, _ := url.Parse(src) destURL, _ := url.Parse(dest) - defaultClient.Jar.SetCookies(destURL, defaultClient.Jar.Cookies(srcURL)) + DefaultClient.Jar.SetCookies(destURL, DefaultClient.Jar.Cookies(srcURL)) } // CheckCookie checks if a cookie exists in the cookie jar for a given host func CheckCookie(urlString string, cookieName string) bool { cookieURL, _ := url.Parse(urlString) found := false - for _, cookie := range defaultClient.Jar.Cookies(cookieURL) { + for _, cookie := range DefaultClient.Jar.Cookies(cookieURL) { if cookie.Name == cookieName { found = true break @@ -106,7 +101,7 @@ func RequestWebPage(urlstring string, body io.Reader, hvals map[string]string, u req.Header.Set(k, v) } - resp, err := defaultClient.Do(req) + resp, err := DefaultClient.Do(req) if err != nil { return "", err } else if resp.StatusCode < 200 || resp.StatusCode >= 400 { @@ -129,8 +124,7 @@ func PullCertificateNames(addr string, ports []int) []string { ctx, cancel := context.WithTimeout(context.Background(), defaultTLSConnectTimeout) defer cancel() // Obtain the connection - d := net.Dialer{} - conn, err := d.DialContext(ctx, "tcp", addr+":"+strconv.Itoa(port)) + conn, err := amassnet.DialContext(ctx, "tcp", addr+":"+strconv.Itoa(port)) if err != nil { continue } diff --git a/net/network.go b/net/network.go index dced81513..54d4d29bb 100644 --- a/net/network.go +++ b/net/network.go @@ -5,10 +5,12 @@ package net import ( "bytes" + "context" "math/big" "net" "strconv" "strings" + "time" ) // IPv4RE is a regular expression that will match an IPv4 address. @@ -17,6 +19,9 @@ const IPv4RE = "((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)[.]){3}(25[0-5]|2[0-4][0- // ReservedCIDRDescription is the description used for reserved address ranges. const ReservedCIDRDescription = "Reserved Network Address Blocks" +// LocalAddr is the global option for specifying the network interface. +var LocalAddr net.Addr + // ReservedCIDRs includes all the networks that are reserved for special use. var ReservedCIDRs = []string{ "192.168.0.0/16", @@ -50,6 +55,26 @@ func init() { } } +// DialContext performs the dial using global variables (e.g. LocalAddr). +func DialContext(ctx context.Context, network, addr string) (net.Conn, error) { + d := &net.Dialer{DualStack: true} + + if LocalAddr != nil && strings.HasPrefix(network, "tcp") { + d.Timeout = 30 * time.Second + d.LocalAddr = &net.TCPAddr{ + IP: net.ParseIP(LocalAddr.String()), + Port: 0, + } + } else if LocalAddr != nil && strings.HasPrefix(network, "udp") { + d.LocalAddr = &net.UDPAddr{ + IP: net.ParseIP(LocalAddr.String()), + Port: 0, + } + } + + return d.DialContext(ctx, network, addr) +} + // IsIPv4 returns true when the provided net.IP address is an IPv4 address. func IsIPv4(ip net.IP) bool { return strings.Count(ip.String(), ":") < 2 diff --git a/resolvers/resolver.go b/resolvers/resolver.go index 0e81caa6f..165f7264a 100644 --- a/resolvers/resolver.go +++ b/resolvers/resolver.go @@ -478,8 +478,6 @@ func (r *BaseResolver) finishProcessing(m *dns.Msg, req *resolveRequest) { } func (r *BaseResolver) tcpExchange(id uint16, req *resolveRequest) { - var d net.Dialer - if len(req.Msg.Question) == 0 { return } @@ -489,7 +487,7 @@ func (r *BaseResolver) tcpExchange(id uint16, req *resolveRequest) { defer cancel() req.Msg = msg - conn, err := d.DialContext(ctx, "tcp", r.address+":"+r.port) + conn, err := amassnet.DialContext(ctx, "tcp", r.address+":"+r.port) if err != nil { estr := fmt.Sprintf("DNS: Failed to obtain TCP connection to %s:%s: %v", r.address, r.port, err) r.returnRequest(req, makeResolveResult(nil, nil, true, estr, NotAvailableRcode)) diff --git a/resolvers/resolver_state.go b/resolvers/resolver_state.go index 1f7f62848..63556769d 100644 --- a/resolvers/resolver_state.go +++ b/resolvers/resolver_state.go @@ -4,9 +4,11 @@ package resolvers import ( + "context" "net" "time" + amassnet "github.com/OWASP/Amass/v3/net" "github.com/miekg/dns" ) @@ -384,9 +386,7 @@ func (r *BaseResolver) periodicRotations(chs *rotationChans) { var err error for { - d := &net.Dialer{} - - current, err = d.Dial("udp", r.address+":"+r.port) + current, err = amassnet.DialContext(context.TODO(), "udp", r.address+":"+r.port) if err == nil { break } diff --git a/resolvers/zone.go b/resolvers/zone.go index df00fccfd..eb0ac3980 100644 --- a/resolvers/zone.go +++ b/resolvers/zone.go @@ -6,12 +6,12 @@ package resolvers import ( "context" "fmt" - "net" "regexp" "strings" "time" "github.com/OWASP/Amass/v3/eventbus" + amassnet "github.com/OWASP/Amass/v3/net" amassdns "github.com/OWASP/Amass/v3/net/dns" "github.com/OWASP/Amass/v3/requests" "github.com/miekg/dns" @@ -26,8 +26,7 @@ func ZoneTransfer(sub, domain, server string) ([]*requests.DNSRequest, error) { ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) defer cancel() - d := net.Dialer{} - conn, err := d.DialContext(ctx, "tcp", server+":53") + conn, err := amassnet.DialContext(ctx, "tcp", server+":53") if err != nil { return results, fmt.Errorf("Zone xfr error: Failed to obtain TCP connection to %s: %v", server+":53", err) }