diff --git a/collectors/dns_processor.go b/collectors/dns_processor.go index cc1f0f04..8ef96a87 100644 --- a/collectors/dns_processor.go +++ b/collectors/dns_processor.go @@ -11,7 +11,6 @@ import ( "github.com/dmachard/go-dnscollector/transformers" "github.com/dmachard/go-logger" "github.com/miekg/dns" - "golang.org/x/net/publicsuffix" ) func GetFakeDns() ([]byte, error) { @@ -149,13 +148,6 @@ func (d *DnsProcessor) Run(sendTo []chan dnsutils.DnsMessage) { // convert latency to human dm.DnsTap.LatencySec = fmt.Sprintf("%.6f", dm.DnsTap.Latency) - // Public suffix - ps, _ := publicsuffix.PublicSuffix(dm.DNS.Qname) - dm.DNS.QnamePublicSuffix = ps - if etpo, err := publicsuffix.EffectiveTLDPlusOne(dm.DNS.Qname); err == nil { - dm.DNS.QnameEffectiveTLDPlusOne = etpo - } - // apply all enabled transformers if subprocessors.ProcessMessage(&dm) == transformers.RETURN_DROP { continue diff --git a/collectors/dnstap_processor.go b/collectors/dnstap_processor.go index 60f9e4ad..6776285f 100644 --- a/collectors/dnstap_processor.go +++ b/collectors/dnstap_processor.go @@ -12,7 +12,6 @@ import ( "github.com/dmachard/go-dnscollector/transformers" "github.com/dmachard/go-dnstap-protobuf" "github.com/dmachard/go-logger" - "golang.org/x/net/publicsuffix" "google.golang.org/protobuf/proto" ) @@ -235,13 +234,6 @@ func (d *DnstapProcessor) Run(sendTo []chan dnsutils.DnsMessage) { // convert latency to human dm.DnsTap.LatencySec = fmt.Sprintf("%.6f", dm.DnsTap.Latency) - // Public suffix - ps, _ := publicsuffix.PublicSuffix(dm.DNS.Qname) - dm.DNS.QnamePublicSuffix = ps - if etpo, err := publicsuffix.EffectiveTLDPlusOne(dm.DNS.Qname); err == nil { - dm.DNS.QnameEffectiveTLDPlusOne = etpo - } - // apply all enabled transformers if subprocessors.ProcessMessage(&dm) == transformers.RETURN_DROP { continue diff --git a/collectors/powerdns_processor.go b/collectors/powerdns_processor.go index 91d43a87..df970547 100644 --- a/collectors/powerdns_processor.go +++ b/collectors/powerdns_processor.go @@ -11,7 +11,6 @@ import ( "github.com/dmachard/go-dnscollector/transformers" "github.com/dmachard/go-logger" powerdns_protobuf "github.com/dmachard/go-powerdns-protobuf" - "golang.org/x/net/publicsuffix" "google.golang.org/protobuf/proto" ) @@ -126,16 +125,8 @@ func (d *PdnsProcessor) Run(sendTo []chan dnsutils.DnsMessage) { dm.DnsTap.TimestampRFC3339 = ts.UTC().Format(time.RFC3339Nano) dm.DNS.Qname = pbdm.Question.GetQName() - // remove ending dot ? - qname := strings.TrimSuffix(dm.DNS.Qname, ".") - dm.DNS.Qname = qname - - ps, _ := publicsuffix.PublicSuffix(qname) - dm.DNS.QnamePublicSuffix = ps - if etpo, err := publicsuffix.EffectiveTLDPlusOne(qname); err == nil { - dm.DNS.QnameEffectiveTLDPlusOne = etpo - } + dm.DNS.Qname = strings.TrimSuffix(dm.DNS.Qname, ".") // get query type dm.DNS.Qtype = dnsutils.RdatatypeToString(int(pbdm.Question.GetQType())) diff --git a/collectors/tail.go b/collectors/tail.go index 0879c9e2..e3306181 100644 --- a/collectors/tail.go +++ b/collectors/tail.go @@ -12,7 +12,6 @@ import ( "github.com/dmachard/go-logger" "github.com/hpcloud/tail" "github.com/miekg/dns" - "golang.org/x/net/publicsuffix" ) type Tail struct { @@ -252,13 +251,6 @@ func (c *Tail) Run() { dm.DNS.Payload, _ = dnspkt.Pack() dm.DNS.Length = len(dm.DNS.Payload) - // Public suffix - ps, _ := publicsuffix.PublicSuffix(dm.DNS.Qname) - dm.DNS.QnamePublicSuffix = ps - if etpo, err := publicsuffix.EffectiveTLDPlusOne(dm.DNS.Qname); err == nil { - dm.DNS.QnameEffectiveTLDPlusOne = etpo - } - // apply all enabled transformers if subprocessors.ProcessMessage(&dm) == transformers.RETURN_DROP { continue diff --git a/config.yml b/config.yml index 34f59438..919ae026 100644 --- a/config.yml +++ b/config.yml @@ -72,14 +72,14 @@ multiplexer: dnstap: listen-ip: 0.0.0.0 listen-port: 6000 + transforms: + normalize: + qname-lowercase: true loggers: - name: console stdout: mode: text - transforms: - normalize: - qname-lowercase: true routes: - from: [ tap ] @@ -414,6 +414,13 @@ multiplexer: # list of transforms to apply on collectors or loggers ################################################ +# # Use this option to add top level domain and tld+1, based on public suffix list https://publicsuffix.org/ +# public-suffix: +# # add top level domain +# add-tld: false +# # add top level domain plus one label +# add-tld-plus-one: false + # # Use this option to protect user privacy # user-privacy: # # IP-Addresses are anonymities by zeroing the host-part of an address. diff --git a/dnsutils/config.go b/dnsutils/config.go index 1d639d2b..aede329e 100644 --- a/dnsutils/config.go +++ b/dnsutils/config.go @@ -40,6 +40,11 @@ type MultiplexRoutes struct { } type ConfigTransformers struct { + PublicSuffix struct { + Enable bool `yaml:"enable"` + AddTld bool `yaml:"add-tld"` + AddTldPlusOne bool `yaml:"add-tld-plus-one"` + } `yaml:"public-suffix"` UserPrivacy struct { Enable bool `yaml:"enable"` AnonymizeIP bool `yaml:"anonymize-ip"` @@ -80,12 +85,16 @@ type ConfigTransformers struct { } func (c *ConfigTransformers) SetDefault() { + c.PublicSuffix.Enable = false + c.PublicSuffix.AddTld = false + c.PublicSuffix.AddTldPlusOne = false + c.Suspicious.Enable = false c.Suspicious.ThresholdQnameLen = 100 c.Suspicious.ThresholdPacketLen = 1000 c.Suspicious.ThresholdSlow = 1.0 c.Suspicious.CommonQtypes = []string{"A", "AAAA", "TXT", "CNAME", "PTR", - "NAPTR", "DNSKEY", "SRV", "SOA", "NS", "MX", "DS"} + "NAPTR", "DNSKEY", "SRV", "SOA", "NS", "MX", "DS", "HTTPS"} c.Suspicious.UnallowedChars = []string{"\"", "==", "/", ":"} c.Suspicious.ThresholdMaxLabels = 10 diff --git a/doc/configuration.md b/doc/configuration.md index 04f69546..0cd06eab 100644 --- a/doc/configuration.md +++ b/doc/configuration.md @@ -15,6 +15,7 @@ incoming traffics. You can take a look to the list of config [`examples`](https: - [Loggers](#loggers) - [Routes](#routes) - [Transforms](#transforms) + - [Public suffix](#public-suffix) - [Normalize](#normalize) - [User privacy](#user-privacy) - [GeoIP Support](#geoip-support) @@ -163,6 +164,22 @@ multiplexer: Some transformations can be done on collectors or loggers. +### Public Suffix + +Option to add top level domain. For example for `books.amazon.co.uk`, the `TLD` +is `co.uk` and the `TLD+1` is `amazon.co.uk`. + +Options: +- `add-tld`: (boolean) add top level domain +- `add-tld-plus-one`: (boolean) add top level domain plus one label + +```yaml +transforms: + public-suffix: + add-tld: false + add-tld-plus-one: false +``` + ### Normalize Option to convert all domain to lowercase. For example: `Wwww.GooGlE.com` will be equal to `www.google.com` diff --git a/loggers/prometheus.go b/loggers/prometheus.go index aaed77b7..21b658a8 100644 --- a/loggers/prometheus.go +++ b/loggers/prometheus.go @@ -616,34 +616,37 @@ func (o *Prometheus) Record(dm dnsutils.DnsMessage) { } // count and top tld - if _, exists := o.tldsUniq[dm.DNS.QnamePublicSuffix]; !exists { - o.tldsUniq[dm.DNS.QnamePublicSuffix] = 1 - o.counterTldsUniq.WithLabelValues().Inc() - } else { - o.tldsUniq[dm.DNS.QnamePublicSuffix] += 1 - } + if dm.DNS.QnamePublicSuffix != "-" { + if _, exists := o.tldsUniq[dm.DNS.QnamePublicSuffix]; !exists { + o.tldsUniq[dm.DNS.QnamePublicSuffix] = 1 + o.counterTldsUniq.WithLabelValues().Inc() + } else { + o.tldsUniq[dm.DNS.QnamePublicSuffix] += 1 + } - if _, exists := o.tlds[dm.DnsTap.Identity]; !exists { - o.tlds[dm.DnsTap.Identity] = make(map[string]int) - } + if _, exists := o.tlds[dm.DnsTap.Identity]; !exists { + o.tlds[dm.DnsTap.Identity] = make(map[string]int) + } - if _, exists := o.tlds[dm.DnsTap.Identity][dm.DNS.QnamePublicSuffix]; !exists { - o.tlds[dm.DnsTap.Identity][dm.DNS.QnamePublicSuffix] = 1 - o.counterTlds.WithLabelValues(dm.DnsTap.Identity).Inc() - } else { - o.tlds[dm.DnsTap.Identity][dm.DNS.QnamePublicSuffix] += 1 - } + if _, exists := o.tlds[dm.DnsTap.Identity][dm.DNS.QnamePublicSuffix]; !exists { + o.tlds[dm.DnsTap.Identity][dm.DNS.QnamePublicSuffix] = 1 + o.counterTlds.WithLabelValues(dm.DnsTap.Identity).Inc() + } else { + o.tlds[dm.DnsTap.Identity][dm.DNS.QnamePublicSuffix] += 1 + } - if _, ok := o.topTlds[dm.DnsTap.Identity]; !ok { - o.topTlds[dm.DnsTap.Identity] = topmap.NewTopMap(o.config.Loggers.Prometheus.TopN) - } - o.topTlds[dm.DnsTap.Identity].Record(dm.DNS.QnamePublicSuffix, o.domains[dm.DnsTap.Identity][dm.DNS.QnamePublicSuffix]) + if _, ok := o.topTlds[dm.DnsTap.Identity]; !ok { + o.topTlds[dm.DnsTap.Identity] = topmap.NewTopMap(o.config.Loggers.Prometheus.TopN) + } + o.topTlds[dm.DnsTap.Identity].Record(dm.DNS.QnamePublicSuffix, o.tlds[dm.DnsTap.Identity][dm.DNS.QnamePublicSuffix]) - o.gaugeTopTlds.Reset() - for s := range o.topTlds { - for _, r := range o.topTlds[s].Get() { - o.gaugeTopTlds.WithLabelValues(s, r.Name).Set(float64(r.Hit)) + o.gaugeTopTlds.Reset() + for s := range o.topTlds { + for _, r := range o.topTlds[s].Get() { + o.gaugeTopTlds.WithLabelValues(s, r.Name).Set(float64(r.Hit)) + } } + } // suspicious domains diff --git a/transformers/publicsuffix.go b/transformers/publicsuffix.go new file mode 100644 index 00000000..8ec2da46 --- /dev/null +++ b/transformers/publicsuffix.go @@ -0,0 +1,49 @@ +package transformers + +import ( + "errors" + "strings" + + "github.com/dmachard/go-dnscollector/dnsutils" + publicsuffixlist "golang.org/x/net/publicsuffix" +) + +type PublicSuffixProcessor struct { + config *dnsutils.ConfigTransformers +} + +func NewPublicSuffixSubprocessor(config *dnsutils.ConfigTransformers) PublicSuffixProcessor { + s := PublicSuffixProcessor{ + config: config, + } + + return s +} + +func (s *PublicSuffixProcessor) IsEnabled() bool { + return s.config.PublicSuffix.Enable +} + +func (s *PublicSuffixProcessor) GetEffectiveTld(qname string) (string, error) { + + // PublicSuffix is case sensitive. + // remove ending dot ? + qname = strings.ToLower(qname) + qname = strings.TrimSuffix(qname, ".") + + // search + etld, icann := publicsuffixlist.PublicSuffix(qname) + if icann { + return etld, nil + } + return "", errors.New("ICANN Unmanaged") +} + +func (s *PublicSuffixProcessor) GetEffectiveTldPlusOne(qname string) (string, error) { + // PublicSuffix is case sensitive. + // remove ending dot ? + qname = strings.ToLower(qname) + qname = strings.TrimSuffix(qname, ".") + + return publicsuffixlist.EffectiveTLDPlusOne(qname) +} diff --git a/transformers/publicsuffix_test.go b/transformers/publicsuffix_test.go new file mode 100644 index 00000000..2f37463d --- /dev/null +++ b/transformers/publicsuffix_test.go @@ -0,0 +1,92 @@ +package transformers + +import ( + "testing" + + "github.com/dmachard/go-dnscollector/dnsutils" +) + +func TestPublicSuffixAddTLD(t *testing.T) { + // enable feature + config := dnsutils.GetFakeConfigTransformers() + config.PublicSuffix.Enable = true + config.PublicSuffix.AddTld = true + + // init the processor + psl := NewPublicSuffixSubprocessor(config) + + tt := []struct { + name string + qname string + want string + }{ + { + name: "get tld", + qname: "www.amazon.fr", + want: "fr", + }, + { + name: "get tld insensitive", + qname: "www.Google.Com", + want: "com", + }, + { + name: "get tld with dot trailing", + qname: "www.amazon.fr.", + want: "fr", + }, + } + + for _, tc := range tt { + t.Run(tc.name, func(t *testing.T) { + tld, err := psl.GetEffectiveTld(tc.qname) + if err != nil { + t.Errorf("Bad TLD with error: %s", err.Error()) + } + if tld != tc.want { + t.Errorf("Bad TLD, got: %s, expected: com", tld) + + } + }) + } +} + +func TestPublicSuffixAddTldPlusOne(t *testing.T) { + // enable feature + config := dnsutils.GetFakeConfigTransformers() + config.PublicSuffix.Enable = true + config.PublicSuffix.AddTld = true + + // init the processor + psl := NewPublicSuffixSubprocessor(config) + + tt := []struct { + name string + qname string + want string + }{ + { + name: "get tld", + qname: "www.amazon.fr", + want: "amazon.fr", + }, + { + name: "get tld insensitive", + qname: "books.amazon.co.uk", + want: "amazon.co.uk", + }, + } + + for _, tc := range tt { + t.Run(tc.name, func(t *testing.T) { + tld, err := psl.GetEffectiveTldPlusOne(tc.qname) + if err != nil { + t.Errorf("Bad TLD with error: %s", err.Error()) + } + if tld != tc.want { + t.Errorf("Bad TLD, got: %s, expected: com", tld) + + } + }) + } +} diff --git a/transformers/subprocessors.go b/transformers/subprocessors.go index 4fbee25d..bd7686b5 100644 --- a/transformers/subprocessors.go +++ b/transformers/subprocessors.go @@ -16,11 +16,12 @@ type Transforms struct { logger *logger.Logger name string - SuspiciousTransform SuspiciousTransform - GeoipTransform GeoIpProcessor - FilteringTransform FilteringProcessor - UserPrivacyTransform UserPrivacyProcessor - NormalizeTransform NormalizeProcessor + SuspiciousTransform SuspiciousTransform + GeoipTransform GeoIpProcessor + FilteringTransform FilteringProcessor + UserPrivacyTransform UserPrivacyProcessor + NormalizeTransform NormalizeProcessor + PublicSuffixTransform PublicSuffixProcessor activeTransforms []func(dm *dnsutils.DnsMessage) int } @@ -32,11 +33,12 @@ func NewTransforms(config *dnsutils.ConfigTransformers, logger *logger.Logger, n logger: logger, name: name, - SuspiciousTransform: NewSuspiciousSubprocessor(config, logger, name), - GeoipTransform: NewDnsGeoIpProcessor(config, logger), - FilteringTransform: NewFilteringProcessor(config, logger, name), - UserPrivacyTransform: NewUserPrivacySubprocessor(config), - NormalizeTransform: NewNormalizeSubprocessor(config), + SuspiciousTransform: NewSuspiciousSubprocessor(config, logger, name), + GeoipTransform: NewDnsGeoIpProcessor(config, logger), + FilteringTransform: NewFilteringProcessor(config, logger, name), + UserPrivacyTransform: NewUserPrivacySubprocessor(config), + NormalizeTransform: NewNormalizeSubprocessor(config), + PublicSuffixTransform: NewPublicSuffixSubprocessor(config), } d.Prepare() @@ -61,6 +63,17 @@ func (p *Transforms) Prepare() error { } } + if p.config.PublicSuffix.Enable { + if p.config.PublicSuffix.AddTld { + p.activeTransforms = append(p.activeTransforms, p.GetEffectiveTld) + p.LogInfo("[public suffix: add tld] enabled") + } + if p.config.PublicSuffix.AddTldPlusOne { + p.activeTransforms = append(p.activeTransforms, p.GetEffectiveTldPlusOne) + p.LogInfo("[public suffix: add tld+1] enabled") + } + } + if p.config.UserPrivacy.Enable { // Apply user privacy on qname and query ip if p.config.UserPrivacy.AnonymizeIP { @@ -122,6 +135,21 @@ func (p *Transforms) geoipTransform(dm *dnsutils.DnsMessage) int { return RETURN_SUCCESS } +func (p *Transforms) GetEffectiveTld(dm *dnsutils.DnsMessage) int { + if etld, err := p.PublicSuffixTransform.GetEffectiveTld(dm.DNS.Qname); err == nil { + dm.DNS.QnamePublicSuffix = etld + } + return RETURN_SUCCESS +} + +func (p *Transforms) GetEffectiveTldPlusOne(dm *dnsutils.DnsMessage) int { + if etld, err := p.PublicSuffixTransform.GetEffectiveTldPlusOne(dm.DNS.Qname); err == nil { + dm.DNS.QnameEffectiveTLDPlusOne = etld + } + + return RETURN_SUCCESS +} + func (p *Transforms) anonymizeIP(dm *dnsutils.DnsMessage) int { dm.NetworkInfo.QueryIp = p.UserPrivacyTransform.AnonymizeIP(dm.NetworkInfo.QueryIp)