Configurable CIDR mask and hash for user-privacy transformer (#528)

* Add flexible config #473
* Update README.md

README.md

@@ -3,8 +3,8 @@
 
 [![Go Report Card](https://goreportcard.com/badge/github.com/dmachard/go-dns-collector)](https://goreportcard.com/report/dmachard/go-dns-collector)
 ![Go version](https://img.shields.io/badge/go%20version-min%201.20-blue)
-![Go tests](https://img.shields.io/badge/go%20tests-366-green)
-![Go lines](https://img.shields.io/badge/go%20lines-32797-red)
+![Go tests](https://img.shields.io/badge/go%20tests-370-green)
+![Go lines](https://img.shields.io/badge/go%20lines-32932-red)
 ![Go Tests](https://github.com/dmachard/go-dns-collector/actions/workflows/testing-go.yml/badge.svg)
 ![Github Actions](https://github.com/dmachard/go-dns-collector/actions/workflows/testing-dnstap.yml/badge.svg)
 ![Github Actions PDNS](https://github.com/dmachard/go-dns-collector/actions/workflows/testing-powerdns.yml/badge.svg)
@@ -57,7 +57,7 @@ Multiplexer
   - *Send to remote host with generic transport protocol*
     - [`TCP`](docs/loggers/logger_tcp.md)
     - [`Syslog`](docs/loggers/logger_syslog.md) with TLS support
-    - [`DNSTap`](docs/loggers/logger_dnstap.md) protobuf messages
+    - [`DNSTap`](docs/loggers/logger_dnstap.md) protobuf messages with TLS support
   - *Send to various sinks*
     - [`Fluentd`](docs/loggers/logger_fluentd.md)
     - [`InfluxDB`](docs/loggers/logger_influxdb.md)

config.yml

@@ -705,10 +705,16 @@ multiplexer:
 # user-privacy:
 #   # IP-Addresses are anonymities by zeroing the host-part of an address.
 #   anonymize-ip: false
+#   # summarize IPv4 down to the /integer level, default is /16
+#   anonymize-v4bits: "/8"
+#   # summarize IPv6 down to the /integer level, default is /64
+#   anonymize-v6bits: "::/64"
 #   # Reduce Qname to second level only, for exemple mail.google.com be replaced by google.com
 #   minimaze-qname: false
-#   # Hash query and response IP
+#   # Hashes the query and response IP with the specified algorithm.
 #   hash-ip: false
+#   # Algorithm to use for IP hashing, currently supported `sha1` (default), `sha256`, `sha512`
+#   hash-ip-algo: sha1
 
 # # Use this option to add top level domain and tld+1, based on public suffix list https://publicsuffix.org/
 # # or convert all domain to lowercase

docs/transformers/transform_userprivacy.md

@@ -9,13 +9,19 @@ For example:
 Options:
 
 - `anonymize-ip`: (boolean) enable or disable anomymiser ip
-- `hash-ip`: (boolean) hash query and response IP with sha1
+- `anonymize-v4bits`: (string) summarize IPv4 down to the /integer level, default is `/16`
+- `anonymize-v6bits`: (string) summarize IPv6 down to the /integer level, default is `::/64`
+- `hash-ip`: (boolean) hashes the query and response IP with the specified algorithm.
+- `hash-ip-algo`: (string) algorithm to use for IP hashing, currently supported `sha1` (default), `sha256`, `sha512`
 - `minimaze-qname`: (boolean) keep only the second level domain
 
 ```yaml
 transforms:
   user-privacy:
     anonymize-ip: false
+    anonymize-v4bits: "/16"
+    anonymize-v6bits: "::/64"
     hash-ip: false
+    hash-ip-algo: "sha1"
     minimaze-qname: false
 ```

pkgconfig/transformers.go

@@ -2,10 +2,13 @@ package pkgconfig
 
 type ConfigTransformers struct {
 	UserPrivacy struct {
-		Enable        bool `yaml:"enable"`
-		AnonymizeIP   bool `yaml:"anonymize-ip"`
-		MinimazeQname bool `yaml:"minimaze-qname"`
-		HashIP        bool `yaml:"hash-ip"`
+		Enable            bool   `yaml:"enable"`
+		AnonymizeIP       bool   `yaml:"anonymize-ip"`
+		AnonymizeIPV4Bits string `yaml:"anonymize-v4bits"`
+		AnonymizeIPV6Bits string `yaml:"anonymize-v6bits"`
+		MinimazeQname     bool   `yaml:"minimaze-qname"`
+		HashIP            bool   `yaml:"hash-ip"`
+		HashIPAlgo        string `yaml:"hash-ip-algo"`
 	} `yaml:"user-privacy"`
 	Normalize struct {
 		Enable         bool `yaml:"enable"`
@@ -79,8 +82,11 @@ func (c *ConfigTransformers) SetDefault() {
 
 	c.UserPrivacy.Enable = false
 	c.UserPrivacy.AnonymizeIP = false
+	c.UserPrivacy.AnonymizeIPV4Bits = "0.0.0.0/16"
+	c.UserPrivacy.AnonymizeIPV6Bits = "::/64"
 	c.UserPrivacy.MinimazeQname = false
 	c.UserPrivacy.HashIP = false
+	c.UserPrivacy.HashIPAlgo = "sha1"
 
 	c.Normalize.Enable = false
 	c.Normalize.QnameLowerCase = false

transformers/userprivacy.go

@@ -2,8 +2,11 @@ package transformers
 
 import (
 	"crypto/sha1"
+	"crypto/sha256"
+	"crypto/sha512"
 	"fmt"
 	"net"
+	"strconv"
 	"strings"
 
 	"github.com/dmachard/go-dnscollector/dnsutils"
@@ -12,10 +15,25 @@ import (
 	"golang.org/x/net/publicsuffix"
 )
 
-var (
-	defaultIPv4Mask = net.IPv4Mask(255, 255, 0, 0)                                                       // /24
-	defaultIPv6Mask = net.IPMask{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0, 0, 0, 0, 0, 0, 0, 0} // /64
-)
+func parseCIDRMask(mask string) (net.IPMask, error) {
+	parts := strings.Split(mask, "/")
+	if len(parts) != 2 {
+		return nil, fmt.Errorf("invalid mask format, expected /integer: %s", mask)
+	}
+
+	ones, err := strconv.Atoi(parts[1])
+	if err != nil {
+		return nil, fmt.Errorf("invalid /%s cidr", mask)
+	}
+
+	if strings.Contains(parts[0], ":") {
+		ipv6Mask := net.CIDRMask(ones, 128)
+		return ipv6Mask, nil
+	}
+
+	ipv4Mask := net.CIDRMask(ones, 32)
+	return ipv4Mask, nil
+}
 
 type UserPrivacyProcessor struct {
 	config      *pkgconfig.ConfigTransformers
@@ -33,21 +51,46 @@ func NewUserPrivacySubprocessor(config *pkgconfig.ConfigTransformers, logger *lo
 ) UserPrivacyProcessor {
 	s := UserPrivacyProcessor{
 		config:      config,
-		v4Mask:      defaultIPv4Mask,
-		v6Mask:      defaultIPv6Mask,
 		instance:    instance,
 		outChannels: outChannels,
 		logInfo:     logInfo,
 		logError:    logError,
 	}
-
+	s.ReadConfig()
 	return s
 }
 
+func (s *UserPrivacyProcessor) ReadConfig() {
+
+	var err error
+	s.v4Mask, err = parseCIDRMask(s.config.UserPrivacy.AnonymizeIPV4Bits)
+	if err != nil {
+		s.LogError("unable to init v4 mask: %v", err)
+	}
+
+	if !strings.Contains(s.config.UserPrivacy.AnonymizeIPV6Bits, ":") {
+		s.LogError("invalid v6 mask, expect format ::/integer")
+	}
+	s.v6Mask, err = parseCIDRMask(s.config.UserPrivacy.AnonymizeIPV6Bits)
+	if err != nil {
+		s.LogError("unable to init v6 mask: %v", err)
+	}
+}
+
 func (s *UserPrivacyProcessor) ReloadConfig(config *pkgconfig.ConfigTransformers) {
 	s.config = config
 }
 
+func (s *UserPrivacyProcessor) LogInfo(msg string, v ...interface{}) {
+	log := fmt.Sprintf("transformer=userprivacy#%d - ", s.instance)
+	s.logInfo(log+msg, v...)
+}
+
+func (s *UserPrivacyProcessor) LogError(msg string, v ...interface{}) {
+	log := fmt.Sprintf("transformer=userprivacy#%d - ", s.instance)
+	s.logError(log+msg, v...)
+}
+
 func (s *UserPrivacyProcessor) MinimazeQname(qname string) string {
 	if etpo, err := publicsuffix.EffectiveTLDPlusOne(qname); err == nil {
 		return etpo
@@ -57,6 +100,14 @@ func (s *UserPrivacyProcessor) MinimazeQname(qname string) string {
 }
 
 func (s *UserPrivacyProcessor) AnonymizeIP(ip string) string {
+	// if mask is nil, something is wrong
+	if s.v4Mask == nil {
+		return ip
+	}
+	if s.v6Mask == nil {
+		return ip
+	}
+
 	ipaddr := net.ParseIP(ip)
 	isipv4 := strings.LastIndex(ip, ".")
 
@@ -70,7 +121,20 @@ func (s *UserPrivacyProcessor) AnonymizeIP(ip string) string {
 }
 
 func (s *UserPrivacyProcessor) HashIP(ip string) string {
-	hash := sha1.New()
-	hash.Write([]byte(ip))
-	return fmt.Sprintf("%x", hash.Sum(nil))
+	switch s.config.UserPrivacy.HashIPAlgo {
+	case "sha1":
+		hash := sha1.New()
+		hash.Write([]byte(ip))
+		return fmt.Sprintf("%x", hash.Sum(nil))
+	case "sha256":
+		hash := sha256.New()
+		hash.Write([]byte(ip))
+		return fmt.Sprintf("%x", hash.Sum(nil))
+	case "sha512":
+		hash := sha512.New()
+		hash.Write([]byte(ip))
+		return fmt.Sprintf("%x", hash.Sum(nil))
+	default:
+		return ip
+	}
 }

transformers/userprivacy_test.go

@@ -8,7 +8,12 @@ import (
 	"github.com/dmachard/go-logger"
 )
 
-func TestReduceQname(t *testing.T) {
+var (
+	TestIP4 = "192.168.1.2"
+	TestIP6 = "fe80::6111:626:c1b2:2353"
+)
+
+func TestUserPrivacy_ReduceQname(t *testing.T) {
 	// enable feature
 	config := pkgconfig.GetFakeConfigTransformers()
 	config.UserPrivacy.Enable = true
@@ -39,27 +44,62 @@ func TestReduceQname(t *testing.T) {
 	}
 }
 
-func TestAnonymizeIPv4(t *testing.T) {
+func TestUserPrivacy_HashIPDefault(t *testing.T) {
 	// enable feature
 	config := pkgconfig.GetFakeConfigTransformers()
 	config.UserPrivacy.Enable = true
-	config.UserPrivacy.AnonymizeIP = true
+	config.UserPrivacy.HashIP = true
+
+	log := logger.New(false)
+	outChans := []chan dnsutils.DNSMessage{}
+
+	// init the processor
+	userPrivacy := NewUserPrivacySubprocessor(config, logger.New(false), "test", 0, outChans, log.Info, log.Error)
+
+	ret := userPrivacy.HashIP(TestIP4)
+	if ret != "c0ca1efec6aaf505e943397662c28f89ac8f3bc2" {
+		t.Errorf("IP hashing failed, got %s", ret)
+	}
+}
+
+func TestUserPrivacy_HashIPSha512(t *testing.T) {
+	// enable feature
+	config := pkgconfig.GetFakeConfigTransformers()
+	config.UserPrivacy.Enable = true
+	config.UserPrivacy.HashIP = true
+	config.UserPrivacy.HashIPAlgo = "sha512"
 
 	log := logger.New(false)
 	outChans := []chan dnsutils.DNSMessage{}
 
 	// init the processor
 	userPrivacy := NewUserPrivacySubprocessor(config, logger.New(false), "test", 0, outChans, log.Info, log.Error)
 
-	ip := "192.168.1.2"
+	ret := userPrivacy.HashIP(TestIP4)
+	if ret != "800e8f97a29404b7031dfb8d7185b2d30a3cd326b535cda3dcec20a0f4749b1099f98e49245d67eb188091adfba9a45dc0c15e612b554ae7181d8f8a479b67a0" {
+		t.Errorf("IP hashing failed, got %s", ret)
+	}
+}
+
+func TestUserPrivacy_AnonymizeIPv4DefaultMask(t *testing.T) {
+	// enable feature
+	config := pkgconfig.GetFakeConfigTransformers()
+	config.UserPrivacy.Enable = true
+	config.UserPrivacy.AnonymizeIP = true
 
-	ret := userPrivacy.AnonymizeIP(ip)
+	log := logger.New(false)
+	outChans := []chan dnsutils.DNSMessage{}
+
+	// init the processor
+	userPrivacy := NewUserPrivacySubprocessor(config, logger.New(false), "test", 0, outChans, log.Info, log.Error)
+
+	ret := userPrivacy.AnonymizeIP(TestIP4)
 	if ret != "192.168.0.0" {
 		t.Errorf("Ipv4 anonymization failed, got %s", ret)
 	}
 }
 
-func TestAnonymizeIPv6(t *testing.T) {
+func TestUserPrivacy_AnonymizeIPv6DefaultMask(t *testing.T) {
 	// enable feature
 	config := pkgconfig.GetFakeConfigTransformers()
 	config.UserPrivacy.Enable = true
@@ -71,10 +111,46 @@ func TestAnonymizeIPv6(t *testing.T) {
 	// init the processor
 	userPrivacy := NewUserPrivacySubprocessor(config, logger.New(false), "test", 0, outChans, log.Info, log.Error)
 
-	ip := "fe80::6111:626:c1b2:2353"
-
-	ret := userPrivacy.AnonymizeIP(ip)
+	ret := userPrivacy.AnonymizeIP(TestIP6)
 	if ret != "fe80::" {
 		t.Errorf("Ipv6 anonymization failed, got %s", ret)
 	}
 }
+
+func TestUserPrivacy_AnonymizeIPv4RemoveIP(t *testing.T) {
+	// enable feature
+	config := pkgconfig.GetFakeConfigTransformers()
+	config.UserPrivacy.Enable = true
+	config.UserPrivacy.AnonymizeIP = true
+	config.UserPrivacy.AnonymizeIPV4Bits = "/0"
+
+	log := logger.New(false)
+	outChans := []chan dnsutils.DNSMessage{}
+
+	// init the processor
+	userPrivacy := NewUserPrivacySubprocessor(config, logger.New(false), "test", 0, outChans, log.Info, log.Error)
+
+	ret := userPrivacy.AnonymizeIP(TestIP4)
+	if ret != "0.0.0.0" {
+		t.Errorf("Ipv4 anonymization failed with mask %s, got %s", config.UserPrivacy.AnonymizeIPV4Bits, ret)
+	}
+}
+
+func TestUserPrivacy_AnonymizeIPv6RemoveIP(t *testing.T) {
+	// enable feature
+	config := pkgconfig.GetFakeConfigTransformers()
+	config.UserPrivacy.Enable = true
+	config.UserPrivacy.AnonymizeIP = true
+	config.UserPrivacy.AnonymizeIPV6Bits = "::/0"
+
+	log := logger.New(false)
+	outChans := []chan dnsutils.DNSMessage{}
+
+	// init the processor
+	userPrivacy := NewUserPrivacySubprocessor(config, logger.New(false), "test", 0, outChans, log.Info, log.Error)
+
+	ret := userPrivacy.AnonymizeIP(TestIP6)
+	if ret != "::" {
+		t.Errorf("Ipv6 anonymization failed, got %s", ret)
+	}
+}