fix dns parser: ignore too short decode error if reply is truncated

README.md

@@ -7,7 +7,7 @@
 
 *NOTE: The code before version 1.x is considered beta quality and is subject to breaking changes.*
 
-`DNS-collector` acts as a passive high speed **aggregator, analyzer, transporter and logging** for your DNS messages, written in **Golang**. The DNS traffic can be collected and aggregated from simultaneously [sources](doc/collectors.md) like DNStap streams, network interface or log files and relays it to multiple other [listeners](doc/loggers.md) with some [transformations](doc/transformers.md) on it ([`Traffic filtering`](doc/transformers.md#dns-filtering), [`User Privacy`](doc/transformers.md#user-privacy), ...) and DNS protocol conversions to ([`Plain Text`](doc/configuration.md#custom-text-format), [`Json`](doc/dnsjson.md), and more... ).
+`DNS-collector` acts as a passive high speed **aggregator, analyzer, transporter and logging** for your DNS messages, written in **Golang**. The DNS traffic can be collected and aggregated from simultaneously [sources](doc/collectors.md) like DNStap streams, network interface or log files and relays it to multiple other [listeners](doc/loggers.md) with some [transformations](doc/transformers.md) on it ([traffic filtering](doc/transformers.md#dns-filtering), [user privacy](doc/transformers.md#user-privacy), ...) and DNS protocol conversions (to [plain text](doc/configuration.md#custom-text-format), [json](doc/dnsjson.md), and more... ).
 
 Additionally, DNS-collector also contains DNS parser with [`EDNS`](doc/dnsparser.md) support.
 
@@ -86,20 +86,33 @@ See the full [configuration guide](doc/configuration.md) for more details.
 
 You will find below some examples of configuration to manage your DNS logs.
 
-- [Capture DNSTap stream and backup-it to text and pcap files](example-config/use-case-1.yml)
-- [Observe DNS metrics with Prometheus and Grafana](example-config/use-case-2.yml)
-- [Transform DNSTap to JSON format](example-config/use-case-3.yml)
-- [Follow DNS traffic with Loki and Grafana](example-config/use-case-4.yml)
-- [Read from UNIX DNSTap socket and forward it to TLS stream](example-config/use-case-5.yml)
-- [Capture DNSTap stream and apply user privacy on it](example-config/use-case-6.yml)
-- [Aggregate several DNSTap stream and forward it to the same file](example-config/use-case-7.yml)
-- [Multiple PowerDNS collectors](example-config/use-case-8.yml)
-- [Filtering incoming traffic with downsample and whitelist of domains](example-config/use-case-9.yml)
-- [Transform all domains to lowercase](example-config/use-case-10.yml)
-- [Add geographical metadata with GeoIP](example-config/use-case-11.yml)
-- [Relays DNSTap stream to multiple remote destination without decoding](example-config/use-case-12.yml)
-- [Save incoming DNStap streams to file (frstrm)](example-config/use-case-13.yml)
-- [Watch for DNStap files as input](example-config/use-case-14.yml)
+
+- Capture DNS traffic with incoming DNSTap streams
+    - [x] [Read from UNIX DNSTap socket and forward it to TLS stream](example-config/use-case-5.yml)
+    - [x] [Transform DNSTap as input to JSON format as output](example-config/use-case-3.yml)
+    - [x] [Relays DNSTap stream to multiple remote destination without decoding](example-config/use-case-12.yml)
+    - [x] [Aggregate several DNSTap stream and forward it to the same file](example-config/use-case-7.yml)
+
+- Capture DNS traffic with PowerDNS
+    - [x] [Capture multiple PowerDNS streams](example-config/use-case-8.yml)
+
+- Observe your DNS traffic from logs
+    - [x] [Observe DNS metrics with Prometheus and Grafana](example-config/use-case-2.yml)
+    - [x] [Follow DNS traffic with Loki and Grafana](example-config/use-case-4.yml)
+
+- Apply some transformations
+    - [x] [Capture DNSTap stream and apply user privacy on it](example-config/use-case-6.yml)
+    - [x] [Filtering incoming traffic with downsample and whitelist of domains](example-config/use-case-9.yml)
+    - [x] [Transform all domains to lowercase](example-config/use-case-10.yml)
+    - [x] [Add geographical metadata with GeoIP](example-config/use-case-11.yml)
+
+- Capture DNS traffic from FRSTRM/dnstap files
+    - [x] [Save incoming DNStap streams to file (frstrm)](example-config/use-case-13.yml)
+    - [x] [Watch for DNStap files as input](example-config/use-case-14.yml)
+
+- Capture DNS traffic from PCAP files
+    - [x] [Capture DNSTap stream and backup-it to text and pcap files](example-config/use-case-1.yml)
+    - [x] [Watch for PCAP files as input and JSON as output](example-config/use-case-15.yml)
 
 ## Contributing
 

collectors/file_ingestor.go

@@ -86,7 +86,9 @@ func (c *FileIngestor) ReadConfig() {
 	c.identity = c.config.GetServerIdentity()
 	c.dnsPort = c.config.Collectors.FileIngestor.PcapDnsPort
 
-	c.LogInfo("watching directory to find [%s] files", c.config.Collectors.FileIngestor.WatchMode)
+	c.LogInfo("watching directory [%s] to find [%s] files",
+		c.config.Collectors.FileIngestor.WatchDir,
+		c.config.Collectors.FileIngestor.WatchMode)
 }
 
 func (c *FileIngestor) LogInfo(msg string, v ...interface{}) {
@@ -356,6 +358,7 @@ func (c *FileIngestor) Run() {
 		// prepare filepath
 		fn := filepath.Join(c.config.Collectors.FileIngestor.WatchDir, entry.Name())
 
+		fmt.Println(c.config.Collectors.FileIngestor.WatchMode)
 		switch c.config.Collectors.FileIngestor.WatchMode {
 		case dnsutils.MODE_PCAP:
 			// process file with pcap extension

config.yml

@@ -69,7 +69,7 @@ multiplexer:
   loggers:
     - name: console
       stdout:
-        mode: json
+        mode: text
 
   routes:
     - from: [ tap ]

dnsutils/dns.go

@@ -271,26 +271,18 @@ func DecodePayload(dm *DnsMessage, header *DnsHeader, config *Config) error {
 		}
 
 		dm.DNS.Qname = dns_qname
-
-		/*	if config.Subprocessors.Normalize.QnameLowerCase {
-					dm.DNS.Qname = strings.ToLower(dns_qname)
-				} else {
-					dm.DNS.Qname = dns_qname
-				}
-
-			// Public suffix
-			ps, _ := publicsuffix.PublicSuffix(dm.DNS.Qname)
-			dm.DNS.QnamePublicSuffix = ps
-			if etpo, err := publicsuffix.EffectiveTLDPlusOne(dm.DNS.Qname); err == nil {
-				dm.DNS.QnameEffectiveTLDPlusOne = etpo
-			}*/
 		dm.DNS.Qtype = RdatatypeToString(dns_rrtype)
 		payload_offset = offsetrr
 	}
 
 	// decode DNS answers
 	if header.Ancount > 0 {
-		if answers, offset, err := DecodeAnswer(header.Ancount, payload_offset, dm.DNS.Payload); err == nil {
+		answers, offset, err := DecodeAnswer(header.Ancount, payload_offset, dm.DNS.Payload)
+		if err == nil {
+			dm.DNS.DnsRRs.Answers = answers
+			payload_offset = offset
+		} else if dm.DNS.Flags.TC && (errors.Is(err, ErrDecodeDnsAnswerTooShort) || errors.Is(err, ErrDecodeDnsAnswerRdataTooShort) || errors.Is(err, ErrDecodeDnsLabelTooShort)) {
+			dm.DNS.MalformedPacket = true
 			dm.DNS.DnsRRs.Answers = answers
 			payload_offset = offset
 		} else {
@@ -304,21 +296,37 @@ func DecodePayload(dm *DnsMessage, header *DnsHeader, config *Config) error {
 		if answers, offsetrr, err := DecodeAnswer(header.Nscount, payload_offset, dm.DNS.Payload); err == nil {
 			dm.DNS.DnsRRs.Nameservers = answers
 			payload_offset = offsetrr
+		} else if dm.DNS.Flags.TC && (errors.Is(err, ErrDecodeDnsAnswerTooShort) || errors.Is(err, ErrDecodeDnsAnswerRdataTooShort) || errors.Is(err, ErrDecodeDnsLabelTooShort)) {
+			dm.DNS.MalformedPacket = true
+			dm.DNS.DnsRRs.Nameservers = answers
+			payload_offset = offsetrr
 		} else {
 			dm.DNS.MalformedPacket = true
 			return &decodingError{part: "authority records", err: err}
 		}
 	}
 	if header.Arcount > 0 {
 		// decode additional answers
-		if answers, _, err := DecodeAnswer(header.Arcount, payload_offset, dm.DNS.Payload); err == nil {
+		answers, _, err := DecodeAnswer(header.Arcount, payload_offset, dm.DNS.Payload)
+		if err == nil {
+			dm.DNS.DnsRRs.Records = answers
+		} else if dm.DNS.Flags.TC && (errors.Is(err, ErrDecodeDnsAnswerTooShort) || errors.Is(err, ErrDecodeDnsAnswerRdataTooShort) || errors.Is(err, ErrDecodeDnsLabelTooShort)) {
+			dm.DNS.MalformedPacket = true
 			dm.DNS.DnsRRs.Records = answers
 		} else {
 			dm.DNS.MalformedPacket = true
 			return &decodingError{part: "additional records", err: err}
 		}
 		// decode EDNS options, if there are any
-		if edns, _, err := DecodeEDNS(header.Arcount, payload_offset, dm.DNS.Payload); err == nil {
+		edns, _, err := DecodeEDNS(header.Arcount, payload_offset, dm.DNS.Payload)
+		if err == nil {
+			dm.EDNS = edns
+		} else if dm.DNS.Flags.TC && (errors.Is(err, ErrDecodeDnsAnswerTooShort) ||
+			errors.Is(err, ErrDecodeDnsAnswerRdataTooShort) ||
+			errors.Is(err, ErrDecodeDnsLabelTooShort) ||
+			errors.Is(err, ErrDecodeEdnsDataTooShort) ||
+			errors.Is(err, ErrDecodeEdnsOptionTooShort)) {
+			dm.DNS.MalformedPacket = true
 			dm.EDNS = edns
 		} else {
 			dm.DNS.MalformedPacket = true

dnsutils/dns_test.go

@@ -2114,3 +2114,159 @@ func TestDecodePayload_AdditionalRRAndEDNS(t *testing.T) {
 	}
 
 }
+
+func TestDecodePayload_Truncated(t *testing.T) {
+	payload := []byte{
+		// header
+		0x77, 0xa0, 0x83, 0x80, 0x00, 0x01, 0x00, 0x23,
+		0x00, 0x00, 0x00, 0x00,
+		// query
+		0x02, 0x41, 0x64, 0x0d,
+		0x6e, 0x6e, 0x6e, 0x6e,
+		0x6e, 0x6e, 0x6e, 0x6e,
+		0x6e, 0x6e, 0x6e, 0x6e,
+		0x6e, 0x02, 0x46, 0x52,
+		0x00, 0x00, 0x01, 0x00, 0x01,
+		// answer 1
+		0xc0, 0x0c, 0x00, 0x01, 0x00, 0x01, 0x00,
+		0x00, 0x00, 0x6b, 0x00, 0x04, 0x0a, 0x01,
+		0x01, 0x01,
+		// answer 2
+		0xc0, 0x0c, 0x00, 0x01, 0x00, 0x01, 0x00,
+		0x00, 0x00, 0x6b, 0x00, 0x04, 0x0a, 0x01,
+		0x01, 0x02,
+		// answer 3
+		0xc0, 0x0c, 0x00, 0x01, 0x00, 0x01, 0x00,
+		0x00, 0x00, 0x6b, 0x00, 0x04, 0x0a, 0x01,
+		0x01, 0x03,
+		// answer 4
+		0xc0, 0x0c, 0x00, 0x01, 0x00, 0x01, 0x00,
+		0x00, 0x00, 0x6b, 0x00, 0x04, 0x0a, 0x01,
+		0x01, 0x04,
+		// answer 5
+		0xc0, 0x0c, 0x00, 0x01, 0x00, 0x01, 0x00,
+		0x00, 0x00, 0x6b, 0x00, 0x04, 0x0a, 0x01,
+		0x01, 0x05,
+		// answer 6
+		0xc0, 0x0c, 0x00, 0x01, 0x00, 0x01, 0x00,
+		0x00, 0x00, 0x6b, 0x00, 0x04, 0x0a, 0x01,
+		0x01, 0x06,
+		// answer 7
+		0xc0, 0x0c, 0x00, 0x01, 0x00, 0x01, 0x00,
+		0x00, 0x00, 0x6b, 0x00, 0x04, 0x0a, 0x01,
+		0x01, 0x07,
+		// answer 8
+		0xc0, 0x0c, 0x00, 0x01, 0x00, 0x01, 0x00,
+		0x00, 0x00, 0x6b, 0x00, 0x04, 0x0a, 0x01,
+		0x01, 0x08,
+		// answer 9
+		0xc0, 0x0c, 0x00, 0x01, 0x00, 0x01, 0x00,
+		0x00, 0x00, 0x6b, 0x00, 0x04, 0x0a, 0x01,
+		0x01, 0x09,
+		// answer 10
+		0xc0, 0x0c, 0x00, 0x01, 0x00, 0x01, 0x00,
+		0x00, 0x00, 0x6b, 0x00, 0x04, 0x0a, 0x01,
+		0x01, 0x0a,
+		// answer 11
+		0xc0, 0x0c, 0x00, 0x01, 0x00, 0x01, 0x00,
+		0x00, 0x00, 0x6b, 0x00, 0x04, 0x0a, 0x01,
+		0x01, 0x0b,
+		// answer 12
+		0xc0, 0x0c, 0x00, 0x01, 0x00, 0x01, 0x00,
+		0x00, 0x00, 0x6b, 0x00, 0x04, 0x0a, 0x01,
+		0x01, 0x0c,
+		// answer 13
+		0xc0, 0x0c, 0x00, 0x01, 0x00, 0x01, 0x00,
+		0x00, 0x00, 0x6b, 0x00, 0x04, 0x0a, 0x01,
+		0x01, 0x0d,
+		// answer 14
+		0xc0, 0x0c, 0x00, 0x01, 0x00, 0x01, 0x00,
+		0x00, 0x00, 0x6b, 0x00, 0x04, 0x0a, 0x01,
+		0x01, 0x0e,
+		// answer 15
+		0xc0, 0x0c, 0x00, 0x01, 0x00, 0x01, 0x00,
+		0x00, 0x00, 0x6b, 0x00, 0x04, 0x0a, 0x01,
+		0x01, 0x0f,
+		// answer 16
+		0xc0, 0x0c, 0x00, 0x01, 0x00, 0x01, 0x00,
+		0x00, 0x00, 0x6b, 0x00, 0x04, 0x0a, 0x01,
+		0x01, 0x10,
+		// answer 17
+		0xc0, 0x0c, 0x00, 0x01, 0x00, 0x01, 0x00,
+		0x00, 0x00, 0x6b, 0x00, 0x04, 0x0a, 0x01,
+		0x01, 0x11,
+		// answer 18
+		0xc0, 0x0c, 0x00, 0x01, 0x00, 0x01, 0x00,
+		0x00, 0x00, 0x6b, 0x00, 0x04, 0x0a, 0x01,
+		0x01, 0x12,
+		// answer 19
+		0xc0, 0x0c, 0x00, 0x01, 0x00, 0x01, 0x00,
+		0x00, 0x00, 0x6b, 0x00, 0x04, 0x0a, 0x01,
+		0x01, 0x13,
+		// answer 20
+		0xc0, 0x0c, 0x00, 0x01, 0x00, 0x01, 0x00,
+		0x00, 0x00, 0x6b, 0x00, 0x04, 0x0a, 0x01,
+		0x01, 0x14,
+		// answer 21
+		0xc0, 0x0c, 0x00, 0x01, 0x00, 0x01, 0x00,
+		0x00, 0x00, 0x6b, 0x00, 0x04, 0x0a, 0x01,
+		0x01, 0x15,
+		// answer 22
+		0xc0, 0x0c, 0x00, 0x01, 0x00, 0x01, 0x00,
+		0x00, 0x00, 0x6b, 0x00, 0x04, 0x0a, 0x01,
+		0x01, 0x16,
+		// answer 23
+		0xc0, 0x0c, 0x00, 0x01, 0x00, 0x01, 0x00,
+		0x00, 0x00, 0x6b, 0x00, 0x04, 0x0a, 0x01,
+		0x01, 0x17,
+		// answer 24
+		0xc0, 0x0c, 0x00, 0x01, 0x00, 0x01, 0x00,
+		0x00, 0x00, 0x6b, 0x00, 0x04, 0x0a, 0x01,
+		0x01, 0x18,
+		// answer 25
+		0xc0, 0x0c, 0x00, 0x01, 0x00, 0x01, 0x00,
+		0x00, 0x00, 0x6b, 0x00, 0x04, 0x0a, 0x01,
+		0x01, 0x19,
+		// answer 26
+		0xc0, 0x0c, 0x00, 0x01, 0x00, 0x01, 0x00,
+		0x00, 0x00, 0x6b, 0x00, 0x04, 0x0a, 0x01,
+		0x01, 0x1a,
+		// answer 27
+		0xc0, 0x0c, 0x00, 0x01, 0x00, 0x01, 0x00,
+		0x00, 0x00, 0x6b, 0x00, 0x04, 0x0a, 0x01,
+		0x01, 0x1b,
+		// answer 28
+		0xc0, 0x0c, 0x00, 0x01, 0x00, 0x01, 0x00,
+		0x00, 0x00, 0x6b, 0x00, 0x04, 0x0a, 0x01,
+		0x01, 0x1c,
+		// answer 29
+		0xc0, 0x0c, 0x00, 0x01, 0x00, 0x01, 0x00,
+		0x00, 0x00, 0x6b, 0x00, 0x04, 0x0a, 0x01,
+		0x01, 0x1d,
+		// answer 30
+		0xc0, 0x0c, 0x00, 0x01, 0x00, 0x01, 0x00,
+		0x00, 0x00, 0x6b, 0x00,
+	}
+
+	dm := DnsMessage{}
+	dm.DNS.Payload = payload
+	dm.DNS.Length = len(payload)
+
+	header, err := DecodeDns(payload)
+	if err != nil {
+		t.Errorf("unexpected error when decoding header: %v", err)
+	}
+
+	if err = DecodePayload(&dm, &header, GetFakeConfig()); err != nil {
+		t.Error("expected no error on decode")
+	}
+
+	if dm.DNS.Flags.TC == false {
+		t.Error("truncated answer expected")
+	}
+
+	if dm.DNS.MalformedPacket != true {
+		t.Errorf("expected packet to be malformed")
+	}
+
+}

example-config/use-case-15.yml

@@ -0,0 +1,27 @@
+# Example 15: Watch for PCAP files as input and convert to JSON
+
+# If turned on, debug messages are printed in the standard output
+global:
+  trace:
+    verbose: true
+
+multiplexer:
+  # Watch in /tmp folder to find pcap files with pcap or pcap.gz extension
+  collectors:
+    - name: pcap
+      file-ingestor:
+        watch-dir: /home/denis/Lab/pcap/
+        watch-mode: pcap
+      transforms:
+        normalize:
+          qname-lowercase: true
+
+  # Redirect output to the console
+  loggers:
+    - name: console
+      stdout:
+        mode: json
+
+  routes:
+    - from: [ pcap ]
+      to: [ console ]