Parse DNS requests over TCP (#524)

Signed-off-by: Daxin Wang <[email protected]>
KindlingProject · May 31, 2023 · 6f31118 · 6f31118
1 parent be8e6e5
commit 6f31118
Show file tree

Hide file tree

Showing 11 changed files with 144 additions and 47 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -10,6 +10,7 @@
 - MySQL CommandLine Case: Ignore quit command and get sql with CLIENT_QUERY_ATTRIBUTES([#523](https://github.com/KindlingProject/kindling/pull/523))
 - ⚠️Breaking change: Refactor the data format of on/off CPU events from "string" to "array". Note that the old data format cannot be parsed using the new version of the front-end.([#512](https://github.com/KindlingProject/kindling/pull/512) [#520](https://github.com/KindlingProject/kindling/pull/520))
 ### Bug fixes
+- Fix the bug where the DNS domain is not obtained when DNS transport over TCP. ([#524](https://github.com/KindlingProject/kindling/pull/524))
 - Fix panic: send on closed channel. ([#519](https://github.com/KindlingProject/kindling/pull/519))
 - Fix the bug that the event detail panel doesn't hide when switching profiles.（[#513](https://github.com/KindlingProject/kindling/pull/513)）
 - Fix span data deduplication issue.（[#511](https://github.com/KindlingProject/kindling/pull/511)）

diff --git a/collector/pkg/component/analyzer/network/network_analyzer.go b/collector/pkg/component/analyzer/network/network_analyzer.go
@@ -442,7 +442,7 @@ func (na *NetworkAnalyzer) parseProtocol(mps *messagePairs, parser *protocol.Pro
 	}
 
 	// Mergable Data
-	requestMsg := protocol.NewRequestMessage(mps.requests.getData())
+	requestMsg := protocol.NewRequestMessage(mps.requests.getData(), mps.requests.event.Ctx.FdInfo.GetProtocol())
 	if !parser.ParseRequest(requestMsg) {
 		// Parse failure
 		return nil
@@ -455,7 +455,7 @@ func (na *NetworkAnalyzer) parseProtocol(mps *messagePairs, parser *protocol.Pro
 		return na.getRecords(mps, parser.GetProtocol(), requestMsg.GetAttributes())
 	}
 
-	responseMsg := protocol.NewResponseMessage(mps.responses.getData(), requestMsg.GetAttributes())
+	responseMsg := protocol.NewResponseMessage(mps.responses.getData(), requestMsg.GetAttributes(), mps.responses.event.Ctx.FdInfo.GetProtocol())
 	if !parser.ParseResponse(responseMsg) {
 		// Parse failure
 		return nil
@@ -471,7 +471,7 @@ func (na *NetworkAnalyzer) parseMultipleRequests(mps *messagePairs, parser *prot
 	parsedReqMsgs := make([]*protocol.PayloadMessage, size)
 	for i := 0; i < size; i++ {
 		req := mps.requests.getEvent(i)
-		requestMsg := protocol.NewRequestMessage(req.GetData())
+		requestMsg := protocol.NewRequestMessage(req.GetData(), mps.requests.event.Ctx.FdInfo.GetProtocol())
 		if !parser.ParseRequest(requestMsg) {
 			// Parse failure
 			return nil
@@ -497,12 +497,12 @@ func (na *NetworkAnalyzer) parseMultipleRequests(mps *messagePairs, parser *prot
 		size := mps.responses.size()
 		for i := 0; i < size; i++ {
 			resp := mps.responses.getEvent(i)
-			responseMsg := protocol.NewResponseMessage(resp.GetData(), model.NewAttributeMap())
+			responseMsg := protocol.NewResponseMessage(resp.GetData(), model.NewAttributeMap(), mps.responses.event.Ctx.FdInfo.GetProtocol())
 			if !parser.ParseResponse(responseMsg) {
 				// Parse failure
 				return nil
 			}
-			// Match Request with repsone
+			// Match Request with response
 			matchIdx := parser.PairMatch(parsedReqMsgs, responseMsg)
 			if matchIdx == -1 {
 				return nil

diff --git a/collector/pkg/component/analyzer/network/network_analyzer_test.go b/collector/pkg/component/analyzer/network/network_analyzer_test.go
@@ -49,6 +49,8 @@ func TestDnsProtocol(t *testing.T) {
 		"dns/server-trace-multi.yml")
 	testProtocol(t, "dns/client-event.yml",
 		"dns/client-trace-sendmmg.yml")
+	testProtocol(t, "dns/client-event-tcp.yml",
+		"dns/client-trace-tcp.yml")
 }
 
 func TestKafkaProtocol(t *testing.T) {

diff --git a/collector/pkg/component/analyzer/network/protocol/dns/dns_request.go b/collector/pkg/component/analyzer/network/protocol/dns/dns_request.go
@@ -2,6 +2,7 @@ package dns
 
 import (
 	"github.com/Kindling-project/kindling/collector/pkg/component/analyzer/network/protocol"
+	"github.com/Kindling-project/kindling/collector/pkg/model"
 	"github.com/Kindling-project/kindling/collector/pkg/model/constlabels"
 )
 
@@ -11,24 +12,29 @@ func fastfailDnsRequest() protocol.FastFailFn {
 	}
 }
 
-/**
-    0  1  2  3  4  5  6  7  8  9  A  B  C  D  E  F
-  +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
-  |                      ID                       |
-  +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
-  |QR|   Opcode  |AA|TC|RD|RA|   Z    |   RCODE   |
-  +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
-  |                    QDCOUNT                    |
-  +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
-  |                    ANCOUNT                    |
-  +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
-  |                    NSCOUNT                    |
-  +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
-  |                    ARCOUNT                    |
-  +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
+/*
+Header format
+
+	  0  1  2  3  4  5  6  7  8  9  A  B  C  D  E  F
+	+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
+	|                      ID                       |
+	+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
+	|QR|   Opcode  |AA|TC|RD|RA|   Z    |   RCODE   |
+	+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
+	|                    QDCOUNT                    |
+	+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
+	|                    ANCOUNT                    |
+	+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
+	|                    NSCOUNT                    |
+	+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
+	|                    ARCOUNT                    |
+	+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
 */
 func parseDnsRequest() protocol.ParsePkgFn {
 	return func(message *protocol.PayloadMessage) (bool, bool) {
+		if message.Protocol == model.L4Proto_TCP {
+			message.Offset += 2
+		}
 		offset := message.Offset
 		id, _ := message.ReadUInt16(offset)
 		flags, _ := message.ReadUInt16(offset + 2)

diff --git a/collector/pkg/component/analyzer/network/protocol/dns/dns_response.go b/collector/pkg/component/analyzer/network/protocol/dns/dns_response.go
@@ -5,6 +5,7 @@ import (
 	"strings"
 
 	"github.com/Kindling-project/kindling/collector/pkg/component/analyzer/network/protocol"
+	"github.com/Kindling-project/kindling/collector/pkg/model"
 	"github.com/Kindling-project/kindling/collector/pkg/model/constlabels"
 )
 
@@ -19,25 +20,29 @@ func fastfailDnsResponse() protocol.FastFailFn {
 	}
 }
 
-/**
-Header
-   0  1  2  3  4  5  6  7  8  9  A  B  C  D  E  F
- +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
- |                      ID                       |
- +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
- |QR|   Opcode  |AA|TC|RD|RA|   Z    |   RCODE   |
- +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
- |                    QDCOUNT                    |
- +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
- |                    ANCOUNT                    |
- +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
- |                    NSCOUNT                    |
- +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
- |                    ARCOUNT                    |
- +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
+/*
+Header format
+
+	  0  1  2  3  4  5  6  7  8  9  A  B  C  D  E  F
+	+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
+	|                      ID                       |
+	+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
+	|QR|   Opcode  |AA|TC|RD|RA|   Z    |   RCODE   |
+	+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
+	|                    QDCOUNT                    |
+	+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
+	|                    ANCOUNT                    |
+	+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
+	|                    NSCOUNT                    |
+	+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
+	|                    ARCOUNT                    |
+	+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
 */
 func parseDnsResponse() protocol.ParsePkgFn {
 	return func(message *protocol.PayloadMessage) (bool, bool) {
+		if message.Protocol == model.L4Proto_TCP {
+			message.Offset += 2
+		}
 		offset := message.Offset
 		id, _ := message.ReadUInt16(offset)
 		flags, _ := message.ReadUInt16(offset + 2)

diff --git a/collector/pkg/component/analyzer/network/protocol/http/http_parser_test.go b/collector/pkg/component/analyzer/network/protocol/http/http_parser_test.go
@@ -5,6 +5,7 @@ import (
 	"testing"
 
 	"github.com/Kindling-project/kindling/collector/pkg/component/analyzer/network/protocol"
+	"github.com/Kindling-project/kindling/collector/pkg/model"
 )
 
 func Test_urlMerge(t *testing.T) {
@@ -93,7 +94,7 @@ func Test_parseHeaders(t *testing.T) {
 		{
 			name: "normal case",
 			args: args{
-				message: protocol.NewRequestMessage([]byte("HTTP/1.1 200 OK\r\nConnection: keep-alive\r\nAPM-AgentID: TTXvC3EQS6KLwxx3eIqINFjAW2olRm+cr8M+yuvwhkY=\r\nTransfer-Encoding: chunked\r\nContent-Type: application/json\r\nAPM-TransactionID: 5e480579c718a4a6498a9")),
+				message: protocol.NewRequestMessage([]byte("HTTP/1.1 200 OK\r\nConnection: keep-alive\r\nAPM-AgentID: TTXvC3EQS6KLwxx3eIqINFjAW2olRm+cr8M+yuvwhkY=\r\nTransfer-Encoding: chunked\r\nContent-Type: application/json\r\nAPM-TransactionID: 5e480579c718a4a6498a9"), model.L4Proto_TCP),
 			},
 			want: map[string]string{
 				"connection":        "keep-alive",
@@ -106,7 +107,7 @@ func Test_parseHeaders(t *testing.T) {
 		{
 			name: "no values",
 			args: args{
-				protocol.NewRequestMessage([]byte("HTTP/1.1 200 OK\r\nConnection: keep-alive\r\nTransfer-Encoding: ")),
+				protocol.NewRequestMessage([]byte("HTTP/1.1 200 OK\r\nConnection: keep-alive\r\nTransfer-Encoding: "), model.L4Proto_TCP),
 			},
 			want: map[string]string{
 				"connection": "keep-alive",
@@ -116,7 +117,7 @@ func Test_parseHeaders(t *testing.T) {
 
 			name: "no spaces",
 			args: args{
-				protocol.NewRequestMessage([]byte("HTTP/1.1 200 OK\r\nConnection: keep-alive\r\nTransfer-Encoding:")),
+				protocol.NewRequestMessage([]byte("HTTP/1.1 200 OK\r\nConnection: keep-alive\r\nTransfer-Encoding:"), model.L4Proto_TCP),
 			},
 			want: map[string]string{
 				"connection": "keep-alive",
@@ -125,7 +126,7 @@ func Test_parseHeaders(t *testing.T) {
 		{
 			name: "no colon",
 			args: args{
-				protocol.NewRequestMessage([]byte("HTTP/1.1 200 OK\r\nConnection: keep-alive\r\nTransfer-Encoding")),
+				protocol.NewRequestMessage([]byte("HTTP/1.1 200 OK\r\nConnection: keep-alive\r\nTransfer-Encoding"), model.L4Proto_TCP),
 			},
 			want: map[string]string{
 				"connection": "keep-alive",

diff --git a/collector/pkg/component/analyzer/network/protocol/protocol_parser.go b/collector/pkg/component/analyzer/network/protocol/protocol_parser.go
@@ -31,21 +31,24 @@ var (
 type PayloadMessage struct {
 	Data         []byte
 	Offset       int
+	Protocol     model.L4Proto
 	attributeMap *model.AttributeMap
 }
 
-func NewRequestMessage(data []byte) *PayloadMessage {
+func NewRequestMessage(data []byte, protocol model.L4Proto) *PayloadMessage {
 	return &PayloadMessage{
 		Data:         data,
 		Offset:       0,
+		Protocol:     protocol,
 		attributeMap: model.NewAttributeMap(),
 	}
 }
 
-func NewResponseMessage(data []byte, attributeMap *model.AttributeMap) *PayloadMessage {
+func NewResponseMessage(data []byte, attributeMap *model.AttributeMap, protocol model.L4Proto) *PayloadMessage {
 	return &PayloadMessage{
 		Data:         data,
 		Offset:       0,
+		Protocol:     protocol,
 		attributeMap: attributeMap,
 	}
 }

diff --git a/collector/pkg/component/analyzer/network/protocol/protocol_parser_test.go b/collector/pkg/component/analyzer/network/protocol/protocol_parser_test.go
@@ -4,12 +4,14 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+
+	"github.com/Kindling-project/kindling/collector/pkg/model"
 )
 
 func TestReadBytes(t *testing.T) {
 	// ff 0 4 t e s t
 	data := []byte{0xff, 0x00, 0x04, 0x74, 0x65, 0x73, 0x74}
-	message := NewRequestMessage(data)
+	message := NewRequestMessage(data, model.L4Proto_TCP)
 
 	tests := []struct {
 		name   string
@@ -41,7 +43,7 @@ func TestReadBytes(t *testing.T) {
 func TestReadInt16(t *testing.T) {
 	// ff 0 4 t e s t
 	data := []byte{0xff, 0x00, 0x04, 0x74, 0x65, 0x73, 0x74}
-	message := NewRequestMessage(data)
+	message := NewRequestMessage(data, model.L4Proto_TCP)
 
 	tests := []struct {
 		name   string
@@ -71,7 +73,7 @@ func TestReadInt16(t *testing.T) {
 func TestReadInt32(t *testing.T) {
 	// ff 0 0 0 4 t e s t
 	data := []byte{0xff, 0x00, 0x00, 0x00, 0x04, 0x74, 0x65, 0x73, 0x74}
-	message := NewRequestMessage(data)
+	message := NewRequestMessage(data, model.L4Proto_TCP)
 
 	tests := []struct {
 		name   string
@@ -101,7 +103,7 @@ func TestReadInt32(t *testing.T) {
 func TestReadNullableString(t *testing.T) {
 	// ff 0 4 t e s t
 	data := []byte{0xff, 0x00, 0x04, 0x74, 0x65, 0x73, 0x74}
-	message := NewRequestMessage(data)
+	message := NewRequestMessage(data, model.L4Proto_TCP)
 
 	tests := []struct {
 		name   string

diff --git a/collector/pkg/component/analyzer/network/protocol/rocketmq/rocketmq_request_test.go b/collector/pkg/component/analyzer/network/protocol/rocketmq/rocketmq_request_test.go
@@ -5,6 +5,7 @@ import (
 	"time"
 
 	"github.com/Kindling-project/kindling/collector/pkg/component/analyzer/network/protocol"
+	"github.com/Kindling-project/kindling/collector/pkg/model"
 )
 
 func Test_parseHeader(t *testing.T) {
@@ -23,7 +24,7 @@ func Test_parseHeader(t *testing.T) {
 					1, 23, 4, 20, 123, 213, 4, 2, 34, 12, 23, 1, 23, 4, 20, 123,
 					213, 4, 2, 34, 12, 0, 0, 0, 0, 20, 123, 213, 4, 254, 34, 12,
 					23, 1, 23, 4, 20, 123, 213, 4, 2, 34, 12, 23, 1, 23, 4, 20,
-					123, 213, 4, 2, 34, 12, 23}),
+					123, 213, 4, 2, 34, 12, 23}, model.L4Proto_TCP),
 				header: &rocketmqHeader{ExtFields: map[string]string{}},
 			},
 		},

diff --git a/collector/pkg/component/analyzer/network/protocol/testdata/dns/client-event-tcp.yml b/collector/pkg/component/analyzer/network/protocol/testdata/dns/client-event-tcp.yml
@@ -0,0 +1,24 @@
+eventCommon:
+  # SYSCALL_EXIT
+  source: 2
+  # CAT_NET
+  category: 3
+  ctx:
+    thread_info:
+      pid: 577
+      tid: 577
+      uid: 101
+      gid: 103
+      comm: "systemd-resolve"
+    fd_info:
+      num: 12
+      # FD_IPV4_SOCK
+      type_fd: 3
+      # TCP
+      protocol: 1
+      # IsServer
+      role: false
+      sip: [16777343]
+      sport: 60129
+      dip: [889192575]
+      dport: 53
diff --git a/collector/pkg/component/analyzer/network/protocol/testdata/dns/client-trace-tcp.yml b/collector/pkg/component/analyzer/network/protocol/testdata/dns/client-trace-tcp.yml
@@ -0,0 +1,52 @@
+trace:
+  key: tcp_not_found
+  requests:
+    - name: "sendmsg"
+      timestamp: 100000000
+      user_attributes:
+        latency: 5000
+        res: 61
+        data:
+          - "hex|003b32f60100000100000000000013616c6572746d616e616765722d6d61696e2d3115616c6572746d616e616765722d6f706572617465640000ff0001"
+  responses:
+    - name: "recvfrom"
+      timestamp: 101000000
+      user_attributes:
+        latency: 20000
+        res: 136
+        data:
+          - "hex|008632f68583000100000001000013616c6572746d616e616765722d6d61696e2d3115616c6572746d616e616765722d6f706572617465640000ff0001000006000100000001004001610c726f6f742d73657276657273036e657400056e73746c640c766572697369676e2d67727303636f6d00789556c8000007080000038400093a8000015180"
+
+  expects:
+    - Timestamp: 99995000
+      Values:
+        request_total_time: 1005000
+        connect_time: 0
+        request_sent_time: 5000
+        waiting_ttfb_time: 980000
+        content_download_time: 20000
+        request_io: 61
+        response_io: 136
+      Labels:
+        comm: "systemd-resolve"
+        pid: 577
+        request_tid: 577
+        response_tid: 577
+        src_ip: "127.0.0.1"
+        src_port: 60129
+        dst_ip: "127.0.0.53"
+        dst_port: 53
+        dnat_ip: ""
+        dnat_port: -1
+        container_id: ""
+        is_slow: false
+        is_server: false
+        protocol: "dns"
+        dns_rcode: 3
+        dns_id: 13046
+        dns_domain: "alertmanager-main-1.alertmanager-operated."
+        is_error: true
+        error_type: 3
+        end_timestamp: 101000000
+        request_payload: ".;2............alertmanager-main-1.alertmanager-operated....."
+        response_payload: "..2............alertmanager-main-1.alertmanager-operated...............@.a.root-servers.net..nstld.verisign-grs.com.x.V...........:...Q."