[Filebeat][suricata] Map x509 for suricata/eve fileset (#20973) (#21018)

* Map x509 for suricata/eve fileset * Fix not_before condition and bump ecs version (cherry picked from commit 70d6bde)
elastic · Sep 10, 2020 · a914041 · a914041
1 parent 1337c65
commit a914041
Show file tree

Hide file tree

Showing 7 changed files with 160 additions and 22 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -641,6 +641,7 @@ field. You can revert this change by configuring tags for the module and omittin
 - Improve Zeek Kerberos module with `x509` ECS mappings {pull}20958[20958]
 - Improve Fortinet firewall module with `x509` ECS mappings {pull}20983[20983]
 - Improve Santa module with `x509` ECS mappings {pull}20976[20976]
+- Improve Suricata Eve module with `x509` ECS mappings {pull}20973[20973]
 
 *Heartbeat*
 

diff --git a/x-pack/filebeat/module/suricata/eve/config/eve.yml b/x-pack/filebeat/module/suricata/eve/config/eve.yml
@@ -384,8 +384,6 @@ processors:
         - {from: suricata.eve.tls.fingerprint, to: tls.server.hash.sha1}
         - {from: suricata.eve.tls.sni, to: tls.client.server_name}
         - {from: suricata.eve.tls.sni, to: destination.domain}
-        - {from: suricata.eve.tls.notbefore, to: tls.server.not_before}
-        - {from: suricata.eve.tls.notafter, to: tls.server.not_after}
         - {from: suricata.eve.tls.ja3s.hash, to: tls.server.ja3s}
         - {from: suricata.eve.tls.ja3.hash, to: tls.client.ja3}
         - {from: suricata.eve.tls.certificate, to: tls.server.certificate}
@@ -404,4 +402,4 @@ processors:
   - add_fields:
       target: ''
       fields:
-        ecs.version: 1.5.0
+        ecs.version: 1.6.0
diff --git a/x-pack/filebeat/module/suricata/eve/ingest/pipeline.yml b/x-pack/filebeat/module/suricata/eve/ingest/pipeline.yml
@@ -234,13 +234,114 @@ processors:
       field: related.hash
       value: "{{tls.server.hash.sha1}}"
       if: "ctx?.tls?.server?.hash?.sha1 != null"
+  - gsub:
+      field: suricata.eve.tls.issuerdn
+      pattern: \\,
+      replacement: ""
+      ignore_missing: true
+  - kv:
+      field: suricata.eve.tls.issuerdn
+      field_split: ', '
+      value_split: '='
+      target_field: suricata.eve.tls.kv_issuerdn
+      ignore_missing: true
+  - rename:
+      field: suricata.eve.tls.kv_issuerdn.C
+      target_field: file.x509.issuer.country
+      ignore_missing: true
+  - rename:
+      field: suricata.eve.tls.kv_issuerdn.CN
+      target_field: file.x509.issuer.common_name
+      ignore_missing: true
+  - rename:
+      field: suricata.eve.tls.kv_issuerdn.L
+      target_field: file.x509.issuer.locality
+      ignore_missing: true
+  - rename:
+      field: suricata.eve.tls.kv_issuerdn.O
+      target_field: file.x509.issuer.organization
+      ignore_missing: true
+  - rename:
+      field: suricata.eve.tls.kv_issuerdn.OU
+      target_field: file.x509.issuer.organizational_unit
+      ignore_missing: true
+  - rename:
+      field: suricata.eve.tls.kv_issuerdn.ST
+      target_field: file.x509.issuer.state_or_province
+      ignore_missing: true
+  - gsub:
+      field: suricata.eve.tls.subject
+      pattern: \\,
+      replacement: ""
+      ignore_missing: true
+  - kv:
+      field: suricata.eve.tls.subject
+      field_split: ', '
+      value_split: '='
+      target_field: suricata.eve.tls.kv_subject
+      ignore_missing: true
+  - rename:
+      field: suricata.eve.tls.kv_subject.C
+      target_field: file.x509.subject.country
+      ignore_missing: true
+  - rename:
+      field: suricata.eve.tls.kv_subject.CN
+      target_field: file.x509.subject.common_name
+      ignore_missing: true
+  - rename:
+      field: suricata.eve.tls.kv_subject.L
+      target_field: file.x509.subject.locality
+      ignore_missing: true
+  - rename:
+      field: suricata.eve.tls.kv_subject.O
+      target_field: file.x509.subject.organization
+      ignore_missing: true
+  - rename:
+      field: suricata.eve.tls.kv_subject.OU
+      target_field: file.x509.subject.organizational_unit
+      ignore_missing: true
+  - rename:
+      field: suricata.eve.tls.kv_subject.ST
+      target_field: file.x509.subject.state_or_province
+      ignore_missing: true
+  - set:
+      field: file.x509.serial_number
+      value: '{{suricata.eve.tls.serial}}'
+      ignore_empty_value: true
+  - gsub:
+      field: file.x509.serial_number
+      pattern: ':'
+      replacement: ''
+      ignore_missing: true
+  - date:
+      field: suricata.eve.tls.notafter
+      target_field: tls.server.not_after
+      formats:
+        - ISO8601
+      if: ctx.suricata?.eve?.tls?.notafter != null
+  - date:
+      field: suricata.eve.tls.notbefore
+      target_field: tls.server.not_before
+      formats:
+        - ISO8601
+      if: ctx.suricata?.eve?.tls?.notbefore != null
+  - set:
+      field: file.x509.not_after
+      value: '{{tls.server.not_after}}'
+      ignore_empty_value: true
+  - set:
+      field: file.x509.not_before
+      value: '{{tls.server.not_before}}'
+      ignore_empty_value: true
   - remove:
       field:
         - suricata.eve.app_proto
         - suricata.eve.flow.end
         - suricata.eve.flow.start
         - suricata.eve.http.http_method
         - suricata.eve.http.http_user_agent
+        - suricata.eve.tls.kv_issuerdn
+        - suricata.eve.tls.kv_subject
       ignore_missing: true
 on_failure:
   - set:

diff --git a/x-pack/filebeat/module/suricata/eve/test/eve-alerts.log b/x-pack/filebeat/module/suricata/eve/test/eve-alerts.log
@@ -18,5 +18,5 @@
 {"timestamp":"2018-10-04T09:35:00.897009+0000","flow_id":112424506237238,"in_iface":"enp0s3","event_type":"alert","src_ip":"192.168.1.146","src_port":52340,"dest_ip":"91.189.91.23","dest_port":80,"proto":"TCP","tx_id":7,"alert":{"action":"allowed","gid":1,"signature_id":2013504,"rev":5,"signature":"ET POLICY GNU\/Linux APT User-Agent Outbound likely related to package management","category":"Not Suspicious Traffic","severity":3},"http":{"hostname":"archive.ubuntu.com","url":"\/ubuntu\/dists\/bionic-updates\/universe\/binary-amd64\/by-hash\/SHA256\/5190f7afbee38b3cb32225db478fdbabd46f76eaa9c5921a13091891bf3e9bbc","http_user_agent":"Debian APT-HTTP\/1.3 (1.6.3ubuntu0.1)","http_method":"GET","protocol":"HTTP\/1.1","status":200,"length":2687},"app_proto":"http","flow":{"pkts_toserver":330,"pkts_toclient":591,"bytes_toserver":23758,"bytes_toclient":884342,"start":"2018-10-04T09:34:58.926006+0000"}}
 {"timestamp":"2018-10-04T09:35:01.362208+0000","flow_id":112424506237238,"in_iface":"enp0s3","event_type":"alert","src_ip":"192.168.1.146","src_port":52340,"dest_ip":"91.189.91.23","dest_port":80,"proto":"TCP","tx_id":8,"alert":{"action":"allowed","gid":1,"signature_id":2013504,"rev":5,"signature":"ET POLICY GNU\/Linux APT User-Agent Outbound likely related to package management","category":"Not Suspicious Traffic","severity":3},"http":{"hostname":"archive.ubuntu.com","url":"\/ubuntu\/dists\/bionic-updates\/universe\/i18n\/by-hash\/SHA256\/9fe539b7036e51327cd85ca5e0a4dd4eb47f69168875de2ac9842a5e36ebd4a4","http_user_agent":"Debian APT-HTTP\/1.3 (1.6.3ubuntu0.1)","http_method":"GET","protocol":"HTTP\/1.1","length":0},"app_proto":"http","flow":{"pkts_toserver":524,"pkts_toclient":979,"bytes_toserver":36819,"bytes_toclient":1467603,"start":"2018-10-04T09:34:58.926006+0000"}}
 {"timestamp":"2018-10-04T09:35:01.575088+0000","flow_id":112424506237238,"in_iface":"enp0s3","event_type":"alert","src_ip":"192.168.1.146","src_port":52340,"dest_ip":"91.189.91.23","dest_port":80,"proto":"TCP","tx_id":9,"alert":{"action":"allowed","gid":1,"signature_id":2013504,"rev":5,"signature":"ET POLICY GNU\/Linux APT User-Agent Outbound likely related to package management","category":"Not Suspicious Traffic","severity":3},"http":{"hostname":"archive.ubuntu.com","url":"\/ubuntu\/dists\/bionic-updates\/multiverse\/binary-amd64\/by-hash\/SHA256\/8ab8cb220c0e50521c589acc2bc2b43a3121210f0b035a0605972bcffd73dd16","http_user_agent":"Debian APT-HTTP\/1.3 (1.6.3ubuntu0.1)","http_method":"GET","protocol":"HTTP\/1.1","length":0},"app_proto":"http","flow":{"pkts_toserver":575,"pkts_toclient":1079,"bytes_toserver":40452,"bytes_toclient":1618380,"start":"2018-10-04T09:34:58.926006+0000"}}
-{"tls":{"ja3s":{"string":"333,55555,66666-22","hash":"0993626a07ad09e1ce91293be7aa5721"},"ja3":{"string":"001,22222-33333-00-44444-66666-333-333-55555-55555-22-55555-44444-22-33-66666-77777-22-88888-99999-333-22-66666-77777-22-99999-96611-22-33-88888-33333-88888-333-22222-33333-333-222-88888-444-99999-22222-666-777-888,22-33-44-55-6,77-88-99-0-11-11-22-33-44-55,0","hash":"d92325c876e7279f4eb8c62415e3a6b7"},"notafter":"2024-07-16T14:52:35","notbefore":"2019-07-17T14:52:35","version":"TLS 1.2","sni":"hostname.domain.net","fingerprint":"00:11:22:33:44:55:66:77:88:99:aa:bb:cc:dd:ee:ff:00:11:22:33","serial":"00:11:22:33:44:55:66:77:88","issuerdn":"CN=UNKNOWN/DC=UNKNOWN/DC=UNKNOWN/C=UNKNOWN/ST=UNKNOWN/O=UNK-NOWN/OU=UNKNOWN","subject":"C=UNKNOWN, ST=UNKNOWN, L=UNKNOWN, O=UNKNOWN, OU=UNKNOWN, CN=hostname.domain.net/[email protected]"},"proto":"TCP","dest_port":9080,"dest_ip":"10.232.0.237","src_port":45884,"src_ip":"10.126.2.140","event_type":"tls","in_iface":"enp5s0","flow_id":1091813059495729,"timestamp":"2018-10-04T09:35:02.796615+0000"}
+{"tls":{"ja3s":{"string":"333,55555,66666-22","hash":"0993626a07ad09e1ce91293be7aa5721"},"ja3":{"string":"001,22222-33333-00-44444-66666-333-333-55555-55555-22-55555-44444-22-33-66666-77777-22-88888-99999-333-22-66666-77777-22-99999-96611-22-33-88888-33333-88888-333-22222-33333-333-222-88888-444-99999-22222-666-777-888,22-33-44-55-6,77-88-99-0-11-11-22-33-44-55,0","hash":"d92325c876e7279f4eb8c62415e3a6b7"},"notafter":"2024-07-16T14:52:35","notbefore":"2019-07-17T14:52:35","version":"TLS 1.2","sni":"hostname.domain.net","fingerprint":"00:11:22:33:44:55:66:77:88:99:aa:bb:cc:dd:ee:ff:00:11:22:33","serial":"00:11:22:33:44:55:66:77:88","issuerdn":"C=US, O=Google Inc, CN=Google Internet Authority G2","subject":"C=US, ST=California, L=Mountain View, O=Google Inc, CN=*.google.com"},"proto":"TCP","dest_port":9080,"dest_ip":"10.232.0.237","src_port":45884,"src_ip":"10.126.2.140","event_type":"tls","in_iface":"enp5s0","flow_id":1091813059495729,"timestamp":"2018-10-04T09:35:02.796615+0000"}
 {"flow":{"start":"2020-06-26T11:00:02.970011-0400","bytes_toclient":4660,"bytes_toserver":1074,"pkts_toclient":8,"pkts_toserver":7},"app_proto":"tls","tls":{"ja3s":{"string":"742,48172,30210-30","hash":"391231ba5675e42807b9e1f457b2614e"},"ja3":{"string":"718,4682-2687-2686-41992-41911-53292-53297-41969-22905-41926-41924-94181-94711-15-23-95-12-11-205,0-33-50-53-6-61-39-23-34-85-81,93-04-52,3-9-3","hash":"3f1ea03f5822e8021b60cc3e4b233181"},"notafter":"2026-06-25T17:36:29","notbefore":"2016-06-27T17:36:29","version":"TLS 1.2","sni":"host.domain.net","fingerprint":"36:3f:ee:2a:1c:fa:de:ad:be:ef:42:99:cf:a9:b0:91:01:eb:a9:cc","serial":"72:A9:2C:51","issuerdn":"C=Unknown, ST=Unknown, L=Unknown, O=Unknown, OU=Unknown, CN=Unknown","subject":"C=Unknown, ST=Unknown, L=Unknown, O=Unknown, OU=Unknown, CN=Unknown"},"alert":{"severity":3,"category":"","signature":"SURICATA TLS on unusual port","rev":1,"signature_id":2610003,"gid":1,"action":"allowed"},"proto":"TCP","dest_port":8443,"dest_ip":"10.128.2.48","src_port":64389,"src_ip":"10.137.3.54","event_type":"alert","in_iface":"enp0s31f6","flow_id":991192778198299,"timestamp":"2020-06-26T11:00:03.342282-0400"}
diff --git a/x-pack/filebeat/module/suricata/eve/test/eve-alerts.log-expected.json b/x-pack/filebeat/module/suricata/eve/test/eve-alerts.log-expected.json
@@ -1549,10 +1549,21 @@
         "event.dataset": "suricata.eve",
         "event.kind": "event",
         "event.module": "suricata",
-        "event.original": "{\"tls\":{\"ja3s\":{\"string\":\"333,55555,66666-22\",\"hash\":\"0993626a07ad09e1ce91293be7aa5721\"},\"ja3\":{\"string\":\"001,22222-33333-00-44444-66666-333-333-55555-55555-22-55555-44444-22-33-66666-77777-22-88888-99999-333-22-66666-77777-22-99999-96611-22-33-88888-33333-88888-333-22222-33333-333-222-88888-444-99999-22222-666-777-888,22-33-44-55-6,77-88-99-0-11-11-22-33-44-55,0\",\"hash\":\"d92325c876e7279f4eb8c62415e3a6b7\"},\"notafter\":\"2024-07-16T14:52:35\",\"notbefore\":\"2019-07-17T14:52:35\",\"version\":\"TLS 1.2\",\"sni\":\"hostname.domain.net\",\"fingerprint\":\"00:11:22:33:44:55:66:77:88:99:aa:bb:cc:dd:ee:ff:00:11:22:33\",\"serial\":\"00:11:22:33:44:55:66:77:88\",\"issuerdn\":\"CN=UNKNOWN/DC=UNKNOWN/DC=UNKNOWN/C=UNKNOWN/ST=UNKNOWN/O=UNK-NOWN/OU=UNKNOWN\",\"subject\":\"C=UNKNOWN, ST=UNKNOWN, L=UNKNOWN, O=UNKNOWN, OU=UNKNOWN, CN=hostname.domain.net/[email protected]\"},\"proto\":\"TCP\",\"dest_port\":9080,\"dest_ip\":\"10.232.0.237\",\"src_port\":45884,\"src_ip\":\"10.126.2.140\",\"event_type\":\"tls\",\"in_iface\":\"enp5s0\",\"flow_id\":1091813059495729,\"timestamp\":\"2018-10-04T09:35:02.796615+0000\"}",
+        "event.original": "{\"tls\":{\"ja3s\":{\"string\":\"333,55555,66666-22\",\"hash\":\"0993626a07ad09e1ce91293be7aa5721\"},\"ja3\":{\"string\":\"001,22222-33333-00-44444-66666-333-333-55555-55555-22-55555-44444-22-33-66666-77777-22-88888-99999-333-22-66666-77777-22-99999-96611-22-33-88888-33333-88888-333-22222-33333-333-222-88888-444-99999-22222-666-777-888,22-33-44-55-6,77-88-99-0-11-11-22-33-44-55,0\",\"hash\":\"d92325c876e7279f4eb8c62415e3a6b7\"},\"notafter\":\"2024-07-16T14:52:35\",\"notbefore\":\"2019-07-17T14:52:35\",\"version\":\"TLS 1.2\",\"sni\":\"hostname.domain.net\",\"fingerprint\":\"00:11:22:33:44:55:66:77:88:99:aa:bb:cc:dd:ee:ff:00:11:22:33\",\"serial\":\"00:11:22:33:44:55:66:77:88\",\"issuerdn\":\"C=US, O=Google Inc, CN=Google Internet Authority G2\",\"subject\":\"C=US, ST=California, L=Mountain View, O=Google Inc, CN=*.google.com\"},\"proto\":\"TCP\",\"dest_port\":9080,\"dest_ip\":\"10.232.0.237\",\"src_port\":45884,\"src_ip\":\"10.126.2.140\",\"event_type\":\"tls\",\"in_iface\":\"enp5s0\",\"flow_id\":1091813059495729,\"timestamp\":\"2018-10-04T09:35:02.796615+0000\"}",
         "event.type": [
             "protocol"
         ],
+        "file.x509.issuer.common_name": "Google Internet Authority G2",
+        "file.x509.issuer.country": "US",
+        "file.x509.issuer.organization": "Google Inc",
+        "file.x509.not_after": "2024-07-16T14:52:35.000Z",
+        "file.x509.not_before": "2019-07-17T14:52:35.000Z",
+        "file.x509.serial_number": "001122334455667788",
+        "file.x509.subject.common_name": "*.google.com",
+        "file.x509.subject.country": "US",
+        "file.x509.subject.locality": "Mountain View",
+        "file.x509.subject.organization": "Google Inc",
+        "file.x509.subject.state_or_province": "California",
         "fileset.name": "eve",
         "input.type": "log",
         "log.offset": 16546,
@@ -1574,7 +1585,7 @@
         "suricata.eve.flow_id": 1091813059495729,
         "suricata.eve.in_iface": "enp5s0",
         "suricata.eve.tls.fingerprint": "00:11:22:33:44:55:66:77:88:99:aa:bb:cc:dd:ee:ff:00:11:22:33",
-        "suricata.eve.tls.issuerdn": "CN=UNKNOWN/DC=UNKNOWN/DC=UNKNOWN/C=UNKNOWN/ST=UNKNOWN/O=UNK-NOWN/OU=UNKNOWN",
+        "suricata.eve.tls.issuerdn": "C=US, O=Google Inc, CN=Google Internet Authority G2",
         "suricata.eve.tls.ja3.hash": "d92325c876e7279f4eb8c62415e3a6b7",
         "suricata.eve.tls.ja3.string": "001,22222-33333-00-44444-66666-333-333-55555-55555-22-55555-44444-22-33-66666-77777-22-88888-99999-333-22-66666-77777-22-99999-96611-22-33-88888-33333-88888-333-22222-33333-333-222-88888-444-99999-22222-666-777-888,22-33-44-55-6,77-88-99-0-11-11-22-33-44-55,0",
         "suricata.eve.tls.ja3s.hash": "0993626a07ad09e1ce91293be7aa5721",
@@ -1583,19 +1594,19 @@
         "suricata.eve.tls.notbefore": "2019-07-17T14:52:35",
         "suricata.eve.tls.serial": "00:11:22:33:44:55:66:77:88",
         "suricata.eve.tls.sni": "hostname.domain.net",
-        "suricata.eve.tls.subject": "C=UNKNOWN, ST=UNKNOWN, L=UNKNOWN, O=UNKNOWN, OU=UNKNOWN, CN=hostname.domain.net/emailAddress=user@domain.com",
+        "suricata.eve.tls.subject": "C=US, ST=California, L=Mountain View, O=Google Inc, CN=*.google.com",
         "suricata.eve.tls.version": "TLS 1.2",
         "tags": [
             "suricata"
         ],
         "tls.client.ja3": "d92325c876e7279f4eb8c62415e3a6b7",
         "tls.client.server_name": "hostname.domain.net",
         "tls.server.hash.sha1": "00112233445566778899AABBCCDDEEFF00112233",
-        "tls.server.issuer": "CN=UNKNOWN/DC=UNKNOWN/DC=UNKNOWN/C=UNKNOWN/ST=UNKNOWN/O=UNK-NOWN/OU=UNKNOWN",
+        "tls.server.issuer": "C=US, O=Google Inc, CN=Google Internet Authority G2",
         "tls.server.ja3s": "0993626a07ad09e1ce91293be7aa5721",
-        "tls.server.not_after": "2024-07-16T14:52:35",
-        "tls.server.not_before": "2019-07-17T14:52:35",
-        "tls.server.subject": "C=UNKNOWN, ST=UNKNOWN, L=UNKNOWN, O=UNKNOWN, OU=UNKNOWN, CN=hostname.domain.net/emailAddress=user@domain.com",
+        "tls.server.not_after": "2024-07-16T14:52:35.000Z",
+        "tls.server.not_before": "2019-07-17T14:52:35.000Z",
+        "tls.server.subject": "C=US, ST=California, L=Mountain View, O=Google Inc, CN=*.google.com",
         "tls.version": "1.2",
         "tls.version_protocol": "tls"
     },
@@ -1620,9 +1631,24 @@
         "event.type": [
             "allowed"
         ],
+        "file.x509.issuer.common_name": "Unknown",
+        "file.x509.issuer.country": "Unknown",
+        "file.x509.issuer.locality": "Unknown",
+        "file.x509.issuer.organization": "Unknown",
+        "file.x509.issuer.organizational_unit": "Unknown",
+        "file.x509.issuer.state_or_province": "Unknown",
+        "file.x509.not_after": "2026-06-25T17:36:29.000Z",
+        "file.x509.not_before": "2016-06-27T17:36:29.000Z",
+        "file.x509.serial_number": "72A92C51",
+        "file.x509.subject.common_name": "Unknown",
+        "file.x509.subject.country": "Unknown",
+        "file.x509.subject.locality": "Unknown",
+        "file.x509.subject.organization": "Unknown",
+        "file.x509.subject.organizational_unit": "Unknown",
+        "file.x509.subject.state_or_province": "Unknown",
         "fileset.name": "eve",
         "input.type": "log",
-        "log.offset": 17606,
+        "log.offset": 17541,
         "message": "",
         "network.bytes": 5734,
         "network.community_id": "1:W6fjhboFUwyEchJ3ELaqSBzDEJE=",
@@ -1672,8 +1698,8 @@
         "tls.server.hash.sha1": "363FEE2A1CFADEADBEEF4299CFA9B09101EBA9CC",
         "tls.server.issuer": "C=Unknown, ST=Unknown, L=Unknown, O=Unknown, OU=Unknown, CN=Unknown",
         "tls.server.ja3s": "391231ba5675e42807b9e1f457b2614e",
-        "tls.server.not_after": "2026-06-25T17:36:29",
-        "tls.server.not_before": "2016-06-27T17:36:29",
+        "tls.server.not_after": "2026-06-25T17:36:29.000Z",
+        "tls.server.not_before": "2016-06-27T17:36:29.000Z",
         "tls.server.subject": "C=Unknown, ST=Unknown, L=Unknown, O=Unknown, OU=Unknown, CN=Unknown",
         "tls.version": "1.2",
         "tls.version_protocol": "tls"