[Zeek] Add additional data sets

packages/zeek/_dev/build/docs/README.md

@@ -110,6 +110,12 @@ contains kerberos data.
 
 {{fields "kerberos"}}
 
+### known_certs
+
+The `known_certs` dataset captures information about SSL/TLS certificates seen on the local network.
+
+{{fields "known_certs"}}
+
 ### known_hosts
 
 The `known_hosts` dataset captures information about SSL/TLS certificates seen on the local network.

packages/zeek/data_stream/known_hosts/_dev/test/pipeline/test-common-config.yml

@@ -0,0 +1,4 @@
+fields:
+  "@timestamp": "2020-04-28T11:07:58.223Z"
+  tags:
+    - preserve_original_event

packages/zeek/data_stream/known_hosts/_dev/test/pipeline/test-known_hosts.log

@@ -0,0 +1,5 @@
+{"ts":"2021-01-03T01:19:26.260073Z","host":"192.168.4.25"}
+{"ts":"2021-01-03T01:19:27.353353Z","host":"192.168.4.29"}
+{"ts":"2021-01-03T01:19:32.488179Z","host":"192.168.4.43"}
+{"ts":"2021-01-03T01:19:58.792683Z","host":"192.168.4.142"}
+{"ts":"2021-01-03T12:17:22.496004Z","host":"192.168.4.115"}

packages/zeek/data_stream/known_hosts/_dev/test/pipeline/test-known_hosts.log-expected.json

@@ -0,0 +1,134 @@
+{
+    "expected": [
+        {
+            "@timestamp": "2021-01-03T01:19:26.260Z",
+            "ecs": {
+                "version": "8.2.0"
+            },
+            "event": {
+                "category": "network",
+                "created": "2020-04-28T11:07:58.223Z",
+                "kind": "info",
+                "original": "{\"ts\":\"2021-01-03T01:19:26.260073Z\",\"host\":\"192.168.4.25\"}"
+            },
+            "host": {
+                "ip": "192.168.4.25"
+            },
+            "network": {
+                "type": "ipv4"
+            },
+            "related": {
+                "ip": [
+                    "192.168.4.25"
+                ]
+            },
+            "tags": [
+                "preserve_original_event"
+            ]
+        },
+        {
+            "@timestamp": "2021-01-03T01:19:27.353Z",
+            "ecs": {
+                "version": "8.2.0"
+            },
+            "event": {
+                "category": "network",
+                "created": "2020-04-28T11:07:58.223Z",
+                "kind": "info",
+                "original": "{\"ts\":\"2021-01-03T01:19:27.353353Z\",\"host\":\"192.168.4.29\"}"
+            },
+            "host": {
+                "ip": "192.168.4.29"
+            },
+            "network": {
+                "type": "ipv4"
+            },
+            "related": {
+                "ip": [
+                    "192.168.4.29"
+                ]
+            },
+            "tags": [
+                "preserve_original_event"
+            ]
+        },
+        {
+            "@timestamp": "2021-01-03T01:19:32.488Z",
+            "ecs": {
+                "version": "8.2.0"
+            },
+            "event": {
+                "category": "network",
+                "created": "2020-04-28T11:07:58.223Z",
+                "kind": "info",
+                "original": "{\"ts\":\"2021-01-03T01:19:32.488179Z\",\"host\":\"192.168.4.43\"}"
+            },
+            "host": {
+                "ip": "192.168.4.43"
+            },
+            "network": {
+                "type": "ipv4"
+            },
+            "related": {
+                "ip": [
+                    "192.168.4.43"
+                ]
+            },
+            "tags": [
+                "preserve_original_event"
+            ]
+        },
+        {
+            "@timestamp": "2021-01-03T01:19:58.792Z",
+            "ecs": {
+                "version": "8.2.0"
+            },
+            "event": {
+                "category": "network",
+                "created": "2020-04-28T11:07:58.223Z",
+                "kind": "info",
+                "original": "{\"ts\":\"2021-01-03T01:19:58.792683Z\",\"host\":\"192.168.4.142\"}"
+            },
+            "host": {
+                "ip": "192.168.4.142"
+            },
+            "network": {
+                "type": "ipv4"
+            },
+            "related": {
+                "ip": [
+                    "192.168.4.142"
+                ]
+            },
+            "tags": [
+                "preserve_original_event"
+            ]
+        },
+        {
+            "@timestamp": "2021-01-03T12:17:22.496Z",
+            "ecs": {
+                "version": "8.2.0"
+            },
+            "event": {
+                "category": "network",
+                "created": "2020-04-28T11:07:58.223Z",
+                "kind": "info",
+                "original": "{\"ts\":\"2021-01-03T12:17:22.496004Z\",\"host\":\"192.168.4.115\"}"
+            },
+            "host": {
+                "ip": "192.168.4.115"
+            },
+            "network": {
+                "type": "ipv4"
+            },
+            "related": {
+                "ip": [
+                    "192.168.4.115"
+                ]
+            },
+            "tags": [
+                "preserve_original_event"
+            ]
+        }
+    ]
+}

packages/zeek/data_stream/known_hosts/_dev/test/system/test-logs-config.yml

@@ -0,0 +1,6 @@
+vars:
+  base_paths:
+    - "{{SERVICE_LOGS_DIR}}"
+input: logfile
+data_stream:
+  vars: ~

packages/zeek/data_stream/known_hosts/agent/stream/log.yml.hbs

@@ -0,0 +1,21 @@
+paths:
+{{#each base_paths}}
+  {{#each ../filenames}}
+ - {{../this}}/{{this}}
+  {{/each}}
+{{/each}}
+exclude_files: [".gz$"]
+tags:
+{{#if preserve_original_event}}
+  - preserve_original_event
+{{/if}}
+{{#each tags as |tag i|}}
+  - {{tag}}
+{{/each}}
+{{#contains "forwarded" tags}}
+publisher_pipeline.disable_host: true
+{{/contains}}
+{{#if processors}}
+processors:
+{{processors}}
+{{/if}}

packages/zeek/data_stream/known_hosts/elasticsearch/ingest_pipeline/default.yml

@@ -0,0 +1,65 @@
+---
+description: Pipeline for normalizing Zeek conn.log
+processors:
+  - rename:
+      field: message
+      target_field: event.original
+  - json:
+      field: event.original
+      target_field: json
+  - drop:
+      description: Drop if no timestamp (invalid json)
+      if: 'ctx?.json?.ts == null'
+
+# Sets event.created from the @timestamp field generated by filebeat before being overwritten further down
+  - set:
+      field: event.created
+      copy_from: "@timestamp"
+  - set:
+      field: ecs.version
+      value: '8.2.0'
+  - set:
+      field: event.kind
+      value: info
+  - set:
+      field: event.category
+      value: network
+  - date:
+      field: json.ts
+      formats:
+        - UNIX
+        - ISO8601
+  - rename:
+      field: json.host
+      target_field: host.ip
+      ignore_missing: true
+  - set:
+      field: network.type
+      value: ipv4
+      if: ctx.host?.ip.contains('.')
+  - set:
+      field: network.type
+      value: ipv6
+      if: ctx.host?.ip.contains(':')
+  - append:
+      field: related.ip
+      value: "{{host.ip}}"
+      if: ctx?.host?.ip != null
+      allow_duplicates: false
+  - geoip:
+      field: host.ip
+      target_field: host.geo
+      ignore_missing: true
+  - remove:
+      field: 
+        - json
+      ignore_missing: true
+  - remove:
+      field: event.original
+      if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
+      ignore_failure: true
+      ignore_missing: true
+on_failure:
+  - set:
+      field: error.message
+      value: "{{ _ingest.on_failure_message }}"