[Zeek] Add additional data sets

packages/zeek/_dev/build/docs/README.md

@@ -122,6 +122,12 @@ The `known_hosts` dataset captures information about SSL/TLS certificates seen o
 
 {{fields "known_hosts"}}
 
+### known_services
+
+The `known_services` dataset captures information about SSL/TLS certificates seen on the local network.
+
+{{fields "known_services"}}
+
 ### modbus
 
 The `modbus` dataset collects the Zeek modbus.log file, which contains

packages/zeek/_dev/deploy/docker/sample_logs/known_services.log

@@ -0,0 +1 @@
+{"ts":"2021-01-03T01:19:36.242774Z","host":"192.168.4.1","port_num":53,"port_proto":"udp","service":["DNS"]}

packages/zeek/data_stream/known_certs/elasticsearch/ingest_pipeline/default.yml

@@ -1,5 +1,5 @@
 ---
-description: Pipeline for normalizing Zeek conn.log
+description: Pipeline for normalizing Zeek known_certs.log
 processors:
   - rename:
       field: message

packages/zeek/data_stream/known_hosts/elasticsearch/ingest_pipeline/default.yml

@@ -1,5 +1,5 @@
 ---
-description: Pipeline for normalizing Zeek conn.log
+description: Pipeline for normalizing Zeek known_hosts.log
 processors:
   - rename:
       field: message

packages/zeek/data_stream/known_services/_dev/test/pipeline/test-common-config.yml

@@ -0,0 +1,4 @@
+fields:
+  "@timestamp": "2020-04-28T11:07:58.223Z"
+  tags:
+    - preserve_original_event

packages/zeek/data_stream/known_services/_dev/test/pipeline/test-known_services.log

@@ -0,0 +1 @@
+{"ts":"2021-01-03T01:19:36.242774Z","host":"192.168.4.1","port_num":53,"port_proto":"udp","service":["DNS"]}

packages/zeek/data_stream/known_services/_dev/test/pipeline/test-known_services.log-expected.json

@@ -0,0 +1,38 @@
+{
+    "expected": [
+        {
+            "@timestamp": "2021-01-03T01:19:36.242Z",
+            "ecs": {
+                "version": "8.2.0"
+            },
+            "event": {
+                "category": "network",
+                "created": "2020-04-28T11:07:58.223Z",
+                "kind": "info",
+                "original": "{\"ts\":\"2021-01-03T01:19:36.242774Z\",\"host\":\"192.168.4.1\",\"port_num\":53,\"port_proto\":\"udp\",\"service\":[\"DNS\"]}"
+            },
+            "host": {
+                "ip": "192.168.4.1"
+            },
+            "network": {
+                "application": [
+                    "DNS"
+                ],
+                "transport": "udp",
+                "type": "ipv4"
+            },
+            "related": {
+                "ip": [
+                    "192.168.4.1"
+                ]
+            },
+            "server": {
+                "ip": "192.168.4.1",
+                "port": 53
+            },
+            "tags": [
+                "preserve_original_event"
+            ]
+        }
+    ]
+}

packages/zeek/data_stream/known_services/_dev/test/system/test-logs-config.yml

@@ -0,0 +1,6 @@
+vars:
+  base_paths:
+    - "{{SERVICE_LOGS_DIR}}"
+input: logfile
+data_stream:
+  vars: ~

packages/zeek/data_stream/known_services/agent/stream/log.yml.hbs

@@ -0,0 +1,21 @@
+paths:
+{{#each base_paths}}
+  {{#each ../filenames}}
+ - {{../this}}/{{this}}
+  {{/each}}
+{{/each}}
+exclude_files: [".gz$"]
+tags:
+{{#if preserve_original_event}}
+  - preserve_original_event
+{{/if}}
+{{#each tags as |tag i|}}
+  - {{tag}}
+{{/each}}
+{{#contains "forwarded" tags}}
+publisher_pipeline.disable_host: true
+{{/contains}}
+{{#if processors}}
+processors:
+{{processors}}
+{{/if}}

packages/zeek/data_stream/known_services/elasticsearch/ingest_pipeline/default.yml

@@ -0,0 +1,97 @@
+---
+description: Pipeline for normalizing Zeek known_services.log
+processors:
+  - rename:
+      field: message
+      target_field: event.original
+  - json:
+      field: event.original
+      target_field: json
+  - drop:
+      description: Drop if no timestamp (invalid json)
+      if: 'ctx?.json?.ts == null'
+
+# Sets event.created from the @timestamp field generated by filebeat before being overwritten further down
+  - set:
+      field: event.created
+      copy_from: "@timestamp"
+  - set:
+      field: ecs.version
+      value: '8.2.0'
+  - set:
+      field: event.kind
+      value: info
+  - set:
+      field: event.category
+      value: network
+  - date:
+      field: json.ts
+      formats:
+        - UNIX
+        - ISO8601
+  - rename:
+      field: json.host
+      target_field: host.ip
+      ignore_missing: true
+  - set:
+      field: network.type
+      value: ipv4
+      if: ctx.host?.ip.contains('.')
+  - set:
+      field: network.type
+      value: ipv6
+      if: ctx.host?.ip.contains(':')
+  - append:
+      field: related.ip
+      value: "{{host.ip}}"
+      if: ctx?.host?.ip != null
+      allow_duplicates: false
+  - geoip:
+      field: host.ip
+      target_field: host.geo
+      ignore_missing: true
+  - set:
+      field: server
+      copy_from: host
+      ignore_empty_value: true
+  - rename:
+      field: json.port_num
+      target_field: server.port
+      ignore_missing: true
+  - geoip:
+      database_file: GeoLite2-ASN.mmdb
+      field: server.ip
+      target_field: server.as
+      properties:
+        - asn
+        - organization_name
+      ignore_missing: true
+  - rename:
+      field: server.as.asn
+      target_field: server.as.number
+      ignore_missing: true
+  - rename:
+      field: server.as.organization_name
+      target_field: server.as.organization.name
+      ignore_missing: true
+  - rename:
+      field: json.port_proto
+      target_field: network.transport
+      ignore_missing: true
+  - rename:
+      field: json.service
+      target_field: network.application
+      ignore_missing: true
+  - remove:
+      field: 
+        - json
+      ignore_missing: true
+  - remove:
+      field: event.original
+      if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
+      ignore_failure: true
+      ignore_missing: true
+on_failure:
+  - set:
+      field: error.message
+      value: "{{ _ingest.on_failure_message }}"