diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 9cde2b1fde8..c0c7e35ab06 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -26,6 +26,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Modify apache/error dataset to follow ECS. {pull}8963[8963] - Rename many `traefik.access.*` fields to map to ECS. {pull}9005[9005] +- Add module zeek. {issue}9931[9931] {pull}10034[10034] - Rename many `kibana.log.*` fields to map to ECS. {pull}9301[9301] *Heartbeat* diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index a096d871733..4137809eb24 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -40,6 +40,7 @@ grouped in the following categories: * <> * <> * <> +* <> -- [[exported-fields-apache]] @@ -12721,3 +12722,654 @@ alias to: source.geo.region_iso_code -- +[[exported-fields-zeek]] +== Zeek fields + +Module for handling logs produced by Zeek/Bro + + + +[float] +== zeek fields + +Fields from Zeek/Bro logs after normalization + + + +*`zeek.session_id`*:: ++ +-- +type: keyword + +-- + +*`zeek.connection.local_orig`*:: ++ +-- +type: boolean + +-- + +*`zeek.connection.local_resp`*:: ++ +-- +type: boolean + +-- + +*`zeek.connection.missed_bytes`*:: ++ +-- +type: long + +-- + +*`zeek.connection.state`*:: ++ +-- +type: keyword + +-- + +*`zeek.connection.history`*:: ++ +-- +type: keyword + +-- + +*`zeek.connection.orig_l2_addr`*:: ++ +-- +type: keyword + +-- + +*`zeek.resp_l2_addr`*:: ++ +-- +type: keyword + +-- + +*`zeek.vlan`*:: ++ +-- +type: keyword + +-- + +*`zeek.inner_vlan`*:: ++ +-- +type: keyword + +-- + +*`zeek.dns.trans_id`*:: ++ +-- +type: integer + +-- + +*`zeek.dns.rtt`*:: ++ +-- +type: double + +-- + +*`zeek.dns.query`*:: ++ +-- +type: keyword + +-- + +*`zeek.dns.qclass`*:: ++ +-- +type: long + +-- + +*`zeek.dns.qclass_name`*:: ++ +-- +type: keyword + +-- + +*`zeek.dns.qtype`*:: ++ +-- +type: long + +-- + +*`zeek.dns.qtype_name`*:: ++ +-- +type: keyword + +-- + +*`zeek.dns.rcode`*:: ++ +-- +type: long + +-- + +*`zeek.dns.rcode_name`*:: ++ +-- +type: keyword + +-- + +*`zeek.dns.AA`*:: ++ +-- +type: boolean + +-- + +*`zeek.dns.TC`*:: ++ +-- +type: boolean + +-- + +*`zeek.dns.RD`*:: ++ +-- +type: boolean + +-- + +*`zeek.dns.RA`*:: ++ +-- +type: boolean + +-- + +*`zeek.dns.answers`*:: ++ +-- +type: keyword + +-- + +*`zeek.dns.TTLs`*:: ++ +-- +type: double + +-- + +*`zeek.dns.rejected`*:: ++ +-- +type: boolean + +-- + +*`zeek.dns.total_answers`*:: ++ +-- +type: integer + +-- + +*`zeek.dns.total_replies`*:: ++ +-- +type: integer + +-- + +*`zeek.dns.saw_query`*:: ++ +-- +type: boolean + +-- + +*`zeek.dns.saw_reply`*:: ++ +-- +type: boolean + +-- + +*`zeek.http.trans_depth`*:: ++ +-- +type: integer + +-- + +*`zeek.http.status_msg`*:: ++ +-- +type: keyword + +-- + +*`zeek.http.info_code`*:: ++ +-- +type: integer + +-- + +*`zeek.http.info_msg`*:: ++ +-- +type: keyword + +-- + +*`zeek.http.filename`*:: ++ +-- +type: keyword + +-- + +*`zeek.http.tags`*:: ++ +-- +type: keyword + +-- + +*`zeek.http.captured_password`*:: ++ +-- +type: boolean + +-- + +*`zeek.http.proxied`*:: ++ +-- +type: keyword + +-- + +*`zeek.http.range_request`*:: ++ +-- +type: boolean + +-- + +*`zeek.http.client_header_names`*:: ++ +-- +type: keyword + +-- + +*`zeek.http.server_header_names`*:: ++ +-- +type: keyword + +-- + +*`zeek.http.orig_fuids`*:: ++ +-- +type: keyword + +-- + +*`zeek.http.orig_mime_types`*:: ++ +-- +type: keyword + +-- + +*`zeek.http.orig_filenames`*:: ++ +-- +type: keyword + +-- + +*`zeek.http.resp_fuids`*:: ++ +-- +type: keyword + +-- + +*`zeek.http.resp_mime_types`*:: ++ +-- +type: keyword + +-- + +*`zeek.http.resp_filenames`*:: ++ +-- +type: keyword + +-- + +*`zeek.http.orig_mime_depth`*:: ++ +-- +type: integer + +-- + +*`zeek.http.resp_mime_depth`*:: ++ +-- +type: integer + +-- + +*`zeek.files.fuid`*:: ++ +-- +type: keyword + +-- + +*`zeek.files.tx_host`*:: ++ +-- +type: ip + +-- + +*`zeek.files.rx_host`*:: ++ +-- +type: ip + +-- + +*`zeek.files.session_ids`*:: ++ +-- +type: keyword + +-- + +*`zeek.files.source`*:: ++ +-- +type: keyword + +-- + +*`zeek.files.depth`*:: ++ +-- +type: long + +-- + +*`zeek.files.analyzers`*:: ++ +-- +type: keyword + +-- + +*`zeek.files.mime_type`*:: ++ +-- +type: keyword + +-- + +*`zeek.files.filename`*:: ++ +-- +type: keyword + +-- + +*`zeek.files.local_orig`*:: ++ +-- +type: boolean + +-- + +*`zeek.files.is_orig`*:: ++ +-- +type: boolean + +-- + +*`zeek.files.duration`*:: ++ +-- +type: double + +-- + +*`zeek.files.seen_bytes`*:: ++ +-- +type: long + +-- + +*`zeek.files.total_bytes`*:: ++ +-- +type: long + +-- + +*`zeek.files.missing_bytes`*:: ++ +-- +type: long + +-- + +*`zeek.files.overflow_bytes`*:: ++ +-- +type: long + +-- + +*`zeek.files.timedout`*:: ++ +-- +type: boolean + +-- + +*`zeek.files.parent_fuid`*:: ++ +-- +type: keyword + +-- + +*`zeek.files.md5`*:: ++ +-- +type: keyword + +-- + +*`zeek.files.sha1`*:: ++ +-- +type: keyword + +-- + +*`zeek.files.sha256`*:: ++ +-- +type: keyword + +-- + +*`zeek.files.extracted`*:: ++ +-- +type: keyword + +-- + +*`zeek.files.extracted_cutoff`*:: ++ +-- +type: boolean + +-- + +*`zeek.files.extracted_size`*:: ++ +-- +type: long + +-- + +*`zeek.files.entropy`*:: ++ +-- +type: double + +-- + +*`zeek.ssl.version`*:: ++ +-- +type: keyword + +-- + +*`zeek.ssl.cipher`*:: ++ +-- +type: keyword + +-- + +*`zeek.ssl.curve`*:: ++ +-- +type: keyword + +-- + +*`zeek.ssl.server_name`*:: ++ +-- +type: keyword + +-- + +*`zeek.ssl.resumed`*:: ++ +-- +type: boolean + +-- + +*`zeek.ssl.next_protocol`*:: ++ +-- +type: keyword + +-- + +*`zeek.ssl.established`*:: ++ +-- +type: boolean + +-- + +*`zeek.ssl.cert_chain`*:: ++ +-- +type: keyword + +-- + +*`zeek.ssl.cert_chain_fuids`*:: ++ +-- +type: keyword + +-- + +*`zeek.ssl.client_cert_chain`*:: ++ +-- +type: keyword + +-- + +*`zeek.ssl.client_cert_chain_fuids`*:: ++ +-- +type: keyword + +-- + +*`zeek.ssl.issuer`*:: ++ +-- +type: keyword + +-- + +*`zeek.ssl.client_issuer`*:: ++ +-- +type: keyword + +-- + +*`zeek.ssl.validation_status`*:: ++ +-- +type: keyword + +-- + +*`zeek.ssl.subject`*:: ++ +-- +type: keyword + +-- + +*`zeek.ssl.client_subject`*:: ++ +-- +type: keyword + +-- + +*`zeek.ssl.last_alert`*:: ++ +-- +type: keyword + +-- + diff --git a/filebeat/docs/images/kibana-zeek.png b/filebeat/docs/images/kibana-zeek.png new file mode 100644 index 00000000000..20aea6164f7 Binary files /dev/null and b/filebeat/docs/images/kibana-zeek.png differ diff --git a/filebeat/docs/modules/zeek.asciidoc b/filebeat/docs/modules/zeek.asciidoc new file mode 100644 index 00000000000..92e284640a5 --- /dev/null +++ b/filebeat/docs/modules/zeek.asciidoc @@ -0,0 +1,43 @@ +//// +This file is generated! See scripts/docs_collector.py +//// + +[[filebeat-module-zeek]] +[role="xpack"] + +:modulename: zeek +:has-dashboards: true + +== Zeek (Bro) Module + +This is a module for Zeek, which used to be called Bro. It parses logs that are in the +https://www.zeek.org/manual/release/logs/index.html[Zeek JSON format]. + +[float] +=== Compatibility + +This module requires the {elasticsearch-plugins}/ingest-geoip.html[ingest-geoip] +and {elasticsearch-plugins}/ingest-user-agent.html[ingest-user-agent] +Elasticsearch plugins. + +This module has been developed against Zeek 2.6.1, but is expected to work +with other versions of Zeek. + +Zeek requires a Unix-like platform, and it currently supports Linux, FreeBSD, and Mac OS X. +Find out how to use Zeek here: https://www.zeek.org/ + +[float] +=== Example dashboard + +This module comes with a sample dashboard. For example: + +[role="screenshot"] +image::./images/kibana-zeek.png[] + + +[float] +=== Fields + +For a description of each field in the module, see the +<> section. + diff --git a/filebeat/docs/modules_list.asciidoc b/filebeat/docs/modules_list.asciidoc index d9cb56fff52..b8712ccdb55 100644 --- a/filebeat/docs/modules_list.asciidoc +++ b/filebeat/docs/modules_list.asciidoc @@ -22,6 +22,7 @@ This file is generated! See scripts/docs_collector.py * <> * <> * <> + * <> -- @@ -46,3 +47,4 @@ include::modules/santa.asciidoc[] include::modules/suricata.asciidoc[] include::modules/system.asciidoc[] include::modules/traefik.asciidoc[] +include::modules/zeek.asciidoc[] diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index 9083e2ba5e5..d79b3b236bd 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -396,6 +396,24 @@ filebeat.modules: # can be added under this section. #input: +#--------------------------------- Zeek Module --------------------------------- +- module: zeek + # All logs + connection: + enabled: true + dns: + enabled: true + http: + enabled: true + files: + enabled: true + ssl: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + #=========================== Filebeat inputs ============================= diff --git a/x-pack/filebeat/include/list.go b/x-pack/filebeat/include/list.go index 718202a1769..529f23b9c1f 100644 --- a/x-pack/filebeat/include/list.go +++ b/x-pack/filebeat/include/list.go @@ -10,4 +10,5 @@ import ( // Import packages that need to register themselves. _ "github.com/elastic/beats/x-pack/filebeat/input/netflow" _ "github.com/elastic/beats/x-pack/filebeat/module/suricata" + _ "github.com/elastic/beats/x-pack/filebeat/module/zeek" ) diff --git a/x-pack/filebeat/module/zeek/README-developer.md b/x-pack/filebeat/module/zeek/README-developer.md new file mode 100644 index 00000000000..a1b431b64a6 --- /dev/null +++ b/x-pack/filebeat/module/zeek/README-developer.md @@ -0,0 +1,70 @@ +# Zeek (Bro) module + +## Caveats + +* Module is to be considered _beta_. + +## Install and Configure Zeek/Bro + +### Install Zeek/Bro (for MacOS with Brew) + +``` +brew install bro +``` + +* Configure it to process network traffic and generate logs. +* Edit `/usr/local/etc/node.cfg` to use the proper network interfaces. +* Edit `/usr/local/etc/network.cfg` to specify local networks accordingly. +* Set `redef LogAscii::use_json=T;` in `/usr/local/share/bro/site/local.bro` to use JSON output. + +### Install Zeek/Bro (for Ubuntu Linux) + +``` +apt install bro +apt install broctl +``` + +* Configure it to process network traffic and generate logs. +* Edit `/etc/bro/node.cfg` to use the proper network interfaces. +* Edit `/etc/bro/network.cfg` to specify local networks accordingly. +* Set `redef LogAscii::use_json=T;` in `/usr/share/bro/site/local.bro` to use JSON output. + +## Start Zeek/Bro + +``` +sudo broctl deploy +``` + +## How to try the module from source + +Clone and build Filebeat + +``` +git clone git@github.com:elastic/beats.git +cd beats/x-pack/filebeat +make mage +mage clean update +mage build +``` + +## Configure Filebeat module and run + +Update filebeat.yml to point to Elasticsearch and Kibana. Setup Filebeat. + +``` +./filebeat setup --modules zeek -e -E setup.dashboards.directory=build/kibana +``` + +Enable the Filebeat zeek module + +``` +./filebeat modules enable zeek +``` + +Start Filebeat + +``` +./filebeat -e +``` + +Now, you should see the Zeek logs and dashboards in Kibana. diff --git a/x-pack/filebeat/module/zeek/README.md b/x-pack/filebeat/module/zeek/README.md new file mode 100644 index 00000000000..44a51dbf456 --- /dev/null +++ b/x-pack/filebeat/module/zeek/README.md @@ -0,0 +1,62 @@ +# Zeek (Bro) module + +## Caveats + +* Module is to be considered _beta_. + +## Install and Configure Zeek/Bro + +### Install Zeek/Bro (for MacOS with Brew) + +``` +brew install bro +``` + +* Configure it to process network traffic and generate logs. +* Edit `/usr/local/etc/node.cfg` to use the proper network interfaces. +* Edit `/usr/local/etc/network.cfg` to specify local networks accordingly. +* Set `redef LogAscii::use_json=T;` in `/usr/local/share/bro/site/local.bro` to use JSON output. + +### Install Zeek/Bro (for Ubuntu Linux) + +``` +apt install bro +apt install broctl +``` + +* Configure it to process network traffic and generate logs. +* Edit `/etc/bro/node.cfg` to use the proper network interfaces. +* Edit `/etc/bro/network.cfg` to specify local networks accordingly. +* Set `redef LogAscii::use_json=T;` in `/usr/share/bro/site/local.bro` to use JSON output. + +## Start Zeek/Bro + +``` +sudo broctl deploy +``` + +## Download and install Filebeat + +Grab the filebeat binary from elastic.co, and install it by following the instructions. + +## Configure Filebeat module and run + +Update filebeat.yml to point to Elasticsearch and Kibana. Setup Filebeat. + +``` +./filebeat setup --modules zeek -e -E setup.dashboards.directory=build/kibana +``` + +Enable the Filebeat zeek module + +``` +./filebeat modules enable zeek +``` + +Start Filebeat + +``` +./filebeat -e +``` + +Now, you should see the Zeek logs and dashboards in Kibana. diff --git a/x-pack/filebeat/module/zeek/_meta/config.yml b/x-pack/filebeat/module/zeek/_meta/config.yml new file mode 100644 index 00000000000..a79fc0456c2 --- /dev/null +++ b/x-pack/filebeat/module/zeek/_meta/config.yml @@ -0,0 +1,16 @@ +- module: zeek + # All logs + connection: + enabled: true + dns: + enabled: true + http: + enabled: true + files: + enabled: true + ssl: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: diff --git a/x-pack/filebeat/module/zeek/_meta/docs.asciidoc b/x-pack/filebeat/module/zeek/_meta/docs.asciidoc new file mode 100644 index 00000000000..59e66c72b9f --- /dev/null +++ b/x-pack/filebeat/module/zeek/_meta/docs.asciidoc @@ -0,0 +1,30 @@ +[role="xpack"] + +:modulename: zeek +:has-dashboards: true + +== Zeek (Bro) Module + +This is a module for Zeek, which used to be called Bro. It parses logs that are in the +https://www.zeek.org/manual/release/logs/index.html[Zeek JSON format]. + +[float] +=== Compatibility + +This module requires the {elasticsearch-plugins}/ingest-geoip.html[ingest-geoip] +and {elasticsearch-plugins}/ingest-user-agent.html[ingest-user-agent] +Elasticsearch plugins. + +This module has been developed against Zeek 2.6.1, but is expected to work +with other versions of Zeek. + +Zeek requires a Unix-like platform, and it currently supports Linux, FreeBSD, and Mac OS X. +Find out how to use Zeek here: https://www.zeek.org/ + +[float] +=== Example dashboard + +This module comes with a sample dashboard. For example: + +[role="screenshot"] +image::./images/kibana-zeek.png[] diff --git a/x-pack/filebeat/module/zeek/_meta/fields.yml b/x-pack/filebeat/module/zeek/_meta/fields.yml new file mode 100644 index 00000000000..60c59f4e75d --- /dev/null +++ b/x-pack/filebeat/module/zeek/_meta/fields.yml @@ -0,0 +1,287 @@ +- key: zeek + title: Zeek + description: > + Module for handling logs produced by Zeek/Bro + fields: + - name: zeek + type: group + description: > + Fields from Zeek/Bro logs after normalization + fields: + - name: session_id + type: keyword + + - name: connection.local_orig + type: boolean + + - name: connection.local_resp + type: boolean + + - name: connection.missed_bytes + type: long + + - name: connection.state + type: keyword + + - name: connection.history + type: keyword + + - name: connection.orig_l2_addr + type: keyword + + - name: resp_l2_addr + type: keyword + + - name: vlan + type: keyword + + - name: inner_vlan + type: keyword + + - name: dns.trans_id + type: integer + + - name: dns.rtt + type: double + + - name: dns.query + type: keyword + + - name: dns.qclass + type: long + + - name: dns.qclass_name + type: keyword + + - name: dns.qtype + type: long + + - name: dns.qtype_name + type: keyword + + - name: dns.rcode + type: long + + - name: dns.rcode_name + type: keyword + + - name: dns.AA + type: boolean + + - name: dns.TC + type: boolean + + - name: dns.RD + type: boolean + + - name: dns.RA + type: boolean + + - name: dns.answers + type: keyword + + - name: dns.TTLs + type: double + + - name: dns.rejected + type: boolean + + - name: dns.total_answers + type: integer + + - name: dns.total_replies + type: integer + + - name: dns.saw_query + type: boolean + + - name: dns.saw_reply + type: boolean + + - name: http.trans_depth + type: integer + + - name: http.status_msg + type: keyword + + - name: http.info_code + type: integer + + - name: http.info_msg + type: keyword + + - name: http.filename + type: keyword + + - name: http.tags + type: keyword + + - name: http.captured_password + type: boolean + + - name: http.proxied + type: keyword + + - name: http.range_request + type: boolean + + - name: http.client_header_names + type: keyword + + - name: http.server_header_names + type: keyword + + - name: http.orig_fuids + type: keyword + + - name: http.orig_mime_types + type: keyword + + - name: http.orig_filenames + type: keyword + + - name: http.resp_fuids + type: keyword + + - name: http.resp_mime_types + type: keyword + + - name: http.resp_filenames + type: keyword + + - name: http.orig_mime_depth + type: integer + + - name: http.resp_mime_depth + type: integer + + - name: files.fuid + type: keyword + + - name: files.tx_host + type: ip + + - name: files.rx_host + type: ip + + - name: files.session_ids + type: keyword + + - name: files.source + type: keyword + + - name: files.depth + type: long + + - names: files.direction + type: keyword + + - name: files.analyzers + type: keyword + + - name: files.mime_type + type: keyword + + - name: files.filename + type: keyword + + - name: files.local_orig + type: boolean + + - name: files.is_orig + type: boolean + + - name: files.duration + type: double + + - name: files.seen_bytes + type: long + + - name: files.total_bytes + type: long + + - name: files.missing_bytes + type: long + + - name: files.overflow_bytes + type: long + + - name: files.timedout + type: boolean + + - name: files.parent_fuid + type: keyword + + - name: files.md5 + type: keyword + + - name: files.sha1 + type: keyword + + - name: files.sha256 + type: keyword + + - name: files.extracted + type: keyword + + - name: files.extracted_cutoff + type: boolean + + - name: files.extracted_size + type: long + + - name: files.entropy + type: double + + - name: ssl.version + type: keyword + + - name: ssl.cipher + type: keyword + + - name: ssl.curve + type: keyword + + - name: ssl.server_name + type: keyword + + - name: ssl.resumed + type: boolean + + - name: ssl.next_protocol + type: keyword + + - name: ssl.established + type: boolean + + - name: ssl.cert_chain + type: keyword + + - name: ssl.cert_chain_fuids + type: keyword + + - name: ssl.client_cert_chain + type: keyword + + - name: ssl.client_cert_chain_fuids + type: keyword + + - name: ssl.issuer + type: keyword + + - name: ssl.client_issuer + type: keyword + + - name: ssl.validation_status + type: keyword + + - name: ssl.subject + type: keyword + + - name: ssl.client_subject + type: keyword + + - name: ssl.last_alert + type: keyword + + diff --git a/x-pack/filebeat/module/zeek/_meta/images/kibana-zeek.png b/x-pack/filebeat/module/zeek/_meta/images/kibana-zeek.png new file mode 100644 index 00000000000..7c8458f19af Binary files /dev/null and b/x-pack/filebeat/module/zeek/_meta/images/kibana-zeek.png differ diff --git a/x-pack/filebeat/module/zeek/_meta/kibana/6/dashboard/Filebeat-Zeek-Overview.json b/x-pack/filebeat/module/zeek/_meta/kibana/6/dashboard/Filebeat-Zeek-Overview.json new file mode 100644 index 00000000000..598b17a56fd --- /dev/null +++ b/x-pack/filebeat/module/zeek/_meta/kibana/6/dashboard/Filebeat-Zeek-Overview.json @@ -0,0 +1,707 @@ +{ + "objects": [ + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "index": "7f83fe80-1947-11e9-84dc-b9c00e3e5a85", + "query": { + "language": "lucene", + "query": "" + } + } + }, + "title": "Destination Geo [SIEM Zeek]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "autoPrecision": true, + "field": "destination.geo.location", + "isFilteredByCollar": true, + "mapCenter": [ + 0, + 0 + ], + "mapZoom": 2, + "precision": 2, + "useGeocentroid": true + }, + "schema": "segment", + "type": "geohash_grid" + } + ], + "params": { + "addTooltip": true, + "colorSchema": "Yellow to Red", + "heatClusterSize": 1.5, + "isDesaturated": true, + "legendPosition": "bottomright", + "mapCenter": [ + 0, + 0 + ], + "mapType": "Scaled Circle Markers", + "mapZoom": 2, + "wms": { + "enabled": false, + "options": { + "format": "image/png", + "transparent": true + }, + "selectedTmsLayer": { + "attribution": "\u003cp\u003e\u0026#169; \u003ca href=\"http://www.openstreetmap.org/copyright\"\u003eOpenStreetMap\u003c/a\u003e contributors | \u003ca href=\"https://www.elastic.co/elastic-maps-service\"\u003eElastic Maps Service\u003c/a\u003e\u003c/p\u003e\u0026#10;", + "id": "road_map", + "maxZoom": 18, + "minZoom": 0, + "subdomains": [], + "url": "https://tiles.maps.elastic.co/v2/default/{z}/{x}/{y}.png?elastic_tile_service_tos=agree\u0026my_app_name=kibana\u0026my_app_version=6.5.4\u0026license=decdfd78-7d5b-47b7-9627-603d9b789d29" + } + } + }, + "title": "Destination Geo [SIEM Zeek]", + "type": "tile_map" + } + }, + "id": "5d95a3e0-1a29-11e9-84b1-a12c578fa9e8", + "type": "visualization", + "updated_at": "2019-01-17T07:27:37.758Z", + "version": 1 + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "index": "7f83fe80-1947-11e9-84dc-b9c00e3e5a85", + "query": { + "language": "lucene", + "query": "" + } + } + }, + "title": "Network Transport [SIEM Zeek]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "field": "network.transport", + "missingBucket": false, + "missingBucketLabel": "Missing", + "order": "desc", + "orderBy": "1", + "otherBucket": false, + "otherBucketLabel": "Other", + "size": 5 + }, + "schema": "segment", + "type": "terms" + } + ], + "params": { + "addLegend": true, + "addTooltip": true, + "isDonut": true, + "labels": { + "last_level": true, + "show": false, + "truncate": 100, + "values": true + }, + "legendPosition": "right", + "type": "pie" + }, + "title": "Network Transport [SIEM Zeek]", + "type": "pie" + } + }, + "id": "c337dbf0-1a29-11e9-84b1-a12c578fa9e8", + "type": "visualization", + "updated_at": "2019-01-17T07:30:28.271Z", + "version": 1 + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "index": "7f83fe80-1947-11e9-84dc-b9c00e3e5a85", + "query": { + "language": "lucene", + "query": "" + } + } + }, + "title": "Network Application [SIEM Zeek]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "field": "network.application", + "missingBucket": false, + "missingBucketLabel": "Missing", + "order": "desc", + "orderBy": "1", + "otherBucket": false, + "otherBucketLabel": "Other", + "size": 10 + }, + "schema": "segment", + "type": "terms" + } + ], + "params": { + "addLegend": true, + "addTooltip": true, + "isDonut": true, + "labels": { + "last_level": true, + "show": false, + "truncate": 100, + "values": true + }, + "legendPosition": "right", + "type": "pie" + }, + "title": "Network Application [SIEM Zeek]", + "type": "pie" + } + }, + "id": "f054ee70-1a29-11e9-84b1-a12c578fa9e8", + "type": "visualization", + "updated_at": "2019-01-17T07:31:43.959Z", + "version": 1 + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "index": "7f83fe80-1947-11e9-84dc-b9c00e3e5a85", + "query": { + "language": "lucene", + "query": "" + } + } + }, + "title": "Network Traffic Direction [SIEM Zeek]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "field": "network.direction", + "missingBucket": false, + "missingBucketLabel": "Missing", + "order": "desc", + "orderBy": "1", + "otherBucket": false, + "otherBucketLabel": "Other", + "size": 5 + }, + "schema": "segment", + "type": "terms" + } + ], + "params": { + "addLegend": true, + "addTooltip": true, + "isDonut": true, + "labels": { + "last_level": true, + "show": false, + "truncate": 100, + "values": true + }, + "legendPosition": "right", + "type": "pie" + }, + "title": "Network Traffic Direction [SIEM Zeek]", + "type": "pie" + } + }, + "id": "15922a40-1a2a-11e9-84b1-a12c578fa9e8", + "type": "visualization", + "updated_at": "2019-01-17T07:32:46.436Z", + "version": 1 + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "index": "7f83fe80-1947-11e9-84dc-b9c00e3e5a85", + "query": { + "language": "lucene", + "query": "" + } + } + }, + "title": "Top DNS Domains [SIEM Zeek]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "field": "zeek.dns.query", + "missingBucket": false, + "missingBucketLabel": "Missing", + "order": "desc", + "orderBy": "1", + "otherBucket": false, + "otherBucketLabel": "Other", + "size": 8 + }, + "schema": "segment", + "type": "terms" + } + ], + "params": { + "addLegend": true, + "addTooltip": true, + "isDonut": true, + "labels": { + "last_level": true, + "show": false, + "truncate": 100, + "values": true + }, + "legendPosition": "right", + "type": "pie" + }, + "title": "Top DNS Domains [SIEM Zeek]", + "type": "pie" + } + }, + "id": "b3705f00-1a2c-11e9-84b1-a12c578fa9e8", + "type": "visualization", + "updated_at": "2019-01-17T07:51:30.288Z", + "version": 1 + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "index": "7f83fe80-1947-11e9-84dc-b9c00e3e5a85", + "query": { + "language": "lucene", + "query": "" + } + } + }, + "title": "Top URL Domain [SIEM Zeek]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "field": "url.domain", + "missingBucket": false, + "missingBucketLabel": "Missing", + "order": "desc", + "orderBy": "1", + "otherBucket": false, + "otherBucketLabel": "Other", + "size": 8 + }, + "schema": "segment", + "type": "terms" + } + ], + "params": { + "addLegend": true, + "addTooltip": true, + "isDonut": true, + "labels": { + "last_level": true, + "show": false, + "truncate": 100, + "values": true + }, + "legendPosition": "right", + "type": "pie" + }, + "title": "Top URL Domain [SIEM Zeek]", + "type": "pie" + } + }, + "id": "ef0cfdc0-1a2c-11e9-84b1-a12c578fa9e8", + "type": "visualization", + "updated_at": "2019-01-17T07:53:10.300Z", + "version": 1 + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "index": "7f83fe80-1947-11e9-84dc-b9c00e3e5a85", + "query": { + "language": "lucene", + "query": "" + } + } + }, + "title": "Top SSL Server [SIEM Zeek]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "field": "zeek.ssl.server_name", + "missingBucket": false, + "missingBucketLabel": "Missing", + "order": "desc", + "orderBy": "1", + "otherBucket": false, + "otherBucketLabel": "Other", + "size": 8 + }, + "schema": "segment", + "type": "terms" + } + ], + "params": { + "addLegend": true, + "addTooltip": true, + "isDonut": true, + "labels": { + "last_level": true, + "show": false, + "truncate": 100, + "values": true + }, + "legendPosition": "right", + "type": "pie" + }, + "title": "Top SSL Server [SIEM Zeek]", + "type": "pie" + } + }, + "id": "13454cb0-1a2d-11e9-84b1-a12c578fa9e8", + "type": "visualization", + "updated_at": "2019-01-17T07:54:11.067Z", + "version": 1 + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "lucene", + "query": "" + } + } + }, + "title": "Time Series Count [SIEM Zeek]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "axis_formatter": "number", + "axis_position": "left", + "axis_scale": "normal", + "background_color_rules": [ + { + "id": "3716ea90-1a2d-11e9-b2af-13b289f0bf65" + } + ], + "bar_color_rules": [ + { + "id": "3822dc50-1a2d-11e9-b2af-13b289f0bf65" + } + ], + "gauge_color_rules": [ + { + "id": "4c1a3ff0-1a2d-11e9-b2af-13b289f0bf65" + } + ], + "gauge_inner_width": 10, + "gauge_style": "half", + "gauge_width": 10, + "id": "61ca57f0-469d-11e7-af02-69e470af7417", + "index_pattern": "filebeat-*", + "interval": "auto", + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "#68BC00", + "fill": 0.5, + "filter": "tags:zeek", + "formatter": "number", + "id": "61ca57f1-469d-11e7-af02-69e470af7417", + "line_width": 1, + "metrics": [ + { + "id": "61ca57f2-469d-11e7-af02-69e470af7417", + "type": "count" + } + ], + "point_size": 1, + "separate_axis": 0, + "split_mode": "filter", + "stacked": "none" + } + ], + "show_grid": 1, + "show_legend": 0, + "time_field": "@timestamp", + "type": "timeseries" + }, + "title": "Time Series Count [SIEM Zeek]", + "type": "metrics" + } + }, + "id": "fad258c0-1078-11e9-b27a-69e6e8b80a25", + "type": "visualization", + "updated_at": "2019-01-17T07:56:26.486Z", + "version": 74 + }, + { + "attributes": { + "description": "", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "lucene", + "query": "" + } + } + }, + "optionsJSON": { + "darkTheme": false, + "hidePanelTitles": false, + "useMargins": true + }, + "panelsJSON": [ + { + "embeddableConfig": { + "mapCenter": [ + 20.3034175184893, + -5.537109375000001 + ], + "mapZoom": 2 + }, + "gridData": { + "h": 18, + "i": "1", + "w": 48, + "x": 0, + "y": 0 + }, + "id": "5d95a3e0-1a29-11e9-84b1-a12c578fa9e8", + "panelIndex": "1", + "type": "visualization", + "version": "6.5.4" + }, + { + "embeddableConfig": { + "vis": { + "legendOpen": true + } + }, + "gridData": { + "h": 10, + "i": "2", + "w": 16, + "x": 0, + "y": 18 + }, + "id": "c337dbf0-1a29-11e9-84b1-a12c578fa9e8", + "panelIndex": "2", + "type": "visualization", + "version": "6.5.4" + }, + { + "embeddableConfig": { + "vis": { + "legendOpen": true + } + }, + "gridData": { + "h": 10, + "i": "3", + "w": 17, + "x": 16, + "y": 18 + }, + "id": "f054ee70-1a29-11e9-84b1-a12c578fa9e8", + "panelIndex": "3", + "type": "visualization", + "version": "6.5.4" + }, + { + "embeddableConfig": { + "vis": { + "legendOpen": true + } + }, + "gridData": { + "h": 10, + "i": "4", + "w": 15, + "x": 33, + "y": 18 + }, + "id": "15922a40-1a2a-11e9-84b1-a12c578fa9e8", + "panelIndex": "4", + "type": "visualization", + "version": "6.5.4" + }, + { + "embeddableConfig": {}, + "gridData": { + "h": 11, + "i": "5", + "w": 16, + "x": 0, + "y": 28 + }, + "id": "b3705f00-1a2c-11e9-84b1-a12c578fa9e8", + "panelIndex": "5", + "type": "visualization", + "version": "6.5.4" + }, + { + "embeddableConfig": {}, + "gridData": { + "h": 11, + "i": "6", + "w": 17, + "x": 16, + "y": 28 + }, + "id": "ef0cfdc0-1a2c-11e9-84b1-a12c578fa9e8", + "panelIndex": "6", + "type": "visualization", + "version": "6.5.4" + }, + { + "embeddableConfig": {}, + "gridData": { + "h": 11, + "i": "7", + "w": 15, + "x": 33, + "y": 28 + }, + "id": "13454cb0-1a2d-11e9-84b1-a12c578fa9e8", + "panelIndex": "7", + "type": "visualization", + "version": "6.5.4" + }, + { + "embeddableConfig": {}, + "gridData": { + "h": 9, + "i": "8", + "w": 48, + "x": 0, + "y": 39 + }, + "id": "fad258c0-1078-11e9-b27a-69e6e8b80a25", + "panelIndex": "8", + "type": "visualization", + "version": "6.5.4" + } + ], + "timeRestore": false, + "title": "Zeek Overview Dashboard [SIEM]", + "version": 1 + }, + "id": "87b0c430-1a2d-11e9-84b1-a12c578fa9e8", + "type": "dashboard", + "updated_at": "2019-01-17T07:57:50.613Z", + "version": 2 + } + ], + "version": "6.5.4" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/zeek/connection/config/connection.yml b/x-pack/filebeat/module/zeek/connection/config/connection.yml new file mode 100644 index 00000000000..b925dc01aec --- /dev/null +++ b/x-pack/filebeat/module/zeek/connection/config/connection.yml @@ -0,0 +1,59 @@ +type: log +paths: +{{ range $i, $path := .paths }} + - {{$path}} +{{ end }} +exclude_files: [".gz$"] +tags: {{.tags}} + +json.keys_under_root: false + +processors: + - drop_fields: + fields: ["json.orig_bytes","json.resp_bytes","json.tunnel_parents"] + - rename: + fields: + - from: "json" + to: "zeek.connection" + + - from: "zeek.connection.duration" + to: "event.duration" + + - from: "zeek.connection.id.orig_h" + to: "source.address" + + - from: "zeek.connection.id.orig_p" + to: "source.port" + + - from: "zeek.connection.id.resp_h" + to: "destination.address" + + - from: "zeek.connection.id.resp_p" + to: "destination.port" + + - from: "zeek.connection.proto" + to: "network.transport" + + - from: "zeek.connection.service" + to: "network.application" + + - from: "zeek.connection.uid" + to: "zeek.session_id" + + - from: "zeek.connection.orig_ip_bytes" + to: "source.bytes" + + - from: "zeek.connection.resp_ip_bytes" + to: "destination.bytes" + + - from: "zeek.connection.orig_pkts" + to: "source.packets" + + - from: "zeek.connection.resp_pkts" + to: "destination.packets" + + - from: "zeek.connection.conn_state" + to: "zeek.connection.state" + + ignore_missing: true + fail_on_error: false diff --git a/x-pack/filebeat/module/zeek/connection/ingest/pipeline.json b/x-pack/filebeat/module/zeek/connection/ingest/pipeline.json new file mode 100644 index 00000000000..862787cd0f7 --- /dev/null +++ b/x-pack/filebeat/module/zeek/connection/ingest/pipeline.json @@ -0,0 +1,61 @@ +{ + "description": "Pipeline for normalizing Zeek conn.log", + "processors": [ + { + "script": { + "lang": "painless", + "source": "ctx.event.created = ctx['@timestamp']; ctx['@timestamp'] = (long)ctx['zeek']['connection']['ts'] * params.multiplier; ctx.zeek.connection.remove('ts');", + "params": { + "multiplier": 1000 + } + } + }, + { + "script": { + "lang": "painless", + "source": "ctx.event.duration = (long)ctx.event.duration * params.multiplier", + "params": { + "multiplier": 1000000000 + }, + "ignore_failure": true + } + }, + { + "script": { + "lang": "painless", + "source": "ctx.event.id = ctx.zeek.session_id + \"-connection\"", + "ignore_failure": true + } + }, + { + "set": { + "field": "source.ip", + "value": "{{source.address}}" + } + }, + { + "set": { + "field": "destination.ip", + "value": "{{destination.address}}" + } + }, + { + "script": { + "lang": "painless", + "source": "if (ctx.zeek.connection.local_orig == true && ctx.zeek.connection.local_resp == true) {ctx.network.direction = \"internal\"} else if (ctx.zeek.connection.local_orig == true && ctx.zeek.connection.local_resp == false) {ctx.network.direction = \"outbound\"} else if (ctx.zeek.connection.local_orig == false && ctx.zeek.connection.local_resp == true) {ctx.network.direction = \"inbound\"} else {ctx.network.direction = \"external\"}" + } + }, + { + "geoip": { + "field": "destination.ip", + "target_field": "destination.geo" + } + }, + { + "geoip": { + "field": "source.ip", + "target_field": "source.geo" + } + } + ] +} \ No newline at end of file diff --git a/x-pack/filebeat/module/zeek/connection/manifest.yml b/x-pack/filebeat/module/zeek/connection/manifest.yml new file mode 100644 index 00000000000..53e7f507cd6 --- /dev/null +++ b/x-pack/filebeat/module/zeek/connection/manifest.yml @@ -0,0 +1,19 @@ +module_version: 1.0 + +var: + - name: paths + default: + - /var/log/bro/current/conn.log + os.linux: + - /var/log/bro/current/conn.log + os.darwin: + - /usr/local/var/logs/current/conn.log + - name: tags + default: [zeek] + +ingest_pipeline: ingest/pipeline.json +input: config/connection.yml + +requires.processors: +- name: geoip + plugin: ingest-geoip diff --git a/x-pack/filebeat/module/zeek/connection/test/connection-json.log b/x-pack/filebeat/module/zeek/connection/test/connection-json.log new file mode 100644 index 00000000000..9e4b15b535a --- /dev/null +++ b/x-pack/filebeat/module/zeek/connection/test/connection-json.log @@ -0,0 +1 @@ +{"ts":1547188415.857497,"uid":"CAcJw21BbVedgFnYH3","id.orig_h":"192.168.86.167","id.orig_p":38339,"id.resp_h":"192.168.86.1","id.resp_p":53,"proto":"udp","service":"dns","duration":0.076967,"orig_bytes":75,"resp_bytes":178,"conn_state":"SF","local_orig":true,"local_resp":true,"missed_bytes":0,"history":"Dd","orig_pkts":1,"orig_ip_bytes":103,"resp_pkts":1,"resp_ip_bytes":206,"tunnel_parents":[]} diff --git a/x-pack/filebeat/module/zeek/connection/test/connection-json.log-expected.json b/x-pack/filebeat/module/zeek/connection/test/connection-json.log-expected.json new file mode 100644 index 00000000000..89b37e6e83e --- /dev/null +++ b/x-pack/filebeat/module/zeek/connection/test/connection-json.log-expected.json @@ -0,0 +1,36 @@ +[ + { + "@timestamp": 1547188415000, + "destination.address": "192.168.86.1", + "destination.bytes": 206, + "destination.ip": "192.168.86.1", + "destination.packets": 1, + "destination.port": 53, + "ecs.version": "1.0.0-beta2", + "event.dataset": "zeek.connection", + "event.duration": 0.0, + "event.id": "CAcJw21BbVedgFnYH3-connection", + "event.module": "zeek", + "fileset.name": "connection", + "input.type": "log", + "log.offset": 0, + "network.application": "dns", + "network.direction": "internal", + "network.transport": "udp", + "service.type": "zeek", + "source.address": "192.168.86.167", + "source.bytes": 103, + "source.ip": "192.168.86.167", + "source.packets": 1, + "source.port": 38339, + "tags": [ + "zeek" + ], + "zeek.connection.history": "Dd", + "zeek.connection.local_orig": true, + "zeek.connection.local_resp": true, + "zeek.connection.missed_bytes": 0, + "zeek.connection.state": "SF", + "zeek.session_id": "CAcJw21BbVedgFnYH3" + } +] \ No newline at end of file diff --git a/x-pack/filebeat/module/zeek/dns/config/dns.yml b/x-pack/filebeat/module/zeek/dns/config/dns.yml new file mode 100644 index 00000000000..d64c92da679 --- /dev/null +++ b/x-pack/filebeat/module/zeek/dns/config/dns.yml @@ -0,0 +1,38 @@ +type: log +paths: +{{ range $i, $path := .paths }} + - {{$path}} +{{ end }} +exclude_files: [".gz$"] +tags: {{.tags}} + +json.keys_under_root: false + +processors: + - drop_fields: + fields: ["json.Z","json.auth","json.addl"] + - rename: + fields: + - from: "json" + to: "zeek.dns" + + - from: "zeek.dns.id.orig_h" + to: "source.address" + + - from: "zeek.dns.id.orig_p" + to: "source.port" + + - from: "zeek.dns.id.resp_h" + to: "destination.address" + + - from: "zeek.dns.id.resp_p" + to: "destination.port" + + - from: "zeek.dns.proto" + to: "network.transport" + + - from: "zeek.dns.uid" + to: "zeek.session_id" + + ignore_missing: true + fail_on_error: false diff --git a/x-pack/filebeat/module/zeek/dns/ingest/pipeline.json b/x-pack/filebeat/module/zeek/dns/ingest/pipeline.json new file mode 100644 index 00000000000..28f4adb5f41 --- /dev/null +++ b/x-pack/filebeat/module/zeek/dns/ingest/pipeline.json @@ -0,0 +1,45 @@ +{ + "description": "Pipeline for normalizing Zeek dns.log", + "processors": [ + { + "script": { + "lang": "painless", + "source": "ctx.event.created = ctx['@timestamp']; ctx['@timestamp'] = (long)ctx['zeek']['dns']['ts'] * params.multiplier; ctx.zeek.dns.remove('ts');", + "params": { + "multiplier": 1000 + } + } + }, + { + "script": { + "lang": "painless", + "source": "ctx.event.id = ctx.zeek.session_id + \"-dns\"", + "ignore_failure": true + } + }, + { + "set": { + "field": "source.ip", + "value": "{{source.address}}" + } + }, + { + "set": { + "field": "destination.ip", + "value": "{{destination.address}}" + } + }, + { + "geoip": { + "field": "destination.ip", + "target_field": "destination.geo" + } + }, + { + "geoip": { + "field": "source.ip", + "target_field": "source.geo" + } + } + ] +} \ No newline at end of file diff --git a/x-pack/filebeat/module/zeek/dns/manifest.yml b/x-pack/filebeat/module/zeek/dns/manifest.yml new file mode 100644 index 00000000000..da306cc5cfe --- /dev/null +++ b/x-pack/filebeat/module/zeek/dns/manifest.yml @@ -0,0 +1,19 @@ +module_version: 1.0 + +var: + - name: paths + default: + - /var/log/bro/current/dns.log + os.linux: + - /var/log/bro/current/dns.log + os.darwin: + - /usr/local/var/logs/current/dns.log + - name: tags + default: [zeek] + +ingest_pipeline: ingest/pipeline.json +input: config/dns.yml + +requires.processors: +- name: geoip + plugin: ingest-geoip diff --git a/x-pack/filebeat/module/zeek/dns/test/dns-json.log b/x-pack/filebeat/module/zeek/dns/test/dns-json.log new file mode 100644 index 00000000000..5a5f2534bba --- /dev/null +++ b/x-pack/filebeat/module/zeek/dns/test/dns-json.log @@ -0,0 +1 @@ +{"ts":1547188415.857497,"uid":"CAcJw21BbVedgFnYH3","id.orig_h":"192.168.86.167","id.orig_p":38339,"id.resp_h":"192.168.86.1","id.resp_p":53,"proto":"udp","trans_id":15209,"rtt":0.076967,"query":"dd625ffb4fc54735b281862aa1cd6cd4.us-west1.gcp.cloud.es.io","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["proxy-production-us-west1.gcp.cloud.es.io","proxy-production-us-west1-v1-009.gcp.cloud.es.io","35.199.178.4"],"TTLs":[119.0,119.0,59.0],"rejected":false} diff --git a/x-pack/filebeat/module/zeek/dns/test/dns-json.log-expected.json b/x-pack/filebeat/module/zeek/dns/test/dns-json.log-expected.json new file mode 100644 index 00000000000..f30c13cfaf6 --- /dev/null +++ b/x-pack/filebeat/module/zeek/dns/test/dns-json.log-expected.json @@ -0,0 +1,48 @@ +[ + { + "@timestamp": 1547188415000, + "destination.address": "192.168.86.1", + "destination.ip": "192.168.86.1", + "destination.port": 53, + "ecs.version": "1.0.0-beta2", + "event.dataset": "zeek.dns", + "event.id": "CAcJw21BbVedgFnYH3-dns", + "event.module": "zeek", + "fileset.name": "dns", + "input.type": "log", + "log.offset": 0, + "network.transport": "udp", + "service.type": "zeek", + "source.address": "192.168.86.167", + "source.ip": "192.168.86.167", + "source.port": 38339, + "tags": [ + "zeek" + ], + "zeek.dns.AA": false, + "zeek.dns.RA": true, + "zeek.dns.RD": true, + "zeek.dns.TC": false, + "zeek.dns.TTLs": [ + 119, + 119, + 59 + ], + "zeek.dns.answers": [ + "proxy-production-us-west1.gcp.cloud.es.io", + "proxy-production-us-west1-v1-009.gcp.cloud.es.io", + "35.199.178.4" + ], + "zeek.dns.qclass": 1, + "zeek.dns.qclass_name": "C_INTERNET", + "zeek.dns.qtype": 1, + "zeek.dns.qtype_name": "A", + "zeek.dns.query": "dd625ffb4fc54735b281862aa1cd6cd4.us-west1.gcp.cloud.es.io", + "zeek.dns.rcode": 0, + "zeek.dns.rcode_name": "NOERROR", + "zeek.dns.rejected": false, + "zeek.dns.rtt": 0.076967, + "zeek.dns.trans_id": 15209, + "zeek.session_id": "CAcJw21BbVedgFnYH3" + } +] \ No newline at end of file diff --git a/x-pack/filebeat/module/zeek/fields.go b/x-pack/filebeat/module/zeek/fields.go new file mode 100644 index 00000000000..932a8585d6f --- /dev/null +++ b/x-pack/filebeat/module/zeek/fields.go @@ -0,0 +1,22 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// Code generated by beats/dev-tools/cmd/asset/asset.go - DO NOT EDIT. + +package zeek + +import ( + "github.com/elastic/beats/libbeat/asset" +) + +func init() { + if err := asset.SetFields("filebeat", "zeek", asset.ModuleFieldsPri, Asset); err != nil { + panic(err) + } +} + +// Asset returns asset data +func Asset() string { + return "eJycl7GOIzcMhns/hV7gNsgBl8JFgEuCVEkTbJVGkCXOjLIaUUdK3vU+fSCNY2M99g2V0vb3kxRFivQn9QKnvXoHeNkplX0OsFd/L58csCWfsse4Vz/vlFLqT3QlgBqQ1GSiCz6OKuDIKhG6YsGpw6nJf/iFcKfU4CE43u+U+qSimeHiSal8SrBXI2FJ7fMdb0r93vRqIJwvZheHZshAKiLNJvh3U2VNc/V49cnA7DFq785mF98vcHpFcrsPrMUYwVZzTwGtCRrJjx9kB8QAJm7ICDh1yWbPDE4fThn4gzBgHB+qOJsMXaeaPGekU5em5kCHz9o4R9vCenQ5fQwmblM+RiAtY13kp0wm8u2F+5hhBFrTlPMH0GE5BFhz3wpIMtdQGwxvXeQV1PULoeX6u8Rw/bHDLll0EruN67D79et2H1Tu+VcZ99dvQk7o10R+BWLZYZ6f/2BRqRD8AzaDuxtD/WZdsphN0PeieVi3i4QgBQ9CCZtXvS7jh8mpeLUvwKec07nvHKQ83Y1ndfKmqq9YYT3zePcW7ot8HFCvavb7jprmkZs1PvgAskJfDm9GQRk11JqUC4HTyTBfDrmd30T45kEwxxpNJo6gCb4V4Cz0YIOHmPUExgG1NpeeiYGOQP9H2ebLULzrEsx+Bl3JPjfnO5WK2jTria0JumNb3HTGdk3D4477XoBCVQ2Ln2oOtuNa2PymJ7wpOZ/ukSQmr1ucID1nCRayoKT4Oh23E5AvqKdlN5LaNtGE07toziyCSwVJBfLHauF71ttF4bkLd4XMKkX35uV/9wtRtP6eS6zNPjlf92sfxw4FHoGGgK89QfkZHBbBW7vwyVB9bHtaa3ZfxOU/mR872M9ffpLS8JbJrNYbiUDbknEYpAm66ti/b22nZ0nMhOm0WXXM4ekIxKImrrD1aQLBP5rGFjoKOrGi57Epa90qIOAyP9gsV3CEt6wTYUaLQWYfOJtD8DxJfVigrO1kvDSRF146WJtqWUy6nd3Kenx65iK+8sVRj+RognftjdTLCiysmHKofy66wurSBMNZmwD0gN/9GwAA///PiHUF" +} diff --git a/x-pack/filebeat/module/zeek/files/config/files.yml b/x-pack/filebeat/module/zeek/files/config/files.yml new file mode 100644 index 00000000000..7148b82a481 --- /dev/null +++ b/x-pack/filebeat/module/zeek/files/config/files.yml @@ -0,0 +1,23 @@ +type: log +paths: +{{ range $i, $path := .paths }} + - {{$path}} +{{ end }} +exclude_files: [".gz$"] +tags: {{.tags}} + +json.keys_under_root: false + +processors: + - drop_fields: + fields: ["json.x509"] + - rename: + fields: + - from: "json" + to: "zeek.files" + + - from: "zeek.files.conn_uids" + to: "zeek.files.session_ids" + + ignore_missing: true + fail_on_error: false diff --git a/x-pack/filebeat/module/zeek/files/ingest/pipeline.json b/x-pack/filebeat/module/zeek/files/ingest/pipeline.json new file mode 100644 index 00000000000..42b6aae2c32 --- /dev/null +++ b/x-pack/filebeat/module/zeek/files/ingest/pipeline.json @@ -0,0 +1,42 @@ +{ + "description": "Pipeline for normalizing Zeek files.log", + "processors": [ + { + "script": { + "lang": "painless", + "source": "ctx.event.created = ctx['@timestamp']; ctx['@timestamp'] = (long)ctx['zeek']['files']['ts'] * params.multiplier; ctx.zeek.files.remove('ts');", + "params": { + "multiplier": 1000 + } + } + }, + { + "script": { + "lang": "painless", + "source": "ctx.zeek.session_id = ctx.zeek.files.session_ids[0];", + "ignore_failure": true + } + }, + { + "script": { + "lang": "painless", + "source": "ctx.zeek.files.rx_host = ctx.zeek.files.rx_hosts[0]; ctx.zeek.files.remove('rx_hosts');", + "ignore_failure": true + } + }, + { + "script": { + "lang": "painless", + "source": "ctx.zeek.files.tx_host = ctx.zeek.files.tx_hosts[0]; ctx.zeek.files.remove('tx_hosts');", + "ignore_failure": true + } + }, + { + "script": { + "lang": "painless", + "source": "ctx.event.id = ctx.zeek.session_id + \"-files\"", + "ignore_failure": true + } + } + ] +} \ No newline at end of file diff --git a/x-pack/filebeat/module/zeek/files/manifest.yml b/x-pack/filebeat/module/zeek/files/manifest.yml new file mode 100644 index 00000000000..9da593ea2ed --- /dev/null +++ b/x-pack/filebeat/module/zeek/files/manifest.yml @@ -0,0 +1,17 @@ +module_version: 1.0 + +var: + - name: paths + default: + - /var/log/bro/current/files.log + os.linux: + - /var/log/bro/current/files.log + os.darwin: + - /usr/local/var/logs/current/files.log + - name: tags + default: [zeek] + +ingest_pipeline: ingest/pipeline.json +input: config/files.yml + +requires.processors: diff --git a/x-pack/filebeat/module/zeek/files/test/files-json.log b/x-pack/filebeat/module/zeek/files/test/files-json.log new file mode 100644 index 00000000000..bd50ab4b5cd --- /dev/null +++ b/x-pack/filebeat/module/zeek/files/test/files-json.log @@ -0,0 +1,3 @@ +{"ts":1547688796.636812,"fuid":"FMkioa222mEuM2RuQ9","tx_hosts":["35.199.178.4"],"rx_hosts":["10.178.98.102"],"conn_uids":["C8I0zn3r9EPbfLgta6"],"source":"SSL","depth":0,"analyzers":["X509","MD5","SHA1"],"mime_type":"application/pkix-cert","duration":0.0,"local_orig":false,"is_orig":false,"seen_bytes":947,"missing_bytes":0,"overflow_bytes":0,"timedout":false,"md5":"79e4a9840d7d3a96d7c04fe2434c892e","sha1":"a8985d3a65e5e5c4b2d7d66d40c6dd2fb19c5436"} +{"ts":1547688801.566262,"fuid":"FShtIS1gydeSFf8M63","tx_hosts":["17.134.127.250"],"rx_hosts":["10.178.98.102"],"conn_uids":["C6sjVo23iNApLnlAt6"],"source":"SSL","depth":0,"analyzers":["X509","MD5","SHA1"],"mime_type":"application/pkix-cert","duration":0.0,"local_orig":false,"is_orig":false,"seen_bytes":2089,"missing_bytes":0,"overflow_bytes":0,"timedout":false,"md5":"b9742f12eb97eff531d94f7800c6706c","sha1":"b88d13fe319d342e7a808ce3a0a1158111fc3c2a"} +{"ts":1547688801.566262,"fuid":"F9ip9a3MDAq3XLBOn2","tx_hosts":["17.134.127.250"],"rx_hosts":["10.178.98.102"],"conn_uids":["C6sjVo23iNApLnlAt6"],"source":"SSL","depth":0,"analyzers":["X509","MD5","SHA1"],"mime_type":"application/pkix-cert","duration":0.0,"local_orig":false,"is_orig":false,"seen_bytes":1092,"missing_bytes":0,"overflow_bytes":0,"timedout":false,"md5":"48f0e38385112eeca5fc9ffd402eaecd","sha1":"8e8321ca08b08e3726fe1d82996884eeb5f0d655"} \ No newline at end of file diff --git a/x-pack/filebeat/module/zeek/files/test/files-json.log-expected.json b/x-pack/filebeat/module/zeek/files/test/files-json.log-expected.json new file mode 100644 index 00000000000..c5d2d872e2f --- /dev/null +++ b/x-pack/filebeat/module/zeek/files/test/files-json.log-expected.json @@ -0,0 +1,78 @@ +[ + { + "@timestamp": 1547688796000, + "ecs.version": "1.0.0-beta2", + "event.dataset": "zeek.files", + "event.id": "C8I0zn3r9EPbfLgta6-files", + "event.module": "zeek", + "fileset.name": "files", + "input.type": "log", + "log.offset": 0, + "service.type": "zeek", + "tags": [ + "zeek" + ], + "zeek.files.analyzers": [ + "X509", + "MD5", + "SHA1" + ], + "zeek.files.depth": 0, + "zeek.files.duration": 0, + "zeek.files.fuid": "FMkioa222mEuM2RuQ9", + "zeek.files.is_orig": false, + "zeek.files.local_orig": false, + "zeek.files.md5": "79e4a9840d7d3a96d7c04fe2434c892e", + "zeek.files.mime_type": "application/pkix-cert", + "zeek.files.missing_bytes": 0, + "zeek.files.overflow_bytes": 0, + "zeek.files.rx_host": "10.178.98.102", + "zeek.files.seen_bytes": 947, + "zeek.files.session_ids": [ + "C8I0zn3r9EPbfLgta6" + ], + "zeek.files.sha1": "a8985d3a65e5e5c4b2d7d66d40c6dd2fb19c5436", + "zeek.files.source": "SSL", + "zeek.files.timedout": false, + "zeek.files.tx_host": "35.199.178.4", + "zeek.session_id": "C8I0zn3r9EPbfLgta6" + }, + { + "@timestamp": 1547688801000, + "ecs.version": "1.0.0-beta2", + "event.dataset": "zeek.files", + "event.id": "C6sjVo23iNApLnlAt6-files", + "event.module": "zeek", + "fileset.name": "files", + "input.type": "log", + "log.offset": 452, + "service.type": "zeek", + "tags": [ + "zeek" + ], + "zeek.files.analyzers": [ + "X509", + "MD5", + "SHA1" + ], + "zeek.files.depth": 0, + "zeek.files.duration": 0, + "zeek.files.fuid": "FShtIS1gydeSFf8M63", + "zeek.files.is_orig": false, + "zeek.files.local_orig": false, + "zeek.files.md5": "b9742f12eb97eff531d94f7800c6706c", + "zeek.files.mime_type": "application/pkix-cert", + "zeek.files.missing_bytes": 0, + "zeek.files.overflow_bytes": 0, + "zeek.files.rx_host": "10.178.98.102", + "zeek.files.seen_bytes": 2089, + "zeek.files.session_ids": [ + "C6sjVo23iNApLnlAt6" + ], + "zeek.files.sha1": "b88d13fe319d342e7a808ce3a0a1158111fc3c2a", + "zeek.files.source": "SSL", + "zeek.files.timedout": false, + "zeek.files.tx_host": "17.134.127.250", + "zeek.session_id": "C6sjVo23iNApLnlAt6" + } +] \ No newline at end of file diff --git a/x-pack/filebeat/module/zeek/http/config/http.yml b/x-pack/filebeat/module/zeek/http/config/http.yml new file mode 100644 index 00000000000..e43398036f3 --- /dev/null +++ b/x-pack/filebeat/module/zeek/http/config/http.yml @@ -0,0 +1,69 @@ +type: log +paths: +{{ range $i, $path := .paths }} + - {{$path}} +{{ end }} +exclude_files: [".gz$"] +tags: {{.tags}} + +json.keys_under_root: false + +processors: + - rename: + fields: + - from: "json" + to: "zeek.http" + + - from: "zeek.http.id.orig_h" + to: "source.address" + + - from: "zeek.http.id.orig_p" + to: "source.port" + + - from: "zeek.http.id.resp_h" + to: "destination.address" + + - from: "zeek.http.id.resp_p" + to: "destination.port" + + - from: "zeek.http.proto" + to: "network.transport" + + - from: "zeek.http.uid" + to: "zeek.session_id" + + - from: "zeek.http.method" + to: "http.request.method" + + - from: "zeek.http.referrer" + to: "http.request.referrer" + + - from: "zeek.http.status_code" + to: "http.response.status_code" + + - from: "zeek.http.version" + to: "http.version" + + - from: "zeek.http.request_body_len" + to: "http.request.body.bytes" + + - from: "zeek.http.response_body_len" + to: "http.response.body.bytes" + + - from: "zeek.http.uri" + to: "url.original" + + - from: "zeek.http.host" + to: "url.domain" + + - from: "zeek.http.username" + to: "url.username" + + - from: "zeek.http.password" + to: "url.password" + + - from: "zeek.http.user_agent" + to: "user_agent.original" + + ignore_missing: true + fail_on_error: false diff --git a/x-pack/filebeat/module/zeek/http/ingest/pipeline.json b/x-pack/filebeat/module/zeek/http/ingest/pipeline.json new file mode 100644 index 00000000000..2da6e89dffc --- /dev/null +++ b/x-pack/filebeat/module/zeek/http/ingest/pipeline.json @@ -0,0 +1,100 @@ +{ + "description": "Pipeline for normalizing Zeek http.log", + "processors": [ + { + "script": { + "lang": "painless", + "source": "ctx.event.created = ctx['@timestamp']; ctx['@timestamp'] = (long)ctx['zeek']['http']['ts'] * params.multiplier; ctx.zeek.http.remove('ts');", + "params": { + "multiplier": 1000 + } + } + }, + { + "script": { + "lang": "painless", + "source": "ctx.event.id = ctx.zeek.session_id + \"-http\"", + "ignore_failure": true + } + }, + { + "set": { + "field": "source.ip", + "value": "{{source.address}}" + } + }, + { + "set": { + "field": "destination.ip", + "value": "{{destination.address}}" + } + }, + { + "set": { + "field": "url.port", + "value": "{{destination.port}}" + } + }, + { + "geoip": { + "field": "destination.ip", + "target_field": "destination.geo" + } + }, + { + "geoip": { + "field": "source.ip", + "target_field": "source.geo" + } + }, + { + "user_agent": { + "field": "user_agent.original", + "target_field": "user_agent", + "ignore_missing": true + } + }, + { + "rename": { + "field": "user_agent.os", + "target_field": "user_agent.temp_os", + "ignore_missing": true + } + }, + { + "rename": { + "field": "user_agent.temp_os", + "target_field": "user_agent.os.full_name", + "ignore_missing": true + } + }, + { + "rename": { + "field": "user_agent.os_name", + "target_field": "user_agent.os.name", + "ignore_missing": true + } + }, + { + "rename": { + "field": "user_agent.os_version", + "target_field": "user_agent.os.version", + "ignore_missing": true + } + }, + { + "rename": { + "field": "user_agent.os_major", + "target_field": "user_agent.os.major", + "ignore_missing": true + } + }, + { + "rename": { + "field": "user_agent.os_minor", + "target_field": "user_agent.os.minor", + "ignore_missing": true + } + } + ] +} \ No newline at end of file diff --git a/x-pack/filebeat/module/zeek/http/manifest.yml b/x-pack/filebeat/module/zeek/http/manifest.yml new file mode 100644 index 00000000000..6ee2cadec4c --- /dev/null +++ b/x-pack/filebeat/module/zeek/http/manifest.yml @@ -0,0 +1,21 @@ +module_version: 1.0 + +var: + - name: paths + default: + - /var/log/bro/current/http.log + os.linux: + - /var/log/bro/current/http.log + os.darwin: + - /usr/local/var/logs/current/http.log + - name: tags + default: [zeek] + +ingest_pipeline: ingest/pipeline.json +input: config/http.yml + +requires.processors: +- name: geoip + plugin: ingest-geoip +- name: user_agent + plugin: ingest-user_agent diff --git a/x-pack/filebeat/module/zeek/http/test/http-json.log b/x-pack/filebeat/module/zeek/http/test/http-json.log new file mode 100644 index 00000000000..733495725a3 --- /dev/null +++ b/x-pack/filebeat/module/zeek/http/test/http-json.log @@ -0,0 +1,2 @@ +{"ts":1547687130.172944,"uid":"CCNp8v1SNzY7v9d1Ih","id.orig_h":"10.178.98.102","id.orig_p":62995,"id.resp_h":"17.253.5.203","id.resp_p":80,"trans_depth":1,"method":"GET","host":"ocsp.apple.com","uri":"/ocsp04-aaica02/ME4wTKADAgEAMEUwQzBBMAkGBSsOAwIaBQAEFNqvF+Za6oA4ceFRLsAWwEInjUhJBBQx6napI3Sl39T97qDBpp7GEQ4R7AIIUP1IOZZ86ns=","version":"1.1","user_agent":"com.apple.trustd/2.0","request_body_len":0,"response_body_len":3735,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["F5zuip1tSwASjNAHy7"],"resp_mime_types":["application/ocsp-response"]} +{"ts":1547707019.757479,"uid":"CMnIaR2V8VXyu7EPs","id.orig_h":"10.20.8.197","id.orig_p":35684,"id.resp_h":"34.206.130.40","id.resp_p":80,"trans_depth":1,"method":"GET","host":"httpbin.org","uri":"/ip","version":"1.1","user_agent":"curl/7.58.0","request_body_len":0,"response_body_len":32,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FwGPlr1GcKUWWdkXoi"],"resp_mime_types":["text/json"]} \ No newline at end of file diff --git a/x-pack/filebeat/module/zeek/http/test/http-json.log-expected.json b/x-pack/filebeat/module/zeek/http/test/http-json.log-expected.json new file mode 100644 index 00000000000..b6d7e0dca5c --- /dev/null +++ b/x-pack/filebeat/module/zeek/http/test/http-json.log-expected.json @@ -0,0 +1,48 @@ +[ + { + "@timestamp": 1547687130000, + "destination.address": "17.253.5.203", + "destination.geo.continent_name": "North America", + "destination.geo.country_iso_code": "US", + "destination.geo.location.lat": 37.751, + "destination.geo.location.lon": -97.822, + "destination.ip": "17.253.5.203", + "destination.port": 80, + "ecs.version": "1.0.0-beta2", + "event.dataset": "zeek.http", + "event.id": "CCNp8v1SNzY7v9d1Ih-http", + "event.module": "zeek", + "fileset.name": "http", + "http.request.body.bytes": 0, + "http.request.method": "GET", + "http.response.body.bytes": 3735, + "http.response.status_code": 200, + "http.version": "1.1", + "input.type": "log", + "log.offset": 0, + "service.type": "zeek", + "source.address": "10.178.98.102", + "source.ip": "10.178.98.102", + "source.port": 62995, + "tags": [ + "zeek" + ], + "url.domain": "ocsp.apple.com", + "url.original": "/ocsp04-aaica02/ME4wTKADAgEAMEUwQzBBMAkGBSsOAwIaBQAEFNqvF+Za6oA4ceFRLsAWwEInjUhJBBQx6napI3Sl39T97qDBpp7GEQ4R7AIIUP1IOZZ86ns=", + "url.port": "80", + "user_agent.device": "Other", + "user_agent.name": "Other", + "user_agent.os.full_name": "Other", + "user_agent.os.name": "Other", + "zeek.http.resp_fuids": [ + "F5zuip1tSwASjNAHy7" + ], + "zeek.http.resp_mime_types": [ + "application/ocsp-response" + ], + "zeek.http.status_msg": "OK", + "zeek.http.tags": [], + "zeek.http.trans_depth": 1, + "zeek.session_id": "CCNp8v1SNzY7v9d1Ih" + } +] \ No newline at end of file diff --git a/x-pack/filebeat/module/zeek/module.yml b/x-pack/filebeat/module/zeek/module.yml new file mode 100644 index 00000000000..0db59890087 --- /dev/null +++ b/x-pack/filebeat/module/zeek/module.yml @@ -0,0 +1,3 @@ +dashboards: +- id: 87b0c430-1a2d-11e9-84b1-a12c578fa9e8 + file: Filebeat-Zeek-Overview.json diff --git a/x-pack/filebeat/module/zeek/ssl/config/ssl.yml b/x-pack/filebeat/module/zeek/ssl/config/ssl.yml new file mode 100644 index 00000000000..efa23ef44ff --- /dev/null +++ b/x-pack/filebeat/module/zeek/ssl/config/ssl.yml @@ -0,0 +1,36 @@ +type: log +paths: +{{ range $i, $path := .paths }} + - {{$path}} +{{ end }} +exclude_files: [".gz$"] +tags: {{.tags}} + +json.keys_under_root: false + +processors: + - rename: + fields: + - from: "json" + to: "zeek.ssl" + + - from: "zeek.ssl.id.orig_h" + to: "source.address" + + - from: "zeek.ssl.id.orig_p" + to: "source.port" + + - from: "zeek.ssl.id.resp_h" + to: "destination.address" + + - from: "zeek.ssl.id.resp_p" + to: "destination.port" + + - from: "zeek.ssl.proto" + to: "network.transport" + + - from: "zeek.ssl.uid" + to: "zeek.session_id" + + ignore_missing: true + fail_on_error: false diff --git a/x-pack/filebeat/module/zeek/ssl/ingest/pipeline.json b/x-pack/filebeat/module/zeek/ssl/ingest/pipeline.json new file mode 100644 index 00000000000..de32cf75099 --- /dev/null +++ b/x-pack/filebeat/module/zeek/ssl/ingest/pipeline.json @@ -0,0 +1,45 @@ +{ + "description": "Pipeline for normalizing Zeek ssl.log", + "processors": [ + { + "script": { + "lang": "painless", + "source": "ctx.event.created = ctx['@timestamp']; ctx['@timestamp'] = (long)ctx['zeek']['ssl']['ts'] * params.multiplier; ctx.zeek.ssl.remove('ts');", + "params": { + "multiplier": 1000 + } + } + }, + { + "script": { + "lang": "painless", + "source": "ctx.event.id = ctx.zeek.session_id + \"-ssl\"", + "ignore_failure": true + } + }, + { + "set": { + "field": "source.ip", + "value": "{{source.address}}" + } + }, + { + "set": { + "field": "destination.ip", + "value": "{{destination.address}}" + } + }, + { + "geoip": { + "field": "destination.ip", + "target_field": "destination.geo" + } + }, + { + "geoip": { + "field": "source.ip", + "target_field": "source.geo" + } + } + ] +} \ No newline at end of file diff --git a/x-pack/filebeat/module/zeek/ssl/manifest.yml b/x-pack/filebeat/module/zeek/ssl/manifest.yml new file mode 100644 index 00000000000..d403fa97311 --- /dev/null +++ b/x-pack/filebeat/module/zeek/ssl/manifest.yml @@ -0,0 +1,19 @@ +module_version: 1.0 + +var: + - name: paths + default: + - /var/log/bro/current/ssl.log + os.linux: + - /var/log/bro/current/ssl.log + os.darwin: + - /usr/local/var/logs/current/ssl.log + - name: tags + default: [zeek] + +ingest_pipeline: ingest/pipeline.json +input: config/ssl.yml + +requires.processors: +- name: geoip + plugin: ingest-geoip diff --git a/x-pack/filebeat/module/zeek/ssl/test/ssl-json.log b/x-pack/filebeat/module/zeek/ssl/test/ssl-json.log new file mode 100644 index 00000000000..78a57e42b16 --- /dev/null +++ b/x-pack/filebeat/module/zeek/ssl/test/ssl-json.log @@ -0,0 +1,3 @@ +{"ts":1547688736.805088,"uid":"CAOvs1BMFCX2Eh0Y3","id.orig_h":"10.178.98.102","id.orig_p":63199,"id.resp_h":"35.199.178.4","id.resp_p":9243,"version":"TLSv12","cipher":"TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384","curve":"secp256r1","server_name":"dd625ffb4fc54735b281862aa1cd6cd4.us-west1.gcp.cloud.es.io","resumed":false,"established":true,"cert_chain_fuids":["FebkbHWVCV8rEEEne","F4BDY41MGUBT6URZMd","FWlfEfiHVkv8evDL3"],"client_cert_chain_fuids":[],"subject":"CN=*.gcp.cloud.es.io,O=Elasticsearch\u005c, Inc.,L=Mountain View,ST=California,C=US","issuer":"CN=DigiCert SHA2 Secure Server CA,O=DigiCert Inc,C=US","validation_status":"ok"} +{"ts":1547688736.80509,"uid":"C3mki91FnnNtm0u1ok","id.orig_h":"10.178.98.102","id.orig_p":63198,"id.resp_h":"35.199.178.4","id.resp_p":9243,"version":"TLSv12","cipher":"TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384","curve":"secp256r1","server_name":"dd625ffb4fc54735b281862aa1cd6cd4.us-west1.gcp.cloud.es.io","resumed":false,"established":true,"cert_chain_fuids":["Fue9H32OmuitQk2zR","FpbiBP215tk2xftxM6","FEdROj1vUzTGw3BIUa"],"client_cert_chain_fuids":[],"subject":"CN=*.gcp.cloud.es.io,O=Elasticsearch\u005c, Inc.,L=Mountain View,ST=California,C=US","issuer":"CN=DigiCert SHA2 Secure Server CA,O=DigiCert Inc,C=US","validation_status":"ok"} +{"ts":1547688736.805527,"uid":"CfGBt82PzCXzHa0iek","id.orig_h":"10.178.98.102","id.orig_p":63197,"id.resp_h":"35.199.178.4","id.resp_p":9243,"version":"TLSv12","cipher":"TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384","curve":"secp256r1","server_name":"dd625ffb4fc54735b281862aa1cd6cd4.us-west1.gcp.cloud.es.io","resumed":false,"established":true,"cert_chain_fuids":["FiFLYv3UjeWyv2gcW","FvSsiB1Xi816EMagI9","FWpPS4mjGaAhTRXLf"],"client_cert_chain_fuids":[],"subject":"CN=*.gcp.cloud.es.io,O=Elasticsearch\u005c, Inc.,L=Mountain View,ST=California,C=US","issuer":"CN=DigiCert SHA2 Secure Server CA,O=DigiCert Inc,C=US","validation_status":"ok"} \ No newline at end of file diff --git a/x-pack/filebeat/module/zeek/ssl/test/ssl-json.log-expected.json b/x-pack/filebeat/module/zeek/ssl/test/ssl-json.log-expected.json new file mode 100644 index 00000000000..3ef9fd2bb8d --- /dev/null +++ b/x-pack/filebeat/module/zeek/ssl/test/ssl-json.log-expected.json @@ -0,0 +1,88 @@ +[ + { + "@timestamp": 1547688736000, + "destination.address": "35.199.178.4", + "destination.geo.city_name": "Mountain View", + "destination.geo.continent_name": "North America", + "destination.geo.country_iso_code": "US", + "destination.geo.location.lat": 37.419200000000004, + "destination.geo.location.lon": -122.0574, + "destination.geo.region_iso_code": "US-CA", + "destination.geo.region_name": "California", + "destination.ip": "35.199.178.4", + "destination.port": 9243, + "ecs.version": "1.0.0-beta2", + "event.dataset": "zeek.ssl", + "event.id": "CAOvs1BMFCX2Eh0Y3-ssl", + "event.module": "zeek", + "fileset.name": "ssl", + "input.type": "log", + "log.offset": 0, + "service.type": "zeek", + "source.address": "10.178.98.102", + "source.ip": "10.178.98.102", + "source.port": 63199, + "tags": [ + "zeek" + ], + "zeek.session_id": "CAOvs1BMFCX2Eh0Y3", + "zeek.ssl.cert_chain_fuids": [ + "FebkbHWVCV8rEEEne", + "F4BDY41MGUBT6URZMd", + "FWlfEfiHVkv8evDL3" + ], + "zeek.ssl.cipher": "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384", + "zeek.ssl.client_cert_chain_fuids": [], + "zeek.ssl.curve": "secp256r1", + "zeek.ssl.established": true, + "zeek.ssl.issuer": "CN=DigiCert SHA2 Secure Server CA,O=DigiCert Inc,C=US", + "zeek.ssl.resumed": false, + "zeek.ssl.server_name": "dd625ffb4fc54735b281862aa1cd6cd4.us-west1.gcp.cloud.es.io", + "zeek.ssl.subject": "CN=*.gcp.cloud.es.io,O=Elasticsearch\\, Inc.,L=Mountain View,ST=California,C=US", + "zeek.ssl.validation_status": "ok", + "zeek.ssl.version": "TLSv12" + }, + { + "@timestamp": 1547688736000, + "destination.address": "35.199.178.4", + "destination.geo.city_name": "Mountain View", + "destination.geo.continent_name": "North America", + "destination.geo.country_iso_code": "US", + "destination.geo.location.lat": 37.419200000000004, + "destination.geo.location.lon": -122.0574, + "destination.geo.region_iso_code": "US-CA", + "destination.geo.region_name": "California", + "destination.ip": "35.199.178.4", + "destination.port": 9243, + "ecs.version": "1.0.0-beta2", + "event.dataset": "zeek.ssl", + "event.id": "C3mki91FnnNtm0u1ok-ssl", + "event.module": "zeek", + "fileset.name": "ssl", + "input.type": "log", + "log.offset": 635, + "service.type": "zeek", + "source.address": "10.178.98.102", + "source.ip": "10.178.98.102", + "source.port": 63198, + "tags": [ + "zeek" + ], + "zeek.session_id": "C3mki91FnnNtm0u1ok", + "zeek.ssl.cert_chain_fuids": [ + "Fue9H32OmuitQk2zR", + "FpbiBP215tk2xftxM6", + "FEdROj1vUzTGw3BIUa" + ], + "zeek.ssl.cipher": "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384", + "zeek.ssl.client_cert_chain_fuids": [], + "zeek.ssl.curve": "secp256r1", + "zeek.ssl.established": true, + "zeek.ssl.issuer": "CN=DigiCert SHA2 Secure Server CA,O=DigiCert Inc,C=US", + "zeek.ssl.resumed": false, + "zeek.ssl.server_name": "dd625ffb4fc54735b281862aa1cd6cd4.us-west1.gcp.cloud.es.io", + "zeek.ssl.subject": "CN=*.gcp.cloud.es.io,O=Elasticsearch\\, Inc.,L=Mountain View,ST=California,C=US", + "zeek.ssl.validation_status": "ok", + "zeek.ssl.version": "TLSv12" + } +] \ No newline at end of file diff --git a/x-pack/filebeat/modules.d/zeek.yml.disabled b/x-pack/filebeat/modules.d/zeek.yml.disabled new file mode 100644 index 00000000000..6cf23b0b823 --- /dev/null +++ b/x-pack/filebeat/modules.d/zeek.yml.disabled @@ -0,0 +1,19 @@ +# Module: zeek +# Docs: https://www.elastic.co/guide/en/beats/filebeat/master/filebeat-module-zeek.html + +- module: zeek + # All logs + connection: + enabled: true + dns: + enabled: true + http: + enabled: true + files: + enabled: true + ssl: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: