From 066a9fd3af9b907a10e14922fd9e857695c8c77c Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Fri, 31 Jul 2020 11:54:03 -0400 Subject: [PATCH] Add event.ingested to all Filebeat modules The event.ingested field defines time at which the event was ingested to Elasticsearch and it added by the Ingest Node pipeline. This field is important when trying to build alerts for activities that may have been reported long after they occurred (@timestamp is much older than event.ingested). This might happen if an agent was offline for a period of time or the processing was delayed. This adds a test to ensure all modules create event.ingested. Closes #20073 --- CHANGELOG.next.asciidoc | 1 + filebeat/module/apache/access/ingest/pipeline.yml | 3 +++ filebeat/module/apache/error/ingest/pipeline.yml | 3 +++ filebeat/module/auditd/log/ingest/pipeline.yml | 3 +++ .../module/elasticsearch/audit/ingest/pipeline.yml | 3 +++ .../elasticsearch/deprecation/ingest/pipeline.yml | 3 +++ .../module/elasticsearch/gc/ingest/pipeline.yml | 3 +++ .../module/elasticsearch/server/ingest/pipeline.yml | 3 +++ .../elasticsearch/slowlog/ingest/pipeline.yml | 3 +++ filebeat/module/haproxy/log/ingest/pipeline.yml | 3 +++ filebeat/module/icinga/debug/ingest/pipeline.yml | 3 +++ filebeat/module/icinga/main/ingest/pipeline.yml | 3 +++ filebeat/module/icinga/startup/ingest/pipeline.yml | 3 +++ filebeat/module/iis/access/ingest/pipeline.yml | 3 +++ filebeat/module/iis/error/ingest/pipeline.yml | 3 +++ filebeat/module/kafka/log/ingest/pipeline.yml | 3 +++ filebeat/module/kibana/log/ingest/pipeline.yml | 3 +++ filebeat/module/logstash/log/ingest/pipeline.yml | 3 +++ .../module/logstash/slowlog/ingest/pipeline.yml | 3 +++ filebeat/module/mongodb/log/ingest/pipeline.yml | 3 +++ filebeat/module/mysql/error/ingest/pipeline.yml | 3 +++ filebeat/module/nats/log/ingest/pipeline.yml | 3 +++ filebeat/module/nginx/access/ingest/pipeline.yml | 5 ++++- filebeat/module/nginx/error/ingest/pipeline.yml | 3 +++ .../nginx/ingress_controller/ingest/pipeline.yml | 3 +++ filebeat/module/postgresql/log/ingest/pipeline.yml | 3 +++ filebeat/module/redis/log/ingest/pipeline.yml | 3 +++ filebeat/module/santa/log/ingest/pipeline.yml | 3 +++ filebeat/module/system/auth/ingest/pipeline.yml | 3 +++ filebeat/module/system/syslog/ingest/pipeline.yml | 3 +++ filebeat/module/traefik/access/ingest/pipeline.yml | 3 +++ filebeat/tests/system/test_modules.py | 4 ++++ .../module/activemq/audit/ingest/pipeline.yml | 3 +++ .../module/activemq/log/ingest/pipeline.yml | 3 +++ .../module/aws/cloudtrail/ingest/pipeline.yml | 5 ++++- .../module/aws/cloudwatch/ingest/pipeline.yml | 3 +++ x-pack/filebeat/module/aws/ec2/ingest/pipeline.yml | 3 +++ x-pack/filebeat/module/aws/elb/ingest/pipeline.yml | 3 +++ .../module/aws/s3access/ingest/pipeline.yml | 3 +++ .../module/azure/activitylogs/ingest/pipeline.yml | 3 +++ .../module/azure/auditlogs/ingest/pipeline.yml | 3 +++ .../module/azure/signinlogs/ingest/pipeline.yml | 3 +++ .../module/checkpoint/firewall/ingest/pipeline.yml | 13 ++++++++----- .../module/fortinet/firewall/ingest/pipeline.yml | 3 +++ .../module/googlecloud/audit/ingest/pipeline.yml | 3 +++ .../module/ibmmq/errorlog/ingest/pipeline.yml | 3 +++ .../module/iptables/log/ingest/pipeline.yml | 3 +++ .../microsoft/defender_atp/ingest/pipeline.yml | 8 ++++---- .../filebeat/module/mssql/log/ingest/pipeline.yml | 3 +++ .../filebeat/module/o365/audit/ingest/pipeline.yml | 3 +++ .../filebeat/module/okta/system/ingest/pipeline.yml | 5 ++++- .../module/rabbitmq/log/ingest/pipeline.yml | 3 +++ .../filebeat/module/sophos/xg/ingest/pipeline.yml | 3 +++ 53 files changed, 167 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index c12c2a9574f..aaa455799b6 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -501,6 +501,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add event.ingested for CrowdStrike module {pull}20138[20138] - Add support for additional fields and FirewallMatchEvent type events in CrowdStrike module {pull}20138[20138] - Add event.ingested for Suricata module {pull}20220[20220] +- Add event.ingested to all Filebeat modules. {pull}20386[20386] *Heartbeat* diff --git a/filebeat/module/apache/access/ingest/pipeline.yml b/filebeat/module/apache/access/ingest/pipeline.yml index 6311bfef12b..a9f23eb4a10 100644 --- a/filebeat/module/apache/access/ingest/pipeline.yml +++ b/filebeat/module/apache/access/ingest/pipeline.yml @@ -1,6 +1,9 @@ description: "Pipeline for parsing Apache HTTP Server access logs. Requires the geoip and user_agent plugins." processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/apache/error/ingest/pipeline.yml b/filebeat/module/apache/error/ingest/pipeline.yml index 967f7a34b69..aad4c3f4a5f 100644 --- a/filebeat/module/apache/error/ingest/pipeline.yml +++ b/filebeat/module/apache/error/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing apache error logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/auditd/log/ingest/pipeline.yml b/filebeat/module/auditd/log/ingest/pipeline.yml index 2b7c114f10a..26a8bf2ab91 100644 --- a/filebeat/module/auditd/log/ingest/pipeline.yml +++ b/filebeat/module/auditd/log/ingest/pipeline.yml @@ -1,6 +1,9 @@ --- description: Pipeline for parsing Linux auditd logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message pattern_definitions: diff --git a/filebeat/module/elasticsearch/audit/ingest/pipeline.yml b/filebeat/module/elasticsearch/audit/ingest/pipeline.yml index 8f1093f5eea..ec3873d2b9f 100644 --- a/filebeat/module/elasticsearch/audit/ingest/pipeline.yml +++ b/filebeat/module/elasticsearch/audit/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing elasticsearch audit logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - rename: field: '@timestamp' target_field: event.created diff --git a/filebeat/module/elasticsearch/deprecation/ingest/pipeline.yml b/filebeat/module/elasticsearch/deprecation/ingest/pipeline.yml index 59b8cf882f9..e1f4838df9b 100644 --- a/filebeat/module/elasticsearch/deprecation/ingest/pipeline.yml +++ b/filebeat/module/elasticsearch/deprecation/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing elasticsearch deprecation logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - rename: field: '@timestamp' target_field: event.created diff --git a/filebeat/module/elasticsearch/gc/ingest/pipeline.yml b/filebeat/module/elasticsearch/gc/ingest/pipeline.yml index fc8ec5c73e3..d0980763ecc 100644 --- a/filebeat/module/elasticsearch/gc/ingest/pipeline.yml +++ b/filebeat/module/elasticsearch/gc/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing Elasticsearch JVM garbage collection logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/elasticsearch/server/ingest/pipeline.yml b/filebeat/module/elasticsearch/server/ingest/pipeline.yml index 6e09a9dbde8..4d4e634cc4b 100644 --- a/filebeat/module/elasticsearch/server/ingest/pipeline.yml +++ b/filebeat/module/elasticsearch/server/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing elasticsearch server logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - rename: field: '@timestamp' target_field: event.created diff --git a/filebeat/module/elasticsearch/slowlog/ingest/pipeline.yml b/filebeat/module/elasticsearch/slowlog/ingest/pipeline.yml index 360e86d9d77..ea501d9b3e0 100644 --- a/filebeat/module/elasticsearch/slowlog/ingest/pipeline.yml +++ b/filebeat/module/elasticsearch/slowlog/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing elasticsearch slow logs. processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - rename: field: '@timestamp' target_field: event.created diff --git a/filebeat/module/haproxy/log/ingest/pipeline.yml b/filebeat/module/haproxy/log/ingest/pipeline.yml index fdcfc828701..d9315df0f02 100644 --- a/filebeat/module/haproxy/log/ingest/pipeline.yml +++ b/filebeat/module/haproxy/log/ingest/pipeline.yml @@ -1,6 +1,9 @@ description: Pipeline for parsing HAProxy http, tcp and default logs. Requires the geoip plugin. processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/icinga/debug/ingest/pipeline.yml b/filebeat/module/icinga/debug/ingest/pipeline.yml index ee25b38e90e..dbe9f1ee39d 100644 --- a/filebeat/module/icinga/debug/ingest/pipeline.yml +++ b/filebeat/module/icinga/debug/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing icinga debug logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/icinga/main/ingest/pipeline.yml b/filebeat/module/icinga/main/ingest/pipeline.yml index 5db480e07ab..654e8c3c4e7 100644 --- a/filebeat/module/icinga/main/ingest/pipeline.yml +++ b/filebeat/module/icinga/main/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing icinga main logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/icinga/startup/ingest/pipeline.yml b/filebeat/module/icinga/startup/ingest/pipeline.yml index 61e0e6fef27..aee7377b140 100644 --- a/filebeat/module/icinga/startup/ingest/pipeline.yml +++ b/filebeat/module/icinga/startup/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing icinga startup logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/iis/access/ingest/pipeline.yml b/filebeat/module/iis/access/ingest/pipeline.yml index 8344cccac1b..84fabdc59b8 100644 --- a/filebeat/module/iis/access/ingest/pipeline.yml +++ b/filebeat/module/iis/access/ingest/pipeline.yml @@ -1,6 +1,9 @@ description: Pipeline for parsing IIS access logs. Requires the geoip and user_agent plugins. processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/iis/error/ingest/pipeline.yml b/filebeat/module/iis/error/ingest/pipeline.yml index 4611744d3c9..a16fde841da 100644 --- a/filebeat/module/iis/error/ingest/pipeline.yml +++ b/filebeat/module/iis/error/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing IIS error logs. Requires the geoip plugin. processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/kafka/log/ingest/pipeline.yml b/filebeat/module/kafka/log/ingest/pipeline.yml index a1072489122..aa72addb642 100644 --- a/filebeat/module/kafka/log/ingest/pipeline.yml +++ b/filebeat/module/kafka/log/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing Kafka log messages processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message trace_match: true diff --git a/filebeat/module/kibana/log/ingest/pipeline.yml b/filebeat/module/kibana/log/ingest/pipeline.yml index 0112e09fcfc..ced76d42c23 100644 --- a/filebeat/module/kibana/log/ingest/pipeline.yml +++ b/filebeat/module/kibana/log/ingest/pipeline.yml @@ -4,6 +4,9 @@ on_failure: field: error.message value: '{{ _ingest.on_failure_message }}' processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - rename: field: '@timestamp' target_field: event.created diff --git a/filebeat/module/logstash/log/ingest/pipeline.yml b/filebeat/module/logstash/log/ingest/pipeline.yml index 0a416e5758e..e7dc228a76d 100644 --- a/filebeat/module/logstash/log/ingest/pipeline.yml +++ b/filebeat/module/logstash/log/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing logstash node logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - rename: field: '@timestamp' target_field: event.created diff --git a/filebeat/module/logstash/slowlog/ingest/pipeline.yml b/filebeat/module/logstash/slowlog/ingest/pipeline.yml index 061a4f8c636..949ffdcb91e 100644 --- a/filebeat/module/logstash/slowlog/ingest/pipeline.yml +++ b/filebeat/module/logstash/slowlog/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing logstash slow logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - rename: field: '@timestamp' target_field: event.created diff --git a/filebeat/module/mongodb/log/ingest/pipeline.yml b/filebeat/module/mongodb/log/ingest/pipeline.yml index 6460a2b02c6..9355e031802 100644 --- a/filebeat/module/mongodb/log/ingest/pipeline.yml +++ b/filebeat/module/mongodb/log/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing MongoDB logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/mysql/error/ingest/pipeline.yml b/filebeat/module/mysql/error/ingest/pipeline.yml index b11f280d1ea..baf4c11aa40 100644 --- a/filebeat/module/mysql/error/ingest/pipeline.yml +++ b/filebeat/module/mysql/error/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing MySQL error logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/nats/log/ingest/pipeline.yml b/filebeat/module/nats/log/ingest/pipeline.yml index 53c4f774b5e..bece77c1b8e 100644 --- a/filebeat/module/nats/log/ingest/pipeline.yml +++ b/filebeat/module/nats/log/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing nats log logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/nginx/access/ingest/pipeline.yml b/filebeat/module/nginx/access/ingest/pipeline.yml index f07e82f2b60..57fe9031b55 100644 --- a/filebeat/module/nginx/access/ingest/pipeline.yml +++ b/filebeat/module/nginx/access/ingest/pipeline.yml @@ -1,6 +1,9 @@ description: Pipeline for parsing Nginx access logs. Requires the geoip and user_agent plugins. processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: @@ -145,7 +148,7 @@ processors: - set: field: event.outcome value: failure - if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code >= 400" + if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code >= 400" - append: field: related.ip value: "{{source.ip}}" diff --git a/filebeat/module/nginx/error/ingest/pipeline.yml b/filebeat/module/nginx/error/ingest/pipeline.yml index 5a33c34710c..05691eeb737 100644 --- a/filebeat/module/nginx/error/ingest/pipeline.yml +++ b/filebeat/module/nginx/error/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing the Nginx error logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml b/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml index 74118b7405e..c9f4a5860c7 100644 --- a/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml +++ b/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml @@ -1,6 +1,9 @@ description: Pipeline for parsing Nginx ingress controller access logs. Requires the geoip and user_agent plugins. processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/postgresql/log/ingest/pipeline.yml b/filebeat/module/postgresql/log/ingest/pipeline.yml index bd208d1eb72..9233ed95c5f 100644 --- a/filebeat/module/postgresql/log/ingest/pipeline.yml +++ b/filebeat/module/postgresql/log/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing PostgreSQL logs. processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message ignore_missing: true diff --git a/filebeat/module/redis/log/ingest/pipeline.yml b/filebeat/module/redis/log/ingest/pipeline.yml index d1c08cab378..472c3398e36 100644 --- a/filebeat/module/redis/log/ingest/pipeline.yml +++ b/filebeat/module/redis/log/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing redis logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/santa/log/ingest/pipeline.yml b/filebeat/module/santa/log/ingest/pipeline.yml index 9b68cce3644..e914253f8ee 100644 --- a/filebeat/module/santa/log/ingest/pipeline.yml +++ b/filebeat/module/santa/log/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing Google Santa logs. processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/system/auth/ingest/pipeline.yml b/filebeat/module/system/auth/ingest/pipeline.yml index 3f45705416a..a958855936a 100644 --- a/filebeat/module/system/auth/ingest/pipeline.yml +++ b/filebeat/module/system/auth/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing system authorisation/secure logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message ignore_missing: true diff --git a/filebeat/module/system/syslog/ingest/pipeline.yml b/filebeat/module/system/syslog/ingest/pipeline.yml index e0c80b9aad6..2963ba410b0 100644 --- a/filebeat/module/system/syslog/ingest/pipeline.yml +++ b/filebeat/module/system/syslog/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing Syslog messages. processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/traefik/access/ingest/pipeline.yml b/filebeat/module/traefik/access/ingest/pipeline.yml index ce489a4a92c..dd5de1b0b0b 100644 --- a/filebeat/module/traefik/access/ingest/pipeline.yml +++ b/filebeat/module/traefik/access/ingest/pipeline.yml @@ -1,6 +1,9 @@ description: Pipeline for parsing Traefik access logs. Requires the geoip and user_agent plugins. processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - dissect: field: message pattern: '%{source.address} %{traefik.access.user_identifier} %{user.name} [%{traefik.access.time}] diff --git a/filebeat/tests/system/test_modules.py b/filebeat/tests/system/test_modules.py index 9360abf51af..b02a98b2f51 100644 --- a/filebeat/tests/system/test_modules.py +++ b/filebeat/tests/system/test_modules.py @@ -161,6 +161,10 @@ def run_on_file(self, module, fileset, test_file, cfgfile): assert obj["event"]["module"] == module, "expected event.module={} but got {}".format( module, obj["event"]["module"]) + # All modules must include a set processor that adds the time that + # the event was ingested to Elasticsearch + assert "ingested" in obj["event"], "missing event.ingested timestamp" + assert "error" not in obj, "not error expected but got: {}".format( obj) diff --git a/x-pack/filebeat/module/activemq/audit/ingest/pipeline.yml b/x-pack/filebeat/module/activemq/audit/ingest/pipeline.yml index afc4c50f3dc..c5cb5ee8ed1 100644 --- a/x-pack/filebeat/module/activemq/audit/ingest/pipeline.yml +++ b/x-pack/filebeat/module/activemq/audit/ingest/pipeline.yml @@ -1,6 +1,9 @@ --- description: Pipeline for parsing ActiveMQ audit logs. processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message pattern_definitions: diff --git a/x-pack/filebeat/module/activemq/log/ingest/pipeline.yml b/x-pack/filebeat/module/activemq/log/ingest/pipeline.yml index c33d77295e5..b84807be893 100644 --- a/x-pack/filebeat/module/activemq/log/ingest/pipeline.yml +++ b/x-pack/filebeat/module/activemq/log/ingest/pipeline.yml @@ -1,6 +1,9 @@ --- description: Pipeline for parsing ActiveMQ logs. processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message pattern_definitions: diff --git a/x-pack/filebeat/module/aws/cloudtrail/ingest/pipeline.yml b/x-pack/filebeat/module/aws/cloudtrail/ingest/pipeline.yml index 36773124439..42395228853 100644 --- a/x-pack/filebeat/module/aws/cloudtrail/ingest/pipeline.yml +++ b/x-pack/filebeat/module/aws/cloudtrail/ingest/pipeline.yml @@ -1,6 +1,9 @@ --- description: Pipeline for AWS CloudTrail Logs processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' - rename: field: "message" target_field: "event.original" @@ -614,7 +617,7 @@ processors: if (ctx.event.action == 'ConsoleLogin' && ctx?.aws?.cloudtrail?.flattened?.response_elements.ConsoleLogin != null) { ctx.event.outcome = Processors.lowercase(ctx.aws.cloudtrail.flattened.response_elements.ConsoleLogin); } - + def hm = new HashMap(params.get(ctx.event.action)); hm.forEach((k, v) -> ctx.event[k] = v); diff --git a/x-pack/filebeat/module/aws/cloudwatch/ingest/pipeline.yml b/x-pack/filebeat/module/aws/cloudwatch/ingest/pipeline.yml index ff7e20d1c3d..05f25463414 100644 --- a/x-pack/filebeat/module/aws/cloudwatch/ingest/pipeline.yml +++ b/x-pack/filebeat/module/aws/cloudwatch/ingest/pipeline.yml @@ -1,6 +1,9 @@ description: "Pipeline for CloudWatch logs" processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/x-pack/filebeat/module/aws/ec2/ingest/pipeline.yml b/x-pack/filebeat/module/aws/ec2/ingest/pipeline.yml index 0ada24c6f77..878aa14aef5 100644 --- a/x-pack/filebeat/module/aws/ec2/ingest/pipeline.yml +++ b/x-pack/filebeat/module/aws/ec2/ingest/pipeline.yml @@ -1,6 +1,9 @@ description: "Pipeline for EC2 logs in CloudWatch" processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/x-pack/filebeat/module/aws/elb/ingest/pipeline.yml b/x-pack/filebeat/module/aws/elb/ingest/pipeline.yml index fbd1195dcae..de772ccdf01 100644 --- a/x-pack/filebeat/module/aws/elb/ingest/pipeline.yml +++ b/x-pack/filebeat/module/aws/elb/ingest/pipeline.yml @@ -1,6 +1,9 @@ description: "Pipeline for ELB logs" processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message # Classic ELB patterns documented in https://docs.aws.amazon.com/elasticloadbalancing/latest/classic/access-log-collection.html diff --git a/x-pack/filebeat/module/aws/s3access/ingest/pipeline.yml b/x-pack/filebeat/module/aws/s3access/ingest/pipeline.yml index efd1a9d358a..dd8613a904a 100644 --- a/x-pack/filebeat/module/aws/s3access/ingest/pipeline.yml +++ b/x-pack/filebeat/module/aws/s3access/ingest/pipeline.yml @@ -1,6 +1,9 @@ description: "Pipeline for s3 server access logs" processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.yml b/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.yml index dac11495608..f8f10132a0d 100644 --- a/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.yml +++ b/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing azure activity logs. processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - rename: field: azure target_field: azure-eventhub diff --git a/x-pack/filebeat/module/azure/auditlogs/ingest/pipeline.yml b/x-pack/filebeat/module/azure/auditlogs/ingest/pipeline.yml index 2bf26322faf..e6a29f6cc13 100644 --- a/x-pack/filebeat/module/azure/auditlogs/ingest/pipeline.yml +++ b/x-pack/filebeat/module/azure/auditlogs/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing azure activity logs. processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - rename: field: azure target_field: azure-eventhub diff --git a/x-pack/filebeat/module/azure/signinlogs/ingest/pipeline.yml b/x-pack/filebeat/module/azure/signinlogs/ingest/pipeline.yml index 9d5351bf36a..77ccfa32dec 100644 --- a/x-pack/filebeat/module/azure/signinlogs/ingest/pipeline.yml +++ b/x-pack/filebeat/module/azure/signinlogs/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing azure signin logs. processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - rename: field: azure target_field: azure-eventhub diff --git a/x-pack/filebeat/module/checkpoint/firewall/ingest/pipeline.yml b/x-pack/filebeat/module/checkpoint/firewall/ingest/pipeline.yml index 9a74b0b7c72..d21d421ce0f 100644 --- a/x-pack/filebeat/module/checkpoint/firewall/ingest/pipeline.yml +++ b/x-pack/filebeat/module/checkpoint/firewall/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing checkpoint firewall logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: @@ -157,7 +160,7 @@ processors: target_field: source.nat.port type: long ignore_failure: true - ignore_missing: true + ignore_missing: true if: "ctx.checkpoint?.xlatesport != '0'" - rename: field: checkpoint.mac_source_address @@ -691,7 +694,7 @@ processors: field: client.nat.port type: long ignore_failure: true - ignore_missing: true + ignore_missing: true - convert: field: client.bytes type: long @@ -711,7 +714,7 @@ processors: field: server.nat.port type: long ignore_failure: true - ignore_missing: true + ignore_missing: true - convert: field: server.bytes type: long @@ -721,7 +724,7 @@ processors: field: server.packets type: long ignore_failure: true - ignore_missing: true + ignore_missing: true - script: lang: painless source: "ctx.network.bytes = ctx.source.bytes + ctx.destination.bytes" @@ -797,4 +800,4 @@ processors: on_failure: - set: field: error.message - value: '{{ _ingest.on_failure_message }}' \ No newline at end of file + value: '{{ _ingest.on_failure_message }}' diff --git a/x-pack/filebeat/module/fortinet/firewall/ingest/pipeline.yml b/x-pack/filebeat/module/fortinet/firewall/ingest/pipeline.yml index 2aaf7065ec1..eeb5368db55 100644 --- a/x-pack/filebeat/module/fortinet/firewall/ingest/pipeline.yml +++ b/x-pack/filebeat/module/fortinet/firewall/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing fortinet firewall logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/x-pack/filebeat/module/googlecloud/audit/ingest/pipeline.yml b/x-pack/filebeat/module/googlecloud/audit/ingest/pipeline.yml index 6c148a0c07c..8e0d3ac6fdb 100644 --- a/x-pack/filebeat/module/googlecloud/audit/ingest/pipeline.yml +++ b/x-pack/filebeat/module/googlecloud/audit/ingest/pipeline.yml @@ -1,6 +1,9 @@ description: Pipeline for Google Cloud audit logs processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' - user_agent: field: user_agent.original ignore_missing: true diff --git a/x-pack/filebeat/module/ibmmq/errorlog/ingest/pipeline.yml b/x-pack/filebeat/module/ibmmq/errorlog/ingest/pipeline.yml index 80db3a86a86..87c3deacb97 100644 --- a/x-pack/filebeat/module/ibmmq/errorlog/ingest/pipeline.yml +++ b/x-pack/filebeat/module/ibmmq/errorlog/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing MQ error logs. processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - gsub: field: message pattern: ^[\-]{5}[a-z0-9\. :]*[\-]{5,} diff --git a/x-pack/filebeat/module/iptables/log/ingest/pipeline.yml b/x-pack/filebeat/module/iptables/log/ingest/pipeline.yml index 4eb24ff7d03..ecaa40ce67c 100644 --- a/x-pack/filebeat/module/iptables/log/ingest/pipeline.yml +++ b/x-pack/filebeat/module/iptables/log/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for IPTables processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/x-pack/filebeat/module/microsoft/defender_atp/ingest/pipeline.yml b/x-pack/filebeat/module/microsoft/defender_atp/ingest/pipeline.yml index 6dedd5e8a1f..392f3a441a7 100644 --- a/x-pack/filebeat/module/microsoft/defender_atp/ingest/pipeline.yml +++ b/x-pack/filebeat/module/microsoft/defender_atp/ingest/pipeline.yml @@ -1,6 +1,9 @@ --- description: Pipeline for parsing microsoft atp logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - remove: field: - message @@ -76,9 +79,6 @@ processors: - set: field: event.provider value: defender_atp -- set: - field: event.ingested - value: '{{_ingest.timestamp}}' - set: field: event.created value: '{{json.alertCreationTime}}' @@ -284,7 +284,7 @@ processors: ## Cleanup ## ############# - remove: - field: + field: - json.alertCreationTime - json.severity - json.relatedUser diff --git a/x-pack/filebeat/module/mssql/log/ingest/pipeline.yml b/x-pack/filebeat/module/mssql/log/ingest/pipeline.yml index 39a10a9ff99..cae8f53ab34 100644 --- a/x-pack/filebeat/module/mssql/log/ingest/pipeline.yml +++ b/x-pack/filebeat/module/mssql/log/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline to parse MSSQL logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/x-pack/filebeat/module/o365/audit/ingest/pipeline.yml b/x-pack/filebeat/module/o365/audit/ingest/pipeline.yml index 98fd4f0ff58..49371346925 100644 --- a/x-pack/filebeat/module/o365/audit/ingest/pipeline.yml +++ b/x-pack/filebeat/module/o365/audit/ingest/pipeline.yml @@ -1,6 +1,9 @@ description: Pipeline for Office 365 Audit logs processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' - user_agent: field: user_agent.original ignore_missing: true diff --git a/x-pack/filebeat/module/okta/system/ingest/pipeline.yml b/x-pack/filebeat/module/okta/system/ingest/pipeline.yml index 78f6fa37047..e3d92540d5f 100644 --- a/x-pack/filebeat/module/okta/system/ingest/pipeline.yml +++ b/x-pack/filebeat/module/okta/system/ingest/pipeline.yml @@ -1,6 +1,9 @@ description: Pipeline for Okta system logs. processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' - user_agent: field: user_agent.original ignore_missing: true @@ -44,7 +47,7 @@ processors: field: destination.as.organization_name target_field: destination.as.organization.name ignore_missing: true - + on_failure: - set: field: error.message diff --git a/x-pack/filebeat/module/rabbitmq/log/ingest/pipeline.yml b/x-pack/filebeat/module/rabbitmq/log/ingest/pipeline.yml index 58097c578d8..e69402c6a95 100644 --- a/x-pack/filebeat/module/rabbitmq/log/ingest/pipeline.yml +++ b/x-pack/filebeat/module/rabbitmq/log/ingest/pipeline.yml @@ -1,6 +1,9 @@ --- description: Pipeline for parsing RabbitMQ logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message pattern_definitions: diff --git a/x-pack/filebeat/module/sophos/xg/ingest/pipeline.yml b/x-pack/filebeat/module/sophos/xg/ingest/pipeline.yml index f408b6f01cd..8102bb92514 100644 --- a/x-pack/filebeat/module/sophos/xg/ingest/pipeline.yml +++ b/x-pack/filebeat/module/sophos/xg/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing sophosxg firewall logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: