diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index c12c2a9574f3..aaa455799b6c 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -501,6 +501,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add event.ingested for CrowdStrike module {pull}20138[20138] - Add support for additional fields and FirewallMatchEvent type events in CrowdStrike module {pull}20138[20138] - Add event.ingested for Suricata module {pull}20220[20220] +- Add event.ingested to all Filebeat modules. {pull}20386[20386] *Heartbeat* diff --git a/filebeat/module/apache/access/ingest/pipeline.yml b/filebeat/module/apache/access/ingest/pipeline.yml index 6311bfef12bd..a9f23eb4a103 100644 --- a/filebeat/module/apache/access/ingest/pipeline.yml +++ b/filebeat/module/apache/access/ingest/pipeline.yml @@ -1,6 +1,9 @@ description: "Pipeline for parsing Apache HTTP Server access logs. Requires the geoip and user_agent plugins." processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/apache/error/ingest/pipeline.yml b/filebeat/module/apache/error/ingest/pipeline.yml index 967f7a34b69c..aad4c3f4a5f5 100644 --- a/filebeat/module/apache/error/ingest/pipeline.yml +++ b/filebeat/module/apache/error/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing apache error logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/auditd/log/ingest/pipeline.yml b/filebeat/module/auditd/log/ingest/pipeline.yml index 2b7c114f10af..26a8bf2ab91f 100644 --- a/filebeat/module/auditd/log/ingest/pipeline.yml +++ b/filebeat/module/auditd/log/ingest/pipeline.yml @@ -1,6 +1,9 @@ --- description: Pipeline for parsing Linux auditd logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message pattern_definitions: diff --git a/filebeat/module/elasticsearch/audit/ingest/pipeline.yml b/filebeat/module/elasticsearch/audit/ingest/pipeline.yml index 8f1093f5eea7..ec3873d2b9fb 100644 --- a/filebeat/module/elasticsearch/audit/ingest/pipeline.yml +++ b/filebeat/module/elasticsearch/audit/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing elasticsearch audit logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - rename: field: '@timestamp' target_field: event.created diff --git a/filebeat/module/elasticsearch/deprecation/ingest/pipeline.yml b/filebeat/module/elasticsearch/deprecation/ingest/pipeline.yml index 59b8cf882f9a..e1f4838df9b5 100644 --- a/filebeat/module/elasticsearch/deprecation/ingest/pipeline.yml +++ b/filebeat/module/elasticsearch/deprecation/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing elasticsearch deprecation logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - rename: field: '@timestamp' target_field: event.created diff --git a/filebeat/module/elasticsearch/gc/ingest/pipeline.yml b/filebeat/module/elasticsearch/gc/ingest/pipeline.yml index fc8ec5c73e35..d0980763ecc0 100644 --- a/filebeat/module/elasticsearch/gc/ingest/pipeline.yml +++ b/filebeat/module/elasticsearch/gc/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing Elasticsearch JVM garbage collection logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/elasticsearch/server/ingest/pipeline.yml b/filebeat/module/elasticsearch/server/ingest/pipeline.yml index 6e09a9dbde89..4d4e634cc4b6 100644 --- a/filebeat/module/elasticsearch/server/ingest/pipeline.yml +++ b/filebeat/module/elasticsearch/server/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing elasticsearch server logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - rename: field: '@timestamp' target_field: event.created diff --git a/filebeat/module/elasticsearch/slowlog/ingest/pipeline.yml b/filebeat/module/elasticsearch/slowlog/ingest/pipeline.yml index 360e86d9d778..ea501d9b3e05 100644 --- a/filebeat/module/elasticsearch/slowlog/ingest/pipeline.yml +++ b/filebeat/module/elasticsearch/slowlog/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing elasticsearch slow logs. processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - rename: field: '@timestamp' target_field: event.created diff --git a/filebeat/module/haproxy/log/ingest/pipeline.yml b/filebeat/module/haproxy/log/ingest/pipeline.yml index fdcfc828701e..d9315df0f024 100644 --- a/filebeat/module/haproxy/log/ingest/pipeline.yml +++ b/filebeat/module/haproxy/log/ingest/pipeline.yml @@ -1,6 +1,9 @@ description: Pipeline for parsing HAProxy http, tcp and default logs. Requires the geoip plugin. processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/icinga/debug/ingest/pipeline.yml b/filebeat/module/icinga/debug/ingest/pipeline.yml index ee25b38e90e5..dbe9f1ee39df 100644 --- a/filebeat/module/icinga/debug/ingest/pipeline.yml +++ b/filebeat/module/icinga/debug/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing icinga debug logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/icinga/main/ingest/pipeline.yml b/filebeat/module/icinga/main/ingest/pipeline.yml index 5db480e07ab3..654e8c3c4e7e 100644 --- a/filebeat/module/icinga/main/ingest/pipeline.yml +++ b/filebeat/module/icinga/main/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing icinga main logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/icinga/startup/ingest/pipeline.yml b/filebeat/module/icinga/startup/ingest/pipeline.yml index 61e0e6fef275..aee7377b1402 100644 --- a/filebeat/module/icinga/startup/ingest/pipeline.yml +++ b/filebeat/module/icinga/startup/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing icinga startup logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/iis/access/ingest/pipeline.yml b/filebeat/module/iis/access/ingest/pipeline.yml index 8344cccac1b8..84fabdc59b8d 100644 --- a/filebeat/module/iis/access/ingest/pipeline.yml +++ b/filebeat/module/iis/access/ingest/pipeline.yml @@ -1,6 +1,9 @@ description: Pipeline for parsing IIS access logs. Requires the geoip and user_agent plugins. processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/iis/error/ingest/pipeline.yml b/filebeat/module/iis/error/ingest/pipeline.yml index 4611744d3c94..a16fde841daf 100644 --- a/filebeat/module/iis/error/ingest/pipeline.yml +++ b/filebeat/module/iis/error/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing IIS error logs. Requires the geoip plugin. processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/kafka/log/ingest/pipeline.yml b/filebeat/module/kafka/log/ingest/pipeline.yml index a10724891225..aa72addb6420 100644 --- a/filebeat/module/kafka/log/ingest/pipeline.yml +++ b/filebeat/module/kafka/log/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing Kafka log messages processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message trace_match: true diff --git a/filebeat/module/kibana/log/ingest/pipeline.yml b/filebeat/module/kibana/log/ingest/pipeline.yml index 0112e09fcfce..ced76d42c23f 100644 --- a/filebeat/module/kibana/log/ingest/pipeline.yml +++ b/filebeat/module/kibana/log/ingest/pipeline.yml @@ -4,6 +4,9 @@ on_failure: field: error.message value: '{{ _ingest.on_failure_message }}' processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - rename: field: '@timestamp' target_field: event.created diff --git a/filebeat/module/logstash/log/ingest/pipeline.yml b/filebeat/module/logstash/log/ingest/pipeline.yml index 0a416e5758e4..e7dc228a76dc 100644 --- a/filebeat/module/logstash/log/ingest/pipeline.yml +++ b/filebeat/module/logstash/log/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing logstash node logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - rename: field: '@timestamp' target_field: event.created diff --git a/filebeat/module/logstash/slowlog/ingest/pipeline.yml b/filebeat/module/logstash/slowlog/ingest/pipeline.yml index 061a4f8c636c..949ffdcb91ef 100644 --- a/filebeat/module/logstash/slowlog/ingest/pipeline.yml +++ b/filebeat/module/logstash/slowlog/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing logstash slow logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - rename: field: '@timestamp' target_field: event.created diff --git a/filebeat/module/mongodb/log/ingest/pipeline.yml b/filebeat/module/mongodb/log/ingest/pipeline.yml index 6460a2b02c66..9355e0318023 100644 --- a/filebeat/module/mongodb/log/ingest/pipeline.yml +++ b/filebeat/module/mongodb/log/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing MongoDB logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/mysql/error/ingest/pipeline.yml b/filebeat/module/mysql/error/ingest/pipeline.yml index b11f280d1ead..baf4c11aa402 100644 --- a/filebeat/module/mysql/error/ingest/pipeline.yml +++ b/filebeat/module/mysql/error/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing MySQL error logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/nats/log/ingest/pipeline.yml b/filebeat/module/nats/log/ingest/pipeline.yml index 53c4f774b5ea..bece77c1b8ea 100644 --- a/filebeat/module/nats/log/ingest/pipeline.yml +++ b/filebeat/module/nats/log/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing nats log logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/nginx/access/ingest/pipeline.yml b/filebeat/module/nginx/access/ingest/pipeline.yml index f07e82f2b60b..57fe9031b557 100644 --- a/filebeat/module/nginx/access/ingest/pipeline.yml +++ b/filebeat/module/nginx/access/ingest/pipeline.yml @@ -1,6 +1,9 @@ description: Pipeline for parsing Nginx access logs. Requires the geoip and user_agent plugins. processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: @@ -145,7 +148,7 @@ processors: - set: field: event.outcome value: failure - if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code >= 400" + if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code >= 400" - append: field: related.ip value: "{{source.ip}}" diff --git a/filebeat/module/nginx/error/ingest/pipeline.yml b/filebeat/module/nginx/error/ingest/pipeline.yml index 5a33c34710cb..05691eeb7375 100644 --- a/filebeat/module/nginx/error/ingest/pipeline.yml +++ b/filebeat/module/nginx/error/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing the Nginx error logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml b/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml index 74118b7405e2..c9f4a5860c7a 100644 --- a/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml +++ b/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml @@ -1,6 +1,9 @@ description: Pipeline for parsing Nginx ingress controller access logs. Requires the geoip and user_agent plugins. processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/postgresql/log/ingest/pipeline.yml b/filebeat/module/postgresql/log/ingest/pipeline.yml index bd208d1eb724..9233ed95c5f4 100644 --- a/filebeat/module/postgresql/log/ingest/pipeline.yml +++ b/filebeat/module/postgresql/log/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing PostgreSQL logs. processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message ignore_missing: true diff --git a/filebeat/module/redis/log/ingest/pipeline.yml b/filebeat/module/redis/log/ingest/pipeline.yml index d1c08cab3787..472c3398e36f 100644 --- a/filebeat/module/redis/log/ingest/pipeline.yml +++ b/filebeat/module/redis/log/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing redis logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/santa/log/ingest/pipeline.yml b/filebeat/module/santa/log/ingest/pipeline.yml index 9b68cce3644e..e914253f8eec 100644 --- a/filebeat/module/santa/log/ingest/pipeline.yml +++ b/filebeat/module/santa/log/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing Google Santa logs. processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/system/auth/ingest/pipeline.yml b/filebeat/module/system/auth/ingest/pipeline.yml index 3f45705416a5..a958855936a9 100644 --- a/filebeat/module/system/auth/ingest/pipeline.yml +++ b/filebeat/module/system/auth/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing system authorisation/secure logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message ignore_missing: true diff --git a/filebeat/module/system/syslog/ingest/pipeline.yml b/filebeat/module/system/syslog/ingest/pipeline.yml index e0c80b9aad66..2963ba410b09 100644 --- a/filebeat/module/system/syslog/ingest/pipeline.yml +++ b/filebeat/module/system/syslog/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing Syslog messages. processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/traefik/access/ingest/pipeline.yml b/filebeat/module/traefik/access/ingest/pipeline.yml index ce489a4a92c2..dd5de1b0b0b6 100644 --- a/filebeat/module/traefik/access/ingest/pipeline.yml +++ b/filebeat/module/traefik/access/ingest/pipeline.yml @@ -1,6 +1,9 @@ description: Pipeline for parsing Traefik access logs. Requires the geoip and user_agent plugins. processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - dissect: field: message pattern: '%{source.address} %{traefik.access.user_identifier} %{user.name} [%{traefik.access.time}] diff --git a/filebeat/tests/system/test_modules.py b/filebeat/tests/system/test_modules.py index 9360abf51af6..b02a98b2f512 100644 --- a/filebeat/tests/system/test_modules.py +++ b/filebeat/tests/system/test_modules.py @@ -161,6 +161,10 @@ def run_on_file(self, module, fileset, test_file, cfgfile): assert obj["event"]["module"] == module, "expected event.module={} but got {}".format( module, obj["event"]["module"]) + # All modules must include a set processor that adds the time that + # the event was ingested to Elasticsearch + assert "ingested" in obj["event"], "missing event.ingested timestamp" + assert "error" not in obj, "not error expected but got: {}".format( obj) diff --git a/x-pack/filebeat/module/activemq/audit/ingest/pipeline.yml b/x-pack/filebeat/module/activemq/audit/ingest/pipeline.yml index afc4c50f3dce..c5cb5ee8ed14 100644 --- a/x-pack/filebeat/module/activemq/audit/ingest/pipeline.yml +++ b/x-pack/filebeat/module/activemq/audit/ingest/pipeline.yml @@ -1,6 +1,9 @@ --- description: Pipeline for parsing ActiveMQ audit logs. processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message pattern_definitions: diff --git a/x-pack/filebeat/module/activemq/log/ingest/pipeline.yml b/x-pack/filebeat/module/activemq/log/ingest/pipeline.yml index c33d77295e5b..b84807be893c 100644 --- a/x-pack/filebeat/module/activemq/log/ingest/pipeline.yml +++ b/x-pack/filebeat/module/activemq/log/ingest/pipeline.yml @@ -1,6 +1,9 @@ --- description: Pipeline for parsing ActiveMQ logs. processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message pattern_definitions: diff --git a/x-pack/filebeat/module/aws/cloudtrail/ingest/pipeline.yml b/x-pack/filebeat/module/aws/cloudtrail/ingest/pipeline.yml index 36773124439e..42395228853d 100644 --- a/x-pack/filebeat/module/aws/cloudtrail/ingest/pipeline.yml +++ b/x-pack/filebeat/module/aws/cloudtrail/ingest/pipeline.yml @@ -1,6 +1,9 @@ --- description: Pipeline for AWS CloudTrail Logs processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' - rename: field: "message" target_field: "event.original" @@ -614,7 +617,7 @@ processors: if (ctx.event.action == 'ConsoleLogin' && ctx?.aws?.cloudtrail?.flattened?.response_elements.ConsoleLogin != null) { ctx.event.outcome = Processors.lowercase(ctx.aws.cloudtrail.flattened.response_elements.ConsoleLogin); } - + def hm = new HashMap(params.get(ctx.event.action)); hm.forEach((k, v) -> ctx.event[k] = v); diff --git a/x-pack/filebeat/module/aws/cloudwatch/ingest/pipeline.yml b/x-pack/filebeat/module/aws/cloudwatch/ingest/pipeline.yml index ff7e20d1c3d1..05f254634141 100644 --- a/x-pack/filebeat/module/aws/cloudwatch/ingest/pipeline.yml +++ b/x-pack/filebeat/module/aws/cloudwatch/ingest/pipeline.yml @@ -1,6 +1,9 @@ description: "Pipeline for CloudWatch logs" processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/x-pack/filebeat/module/aws/ec2/ingest/pipeline.yml b/x-pack/filebeat/module/aws/ec2/ingest/pipeline.yml index 0ada24c6f77d..878aa14aef5a 100644 --- a/x-pack/filebeat/module/aws/ec2/ingest/pipeline.yml +++ b/x-pack/filebeat/module/aws/ec2/ingest/pipeline.yml @@ -1,6 +1,9 @@ description: "Pipeline for EC2 logs in CloudWatch" processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/x-pack/filebeat/module/aws/elb/ingest/pipeline.yml b/x-pack/filebeat/module/aws/elb/ingest/pipeline.yml index fbd1195dcae4..de772ccdf018 100644 --- a/x-pack/filebeat/module/aws/elb/ingest/pipeline.yml +++ b/x-pack/filebeat/module/aws/elb/ingest/pipeline.yml @@ -1,6 +1,9 @@ description: "Pipeline for ELB logs" processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message # Classic ELB patterns documented in https://docs.aws.amazon.com/elasticloadbalancing/latest/classic/access-log-collection.html diff --git a/x-pack/filebeat/module/aws/s3access/ingest/pipeline.yml b/x-pack/filebeat/module/aws/s3access/ingest/pipeline.yml index efd1a9d358a8..dd8613a904ad 100644 --- a/x-pack/filebeat/module/aws/s3access/ingest/pipeline.yml +++ b/x-pack/filebeat/module/aws/s3access/ingest/pipeline.yml @@ -1,6 +1,9 @@ description: "Pipeline for s3 server access logs" processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.yml b/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.yml index dac114956081..f8f10132a0de 100644 --- a/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.yml +++ b/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing azure activity logs. processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - rename: field: azure target_field: azure-eventhub diff --git a/x-pack/filebeat/module/azure/auditlogs/ingest/pipeline.yml b/x-pack/filebeat/module/azure/auditlogs/ingest/pipeline.yml index 2bf26322faf3..e6a29f6cc13d 100644 --- a/x-pack/filebeat/module/azure/auditlogs/ingest/pipeline.yml +++ b/x-pack/filebeat/module/azure/auditlogs/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing azure activity logs. processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - rename: field: azure target_field: azure-eventhub diff --git a/x-pack/filebeat/module/azure/signinlogs/ingest/pipeline.yml b/x-pack/filebeat/module/azure/signinlogs/ingest/pipeline.yml index 9d5351bf36ab..77ccfa32decd 100644 --- a/x-pack/filebeat/module/azure/signinlogs/ingest/pipeline.yml +++ b/x-pack/filebeat/module/azure/signinlogs/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing azure signin logs. processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - rename: field: azure target_field: azure-eventhub diff --git a/x-pack/filebeat/module/checkpoint/firewall/ingest/pipeline.yml b/x-pack/filebeat/module/checkpoint/firewall/ingest/pipeline.yml index 9a74b0b7c724..d21d421ce0ff 100644 --- a/x-pack/filebeat/module/checkpoint/firewall/ingest/pipeline.yml +++ b/x-pack/filebeat/module/checkpoint/firewall/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing checkpoint firewall logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: @@ -157,7 +160,7 @@ processors: target_field: source.nat.port type: long ignore_failure: true - ignore_missing: true + ignore_missing: true if: "ctx.checkpoint?.xlatesport != '0'" - rename: field: checkpoint.mac_source_address @@ -691,7 +694,7 @@ processors: field: client.nat.port type: long ignore_failure: true - ignore_missing: true + ignore_missing: true - convert: field: client.bytes type: long @@ -711,7 +714,7 @@ processors: field: server.nat.port type: long ignore_failure: true - ignore_missing: true + ignore_missing: true - convert: field: server.bytes type: long @@ -721,7 +724,7 @@ processors: field: server.packets type: long ignore_failure: true - ignore_missing: true + ignore_missing: true - script: lang: painless source: "ctx.network.bytes = ctx.source.bytes + ctx.destination.bytes" @@ -797,4 +800,4 @@ processors: on_failure: - set: field: error.message - value: '{{ _ingest.on_failure_message }}' \ No newline at end of file + value: '{{ _ingest.on_failure_message }}' diff --git a/x-pack/filebeat/module/fortinet/firewall/ingest/pipeline.yml b/x-pack/filebeat/module/fortinet/firewall/ingest/pipeline.yml index 2aaf7065ec1d..eeb5368db552 100644 --- a/x-pack/filebeat/module/fortinet/firewall/ingest/pipeline.yml +++ b/x-pack/filebeat/module/fortinet/firewall/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing fortinet firewall logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/x-pack/filebeat/module/googlecloud/audit/ingest/pipeline.yml b/x-pack/filebeat/module/googlecloud/audit/ingest/pipeline.yml index 6c148a0c07c4..8e0d3ac6fdb7 100644 --- a/x-pack/filebeat/module/googlecloud/audit/ingest/pipeline.yml +++ b/x-pack/filebeat/module/googlecloud/audit/ingest/pipeline.yml @@ -1,6 +1,9 @@ description: Pipeline for Google Cloud audit logs processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' - user_agent: field: user_agent.original ignore_missing: true diff --git a/x-pack/filebeat/module/ibmmq/errorlog/ingest/pipeline.yml b/x-pack/filebeat/module/ibmmq/errorlog/ingest/pipeline.yml index 80db3a86a86f..87c3deacb97c 100644 --- a/x-pack/filebeat/module/ibmmq/errorlog/ingest/pipeline.yml +++ b/x-pack/filebeat/module/ibmmq/errorlog/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing MQ error logs. processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - gsub: field: message pattern: ^[\-]{5}[a-z0-9\. :]*[\-]{5,} diff --git a/x-pack/filebeat/module/iptables/log/ingest/pipeline.yml b/x-pack/filebeat/module/iptables/log/ingest/pipeline.yml index 4eb24ff7d033..ecaa40ce67c6 100644 --- a/x-pack/filebeat/module/iptables/log/ingest/pipeline.yml +++ b/x-pack/filebeat/module/iptables/log/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for IPTables processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/x-pack/filebeat/module/microsoft/defender_atp/ingest/pipeline.yml b/x-pack/filebeat/module/microsoft/defender_atp/ingest/pipeline.yml index 6dedd5e8a1fc..392f3a441a73 100644 --- a/x-pack/filebeat/module/microsoft/defender_atp/ingest/pipeline.yml +++ b/x-pack/filebeat/module/microsoft/defender_atp/ingest/pipeline.yml @@ -1,6 +1,9 @@ --- description: Pipeline for parsing microsoft atp logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - remove: field: - message @@ -76,9 +79,6 @@ processors: - set: field: event.provider value: defender_atp -- set: - field: event.ingested - value: '{{_ingest.timestamp}}' - set: field: event.created value: '{{json.alertCreationTime}}' @@ -284,7 +284,7 @@ processors: ## Cleanup ## ############# - remove: - field: + field: - json.alertCreationTime - json.severity - json.relatedUser diff --git a/x-pack/filebeat/module/mssql/log/ingest/pipeline.yml b/x-pack/filebeat/module/mssql/log/ingest/pipeline.yml index 39a10a9ff99e..cae8f53ab347 100644 --- a/x-pack/filebeat/module/mssql/log/ingest/pipeline.yml +++ b/x-pack/filebeat/module/mssql/log/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline to parse MSSQL logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/x-pack/filebeat/module/o365/audit/ingest/pipeline.yml b/x-pack/filebeat/module/o365/audit/ingest/pipeline.yml index 98fd4f0ff588..493713469254 100644 --- a/x-pack/filebeat/module/o365/audit/ingest/pipeline.yml +++ b/x-pack/filebeat/module/o365/audit/ingest/pipeline.yml @@ -1,6 +1,9 @@ description: Pipeline for Office 365 Audit logs processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' - user_agent: field: user_agent.original ignore_missing: true diff --git a/x-pack/filebeat/module/okta/system/ingest/pipeline.yml b/x-pack/filebeat/module/okta/system/ingest/pipeline.yml index 78f6fa370477..e3d92540d5f0 100644 --- a/x-pack/filebeat/module/okta/system/ingest/pipeline.yml +++ b/x-pack/filebeat/module/okta/system/ingest/pipeline.yml @@ -1,6 +1,9 @@ description: Pipeline for Okta system logs. processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' - user_agent: field: user_agent.original ignore_missing: true @@ -44,7 +47,7 @@ processors: field: destination.as.organization_name target_field: destination.as.organization.name ignore_missing: true - + on_failure: - set: field: error.message diff --git a/x-pack/filebeat/module/rabbitmq/log/ingest/pipeline.yml b/x-pack/filebeat/module/rabbitmq/log/ingest/pipeline.yml index 58097c578d8c..e69402c6a953 100644 --- a/x-pack/filebeat/module/rabbitmq/log/ingest/pipeline.yml +++ b/x-pack/filebeat/module/rabbitmq/log/ingest/pipeline.yml @@ -1,6 +1,9 @@ --- description: Pipeline for parsing RabbitMQ logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message pattern_definitions: diff --git a/x-pack/filebeat/module/sophos/xg/ingest/pipeline.yml b/x-pack/filebeat/module/sophos/xg/ingest/pipeline.yml index f408b6f01cd0..8102bb925148 100644 --- a/x-pack/filebeat/module/sophos/xg/ingest/pipeline.yml +++ b/x-pack/filebeat/module/sophos/xg/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing sophosxg firewall logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: