diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 20297a22f6bf..6f6ffd6627bf 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -506,6 +506,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add support for additional fields and FirewallMatchEvent type events in CrowdStrike module {pull}20138[20138] - Add event.ingested for Suricata module {pull}20220[20220] - Add support for custom header and headersecret for filebeat http_endpoint input {pull}20435[20435] +- Add event.ingested to all Filebeat modules. {pull}20386[20386] *Heartbeat* diff --git a/filebeat/module/apache/access/ingest/pipeline.yml b/filebeat/module/apache/access/ingest/pipeline.yml index 6311bfef12bd..a9f23eb4a103 100644 --- a/filebeat/module/apache/access/ingest/pipeline.yml +++ b/filebeat/module/apache/access/ingest/pipeline.yml @@ -1,6 +1,9 @@ description: "Pipeline for parsing Apache HTTP Server access logs. Requires the geoip and user_agent plugins." processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/apache/error/ingest/pipeline.yml b/filebeat/module/apache/error/ingest/pipeline.yml index 967f7a34b69c..aad4c3f4a5f5 100644 --- a/filebeat/module/apache/error/ingest/pipeline.yml +++ b/filebeat/module/apache/error/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing apache error logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/auditd/log/ingest/pipeline.yml b/filebeat/module/auditd/log/ingest/pipeline.yml index 2b7c114f10af..26a8bf2ab91f 100644 --- a/filebeat/module/auditd/log/ingest/pipeline.yml +++ b/filebeat/module/auditd/log/ingest/pipeline.yml @@ -1,6 +1,9 @@ --- description: Pipeline for parsing Linux auditd logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message pattern_definitions: diff --git a/filebeat/module/elasticsearch/audit/ingest/pipeline.yml b/filebeat/module/elasticsearch/audit/ingest/pipeline.yml index 8f1093f5eea7..ec3873d2b9fb 100644 --- a/filebeat/module/elasticsearch/audit/ingest/pipeline.yml +++ b/filebeat/module/elasticsearch/audit/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing elasticsearch audit logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - rename: field: '@timestamp' target_field: event.created diff --git a/filebeat/module/elasticsearch/deprecation/ingest/pipeline.yml b/filebeat/module/elasticsearch/deprecation/ingest/pipeline.yml index 59b8cf882f9a..e1f4838df9b5 100644 --- a/filebeat/module/elasticsearch/deprecation/ingest/pipeline.yml +++ b/filebeat/module/elasticsearch/deprecation/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing elasticsearch deprecation logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - rename: field: '@timestamp' target_field: event.created diff --git a/filebeat/module/elasticsearch/gc/ingest/pipeline.yml b/filebeat/module/elasticsearch/gc/ingest/pipeline.yml index fc8ec5c73e35..d0980763ecc0 100644 --- a/filebeat/module/elasticsearch/gc/ingest/pipeline.yml +++ b/filebeat/module/elasticsearch/gc/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing Elasticsearch JVM garbage collection logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/elasticsearch/server/ingest/pipeline.yml b/filebeat/module/elasticsearch/server/ingest/pipeline.yml index 6e09a9dbde89..4d4e634cc4b6 100644 --- a/filebeat/module/elasticsearch/server/ingest/pipeline.yml +++ b/filebeat/module/elasticsearch/server/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing elasticsearch server logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - rename: field: '@timestamp' target_field: event.created diff --git a/filebeat/module/elasticsearch/slowlog/ingest/pipeline.yml b/filebeat/module/elasticsearch/slowlog/ingest/pipeline.yml index 360e86d9d778..ea501d9b3e05 100644 --- a/filebeat/module/elasticsearch/slowlog/ingest/pipeline.yml +++ b/filebeat/module/elasticsearch/slowlog/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing elasticsearch slow logs. processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - rename: field: '@timestamp' target_field: event.created diff --git a/filebeat/module/haproxy/log/ingest/pipeline.yml b/filebeat/module/haproxy/log/ingest/pipeline.yml index fdcfc828701e..d9315df0f024 100644 --- a/filebeat/module/haproxy/log/ingest/pipeline.yml +++ b/filebeat/module/haproxy/log/ingest/pipeline.yml @@ -1,6 +1,9 @@ description: Pipeline for parsing HAProxy http, tcp and default logs. Requires the geoip plugin. processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/icinga/debug/ingest/pipeline.yml b/filebeat/module/icinga/debug/ingest/pipeline.yml index ee25b38e90e5..dbe9f1ee39df 100644 --- a/filebeat/module/icinga/debug/ingest/pipeline.yml +++ b/filebeat/module/icinga/debug/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing icinga debug logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/icinga/main/ingest/pipeline.yml b/filebeat/module/icinga/main/ingest/pipeline.yml index 5db480e07ab3..654e8c3c4e7e 100644 --- a/filebeat/module/icinga/main/ingest/pipeline.yml +++ b/filebeat/module/icinga/main/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing icinga main logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/icinga/startup/ingest/pipeline.yml b/filebeat/module/icinga/startup/ingest/pipeline.yml index 61e0e6fef275..aee7377b1402 100644 --- a/filebeat/module/icinga/startup/ingest/pipeline.yml +++ b/filebeat/module/icinga/startup/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing icinga startup logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/iis/access/ingest/pipeline.yml b/filebeat/module/iis/access/ingest/pipeline.yml index 8344cccac1b8..84fabdc59b8d 100644 --- a/filebeat/module/iis/access/ingest/pipeline.yml +++ b/filebeat/module/iis/access/ingest/pipeline.yml @@ -1,6 +1,9 @@ description: Pipeline for parsing IIS access logs. Requires the geoip and user_agent plugins. processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/iis/error/ingest/pipeline.yml b/filebeat/module/iis/error/ingest/pipeline.yml index 4611744d3c94..a16fde841daf 100644 --- a/filebeat/module/iis/error/ingest/pipeline.yml +++ b/filebeat/module/iis/error/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing IIS error logs. Requires the geoip plugin. processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/kafka/log/ingest/pipeline.yml b/filebeat/module/kafka/log/ingest/pipeline.yml index a10724891225..aa72addb6420 100644 --- a/filebeat/module/kafka/log/ingest/pipeline.yml +++ b/filebeat/module/kafka/log/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing Kafka log messages processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message trace_match: true diff --git a/filebeat/module/kibana/log/ingest/pipeline.yml b/filebeat/module/kibana/log/ingest/pipeline.yml index 0112e09fcfce..ced76d42c23f 100644 --- a/filebeat/module/kibana/log/ingest/pipeline.yml +++ b/filebeat/module/kibana/log/ingest/pipeline.yml @@ -4,6 +4,9 @@ on_failure: field: error.message value: '{{ _ingest.on_failure_message }}' processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - rename: field: '@timestamp' target_field: event.created diff --git a/filebeat/module/logstash/log/ingest/pipeline.yml b/filebeat/module/logstash/log/ingest/pipeline.yml index 0a416e5758e4..e7dc228a76dc 100644 --- a/filebeat/module/logstash/log/ingest/pipeline.yml +++ b/filebeat/module/logstash/log/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing logstash node logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - rename: field: '@timestamp' target_field: event.created diff --git a/filebeat/module/logstash/slowlog/ingest/pipeline.yml b/filebeat/module/logstash/slowlog/ingest/pipeline.yml index 061a4f8c636c..949ffdcb91ef 100644 --- a/filebeat/module/logstash/slowlog/ingest/pipeline.yml +++ b/filebeat/module/logstash/slowlog/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing logstash slow logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - rename: field: '@timestamp' target_field: event.created diff --git a/filebeat/module/mongodb/log/ingest/pipeline.yml b/filebeat/module/mongodb/log/ingest/pipeline.yml index 6460a2b02c66..9355e0318023 100644 --- a/filebeat/module/mongodb/log/ingest/pipeline.yml +++ b/filebeat/module/mongodb/log/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing MongoDB logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/mysql/error/ingest/pipeline.yml b/filebeat/module/mysql/error/ingest/pipeline.yml index b11f280d1ead..baf4c11aa402 100644 --- a/filebeat/module/mysql/error/ingest/pipeline.yml +++ b/filebeat/module/mysql/error/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing MySQL error logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/mysql/slowlog/ingest/pipeline.json b/filebeat/module/mysql/slowlog/ingest/pipeline.json index 93ce577a3304..d3fbe49707cb 100644 --- a/filebeat/module/mysql/slowlog/ingest/pipeline.json +++ b/filebeat/module/mysql/slowlog/ingest/pipeline.json @@ -1,6 +1,11 @@ { "description": "Pipeline for parsing MySQL slow logs.", "processors": [{ + "set": { + "field": "event.ingested", + "value": "{{_ingest.timestamp}}" + } + }, { "grok": { "field": "message", "patterns":[ diff --git a/filebeat/module/nats/log/ingest/pipeline.yml b/filebeat/module/nats/log/ingest/pipeline.yml index 53c4f774b5ea..bece77c1b8ea 100644 --- a/filebeat/module/nats/log/ingest/pipeline.yml +++ b/filebeat/module/nats/log/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing nats log logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/nginx/access/ingest/pipeline.yml b/filebeat/module/nginx/access/ingest/pipeline.yml index f07e82f2b60b..57fe9031b557 100644 --- a/filebeat/module/nginx/access/ingest/pipeline.yml +++ b/filebeat/module/nginx/access/ingest/pipeline.yml @@ -1,6 +1,9 @@ description: Pipeline for parsing Nginx access logs. Requires the geoip and user_agent plugins. processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: @@ -145,7 +148,7 @@ processors: - set: field: event.outcome value: failure - if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code >= 400" + if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code >= 400" - append: field: related.ip value: "{{source.ip}}" diff --git a/filebeat/module/nginx/error/ingest/pipeline.yml b/filebeat/module/nginx/error/ingest/pipeline.yml index 5a33c34710cb..05691eeb7375 100644 --- a/filebeat/module/nginx/error/ingest/pipeline.yml +++ b/filebeat/module/nginx/error/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing the Nginx error logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml b/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml index 74118b7405e2..c9f4a5860c7a 100644 --- a/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml +++ b/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml @@ -1,6 +1,9 @@ description: Pipeline for parsing Nginx ingress controller access logs. Requires the geoip and user_agent plugins. processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/osquery/result/ingest/pipeline.json b/filebeat/module/osquery/result/ingest/pipeline.json index cbc45c202f90..c14b9664d1e6 100644 --- a/filebeat/module/osquery/result/ingest/pipeline.json +++ b/filebeat/module/osquery/result/ingest/pipeline.json @@ -2,6 +2,11 @@ "description": "Pipeline for parsing osquery result logs", "processors": [ { + "set":{ + "field": "event.ingested", + "value": "{{_ingest.timestamp}}" + } + }, { "rename": { "field": "@timestamp", "target_field": "event.created" diff --git a/filebeat/module/postgresql/log/ingest/pipeline.yml b/filebeat/module/postgresql/log/ingest/pipeline.yml index bd208d1eb724..9233ed95c5f4 100644 --- a/filebeat/module/postgresql/log/ingest/pipeline.yml +++ b/filebeat/module/postgresql/log/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing PostgreSQL logs. processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message ignore_missing: true diff --git a/filebeat/module/redis/log/ingest/pipeline.yml b/filebeat/module/redis/log/ingest/pipeline.yml index d1c08cab3787..472c3398e36f 100644 --- a/filebeat/module/redis/log/ingest/pipeline.yml +++ b/filebeat/module/redis/log/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing redis logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/santa/log/ingest/pipeline.yml b/filebeat/module/santa/log/ingest/pipeline.yml index 9b68cce3644e..e914253f8eec 100644 --- a/filebeat/module/santa/log/ingest/pipeline.yml +++ b/filebeat/module/santa/log/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing Google Santa logs. processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/system/auth/ingest/pipeline.yml b/filebeat/module/system/auth/ingest/pipeline.yml index 3f45705416a5..a958855936a9 100644 --- a/filebeat/module/system/auth/ingest/pipeline.yml +++ b/filebeat/module/system/auth/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing system authorisation/secure logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message ignore_missing: true diff --git a/filebeat/module/system/syslog/ingest/pipeline.yml b/filebeat/module/system/syslog/ingest/pipeline.yml index e0c80b9aad66..2963ba410b09 100644 --- a/filebeat/module/system/syslog/ingest/pipeline.yml +++ b/filebeat/module/system/syslog/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing Syslog messages. processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/traefik/access/ingest/pipeline.yml b/filebeat/module/traefik/access/ingest/pipeline.yml index ce489a4a92c2..dd5de1b0b0b6 100644 --- a/filebeat/module/traefik/access/ingest/pipeline.yml +++ b/filebeat/module/traefik/access/ingest/pipeline.yml @@ -1,6 +1,9 @@ description: Pipeline for parsing Traefik access logs. Requires the geoip and user_agent plugins. processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - dissect: field: message pattern: '%{source.address} %{traefik.access.user_identifier} %{user.name} [%{traefik.access.time}] diff --git a/filebeat/tests/system/test_modules.py b/filebeat/tests/system/test_modules.py index 9360abf51af6..b02a98b2f512 100644 --- a/filebeat/tests/system/test_modules.py +++ b/filebeat/tests/system/test_modules.py @@ -161,6 +161,10 @@ def run_on_file(self, module, fileset, test_file, cfgfile): assert obj["event"]["module"] == module, "expected event.module={} but got {}".format( module, obj["event"]["module"]) + # All modules must include a set processor that adds the time that + # the event was ingested to Elasticsearch + assert "ingested" in obj["event"], "missing event.ingested timestamp" + assert "error" not in obj, "not error expected but got: {}".format( obj) diff --git a/libbeat/tests/system/beat/compose.py b/libbeat/tests/system/beat/compose.py index f50ab299dff9..9e9f36d849bd 100644 --- a/libbeat/tests/system/beat/compose.py +++ b/libbeat/tests/system/beat/compose.py @@ -1,8 +1,11 @@ +import io +import logging import os import sys import tarfile import time -import io + +from contextlib import contextmanager INTEGRATION_TESTS = os.environ.get('INTEGRATION_TESTS', False) @@ -54,9 +57,12 @@ def is_healthy(container): return container.inspect()['State']['Health']['Status'] == 'healthy' project = cls.compose_project() - project.pull( - ignore_pull_failures=True, - service_names=cls.COMPOSE_SERVICES) + + with disabled_logger('compose.service'): + project.pull( + ignore_pull_failures=True, + service_names=cls.COMPOSE_SERVICES) + project.up( strategy=ConvergenceStrategy.always, service_names=cls.COMPOSE_SERVICES, @@ -231,3 +237,14 @@ def service_log_contains(cls, service, msg): if line.find(msg.encode("utf-8")) >= 0: counter += 1 return counter > 0 + + +@contextmanager +def disabled_logger(name): + logger = logging.getLogger(name) + old_level = logger.getEffectiveLevel() + logger.setLevel(logging.CRITICAL) + try: + yield logger + finally: + logger.setLevel(old_level) diff --git a/libbeat/tests/system/requirements.txt b/libbeat/tests/system/requirements.txt index 0d8d0f081a25..0829cbd01640 100644 --- a/libbeat/tests/system/requirements.txt +++ b/libbeat/tests/system/requirements.txt @@ -8,7 +8,7 @@ docker-compose==1.25.3 docker-pycreds==0.4.0 dockerpty==0.4.1 docopt==0.6.2 -elasticsearch==7.1.0 +elasticsearch==7.8.1 enum34==1.1.6 idna==2.6 ipaddress==1.0.19 @@ -19,7 +19,6 @@ nose==1.3.7 nose-timer==0.7.1 pycodestyle==2.4.0 PyYAML==4.2b1 -Pillow>=7.1.0 redis==2.10.6 requests==2.20.0 six==1.11.0 diff --git a/metricbeat/module/elasticsearch/test_elasticsearch.py b/metricbeat/module/elasticsearch/test_elasticsearch.py index f8e2762b8cf7..88e064c6d11e 100644 --- a/metricbeat/module/elasticsearch/test_elasticsearch.py +++ b/metricbeat/module/elasticsearch/test_elasticsearch.py @@ -24,7 +24,7 @@ class Test(metricbeat.BaseTest): def setUp(self): super(Test, self).setUp() self.es = Elasticsearch(self.get_hosts()) - self.ml_es = client.xpack.ml.MlClient(self.es) + self.ml_es = client.ml.MlClient(self.es) es_version = self.get_version() if es_version["major"] < 7: diff --git a/x-pack/elastic-agent/pkg/agent/operation/monitoring.go b/x-pack/elastic-agent/pkg/agent/operation/monitoring.go index 62372cc3f543..cd9d3d95a2cc 100644 --- a/x-pack/elastic-agent/pkg/agent/operation/monitoring.go +++ b/x-pack/elastic-agent/pkg/agent/operation/monitoring.go @@ -190,10 +190,10 @@ func (o *Operator) getMonitoringFilebeatConfig(output interface{}) (map[string]i "processors": []map[string]interface{}{ { "add_fields": map[string]interface{}{ - "target": "dataset", + "target": "data_stream", "fields": map[string]interface{}{ "type": "logs", - "name": "elastic.agent", + "dataset": "elastic.agent", "namespace": "default", }, }, @@ -224,10 +224,10 @@ func (o *Operator) getMonitoringFilebeatConfig(output interface{}) (map[string]i "processors": []map[string]interface{}{ { "add_fields": map[string]interface{}{ - "target": "dataset", + "target": "data_stream", "fields": map[string]interface{}{ "type": "logs", - "name": fmt.Sprintf("elastic.agent.%s", name), + "dataset": fmt.Sprintf("elastic.agent.%s", name), "namespace": "default", }, }, @@ -274,10 +274,10 @@ func (o *Operator) getMonitoringMetricbeatConfig(output interface{}) (map[string "processors": []map[string]interface{}{ { "add_fields": map[string]interface{}{ - "target": "dataset", + "target": "data_stream", "fields": map[string]interface{}{ "type": "metrics", - "name": fmt.Sprintf("elastic.agent.%s", name), + "dataset": fmt.Sprintf("elastic.agent.%s", name), "namespace": "default", }, }, diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/constraints_config-filebeat.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/constraints_config-filebeat.yml index 15f6b71a9531..15013da3377b 100644 --- a/x-pack/elastic-agent/pkg/agent/program/testdata/constraints_config-filebeat.yml +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/constraints_config-filebeat.yml @@ -7,10 +7,10 @@ filebeat: index: logs-generic-default processors: - add_fields: - target: "dataset" + target: "data_stream" fields: type: logs - name: generic + dataset: generic namespace: default - add_fields: target: "event" diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/enabled_output_true-filebeat.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/enabled_output_true-filebeat.yml index c2e8c0d26ec0..8edc27061b00 100644 --- a/x-pack/elastic-agent/pkg/agent/program/testdata/enabled_output_true-filebeat.yml +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/enabled_output_true-filebeat.yml @@ -7,10 +7,10 @@ filebeat: index: logs-generic-default processors: - add_fields: - target: "dataset" + target: "data_stream" fields: type: logs - name: generic + dataset: generic namespace: default - add_fields: target: "event" diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/enabled_true-filebeat.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/enabled_true-filebeat.yml index 1da1c701d81c..8bd5d93a3b97 100644 --- a/x-pack/elastic-agent/pkg/agent/program/testdata/enabled_true-filebeat.yml +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/enabled_true-filebeat.yml @@ -8,10 +8,10 @@ filebeat: index: logs-generic-default processors: - add_fields: - target: "dataset" + target: "data_stream" fields: type: logs - name: generic + dataset: generic namespace: default - add_fields: target: "event" diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-filebeat.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-filebeat.yml index 0fb1a4356b55..b996e13b5318 100644 --- a/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-filebeat.yml +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-filebeat.yml @@ -9,10 +9,10 @@ filebeat: var: value processors: - add_fields: - target: "dataset" + target: "data_stream" fields: type: logs - name: generic + dataset: generic namespace: default - add_fields: target: "event" @@ -27,10 +27,10 @@ filebeat: var: value processors: - add_fields: - target: "dataset" + target: "data_stream" fields: type: testtype - name: generic + dataset: generic namespace: default - add_fields: target: "event" diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-metricbeat.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-metricbeat.yml index 67a3815e4a76..c62882ff6da6 100644 --- a/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-metricbeat.yml +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-metricbeat.yml @@ -6,10 +6,10 @@ metricbeat: hosts: ["http://127.0.0.1:8080"] processors: - add_fields: - target: "dataset" + target: "data_stream" fields: type: metrics - name: docker.status + dataset: docker.status namespace: default - add_fields: target: "event" @@ -21,10 +21,10 @@ metricbeat: hosts: ["http://127.0.0.1:8080"] processors: - add_fields: - target: "dataset" + target: "data_stream" fields: type: metrics - name: generic + dataset: generic namespace: default - add_fields: target: "event" @@ -39,10 +39,10 @@ metricbeat: fields: should_be: first - add_fields: - target: "dataset" + target: "data_stream" fields: type: metrics - name: generic + dataset: generic namespace: testing - add_fields: target: "event" diff --git a/x-pack/elastic-agent/pkg/agent/transpiler/rules.go b/x-pack/elastic-agent/pkg/agent/transpiler/rules.go index fe98386a150c..a9fc8f4497da 100644 --- a/x-pack/elastic-agent/pkg/agent/transpiler/rules.go +++ b/x-pack/elastic-agent/pkg/agent/transpiler/rules.go @@ -632,16 +632,18 @@ func (r *InjectStreamProcessorRule) Apply(ast *AST) error { return errors.New("InjectStreamProcessorRule: processors is not a list") } + // datastream processorMap := &Dict{value: make([]Node, 0)} - processorMap.value = append(processorMap.value, &Key{name: "target", value: &StrVal{value: "dataset"}}) + processorMap.value = append(processorMap.value, &Key{name: "target", value: &StrVal{value: "data_stream"}}) processorMap.value = append(processorMap.value, &Key{name: "fields", value: &Dict{value: []Node{ &Key{name: "type", value: &StrVal{value: datasetType}}, &Key{name: "namespace", value: &StrVal{value: namespace}}, - &Key{name: "name", value: &StrVal{value: dataset}}, + &Key{name: "dataset", value: &StrVal{value: dataset}}, }}}) addFieldsMap := &Dict{value: []Node{&Key{"add_fields", processorMap}}} processorsList.value = mergeStrategy(r.OnConflict).InjectItem(processorsList.value, addFieldsMap) + // event processorMap = &Dict{value: make([]Node, 0)} processorMap.value = append(processorMap.value, &Key{name: "target", value: &StrVal{value: "event"}}) processorMap.value = append(processorMap.value, &Key{name: "fields", value: &Dict{value: []Node{ diff --git a/x-pack/filebeat/module/activemq/audit/ingest/pipeline.yml b/x-pack/filebeat/module/activemq/audit/ingest/pipeline.yml index afc4c50f3dce..c5cb5ee8ed14 100644 --- a/x-pack/filebeat/module/activemq/audit/ingest/pipeline.yml +++ b/x-pack/filebeat/module/activemq/audit/ingest/pipeline.yml @@ -1,6 +1,9 @@ --- description: Pipeline for parsing ActiveMQ audit logs. processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message pattern_definitions: diff --git a/x-pack/filebeat/module/activemq/log/ingest/pipeline.yml b/x-pack/filebeat/module/activemq/log/ingest/pipeline.yml index c33d77295e5b..b84807be893c 100644 --- a/x-pack/filebeat/module/activemq/log/ingest/pipeline.yml +++ b/x-pack/filebeat/module/activemq/log/ingest/pipeline.yml @@ -1,6 +1,9 @@ --- description: Pipeline for parsing ActiveMQ logs. processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message pattern_definitions: diff --git a/x-pack/filebeat/module/aws/cloudtrail/ingest/pipeline.yml b/x-pack/filebeat/module/aws/cloudtrail/ingest/pipeline.yml index 36773124439e..42395228853d 100644 --- a/x-pack/filebeat/module/aws/cloudtrail/ingest/pipeline.yml +++ b/x-pack/filebeat/module/aws/cloudtrail/ingest/pipeline.yml @@ -1,6 +1,9 @@ --- description: Pipeline for AWS CloudTrail Logs processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' - rename: field: "message" target_field: "event.original" @@ -614,7 +617,7 @@ processors: if (ctx.event.action == 'ConsoleLogin' && ctx?.aws?.cloudtrail?.flattened?.response_elements.ConsoleLogin != null) { ctx.event.outcome = Processors.lowercase(ctx.aws.cloudtrail.flattened.response_elements.ConsoleLogin); } - + def hm = new HashMap(params.get(ctx.event.action)); hm.forEach((k, v) -> ctx.event[k] = v); diff --git a/x-pack/filebeat/module/aws/cloudwatch/ingest/pipeline.yml b/x-pack/filebeat/module/aws/cloudwatch/ingest/pipeline.yml index ff7e20d1c3d1..05f254634141 100644 --- a/x-pack/filebeat/module/aws/cloudwatch/ingest/pipeline.yml +++ b/x-pack/filebeat/module/aws/cloudwatch/ingest/pipeline.yml @@ -1,6 +1,9 @@ description: "Pipeline for CloudWatch logs" processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/x-pack/filebeat/module/aws/ec2/ingest/pipeline.yml b/x-pack/filebeat/module/aws/ec2/ingest/pipeline.yml index 0ada24c6f77d..878aa14aef5a 100644 --- a/x-pack/filebeat/module/aws/ec2/ingest/pipeline.yml +++ b/x-pack/filebeat/module/aws/ec2/ingest/pipeline.yml @@ -1,6 +1,9 @@ description: "Pipeline for EC2 logs in CloudWatch" processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/x-pack/filebeat/module/aws/elb/ingest/pipeline.yml b/x-pack/filebeat/module/aws/elb/ingest/pipeline.yml index fbd1195dcae4..de772ccdf018 100644 --- a/x-pack/filebeat/module/aws/elb/ingest/pipeline.yml +++ b/x-pack/filebeat/module/aws/elb/ingest/pipeline.yml @@ -1,6 +1,9 @@ description: "Pipeline for ELB logs" processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message # Classic ELB patterns documented in https://docs.aws.amazon.com/elasticloadbalancing/latest/classic/access-log-collection.html diff --git a/x-pack/filebeat/module/aws/s3access/ingest/pipeline.yml b/x-pack/filebeat/module/aws/s3access/ingest/pipeline.yml index efd1a9d358a8..dd8613a904ad 100644 --- a/x-pack/filebeat/module/aws/s3access/ingest/pipeline.yml +++ b/x-pack/filebeat/module/aws/s3access/ingest/pipeline.yml @@ -1,6 +1,9 @@ description: "Pipeline for s3 server access logs" processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/x-pack/filebeat/module/aws/vpcflow/ingest/pipeline.yml b/x-pack/filebeat/module/aws/vpcflow/ingest/pipeline.yml index 0ad04419cbd7..bd9b1d32769b 100644 --- a/x-pack/filebeat/module/aws/vpcflow/ingest/pipeline.yml +++ b/x-pack/filebeat/module/aws/vpcflow/ingest/pipeline.yml @@ -1,6 +1,10 @@ description: Pipeline for AWS VPC Flow Logs processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' + # Convert Unix epoch to timestamp - date: field: "aws.vpcflow.end" diff --git a/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.yml b/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.yml index dac114956081..f8f10132a0de 100644 --- a/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.yml +++ b/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing azure activity logs. processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - rename: field: azure target_field: azure-eventhub diff --git a/x-pack/filebeat/module/azure/auditlogs/ingest/pipeline.yml b/x-pack/filebeat/module/azure/auditlogs/ingest/pipeline.yml index 2bf26322faf3..e6a29f6cc13d 100644 --- a/x-pack/filebeat/module/azure/auditlogs/ingest/pipeline.yml +++ b/x-pack/filebeat/module/azure/auditlogs/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing azure activity logs. processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - rename: field: azure target_field: azure-eventhub diff --git a/x-pack/filebeat/module/azure/signinlogs/ingest/pipeline.yml b/x-pack/filebeat/module/azure/signinlogs/ingest/pipeline.yml index 9d5351bf36ab..77ccfa32decd 100644 --- a/x-pack/filebeat/module/azure/signinlogs/ingest/pipeline.yml +++ b/x-pack/filebeat/module/azure/signinlogs/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing azure signin logs. processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - rename: field: azure target_field: azure-eventhub diff --git a/x-pack/filebeat/module/barracuda/waf/ingest/pipeline.yml b/x-pack/filebeat/module/barracuda/waf/ingest/pipeline.yml index dffea9720867..1616836706d6 100644 --- a/x-pack/filebeat/module/barracuda/waf/ingest/pipeline.yml +++ b/x-pack/filebeat/module/barracuda/waf/ingest/pipeline.yml @@ -2,6 +2,10 @@ description: Pipeline for Barracuda Web Application Firewall processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' + # User agent - user_agent: field: user_agent.original diff --git a/x-pack/filebeat/module/bluecoat/director/ingest/pipeline.yml b/x-pack/filebeat/module/bluecoat/director/ingest/pipeline.yml index e26891a1ad08..8f8064017d48 100644 --- a/x-pack/filebeat/module/bluecoat/director/ingest/pipeline.yml +++ b/x-pack/filebeat/module/bluecoat/director/ingest/pipeline.yml @@ -2,6 +2,10 @@ description: Pipeline for Blue Coat Director processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' + # User agent - user_agent: field: user_agent.original diff --git a/x-pack/filebeat/module/cef/log/ingest/pipeline.yml b/x-pack/filebeat/module/cef/log/ingest/pipeline.yml index 75a86ea27588..7dab1ca33821 100644 --- a/x-pack/filebeat/module/cef/log/ingest/pipeline.yml +++ b/x-pack/filebeat/module/cef/log/ingest/pipeline.yml @@ -2,6 +2,10 @@ description: Pipeline for Filebeat CEF processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' + # IP Geolocation Lookup - geoip: field: source.ip diff --git a/x-pack/filebeat/module/checkpoint/firewall/ingest/pipeline.yml b/x-pack/filebeat/module/checkpoint/firewall/ingest/pipeline.yml index 9a74b0b7c724..d21d421ce0ff 100644 --- a/x-pack/filebeat/module/checkpoint/firewall/ingest/pipeline.yml +++ b/x-pack/filebeat/module/checkpoint/firewall/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing checkpoint firewall logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: @@ -157,7 +160,7 @@ processors: target_field: source.nat.port type: long ignore_failure: true - ignore_missing: true + ignore_missing: true if: "ctx.checkpoint?.xlatesport != '0'" - rename: field: checkpoint.mac_source_address @@ -691,7 +694,7 @@ processors: field: client.nat.port type: long ignore_failure: true - ignore_missing: true + ignore_missing: true - convert: field: client.bytes type: long @@ -711,7 +714,7 @@ processors: field: server.nat.port type: long ignore_failure: true - ignore_missing: true + ignore_missing: true - convert: field: server.bytes type: long @@ -721,7 +724,7 @@ processors: field: server.packets type: long ignore_failure: true - ignore_missing: true + ignore_missing: true - script: lang: painless source: "ctx.network.bytes = ctx.source.bytes + ctx.destination.bytes" @@ -797,4 +800,4 @@ processors: on_failure: - set: field: error.message - value: '{{ _ingest.on_failure_message }}' \ No newline at end of file + value: '{{ _ingest.on_failure_message }}' diff --git a/x-pack/filebeat/module/cisco/ios/ingest/pipeline.yml b/x-pack/filebeat/module/cisco/ios/ingest/pipeline.yml index 6ffe20df8f5f..a09d2b31c5e9 100644 --- a/x-pack/filebeat/module/cisco/ios/ingest/pipeline.yml +++ b/x-pack/filebeat/module/cisco/ios/ingest/pipeline.yml @@ -1,6 +1,9 @@ description: Pipeline for Cisco IOS logs. processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' # IP Geolocation Lookup - geoip: field: source.ip diff --git a/x-pack/filebeat/module/cisco/shared/ingest/asa-ftd-pipeline.yml b/x-pack/filebeat/module/cisco/shared/ingest/asa-ftd-pipeline.yml index d34f3562e684..28a2750b6d49 100644 --- a/x-pack/filebeat/module/cisco/shared/ingest/asa-ftd-pipeline.yml +++ b/x-pack/filebeat/module/cisco/shared/ingest/asa-ftd-pipeline.yml @@ -1,6 +1,9 @@ --- description: "Pipeline for Cisco {< .internal_PREFIX >} logs" processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' # # Parse the syslog header # diff --git a/x-pack/filebeat/module/coredns/log/ingest/pipeline-entry.yml b/x-pack/filebeat/module/coredns/log/ingest/pipeline-entry.yml index f25d34178363..0a14b12f4c1f 100644 --- a/x-pack/filebeat/module/coredns/log/ingest/pipeline-entry.yml +++ b/x-pack/filebeat/module/coredns/log/ingest/pipeline-entry.yml @@ -1,6 +1,9 @@ --- description: Pipeline for normalizing Kubernetes CoreDNS logs. processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' - pipeline: if: ctx.message.charAt(0) == (char)("{") name: '{< IngestPipeline "pipeline-json" >}' diff --git a/x-pack/filebeat/module/cylance/protect/ingest/pipeline.yml b/x-pack/filebeat/module/cylance/protect/ingest/pipeline.yml index d6bca1e8c47c..286058aea621 100644 --- a/x-pack/filebeat/module/cylance/protect/ingest/pipeline.yml +++ b/x-pack/filebeat/module/cylance/protect/ingest/pipeline.yml @@ -2,6 +2,10 @@ description: Pipeline for CylanceProtect processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' + # User agent - user_agent: field: user_agent.original diff --git a/x-pack/filebeat/module/envoyproxy/log/ingest/pipeline-entry.yml b/x-pack/filebeat/module/envoyproxy/log/ingest/pipeline-entry.yml index 2bc7e14fb4f4..296d932f2cef 100644 --- a/x-pack/filebeat/module/envoyproxy/log/ingest/pipeline-entry.yml +++ b/x-pack/filebeat/module/envoyproxy/log/ingest/pipeline-entry.yml @@ -1,5 +1,8 @@ description: Pipeline for normalizing envoyproxy logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - pipeline: if: ctx.message.charAt(0) != (char)("{") name: '{< IngestPipeline "pipeline-plaintext" >}' diff --git a/x-pack/filebeat/module/f5/bigipapm/ingest/pipeline.yml b/x-pack/filebeat/module/f5/bigipapm/ingest/pipeline.yml index 0ea72c6ba4dc..2de20fc1a500 100644 --- a/x-pack/filebeat/module/f5/bigipapm/ingest/pipeline.yml +++ b/x-pack/filebeat/module/f5/bigipapm/ingest/pipeline.yml @@ -2,6 +2,10 @@ description: Pipeline for Big-IP Access Policy Manager processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' + # User agent - user_agent: field: user_agent.original diff --git a/x-pack/filebeat/module/fortinet/clientendpoint/ingest/pipeline.yml b/x-pack/filebeat/module/fortinet/clientendpoint/ingest/pipeline.yml index 1897a785e50d..1fd14e58bd61 100644 --- a/x-pack/filebeat/module/fortinet/clientendpoint/ingest/pipeline.yml +++ b/x-pack/filebeat/module/fortinet/clientendpoint/ingest/pipeline.yml @@ -2,6 +2,10 @@ description: Pipeline for Fortinet FortiClient Endpoint Security processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' + # User agent - user_agent: field: user_agent.original diff --git a/x-pack/filebeat/module/fortinet/firewall/ingest/pipeline.yml b/x-pack/filebeat/module/fortinet/firewall/ingest/pipeline.yml index 2aaf7065ec1d..eeb5368db552 100644 --- a/x-pack/filebeat/module/fortinet/firewall/ingest/pipeline.yml +++ b/x-pack/filebeat/module/fortinet/firewall/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing fortinet firewall logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/x-pack/filebeat/module/googlecloud/audit/ingest/pipeline.yml b/x-pack/filebeat/module/googlecloud/audit/ingest/pipeline.yml index 6c148a0c07c4..8e0d3ac6fdb7 100644 --- a/x-pack/filebeat/module/googlecloud/audit/ingest/pipeline.yml +++ b/x-pack/filebeat/module/googlecloud/audit/ingest/pipeline.yml @@ -1,6 +1,9 @@ description: Pipeline for Google Cloud audit logs processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' - user_agent: field: user_agent.original ignore_missing: true diff --git a/x-pack/filebeat/module/googlecloud/firewall/ingest/pipeline.yml b/x-pack/filebeat/module/googlecloud/firewall/ingest/pipeline.yml index 8d68de684a6f..b01435b7b62f 100644 --- a/x-pack/filebeat/module/googlecloud/firewall/ingest/pipeline.yml +++ b/x-pack/filebeat/module/googlecloud/firewall/ingest/pipeline.yml @@ -1,6 +1,10 @@ description: Pipeline for Google Cloud Firewall Logs processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' + # IP Geolocation Lookup - geoip: field: source.ip diff --git a/x-pack/filebeat/module/googlecloud/vpcflow/ingest/pipeline.yml b/x-pack/filebeat/module/googlecloud/vpcflow/ingest/pipeline.yml index 161de8ea0317..a8af06f2f4b6 100644 --- a/x-pack/filebeat/module/googlecloud/vpcflow/ingest/pipeline.yml +++ b/x-pack/filebeat/module/googlecloud/vpcflow/ingest/pipeline.yml @@ -1,6 +1,10 @@ description: Pipeline for Google Cloud VPC Flow Logs processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' + # IP Geolocation Lookup - geoip: field: source.ip diff --git a/x-pack/filebeat/module/ibmmq/errorlog/ingest/pipeline.yml b/x-pack/filebeat/module/ibmmq/errorlog/ingest/pipeline.yml index 80db3a86a86f..87c3deacb97c 100644 --- a/x-pack/filebeat/module/ibmmq/errorlog/ingest/pipeline.yml +++ b/x-pack/filebeat/module/ibmmq/errorlog/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing MQ error logs. processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - gsub: field: message pattern: ^[\-]{5}[a-z0-9\. :]*[\-]{5,} diff --git a/x-pack/filebeat/module/imperva/securesphere/ingest/pipeline.yml b/x-pack/filebeat/module/imperva/securesphere/ingest/pipeline.yml index 4a84f2a8bc89..63671e09e979 100644 --- a/x-pack/filebeat/module/imperva/securesphere/ingest/pipeline.yml +++ b/x-pack/filebeat/module/imperva/securesphere/ingest/pipeline.yml @@ -2,6 +2,10 @@ description: Pipeline for Imperva SecureSphere processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' + # User agent - user_agent: field: user_agent.original diff --git a/x-pack/filebeat/module/infoblox/nios/ingest/pipeline.yml b/x-pack/filebeat/module/infoblox/nios/ingest/pipeline.yml index 5693b4aea498..cc784492797e 100644 --- a/x-pack/filebeat/module/infoblox/nios/ingest/pipeline.yml +++ b/x-pack/filebeat/module/infoblox/nios/ingest/pipeline.yml @@ -2,6 +2,10 @@ description: Pipeline for Infoblox NIOS processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' + # User agent - user_agent: field: user_agent.original diff --git a/x-pack/filebeat/module/iptables/log/ingest/pipeline.yml b/x-pack/filebeat/module/iptables/log/ingest/pipeline.yml index 4eb24ff7d033..ecaa40ce67c6 100644 --- a/x-pack/filebeat/module/iptables/log/ingest/pipeline.yml +++ b/x-pack/filebeat/module/iptables/log/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for IPTables processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/x-pack/filebeat/module/microsoft/defender_atp/config/atp.yml b/x-pack/filebeat/module/microsoft/defender_atp/config/atp.yml index 5210fc53e759..5108ebdad073 100644 --- a/x-pack/filebeat/module/microsoft/defender_atp/config/atp.yml +++ b/x-pack/filebeat/module/microsoft/defender_atp/config/atp.yml @@ -11,7 +11,7 @@ url: {{ .url }} oauth2: {{ .oauth2 | tojson }} oauth2.provider: azure oauth2.azure.resource: https://api.securitycenter.windows.com/ - +http_headers: {{ .http_headers | tojson }} date_cursor.field: lastUpdateTime date_cursor.url_field: '$filter' date_cursor.value_template: {{ .date_cursor.value_template }} diff --git a/x-pack/filebeat/module/microsoft/defender_atp/ingest/pipeline.yml b/x-pack/filebeat/module/microsoft/defender_atp/ingest/pipeline.yml index 6dedd5e8a1fc..392f3a441a73 100644 --- a/x-pack/filebeat/module/microsoft/defender_atp/ingest/pipeline.yml +++ b/x-pack/filebeat/module/microsoft/defender_atp/ingest/pipeline.yml @@ -1,6 +1,9 @@ --- description: Pipeline for parsing microsoft atp logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - remove: field: - message @@ -76,9 +79,6 @@ processors: - set: field: event.provider value: defender_atp -- set: - field: event.ingested - value: '{{_ingest.timestamp}}' - set: field: event.created value: '{{json.alertCreationTime}}' @@ -284,7 +284,7 @@ processors: ## Cleanup ## ############# - remove: - field: + field: - json.alertCreationTime - json.severity - json.relatedUser diff --git a/x-pack/filebeat/module/microsoft/defender_atp/manifest.yml b/x-pack/filebeat/module/microsoft/defender_atp/manifest.yml index 99cca9da1af1..22db34487109 100644 --- a/x-pack/filebeat/module/microsoft/defender_atp/manifest.yml +++ b/x-pack/filebeat/module/microsoft/defender_atp/manifest.yml @@ -6,14 +6,17 @@ var: - name: interval default: 5m - name: date_cursor - default: + default: value_template: "lastUpdateTime gt {{.}}" - name: tags default: [defender-atp, forwarded] + - name: http_headers + default: + User-Agent: MdatpPartner-Elastic-Filebeat/1.0.0 - name: url default: "https://api.securitycenter.windows.com/api/alerts?$expand=evidence" - name: oauth2 - + ingest_pipeline: ingest/pipeline.yml input: config/atp.yml diff --git a/x-pack/filebeat/module/microsoft/dhcp/ingest/pipeline.yml b/x-pack/filebeat/module/microsoft/dhcp/ingest/pipeline.yml index 184e6c3e4a92..fd43032ff6eb 100644 --- a/x-pack/filebeat/module/microsoft/dhcp/ingest/pipeline.yml +++ b/x-pack/filebeat/module/microsoft/dhcp/ingest/pipeline.yml @@ -2,6 +2,10 @@ description: Pipeline for Microsoft DHCP processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' + # User agent - user_agent: field: user_agent.original diff --git a/x-pack/filebeat/module/misp/threat/ingest/pipeline.json b/x-pack/filebeat/module/misp/threat/ingest/pipeline.json index 0d710feeb243..59abc2fc21e2 100644 --- a/x-pack/filebeat/module/misp/threat/ingest/pipeline.json +++ b/x-pack/filebeat/module/misp/threat/ingest/pipeline.json @@ -1,6 +1,12 @@ { "description": "Pipeline for normalizing MISP threat", "processors": [ + { + "set": { + "field": "event.ingested", + "value": "{{_ingest.timestamp}}" + } + }, { "geoip": { "field": "destination.ip", diff --git a/x-pack/filebeat/module/mssql/log/ingest/pipeline.yml b/x-pack/filebeat/module/mssql/log/ingest/pipeline.yml index 39a10a9ff99e..cae8f53ab347 100644 --- a/x-pack/filebeat/module/mssql/log/ingest/pipeline.yml +++ b/x-pack/filebeat/module/mssql/log/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline to parse MSSQL logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/x-pack/filebeat/module/netscout/sightline/ingest/pipeline.yml b/x-pack/filebeat/module/netscout/sightline/ingest/pipeline.yml index 66f9ab7bcc17..5525c2ba70f2 100644 --- a/x-pack/filebeat/module/netscout/sightline/ingest/pipeline.yml +++ b/x-pack/filebeat/module/netscout/sightline/ingest/pipeline.yml @@ -2,6 +2,10 @@ description: Pipeline for Arbor Peakflow SP processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' + # User agent - user_agent: field: user_agent.original diff --git a/x-pack/filebeat/module/o365/audit/ingest/pipeline.yml b/x-pack/filebeat/module/o365/audit/ingest/pipeline.yml index 98fd4f0ff588..493713469254 100644 --- a/x-pack/filebeat/module/o365/audit/ingest/pipeline.yml +++ b/x-pack/filebeat/module/o365/audit/ingest/pipeline.yml @@ -1,6 +1,9 @@ description: Pipeline for Office 365 Audit logs processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' - user_agent: field: user_agent.original ignore_missing: true diff --git a/x-pack/filebeat/module/okta/system/ingest/pipeline.yml b/x-pack/filebeat/module/okta/system/ingest/pipeline.yml index 78f6fa370477..e3d92540d5f0 100644 --- a/x-pack/filebeat/module/okta/system/ingest/pipeline.yml +++ b/x-pack/filebeat/module/okta/system/ingest/pipeline.yml @@ -1,6 +1,9 @@ description: Pipeline for Okta system logs. processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' - user_agent: field: user_agent.original ignore_missing: true @@ -44,7 +47,7 @@ processors: field: destination.as.organization_name target_field: destination.as.organization.name ignore_missing: true - + on_failure: - set: field: error.message diff --git a/x-pack/filebeat/module/panw/panos/ingest/pipeline.yml b/x-pack/filebeat/module/panw/panos/ingest/pipeline.yml index 7cc44f287b69..25d1ba1681bf 100644 --- a/x-pack/filebeat/module/panw/panos/ingest/pipeline.yml +++ b/x-pack/filebeat/module/panw/panos/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: "Pipeline for Palo Alto Networks PAN-OS Logs" processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' # keep message as log.original. - rename: diff --git a/x-pack/filebeat/module/rabbitmq/log/ingest/pipeline.yml b/x-pack/filebeat/module/rabbitmq/log/ingest/pipeline.yml index 58097c578d8c..e69402c6a953 100644 --- a/x-pack/filebeat/module/rabbitmq/log/ingest/pipeline.yml +++ b/x-pack/filebeat/module/rabbitmq/log/ingest/pipeline.yml @@ -1,6 +1,9 @@ --- description: Pipeline for parsing RabbitMQ logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message pattern_definitions: diff --git a/x-pack/filebeat/module/rapid7/nexpose/ingest/pipeline.yml b/x-pack/filebeat/module/rapid7/nexpose/ingest/pipeline.yml index d558e7071eaf..816d612b6a70 100644 --- a/x-pack/filebeat/module/rapid7/nexpose/ingest/pipeline.yml +++ b/x-pack/filebeat/module/rapid7/nexpose/ingest/pipeline.yml @@ -2,6 +2,10 @@ description: Pipeline for Rapid7 NeXpose processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' + # User agent - user_agent: field: user_agent.original diff --git a/x-pack/filebeat/module/sonicwall/firewall/ingest/pipeline.yml b/x-pack/filebeat/module/sonicwall/firewall/ingest/pipeline.yml index 75670b6f441a..fdfb0f7f9a06 100644 --- a/x-pack/filebeat/module/sonicwall/firewall/ingest/pipeline.yml +++ b/x-pack/filebeat/module/sonicwall/firewall/ingest/pipeline.yml @@ -2,6 +2,10 @@ description: Pipeline for Sonicwall-FW processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' + # User agent - user_agent: field: user_agent.original diff --git a/x-pack/filebeat/module/sophos/xg/ingest/pipeline.yml b/x-pack/filebeat/module/sophos/xg/ingest/pipeline.yml index f408b6f01cd0..8102bb925148 100644 --- a/x-pack/filebeat/module/sophos/xg/ingest/pipeline.yml +++ b/x-pack/filebeat/module/sophos/xg/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing sophosxg firewall logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/x-pack/filebeat/module/squid/log/ingest/pipeline.yml b/x-pack/filebeat/module/squid/log/ingest/pipeline.yml index caeba41fcbc5..574cfafde0a2 100644 --- a/x-pack/filebeat/module/squid/log/ingest/pipeline.yml +++ b/x-pack/filebeat/module/squid/log/ingest/pipeline.yml @@ -2,6 +2,10 @@ description: Pipeline for Squid processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' + # User agent - user_agent: field: user_agent.original diff --git a/x-pack/filebeat/module/tomcat/log/ingest/pipeline.yml b/x-pack/filebeat/module/tomcat/log/ingest/pipeline.yml index e5cd87682ea3..16a25fde6f2c 100644 --- a/x-pack/filebeat/module/tomcat/log/ingest/pipeline.yml +++ b/x-pack/filebeat/module/tomcat/log/ingest/pipeline.yml @@ -2,6 +2,10 @@ description: Pipeline for Apache Tomcat processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' + # User agent - user_agent: field: user_agent.original diff --git a/x-pack/filebeat/module/zeek/capture_loss/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/capture_loss/ingest/pipeline.yml index 3c6171bc0451..76e5178572e3 100644 --- a/x-pack/filebeat/module/zeek/capture_loss/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/capture_loss/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek capture_loss.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.capture_loss.ts formats: diff --git a/x-pack/filebeat/module/zeek/connection/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/connection/ingest/pipeline.yml index b660079324ab..a243e057420d 100644 --- a/x-pack/filebeat/module/zeek/connection/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/connection/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek conn.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.connection.ts formats: diff --git a/x-pack/filebeat/module/zeek/dce_rpc/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/dce_rpc/ingest/pipeline.yml index 1ecda252cc85..e77cd5646f03 100644 --- a/x-pack/filebeat/module/zeek/dce_rpc/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/dce_rpc/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek dce_rpc.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.dce_rpc.ts formats: diff --git a/x-pack/filebeat/module/zeek/dhcp/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/dhcp/ingest/pipeline.yml index 49df687ecc34..49216c077c27 100644 --- a/x-pack/filebeat/module/zeek/dhcp/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/dhcp/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek dhcp.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.dhcp.ts formats: diff --git a/x-pack/filebeat/module/zeek/dnp3/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/dnp3/ingest/pipeline.yml index ad4670dc3502..f1a1e527cfc2 100644 --- a/x-pack/filebeat/module/zeek/dnp3/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/dnp3/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek dnp3.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.dnp3.ts formats: diff --git a/x-pack/filebeat/module/zeek/dns/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/dns/ingest/pipeline.yml index db603d93dbbb..77ea898c66bb 100644 --- a/x-pack/filebeat/module/zeek/dns/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/dns/ingest/pipeline.yml @@ -2,6 +2,10 @@ description: Pipeline for Filebeat Zeek dns.log processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' + # IP Geolocation Lookup - geoip: field: source.ip diff --git a/x-pack/filebeat/module/zeek/dpd/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/dpd/ingest/pipeline.yml index f30ff172fa8e..32d1852c3e2c 100644 --- a/x-pack/filebeat/module/zeek/dpd/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/dpd/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek dpd.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.dpd.ts formats: diff --git a/x-pack/filebeat/module/zeek/files/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/files/ingest/pipeline.yml index 0d5abf9bdda9..754720e92095 100644 --- a/x-pack/filebeat/module/zeek/files/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/files/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek files.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.files.ts formats: @@ -47,7 +50,7 @@ processors: - set: field: client.ip value: "{{zeek.files.rx_host}}" - if: "ctx?.zeek?.files?.rx_host != null" + if: "ctx?.zeek?.files?.rx_host != null" - append: field: related.hash value: "{{file.hash.md5}}" diff --git a/x-pack/filebeat/module/zeek/ftp/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/ftp/ingest/pipeline.yml index 7c15dce3ac52..f1f7d0b4f522 100644 --- a/x-pack/filebeat/module/zeek/ftp/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/ftp/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek ftp.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.ftp.ts formats: diff --git a/x-pack/filebeat/module/zeek/http/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/http/ingest/pipeline.yml index a382c25a74db..a2c4a85b9941 100644 --- a/x-pack/filebeat/module/zeek/http/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/http/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek http.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.http.ts formats: diff --git a/x-pack/filebeat/module/zeek/intel/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/intel/ingest/pipeline.yml index 6a2bd6382ad7..c6e64e016b85 100644 --- a/x-pack/filebeat/module/zeek/intel/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/intel/ingest/pipeline.yml @@ -1,9 +1,12 @@ --- description: Pipeline for normalizing Zeek intel.log. processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' - set: field: event.created - value: "{{_ingest.timestamp}}" + value: '{{@timestamp}}' # IP Geolocation Lookup - geoip: diff --git a/x-pack/filebeat/module/zeek/irc/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/irc/ingest/pipeline.yml index ec04f4e7c933..dd1e37a7035e 100644 --- a/x-pack/filebeat/module/zeek/irc/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/irc/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek irc.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.irc.ts formats: diff --git a/x-pack/filebeat/module/zeek/kerberos/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/kerberos/ingest/pipeline.yml index 05005491115d..3604287cb5e7 100644 --- a/x-pack/filebeat/module/zeek/kerberos/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/kerberos/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek kerberos.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.kerberos.ts formats: diff --git a/x-pack/filebeat/module/zeek/modbus/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/modbus/ingest/pipeline.yml index d053a541ef51..d918b2de09a2 100644 --- a/x-pack/filebeat/module/zeek/modbus/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/modbus/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek modbus.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.modbus.ts formats: diff --git a/x-pack/filebeat/module/zeek/mysql/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/mysql/ingest/pipeline.yml index ca2c6c571726..ce2de3535495 100644 --- a/x-pack/filebeat/module/zeek/mysql/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/mysql/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek mysql.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.mysql.ts formats: diff --git a/x-pack/filebeat/module/zeek/notice/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/notice/ingest/pipeline.yml index c4dee6b78f23..c741d355361f 100644 --- a/x-pack/filebeat/module/zeek/notice/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/notice/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek notice.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.notice.ts formats: diff --git a/x-pack/filebeat/module/zeek/ntlm/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/ntlm/ingest/pipeline.yml index 9f76d461392b..690fd54a54ba 100644 --- a/x-pack/filebeat/module/zeek/ntlm/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/ntlm/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek ntlm.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.ntlm.ts formats: diff --git a/x-pack/filebeat/module/zeek/ocsp/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/ocsp/ingest/pipeline.yml index 63a878825d73..462c1f366120 100644 --- a/x-pack/filebeat/module/zeek/ocsp/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/ocsp/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek ocsp.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.ocsp.ts formats: diff --git a/x-pack/filebeat/module/zeek/pe/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/pe/ingest/pipeline.yml index 6a7fa7dca872..6e1272a8ab2a 100644 --- a/x-pack/filebeat/module/zeek/pe/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/pe/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek pe.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.pe.ts formats: diff --git a/x-pack/filebeat/module/zeek/radius/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/radius/ingest/pipeline.yml index c69dfaefbb42..acc7fad2f030 100644 --- a/x-pack/filebeat/module/zeek/radius/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/radius/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek radius.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.radius.ts formats: diff --git a/x-pack/filebeat/module/zeek/rdp/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/rdp/ingest/pipeline.yml index d6b70dd92e67..bbe4abcee9fa 100644 --- a/x-pack/filebeat/module/zeek/rdp/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/rdp/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek rdp.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.rdp.ts formats: diff --git a/x-pack/filebeat/module/zeek/rfb/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/rfb/ingest/pipeline.yml index 8cf2cebdf4dd..2ce5fda4e16b 100644 --- a/x-pack/filebeat/module/zeek/rfb/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/rfb/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek rfb.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.rfb.ts formats: diff --git a/x-pack/filebeat/module/zeek/sip/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/sip/ingest/pipeline.yml index 9982cb82d872..d8408c511333 100644 --- a/x-pack/filebeat/module/zeek/sip/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/sip/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek sip.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.sip.ts formats: diff --git a/x-pack/filebeat/module/zeek/smb_cmd/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/smb_cmd/ingest/pipeline.yml index 838e9f2e8bcc..0a853104351e 100644 --- a/x-pack/filebeat/module/zeek/smb_cmd/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/smb_cmd/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek smb_cmd.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.smb_cmd.ts formats: diff --git a/x-pack/filebeat/module/zeek/smb_files/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/smb_files/ingest/pipeline.yml index b2c7f52a29bc..b1c0d3a69920 100644 --- a/x-pack/filebeat/module/zeek/smb_files/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/smb_files/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek smb_files.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.smb_files.ts formats: diff --git a/x-pack/filebeat/module/zeek/smb_mapping/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/smb_mapping/ingest/pipeline.yml index b5752120267c..e116e1bfb600 100644 --- a/x-pack/filebeat/module/zeek/smb_mapping/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/smb_mapping/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek smb_mapping.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.smb_mapping.ts formats: diff --git a/x-pack/filebeat/module/zeek/smtp/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/smtp/ingest/pipeline.yml index 4424d3674ff0..03e2ffb6a250 100644 --- a/x-pack/filebeat/module/zeek/smtp/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/smtp/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek smtp.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.smtp.ts formats: diff --git a/x-pack/filebeat/module/zeek/snmp/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/snmp/ingest/pipeline.yml index f0070ef790dd..1aefc539733d 100644 --- a/x-pack/filebeat/module/zeek/snmp/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/snmp/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek snmp.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.snmp.ts formats: diff --git a/x-pack/filebeat/module/zeek/socks/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/socks/ingest/pipeline.yml index 04a84b131777..210d97710239 100644 --- a/x-pack/filebeat/module/zeek/socks/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/socks/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek socks.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.socks.ts formats: diff --git a/x-pack/filebeat/module/zeek/ssh/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/ssh/ingest/pipeline.yml index 019a44b89e0f..26980d26f3da 100644 --- a/x-pack/filebeat/module/zeek/ssh/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/ssh/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek ssh.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.ssh.ts formats: diff --git a/x-pack/filebeat/module/zeek/ssl/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/ssl/ingest/pipeline.yml index bbeaa24d1bd4..c40801f4243f 100644 --- a/x-pack/filebeat/module/zeek/ssl/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/ssl/ingest/pipeline.yml @@ -2,8 +2,11 @@ description: Pipeline for normalizing Zeek ssl.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.ssl.ts formats: diff --git a/x-pack/filebeat/module/zeek/stats/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/stats/ingest/pipeline.yml index c0347161190d..04e851e14a90 100644 --- a/x-pack/filebeat/module/zeek/stats/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/stats/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek stats.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.stats.ts formats: diff --git a/x-pack/filebeat/module/zeek/syslog/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/syslog/ingest/pipeline.yml index 7fd848682b16..5f3432ec4888 100644 --- a/x-pack/filebeat/module/zeek/syslog/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/syslog/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek syslog.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.syslog.ts formats: diff --git a/x-pack/filebeat/module/zeek/traceroute/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/traceroute/ingest/pipeline.yml index 6fa5a0bc9937..f4744c540d71 100644 --- a/x-pack/filebeat/module/zeek/traceroute/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/traceroute/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek traceroute.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.traceroute.ts formats: diff --git a/x-pack/filebeat/module/zeek/tunnel/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/tunnel/ingest/pipeline.yml index 402bce5fa5d3..9ca83da33051 100644 --- a/x-pack/filebeat/module/zeek/tunnel/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/tunnel/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek tunnel.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.tunnel.ts formats: diff --git a/x-pack/filebeat/module/zeek/weird/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/weird/ingest/pipeline.yml index e0325d9a1c53..d791eb77a09c 100644 --- a/x-pack/filebeat/module/zeek/weird/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/weird/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek weird.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.weird.ts formats: diff --git a/x-pack/filebeat/module/zeek/x509/ingest/pipeline.json b/x-pack/filebeat/module/zeek/x509/ingest/pipeline.json index e35b8bbbafcb..aeb1dbf36fba 100644 --- a/x-pack/filebeat/module/zeek/x509/ingest/pipeline.json +++ b/x-pack/filebeat/module/zeek/x509/ingest/pipeline.json @@ -3,10 +3,16 @@ "processors": [ { "set": { - "field": "event.created", + "field": "event.ingested", "value": "{{_ingest.timestamp}}" } }, + { + "set": { + "field": "event.created", + "value": "{{@timestamp}}" + } + }, { "date": { "field": "zeek.x509.ts", diff --git a/x-pack/filebeat/module/zscaler/zia/ingest/pipeline.yml b/x-pack/filebeat/module/zscaler/zia/ingest/pipeline.yml index 3354fb0674a1..884dd6392a5d 100644 --- a/x-pack/filebeat/module/zscaler/zia/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zscaler/zia/ingest/pipeline.yml @@ -2,6 +2,10 @@ description: Pipeline for Zscaler NSS processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' + # User agent - user_agent: field: user_agent.original