diff --git a/filebeat/module/postgresql/log/ingest/pipeline.json b/filebeat/module/postgresql/log/ingest/pipeline.json deleted file mode 100644 index 03f17294633e..000000000000 --- a/filebeat/module/postgresql/log/ingest/pipeline.json +++ /dev/null @@ -1,49 +0,0 @@ -{ - "description": "Pipeline for parsing PostgreSQL logs.", - "processors": [ - { - "grok": { - "field": "message", - "ignore_missing": true, - "patterns": [ - "^%{DATETIME:postgresql.log.timestamp} (\\[%{NUMBER:process.pid:long}(-%{BASE16FLOAT:postgresql.log.core_id:long})?\\] ((\\[%{USERNAME:user.name}\\]@\\[%{POSTGRESQL_DB_NAME:postgresql.log.database}\\]|%{USERNAME:user.name}@%{POSTGRESQL_DB_NAME:postgresql.log.database}) )?)?%{WORD:log.level}: (?:%{NUMBER:postgresql.log.error.code:long}|%{SPACE})(duration: %{NUMBER:temp.duration:float} ms %{POSTGRESQL_QUERY_STEP}: %{GREEDYDATA:postgresql.log.query}|: %{GREEDYDATA:message}|%{GREEDYDATA:message})" - ], - "pattern_definitions": { - "DATETIME": "[-0-9]+ %{TIME} %{WORD:event.timezone}", - "GREEDYDATA": "(.|\n|\t)*", - "POSTGRESQL_DB_NAME": "[a-zA-Z0-9_]+[a-zA-Z0-9_\\$]*", - "POSTGRESQL_QUERY_STEP": "%{WORD:postgresql.log.query_step}(?: | %{WORD:postgresql.log.query_name})?" - } - } - }, - { - "date": { - "field": "postgresql.log.timestamp", - "target_field": "@timestamp", - "formats": [ - "yyyy-MM-dd HH:mm:ss.SSS zz", "yyyy-MM-dd HH:mm:ss zz" - ] - } - }, { - "script": { - "lang": "painless", - "source": "ctx.event.duration = Math.round(ctx.temp.duration * params.scale)", - "params": { "scale": 1000000 }, - "if": "ctx.temp?.duration != null" - } - }, { - "remove": { - "field": "temp.duration", - "ignore_missing": true - } - } - ], - "on_failure": [ - { - "set": { - "field": "error.message", - "value": "{{ _ingest.on_failure_message }}" - } - } - ] -} diff --git a/filebeat/module/postgresql/log/test/postgresql-10-default.log-expected.json b/filebeat/module/postgresql/log/test/postgresql-10-default.log-expected.json index 3e9ddde02750..3f1f0fe1d582 100644 --- a/filebeat/module/postgresql/log/test/postgresql-10-default.log-expected.json +++ b/filebeat/module/postgresql/log/test/postgresql-10-default.log-expected.json @@ -1,9 +1,16 @@ [ { "@timestamp": "2020-04-15T10:02:55.244Z", + "event.category": [ + "database" + ], "event.dataset": "postgresql.log", + "event.kind": "event", "event.module": "postgresql", "event.timezone": "CEST", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "LOG", @@ -15,9 +22,16 @@ }, { "@timestamp": "2020-04-15T10:02:55.247Z", + "event.category": [ + "database" + ], "event.dataset": "postgresql.log", + "event.kind": "event", "event.module": "postgresql", "event.timezone": "CEST", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "LOG", @@ -29,9 +43,16 @@ }, { "@timestamp": "2020-04-15T10:04:45.416Z", + "event.category": [ + "database" + ], "event.dataset": "postgresql.log", + "event.kind": "event", "event.module": "postgresql", "event.timezone": "CEST", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "FATAL", @@ -43,9 +64,16 @@ }, { "@timestamp": "2020-04-15T10:04:45.416Z", + "event.category": [ + "database" + ], "event.dataset": "postgresql.log", + "event.kind": "event", "event.module": "postgresql", "event.timezone": "CEST", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.flags": [ @@ -60,9 +88,16 @@ }, { "@timestamp": "2020-04-15T10:04:45.416Z", + "event.category": [ + "database" + ], "event.dataset": "postgresql.log", + "event.kind": "event", "event.module": "postgresql", "event.timezone": "CEST", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "LOG", @@ -74,9 +109,16 @@ }, { "@timestamp": "2020-04-15T10:06:36.719Z", + "event.category": [ + "database" + ], "event.dataset": "postgresql.log", + "event.kind": "event", "event.module": "postgresql", "event.timezone": "CEST", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "ERROR", @@ -88,9 +130,16 @@ }, { "@timestamp": "2020-04-15T10:56:29.569Z", + "event.category": [ + "database" + ], "event.dataset": "postgresql.log", + "event.kind": "event", "event.module": "postgresql", "event.timezone": "CEST", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.flags": [ diff --git a/filebeat/module/postgresql/log/test/postgresql-10-min-duration-statement.log-expected.json b/filebeat/module/postgresql/log/test/postgresql-10-min-duration-statement.log-expected.json index 591e1b69dbe1..d179a88067c8 100644 --- a/filebeat/module/postgresql/log/test/postgresql-10-min-duration-statement.log-expected.json +++ b/filebeat/module/postgresql/log/test/postgresql-10-min-duration-statement.log-expected.json @@ -1,9 +1,16 @@ [ { "@timestamp": "2019-09-22T06:28:24.000Z", + "event.category": [ + "database" + ], "event.dataset": "postgresql.log", + "event.kind": "event", "event.module": "postgresql", "event.timezone": "UTC", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "DETAIL", @@ -14,10 +21,17 @@ }, { "@timestamp": "2019-09-22T06:28:24.000Z", + "event.category": [ + "database" + ], "event.dataset": "postgresql.log", - "event.duration": 112336998, + "event.duration": 112337000, + "event.kind": "event", "event.module": "postgresql", "event.timezone": "UTC", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "LOG", @@ -31,9 +45,16 @@ }, { "@timestamp": "2019-09-22T06:28:24.000Z", + "event.category": [ + "database" + ], "event.dataset": "postgresql.log", + "event.kind": "event", "event.module": "postgresql", "event.timezone": "UTC", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "DETAIL", @@ -44,10 +65,17 @@ }, { "@timestamp": "2019-09-22T06:28:24.000Z", + "event.category": [ + "database" + ], "event.dataset": "postgresql.log", - "event.duration": 2474306885, + "event.duration": 2474306816, + "event.kind": "event", "event.module": "postgresql", "event.timezone": "UTC", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "LOG", @@ -61,9 +89,16 @@ }, { "@timestamp": "2019-09-22T06:28:24.000Z", + "event.category": [ + "database" + ], "event.dataset": "postgresql.log", + "event.kind": "event", "event.module": "postgresql", "event.timezone": "UTC", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "DETAIL", @@ -74,10 +109,17 @@ }, { "@timestamp": "2019-09-22T06:28:24.000Z", + "event.category": [ + "database" + ], "event.dataset": "postgresql.log", "event.duration": 18327000, + "event.kind": "event", "event.module": "postgresql", "event.timezone": "UTC", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.flags": [ diff --git a/filebeat/module/postgresql/log/test/postgresql-12-default.log-expected.json b/filebeat/module/postgresql/log/test/postgresql-12-default.log-expected.json index feedc6945de1..96918abc842a 100644 --- a/filebeat/module/postgresql/log/test/postgresql-12-default.log-expected.json +++ b/filebeat/module/postgresql/log/test/postgresql-12-default.log-expected.json @@ -1,9 +1,16 @@ [ { "@timestamp": "2020-04-16T09:45:11.844Z", + "event.category": [ + "database" + ], "event.dataset": "postgresql.log", + "event.kind": "event", "event.module": "postgresql", "event.timezone": "CEST", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "LOG", @@ -15,9 +22,16 @@ }, { "@timestamp": "2020-04-16T09:45:11.844Z", + "event.category": [ + "database" + ], "event.dataset": "postgresql.log", + "event.kind": "event", "event.module": "postgresql", "event.timezone": "CEST", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "LOG", @@ -29,9 +43,16 @@ }, { "@timestamp": "2020-04-16T09:45:11.844Z", + "event.category": [ + "database" + ], "event.dataset": "postgresql.log", + "event.kind": "event", "event.module": "postgresql", "event.timezone": "CEST", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "LOG", @@ -43,9 +64,16 @@ }, { "@timestamp": "2020-04-16T09:45:11.846Z", + "event.category": [ + "database" + ], "event.dataset": "postgresql.log", + "event.kind": "event", "event.module": "postgresql", "event.timezone": "CEST", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "LOG", @@ -57,9 +85,16 @@ }, { "@timestamp": "2020-04-16T09:45:11.861Z", + "event.category": [ + "database" + ], "event.dataset": "postgresql.log", + "event.kind": "event", "event.module": "postgresql", "event.timezone": "CEST", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "LOG", @@ -71,9 +106,16 @@ }, { "@timestamp": "2020-04-16T09:45:11.864Z", + "event.category": [ + "database" + ], "event.dataset": "postgresql.log", + "event.kind": "event", "event.module": "postgresql", "event.timezone": "CEST", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "LOG", @@ -85,9 +127,16 @@ }, { "@timestamp": "2020-04-16T10:22:22.579Z", + "event.category": [ + "database" + ], "event.dataset": "postgresql.log", + "event.kind": "event", "event.module": "postgresql", "event.timezone": "CEST", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "LOG", @@ -99,9 +148,16 @@ }, { "@timestamp": "2020-04-16T10:22:22.582Z", + "event.category": [ + "database" + ], "event.dataset": "postgresql.log", + "event.kind": "event", "event.module": "postgresql", "event.timezone": "CEST", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "LOG", @@ -113,9 +169,16 @@ }, { "@timestamp": "2020-04-16T10:22:22.582Z", + "event.category": [ + "database" + ], "event.dataset": "postgresql.log", + "event.kind": "event", "event.module": "postgresql", "event.timezone": "CEST", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "LOG", @@ -127,9 +190,16 @@ }, { "@timestamp": "2020-04-16T10:22:22.596Z", + "event.category": [ + "database" + ], "event.dataset": "postgresql.log", + "event.kind": "event", "event.module": "postgresql", "event.timezone": "CEST", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "LOG", diff --git a/filebeat/module/postgresql/log/test/postgresql-12-min-duration-statement.log-expected.json b/filebeat/module/postgresql/log/test/postgresql-12-min-duration-statement.log-expected.json index e39afe401d3f..74961ac0e741 100644 --- a/filebeat/module/postgresql/log/test/postgresql-12-min-duration-statement.log-expected.json +++ b/filebeat/module/postgresql/log/test/postgresql-12-min-duration-statement.log-expected.json @@ -1,10 +1,17 @@ [ { "@timestamp": "2020-04-16T10:48:36.677Z", + "event.category": [ + "database" + ], "event.dataset": "postgresql.log", "event.duration": 327000, + "event.kind": "event", "event.module": "postgresql", "event.timezone": "CEST", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "LOG", @@ -18,10 +25,17 @@ }, { "@timestamp": "2020-04-16T10:48:40.316Z", + "event.category": [ + "database" + ], "event.dataset": "postgresql.log", "event.duration": 320000, + "event.kind": "event", "event.module": "postgresql", "event.timezone": "CEST", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "LOG", @@ -35,9 +49,16 @@ }, { "@timestamp": "2020-04-16T10:48:44.696Z", + "event.category": [ + "database" + ], "event.dataset": "postgresql.log", + "event.kind": "event", "event.module": "postgresql", "event.timezone": "CEST", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "ERROR", @@ -49,9 +70,16 @@ }, { "@timestamp": "2020-04-16T10:48:44.696Z", + "event.category": [ + "database" + ], "event.dataset": "postgresql.log", + "event.kind": "event", "event.module": "postgresql", "event.timezone": "CEST", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "STATEMENT", @@ -63,10 +91,17 @@ }, { "@timestamp": "2020-04-16T10:49:16.871Z", + "event.category": [ + "database" + ], "event.dataset": "postgresql.log", "event.duration": 3431000, + "event.kind": "event", "event.module": "postgresql", "event.timezone": "CEST", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.flags": [ @@ -83,9 +118,16 @@ }, { "@timestamp": "2020-04-16T10:49:19.866Z", + "event.category": [ + "database" + ], "event.dataset": "postgresql.log", + "event.kind": "event", "event.module": "postgresql", "event.timezone": "CEST", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "ERROR", @@ -97,10 +139,17 @@ }, { "@timestamp": "2020-04-16T10:49:54.907Z", + "event.category": [ + "database" + ], "event.dataset": "postgresql.log", "event.duration": 3039000, + "event.kind": "event", "event.module": "postgresql", "event.timezone": "CEST", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.flags": [ @@ -117,10 +166,17 @@ }, { "@timestamp": "2020-04-16T10:49:55.464Z", + "event.category": [ + "database" + ], "event.dataset": "postgresql.log", "event.duration": 179000, + "event.kind": "event", "event.module": "postgresql", "event.timezone": "CEST", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "LOG", @@ -134,10 +190,17 @@ }, { "@timestamp": "2020-04-16T10:50:05.322Z", + "event.category": [ + "database" + ], "event.dataset": "postgresql.log", "event.duration": 1661000, + "event.kind": "event", "event.module": "postgresql", "event.timezone": "CEST", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "LOG", @@ -151,10 +214,17 @@ }, { "@timestamp": "2020-04-16T10:50:06.741Z", + "event.category": [ + "database" + ], "event.dataset": "postgresql.log", "event.duration": 144000, + "event.kind": "event", "event.module": "postgresql", "event.timezone": "CEST", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "LOG",