diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 61910847ffef..82d8b24bed49 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -66,6 +66,8 @@ https://github.com/elastic/beats/compare/v6.0.0-beta1...master[Check the HEAD di *Filebeat* +- Add PostgreSQL module with slowlog support. {pull}4763[4763] + *Heartbeat* *Metricbeat* diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index 54ef5bc482d5..4e1cf9673fca 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -22,6 +22,7 @@ grouped in the following categories: * <> * <> * <> +* <> * <> * <> @@ -1277,6 +1278,92 @@ type: text The error message +[[exported-fields-postgresql]] +== PostgreSQL fields + +Module for parsing the PostgreSQL log files. + + + +[float] +== postgresql fields + +Fields from PostgreSQL logs. + + + +[float] +== log fields + +Fields from the PostgreSQL log files. + + + +[float] +=== `postgresql.log.timestamp` + +The timestamp from the log line. + + +[float] +=== `postgresql.log.timezone` + +The timezone of timestamp. + + +[float] +=== `postgresql.log.thread_id` + +type: long + +Processs id + + +[float] +=== `postgresql.log.user` + +example: admin + +Name of user + +[float] +=== `postgresql.log.database` + +example: mydb + +Name of database + +[float] +=== `postgresql.log.level` + +example: FATAL + +The log level. + +[float] +=== `postgresql.log.duration` + +type: float + +example: 30.0 + +Duration of a query. + +[float] +=== `postgresql.log.query` + +example: SELECT * FROM users; + +Query statment. + +[float] +=== `postgresql.log.message` + +type: text + +The logged message. + + [[exported-fields-redis]] == Redis fields diff --git a/filebeat/docs/images/filebeat-postgresql-overview.png b/filebeat/docs/images/filebeat-postgresql-overview.png new file mode 100644 index 000000000000..b4fc428c4d02 Binary files /dev/null and b/filebeat/docs/images/filebeat-postgresql-overview.png differ diff --git a/filebeat/docs/images/filebeat-postgresql-slowlog-overview.png b/filebeat/docs/images/filebeat-postgresql-slowlog-overview.png new file mode 100644 index 000000000000..944a25778522 Binary files /dev/null and b/filebeat/docs/images/filebeat-postgresql-slowlog-overview.png differ diff --git a/filebeat/docs/images/filebeat-postgresql.png b/filebeat/docs/images/filebeat-postgresql.png new file mode 100644 index 000000000000..5772eb8bfa3d Binary files /dev/null and b/filebeat/docs/images/filebeat-postgresql.png differ diff --git a/filebeat/docs/modules/postgresql.asciidoc b/filebeat/docs/modules/postgresql.asciidoc new file mode 100644 index 000000000000..d11930a3df5d --- /dev/null +++ b/filebeat/docs/modules/postgresql.asciidoc @@ -0,0 +1,43 @@ +//// +This file is generated! See scripts/docs_collector.py +//// + +[[filebeat-module-postgresql]] +== PostgreSQL module + +This module collects and parses the logs created by https://www.postgresql.org/[PostgreSQL]. + +[float] +=== Compatibility + +The PostgreSQL module was tested with logs from versions 9.5 on Ubuntu and 9.6 on Debian. + +[float] +=== Dashboard + +This module comes with two sample dashboards. + +The first dashboard is for regulars logs. + +image::./images/filebeat-postgresql-overview.png[] + +The second one shows the slowlogs of PostgreSQL. + +image::./images/filebeat-postgresql-postgresql-overview.png[] + +[float] +=== Logs fileset settings + +[float] +==== var.paths + +An array of paths where to look for the log files. If left empty, Filebeat +will choose the paths depending on your operating systems. + + +[float] +=== Fields + +For a description of each field in the metricset, see the +<> section. + diff --git a/filebeat/docs/modules_list.asciidoc b/filebeat/docs/modules_list.asciidoc index f8ce7354cdc8..1dd629002500 100644 --- a/filebeat/docs/modules_list.asciidoc +++ b/filebeat/docs/modules_list.asciidoc @@ -8,6 +8,7 @@ This file is generated! See scripts/docs_collector.py * <> * <> * <> + * <> * <> * <> @@ -20,5 +21,6 @@ include::modules/auditd.asciidoc[] include::modules/icinga.asciidoc[] include::modules/mysql.asciidoc[] include::modules/nginx.asciidoc[] +include::modules/postgresql.asciidoc[] include::modules/redis.asciidoc[] include::modules/system.asciidoc[] diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml index 20ee62930e2e..dcd7d03653be 100644 --- a/filebeat/filebeat.reference.yml +++ b/filebeat/filebeat.reference.yml @@ -166,6 +166,20 @@ filebeat.modules: # can be added under this section. #prospector: +#----------------------------- PostgreSQL Module ----------------------------- +#- module: postgresql + # Logs + #log: + #enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + + # Prospector configuration (advanced). Any prospector configuration option + # can be added under this section. + #prospector: + #-------------------------------- Redis Module ------------------------------- #- module: redis # Main logs diff --git a/filebeat/module/postgresql/_meta/config.reference.yml b/filebeat/module/postgresql/_meta/config.reference.yml new file mode 100644 index 000000000000..fa500fda4b29 --- /dev/null +++ b/filebeat/module/postgresql/_meta/config.reference.yml @@ -0,0 +1,12 @@ +#- module: postgresql + # Logs + #log: + #enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + + # Prospector configuration (advanced). Any prospector configuration option + # can be added under this section. + #prospector: diff --git a/filebeat/module/postgresql/_meta/config.yml b/filebeat/module/postgresql/_meta/config.yml new file mode 100644 index 000000000000..c82734a9570f --- /dev/null +++ b/filebeat/module/postgresql/_meta/config.yml @@ -0,0 +1,8 @@ +- module: postgresql + # All logs + log: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: diff --git a/filebeat/module/postgresql/_meta/docs.asciidoc b/filebeat/module/postgresql/_meta/docs.asciidoc new file mode 100644 index 000000000000..c67350d08bce --- /dev/null +++ b/filebeat/module/postgresql/_meta/docs.asciidoc @@ -0,0 +1,30 @@ +== PostgreSQL module + +This module collects and parses the logs created by https://www.postgresql.org/[PostgreSQL]. + +[float] +=== Compatibility + +The PostgreSQL module was tested with logs from versions 9.5 on Ubuntu and 9.6 on Debian. + +[float] +=== Dashboard + +This module comes with two sample dashboards. + +The first dashboard is for regulars logs. + +image::./images/filebeat-postgresql-overview.png[] + +The second one shows the slowlogs of PostgreSQL. + +image::./images/filebeat-postgresql-postgresql-overview.png[] + +[float] +=== Logs fileset settings + +[float] +==== var.paths + +An array of paths where to look for the log files. If left empty, Filebeat +will choose the paths depending on your operating systems. diff --git a/filebeat/module/postgresql/_meta/fields.yml b/filebeat/module/postgresql/_meta/fields.yml new file mode 100644 index 000000000000..5c773288ab6d --- /dev/null +++ b/filebeat/module/postgresql/_meta/fields.yml @@ -0,0 +1,11 @@ +- key: postgresql + title: "PostgreSQL" + description: > + Module for parsing the PostgreSQL log files. + short_config: true + fields: + - name: postgresql + type: group + description: > + Fields from PostgreSQL logs. + fields: diff --git a/filebeat/module/postgresql/_meta/kibana/default/dashboard/Filebeat-Postgresql-overview.json b/filebeat/module/postgresql/_meta/kibana/default/dashboard/Filebeat-Postgresql-overview.json new file mode 100644 index 000000000000..c63dea005ff9 --- /dev/null +++ b/filebeat/module/postgresql/_meta/kibana/default/dashboard/Filebeat-Postgresql-overview.json @@ -0,0 +1,64 @@ +{ + "objects": [ + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"filter\":[]}" + }, + "savedSearchId": "ef9da9b0-776c-11e7-9e03-393c0d1aa325", + "title": "PostgreSQL Log Level Count", + "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}", + "version": 1, + "visState": "{\"title\":\"PostgreSQL Log Level Count\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"postgresql.log.level\",\"size\":12,\"order\":\"desc\",\"orderBy\":\"1\"}}]}" + }, + "id": "28563b50-776d-11e7-9e03-393c0d1aa325", + "type": "visualization", + "version": 1 + }, + { + "attributes": { + "columns": [ + "postgresql.log.user", + "postgresql.log.database", + "postgresql.log.level", + "postgresql.log.message", + "postgresql.log.query" + ], + "description": "", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"a0b887a0-763d-11e7-9e03-393c0d1aa325\",\"highlightAll\":true,\"version\":true,\"query\":{\"query\":\"postgresql.log.level:*\",\"language\":\"lucene\"},\"filter\":[]}" + }, + "sort": [ + "@timestamp", + "desc" + ], + "title": "PostgreSQL All Logs", + "version": 1 + }, + "id": "ef9da9b0-776c-11e7-9e03-393c0d1aa325", + "type": "search", + "version": 1 + }, + { + "attributes": { + "description": "", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[],\"highlightAll\":true,\"version\":true}" + }, + "optionsJSON": "{\"darkTheme\":false}", + "panelsJSON": "[{\"size_x\":6,\"size_y\":3,\"panelIndex\":1,\"type\":\"visualization\",\"id\":\"28563b50-776d-11e7-9e03-393c0d1aa325\",\"col\":1,\"row\":4},{\"size_x\":12,\"size_y\":3,\"panelIndex\":2,\"type\":\"search\",\"id\":\"ef9da9b0-776c-11e7-9e03-393c0d1aa325\",\"col\":1,\"row\":1,\"columns\":[\"postgresql.log.user\",\"postgresql.log.database\",\"postgresql.log.level\",\"postgresql.log.message\",\"postgresql.log.query\"],\"sort\":[\"@timestamp\",\"desc\"]}]", + "timeRestore": false, + "title": "PostgreSQL All Logs", + "uiStateJSON": "{\"P-1\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}}", + "version": 1 + }, + "id": "a21284c0-776e-11e7-9e03-393c0d1aa325", + "type": "dashboard", + "version": 1 + } + ], + "version": "6.0.0-beta1-SNAPSHOT" +} \ No newline at end of file diff --git a/filebeat/module/postgresql/_meta/kibana/default/dashboard/Filebeat-Postgresql-slowlogs.json b/filebeat/module/postgresql/_meta/kibana/default/dashboard/Filebeat-Postgresql-slowlogs.json new file mode 100644 index 000000000000..6268c9a5cc8c --- /dev/null +++ b/filebeat/module/postgresql/_meta/kibana/default/dashboard/Filebeat-Postgresql-slowlogs.json @@ -0,0 +1,87 @@ +{ + "objects": [ + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"filter\":[],\"query\":{\"language\":\"lucene\"}}" + }, + "savedSearchId": "d338f1e0-776b-11e7-9e03-393c0d1aa325", + "title": "PostgreSQL Query Count and Duration", + "uiStateJSON": "{\"vis\":{\"colors\":{\"Sum of query duration\":\"#6ED0E0\",\"Number of queries\":\"#0A437C\"},\"legendOpen\":true}}", + "version": 1, + "visState": "{\"title\":\"PostgreSQL Query Count and Duration\",\"type\":\"histogram\",\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"truncate\":100},\"title\":{\"text\":\"@timestamp per minute\"}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"\"}}],\"seriesParams\":[{\"show\":true,\"mode\":\"normal\",\"type\":\"histogram\",\"drawLinesBetweenPoints\":true,\"showCircles\":true,\"interpolate\":\"linear\",\"lineWidth\":2,\"data\":{\"id\":\"4\",\"label\":\"Number of queries\"},\"valueAxis\":\"ValueAxis-1\"},{\"show\":true,\"mode\":\"normal\",\"type\":\"histogram\",\"drawLinesBetweenPoints\":true,\"showCircles\":true,\"interpolate\":\"linear\",\"lineWidth\":2,\"data\":{\"id\":\"2\",\"label\":\"Sum of query duration\"},\"valueAxis\":\"ValueAxis-1\"}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false},\"aggs\":[{\"id\":\"3\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"m\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"4\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"Number of queries\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"postgresql.log.duration\",\"customLabel\":\"Sum of query duration\"}}]}" + }, + "id": "63bb6bd0-776c-11e7-9e03-393c0d1aa325", + "type": "visualization", + "version": 2 + }, + { + "attributes": { + "columns": [ + "postgresql.log.user", + "postgresql.log.database", + "postgresql.log.duration", + "postgresql.log.query" + ], + "description": "", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"a0b887a0-763d-11e7-9e03-393c0d1aa325\",\"highlightAll\":true,\"version\":true,\"query\":{\"query\":\"postgresql.log.duration:\u003e30\",\"language\":\"lucene\"},\"filter\":[]}" + }, + "sort": [ + "@timestamp", + "desc" + ], + "title": "Slow PostgreSQL Queries", + "version": 1 + }, + "id": "653bbfb0-776b-11e7-9e03-393c0d1aa325", + "type": "search", + "version": 3 + }, + { + "attributes": { + "columns": [ + "postgresql.log.user", + "postgresql.log.database", + "postgresql.log.duration", + "postgresql.log.query" + ], + "description": "", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"a0b887a0-763d-11e7-9e03-393c0d1aa325\",\"highlightAll\":true,\"version\":true,\"query\":{\"query\":\"postgresql.log.duration:*\",\"language\":\"lucene\"},\"filter\":[]}" + }, + "sort": [ + "@timestamp", + "desc" + ], + "title": "PostgreSQL Query Durations", + "version": 1 + }, + "id": "d338f1e0-776b-11e7-9e03-393c0d1aa325", + "type": "search", + "version": 2 + }, + { + "attributes": { + "description": "", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"language\":\"lucene\",\"query\":\"postgresql.log.query:*\"},\"filter\":[],\"highlightAll\":true,\"version\":true}" + }, + "optionsJSON": "{\"darkTheme\":false}", + "panelsJSON": "[{\"col\":1,\"id\":\"63bb6bd0-776c-11e7-9e03-393c0d1aa325\",\"panelIndex\":1,\"row\":1,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":7,\"columns\":[\"postgresql.log.user\",\"postgresql.log.database\",\"postgresql.log.duration\",\"postgresql.log.query\"],\"id\":\"653bbfb0-776b-11e7-9e03-393c0d1aa325\",\"panelIndex\":2,\"row\":1,\"size_x\":6,\"size_y\":3,\"sort\":[\"@timestamp\",\"desc\"],\"type\":\"search\"},{\"size_x\":12,\"size_y\":3,\"panelIndex\":3,\"type\":\"search\",\"id\":\"d338f1e0-776b-11e7-9e03-393c0d1aa325\",\"col\":1,\"row\":4,\"columns\":[\"postgresql.log.user\",\"postgresql.log.database\",\"postgresql.log.duration\",\"postgresql.log.query\"],\"sort\":[\"@timestamp\",\"desc\"]}]", + "timeRestore": false, + "title": "PostgreSQL Query Duration Overview", + "uiStateJSON": "{}", + "version": 1 + }, + "id": "92370770-776d-11e7-9e03-393c0d1aa325", + "type": "dashboard", + "version": 1 + } + ], + "version": "6.0.0-beta1-SNAPSHOT" +} \ No newline at end of file diff --git a/filebeat/module/postgresql/log/_meta/fields.yml b/filebeat/module/postgresql/log/_meta/fields.yml new file mode 100644 index 000000000000..06e01ca78629 --- /dev/null +++ b/filebeat/module/postgresql/log/_meta/fields.yml @@ -0,0 +1,40 @@ +- name: log + type: group + description: > + Fields from the PostgreSQL log files. + fields: + - name: timestamp + description: > + The timestamp from the log line. + - name: timezone + description: > + The timezone of timestamp. + - name: thread_id + type: long + description: > + Processs id + - name: user + example: "admin" + description: + Name of user + - name: database + example: "mydb" + description: + Name of database + - name: level + example: "FATAL" + description: + The log level. + - name: duration + type: float + example: "30.0" + description: + Duration of a query. + - name: query + example: "SELECT * FROM users;" + description: + Query statment. + - name: message + type: text + description: > + The logged message. diff --git a/filebeat/module/postgresql/log/config/log.yml b/filebeat/module/postgresql/log/config/log.yml new file mode 100644 index 000000000000..5f236667e90c --- /dev/null +++ b/filebeat/module/postgresql/log/config/log.yml @@ -0,0 +1,10 @@ +type: log +paths: +{{ range $i, $path := .paths }} + - {{$path}} +{{ end }} +exclude_files: [".gz$"] +multiline: + pattern: '^[-0-9]* ' + negate: true + match: after diff --git a/filebeat/module/postgresql/log/ingest/pipeline.json b/filebeat/module/postgresql/log/ingest/pipeline.json new file mode 100644 index 000000000000..17e461d897e5 --- /dev/null +++ b/filebeat/module/postgresql/log/ingest/pipeline.json @@ -0,0 +1,41 @@ +{ + "description": "Pipeline for parsing PostgreSQL logs.", + "processors": [ + { + "grok": { + "field": "message", + "ignore_missing": true, + "patterns": [ + "%{LOCALDATETIME:postgresql.log.timestamp} %{WORD:postgresql.log.timezone} \\[%{NUMBER:postgresql.log.thread_id}\\] %{USERNAME:postgresql.log.user}@%{HOSTNAME:postgresql.log.database} %{WORD:postgresql.log.level}: duration: %{NUMBER:postgresql.log.duration} ms statement: %{MULTILINEQUERY:postgresql.log.query}", + "%{LOCALDATETIME:postgresql.log.timestamp} %{WORD:postgresql.log.timezone} \\[%{NUMBER:postgresql.log.thread_id}\\] \\[%{USERNAME:postgresql.log.user}\\]@\\[%{HOSTNAME:postgresql.log.database}\\] %{WORD:postgresql.log.level}: duration: %{NUMBER:postgresql.log.duration} ms statement: %{MULTILINEQUERY:postgresql.log.query}", + "%{LOCALDATETIME:postgresql.log.timestamp} %{WORD:postgresql.log.timezone} \\[%{NUMBER:postgresql.log.thread_id}\\] %{USERNAME:postgresql.log.user}@%{HOSTNAME:postgresql.log.database} %{WORD:postgresql.log.level}: ?%{GREEDYDATA:postgresql.log.message}", + "%{LOCALDATETIME:postgresql.log.timestamp} %{WORD:postgresql.log.timezone} \\[%{NUMBER:postgresql.log.thread_id}\\] \\[%{USERNAME:postgresql.log.user}\\]@\\[%{HOSTNAME:postgresql.log.database}\\] %{WORD:postgresql.log.level}: ?%{GREEDYDATA:postgresql.log.message}", + "%{LOCALDATETIME:postgresql.log.timestamp} %{WORD:postgresql.log.timezone} \\[%{NUMBER:postgresql.log.thread_id}\\] %{WORD:postgresql.log.level}: ?%{GREEDYDATA:postgresql.log.message}" + ], + "pattern_definitions": { + "LOCALDATETIME": "[-0-9]+ %{TIME}", + "GREEDYDATA": ".*", + "MULTILINEQUERY" : "(.|\n|\t)*?;$" + } + } + }, + { + "date": { + "field": "postgresql.log.timestamp", + "target_field": "@timestamp", + "formats": [ + "yyyy-MM-dd HH:mm:ss.SSS" + ], + "ignore_failure": true + } + } + ], + "on_failure": [ + { + "set": { + "field": "error.message", + "value": "{{ _ingest.on_failure_message }}" + } + } + ] +} diff --git a/filebeat/module/postgresql/log/manifest.yml b/filebeat/module/postgresql/log/manifest.yml new file mode 100644 index 000000000000..bd9631f945f6 --- /dev/null +++ b/filebeat/module/postgresql/log/manifest.yml @@ -0,0 +1,13 @@ +module_version: "1.0" + +var: + - name: paths + default: + - /var/log/postgresql/postgresql-*-*.log* + os.darwin: + - /usr/local/var/postgres/*.log* + os.windows: + - "c:/Program Files/PostgreSQL/*/logs/*.log*" + +ingest_pipeline: ingest/pipeline.json +prospector: config/log.yml diff --git a/filebeat/module/postgresql/log/test/postgresql-9.6-debian-with-slowlog.log b/filebeat/module/postgresql/log/test/postgresql-9.6-debian-with-slowlog.log new file mode 100644 index 000000000000..1903a2a16db8 --- /dev/null +++ b/filebeat/module/postgresql/log/test/postgresql-9.6-debian-with-slowlog.log @@ -0,0 +1,61 @@ +2017-07-31 13:36:42.585 CEST [4974] LOG: database system was shut down at 2017-06-17 16:58:04 CEST +2017-07-31 13:36:42.605 CEST [4974] LOG: MultiXact member wraparound protections are now enabled +2017-07-31 13:36:42.615 CEST [4978] LOG: autovacuum launcher started +2017-07-31 13:36:42.616 CEST [4973] LOG: database system is ready to accept connections +2017-07-31 13:36:42.956 CEST [4980] [unknown]@[unknown] LOG: incomplete startup packet +2017-07-31 13:36:43.557 CEST [4983] postgres@postgres LOG: duration: 37.118 ms statement: SELECT d.datname as "Name", + pg_catalog.pg_get_userbyid(d.datdba) as "Owner", + pg_catalog.pg_encoding_to_char(d.encoding) as "Encoding", + d.datcollate as "Collate", + d.datctype as "Ctype", + pg_catalog.array_to_string(d.datacl, E'\n') AS "Access privileges" + FROM pg_catalog.pg_database d + ORDER BY 1; +2017-07-31 13:36:44.104 CEST [4986] postgres@postgres LOG: duration: 2.895 ms statement: SELECT d.datname as "Name", + pg_catalog.pg_get_userbyid(d.datdba) as "Owner", + pg_catalog.pg_encoding_to_char(d.encoding) as "Encoding", + d.datcollate as "Collate", + d.datctype as "Ctype", + pg_catalog.array_to_string(d.datacl, E'\n') AS "Access privileges" + FROM pg_catalog.pg_database d + ORDER BY 1; +2017-07-31 13:36:44.642 CEST [4989] postgres@postgres LOG: duration: 2.809 ms statement: SELECT d.datname as "Name", + pg_catalog.pg_get_userbyid(d.datdba) as "Owner", + pg_catalog.pg_encoding_to_char(d.encoding) as "Encoding", + d.datcollate as "Collate", + d.datctype as "Ctype", + pg_catalog.array_to_string(d.datacl, E'\n') AS "Access privileges" + FROM pg_catalog.pg_database d + ORDER BY 1; +2017-07-31 13:39:16.249 CEST [5407] postgres@users FATAL: database "users" does not exist +2017-07-31 13:39:17.945 CEST [5500] postgres@user FATAL: database "user" does not exist +2017-07-31 13:39:21.025 CEST [5404] postgres@postgres LOG: duration: 37.598 ms statement: SELECT n.nspname as "Schema", + c.relname as "Name", + CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' END as "Type", + pg_catalog.pg_get_userbyid(c.relowner) as "Owner" + FROM pg_catalog.pg_class c + LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace + WHERE c.relkind IN ('r','') + AND n.nspname <> 'pg_catalog' + AND n.nspname <> 'information_schema' + AND n.nspname !~ '^pg_toast' + AND pg_catalog.pg_table_is_visible(c.oid) + ORDER BY 1,2; +2017-07-31 13:39:31.619 CEST [5502] postgres@clients LOG: duration: 9.482 ms statement: select * from clients; +2017-07-31 13:39:40.147 CEST [5502] postgres@clients LOG: duration: 0.765 ms statement: select id from clients; +2017-07-31 13:40:54.310 CEST [5502] postgres@clients LOG: duration: 26.082 ms statement: SELECT n.nspname as "Schema", + c.relname as "Name", + CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' END as "Type", + pg_catalog.pg_get_userbyid(c.relowner) as "Owner" + FROM pg_catalog.pg_class c + LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace + WHERE c.relkind IN ('r','') + AND n.nspname <> 'pg_catalog' + AND n.nspname <> 'information_schema' + AND n.nspname !~ '^pg_toast' + AND pg_catalog.pg_table_is_visible(c.oid) + ORDER BY 1,2; +2017-07-31 13:43:22.645 CEST [5502] postgres@clients LOG: duration: 36.162 ms statement: create table cats(name varchar(50) primary key, toy varchar (50) not null, born timestamp not null); +2017-07-31 13:46:02.670 CEST [5502] postgres@clients LOG: duration: 10.540 ms statement: insert into cats(name, toy, born) values('kate', 'ball', now()); +2017-07-31 13:46:23.016 CEST [5502] postgres@clients LOG: duration: 5.156 ms statement: insert into cats(name, toy, born) values('frida', 'horse', now()); +2017-07-31 13:46:55.637 CEST [5502] postgres@clients LOG: duration: 25.871 ms statement: create table dogs(name varchar(50) primary key, owner varchar (50) not null, born timestamp not null); diff --git a/filebeat/module/postgresql/log/test/postgresql-9.6-debian-with-slowlog.log-expected.json b/filebeat/module/postgresql/log/test/postgresql-9.6-debian-with-slowlog.log-expected.json new file mode 100644 index 000000000000..91f5e3468342 --- /dev/null +++ b/filebeat/module/postgresql/log/test/postgresql-9.6-debian-with-slowlog.log-expected.json @@ -0,0 +1,814 @@ +[ + { + "_index": "filebeat-7.0.0-alpha1-2017.08.01", + "_type": "doc", + "_id": "t8S2ml0BVlT2mTAm4zRv", + "_version": 1, + "_score": null, + "_source": { + "@timestamp": "2017-07-31T13:36:42.585Z", + "offset": 100, + "postgresql": { + "log": { + "thread_id": "4974", + "level": "LOG", + "timezone": "CEST", + "message": "database system was shut down at 2017-06-17 16:58:04 CEST", + "timestamp": "2017-07-31 13:36:42.585" + } + }, + "beat": { + "hostname": "baldur", + "name": "baldur", + "version": "7.0.0-alpha1" + }, + "prospector": { + "type": "log" + }, + "source": "/home/n/go/src/github.com/elastic/beats/filebeat/module/postgresql/log/test/postgresql-9.6-debian.log", + "message": "2017-07-31 13:36:42.585 CEST [4974] LOG: database system was shut down at 2017-06-17 16:58:04 CEST", + "fileset": { + "module": "postgresql", + "name": "log" + } + }, + "fields": { + "@timestamp": [ + "2017-07-31T13:36:42.585Z" + ] + }, + "sort": [ + 1501508202585 + ] + }, + + { + "_index": "filebeat-7.0.0-alpha1-2017.08.01", + "_type": "doc", + "_id": "tcS2ml0BVlT2mTAm4jSf", + "_version": 1, + "_score": null, + "_source": { + "@timestamp": "2017-07-31T13:36:42.605Z", + "offset": 198, + "postgresql": { + "log": { + "thread_id": "4974", + "level": "LOG", + "timezone": "CEST", + "message": "MultiXact member wraparound protections are now enabled", + "timestamp": "2017-07-31 13:36:42.605" + } + }, + "beat": { + "hostname": "baldur", + "name": "baldur", + "version": "7.0.0-alpha1" + }, + "prospector": { + "type": "log" + }, + "source": "/home/n/go/src/github.com/elastic/beats/filebeat/module/postgresql/log/test/postgresql-9.6-debian.log", + "message": "2017-07-31 13:36:42.605 CEST [4974] LOG: MultiXact member wraparound protections are now enabled", + "fileset": { + "module": "postgresql", + "name": "log" + } + }, + "fields": { + "@timestamp": [ + "2017-07-31T13:36:42.605Z" + ] + }, + "sort": [ + 1501508202605 + ] + }, + + { + "_index": "filebeat-7.0.0-alpha1-2017.08.01", + "_type": "doc", + "_id": "tsS2ml0BVlT2mTAm4jSf", + "_version": 1, + "_score": null, + "_source": { + "@timestamp": "2017-07-31T13:36:42.615Z", + "offset": 268, + "postgresql": { + "log": { + "thread_id": "4978", + "level": "LOG", + "timezone": "CEST", + "message": "autovacuum launcher started", + "timestamp": "2017-07-31 13:36:42.615" + } + }, + "beat": { + "hostname": "baldur", + "name": "baldur", + "version": "7.0.0-alpha1" + }, + "prospector": { + "type": "log" + }, + "source": "/home/n/go/src/github.com/elastic/beats/filebeat/module/postgresql/log/test/postgresql-9.6-debian.log", + "message": "2017-07-31 13:36:42.615 CEST [4978] LOG: autovacuum launcher started", + "fileset": { + "module": "postgresql", + "name": "log" + } + }, + "fields": { + "@timestamp": [ + "2017-07-31T13:36:42.615Z" + ] + }, + "sort": [ + 1501508202615 + ] + }, + + { + "_index": "filebeat-7.0.0-alpha1-2017.08.01", + "_type": "doc", + "_id": "uMS2ml0BVlT2mTAm4zST", + "_version": 1, + "_score": null, + "_source": { + "@timestamp": "2017-07-31T13:36:42.616Z", + "offset": 357, + "postgresql": { + "log": { + "thread_id": "4973", + "level": "LOG", + "timezone": "CEST", + "message": "database system is ready to accept connections", + "timestamp": "2017-07-31 13:36:42.616" + } + }, + "beat": { + "hostname": "baldur", + "name": "baldur", + "version": "7.0.0-alpha1" + }, + "prospector": { + "type": "log" + }, + "source": "/home/n/go/src/github.com/elastic/beats/filebeat/module/postgresql/log/test/postgresql-9.6-debian.log", + "message": "2017-07-31 13:36:42.616 CEST [4973] LOG: database system is ready to accept connections", + "fileset": { + "module": "postgresql", + "name": "log" + } + }, + "fields": { + "@timestamp": [ + "2017-07-31T13:36:42.616Z" + ] + }, + "sort": [ + 1501508202616 + ] + }, + + { + "_index": "filebeat-7.0.0-alpha1-2017.08.01", + "_type": "doc", + "_id": "ucS2ml0BVlT2mTAm4zST", + "_version": 1, + "_score": null, + "_source": { + "@timestamp": "2017-07-31T13:36:42.956Z", + "offset": 445, + "postgresql": { + "log": { + "thread_id": "4980", + "database": "unknown", + "level": "LOG", + "timezone": "CEST", + "message": "incomplete startup packet", + "user": "unknown", + "timestamp": "2017-07-31 13:36:42.956" + } + }, + "beat": { + "hostname": "baldur", + "name": "baldur", + "version": "7.0.0-alpha1" + }, + "prospector": { + "type": "log" + }, + "source": "/home/n/go/src/github.com/elastic/beats/filebeat/module/postgresql/log/test/postgresql-9.6-debian.log", + "fileset": { + "module": "postgresql", + "name": "log" + }, + "message": "2017-07-31 13:36:42.956 CEST [4980] [unknown]@[unknown] LOG: incomplete startup packet" + }, + "fields": { + "@timestamp": [ + "2017-07-31T13:36:42.956Z" + ] + }, + "sort": [ + 1501508202956 + ] + }, + + { + "_index": "filebeat-7.0.0-alpha1-2017.08.01", + "_type": "doc", + "_id": "usS2ml0BVlT2mTAm4zST", + "_version": 1, + "_score": null, + "_source": { + "@timestamp": "2017-07-31T13:36:43.557Z", + "offset": 873, + "postgresql": { + "log": { + "duration": "37.118", + "thread_id": "4983", + "database": "postgres", + "level": "LOG", + "timezone": "CEST", + "query": "SELECT d.datname as \"Name\",\n\t pg_catalog.pg_get_userbyid(d.datdba) as \"Owner\",\n\t pg_catalog.pg_encoding_to_char(d.encoding) as \"Encoding\",\n\t d.datcollate as \"Collate\",\n\t d.datctype as \"Ctype\",\n\t pg_catalog.array_to_string(d.datacl, E'\\n') AS \"Access privileges\"\n\tFROM pg_catalog.pg_database d\n\tORDER BY 1;", + "user": "postgres", + "timestamp": "2017-07-31 13:36:43.557" + } + }, + "beat": { + "hostname": "baldur", + "name": "baldur", + "version": "7.0.0-alpha1" + }, + "prospector": { + "type": "log" + }, + "source": "/home/n/go/src/github.com/elastic/beats/filebeat/module/postgresql/log/test/postgresql-9.6-debian.log", + "fileset": { + "module": "postgresql", + "name": "log" + }, + "message": "2017-07-31 13:36:43.557 CEST [4983] postgres@postgres LOG: duration: 37.118 ms statement: SELECT d.datname as \"Name\",\n\t pg_catalog.pg_get_userbyid(d.datdba) as \"Owner\",\n\t pg_catalog.pg_encoding_to_char(d.encoding) as \"Encoding\",\n\t d.datcollate as \"Collate\",\n\t d.datctype as \"Ctype\",\n\t pg_catalog.array_to_string(d.datacl, E'\\n') AS \"Access privileges\"\n\tFROM pg_catalog.pg_database d\n\tORDER BY 1;" + }, + "fields": { + "@timestamp": [ + "2017-07-31T13:36:43.557Z" + ] + }, + "sort": [ + 1501508203557 + ] + }, + + { + "_index": "filebeat-7.0.0-alpha1-2017.08.01", + "_type": "doc", + "_id": "u8S2ml0BVlT2mTAm4zST", + "_version": 1, + "_score": null, + "_source": { + "@timestamp": "2017-07-31T13:36:44.104Z", + "offset": 1300, + "postgresql": { + "log": { + "duration": "2.895", + "thread_id": "4986", + "database": "postgres", + "level": "LOG", + "timezone": "CEST", + "query": "SELECT d.datname as \"Name\",\n\t pg_catalog.pg_get_userbyid(d.datdba) as \"Owner\",\n\t pg_catalog.pg_encoding_to_char(d.encoding) as \"Encoding\",\n\t d.datcollate as \"Collate\",\n\t d.datctype as \"Ctype\",\n\t pg_catalog.array_to_string(d.datacl, E'\\n') AS \"Access privileges\"\n\tFROM pg_catalog.pg_database d\n\tORDER BY 1;", + "user": "postgres", + "timestamp": "2017-07-31 13:36:44.104" + } + }, + "beat": { + "hostname": "baldur", + "name": "baldur", + "version": "7.0.0-alpha1" + }, + "prospector": { + "type": "log" + }, + "source": "/home/n/go/src/github.com/elastic/beats/filebeat/module/postgresql/log/test/postgresql-9.6-debian.log", + "message": "2017-07-31 13:36:44.104 CEST [4986] postgres@postgres LOG: duration: 2.895 ms statement: SELECT d.datname as \"Name\",\n\t pg_catalog.pg_get_userbyid(d.datdba) as \"Owner\",\n\t pg_catalog.pg_encoding_to_char(d.encoding) as \"Encoding\",\n\t d.datcollate as \"Collate\",\n\t d.datctype as \"Ctype\",\n\t pg_catalog.array_to_string(d.datacl, E'\\n') AS \"Access privileges\"\n\tFROM pg_catalog.pg_database d\n\tORDER BY 1;", + "fileset": { + "module": "postgresql", + "name": "log" + } + }, + "fields": { + "@timestamp": [ + "2017-07-31T13:36:44.104Z" + ] + }, + "sort": [ + 1501508204104 + ] + }, + + { + "_index": "filebeat-7.0.0-alpha1-2017.08.01", + "_type": "doc", + "_id": "vMS2ml0BVlT2mTAm4zST", + "_version": 1, + "_score": null, + "_source": { + "@timestamp": "2017-07-31T13:36:44.642Z", + "offset": 1727, + "postgresql": { + "log": { + "duration": "2.809", + "thread_id": "4989", + "database": "postgres", + "level": "LOG", + "timezone": "CEST", + "query": "SELECT d.datname as \"Name\",\n\t pg_catalog.pg_get_userbyid(d.datdba) as \"Owner\",\n\t pg_catalog.pg_encoding_to_char(d.encoding) as \"Encoding\",\n\t d.datcollate as \"Collate\",\n\t d.datctype as \"Ctype\",\n\t pg_catalog.array_to_string(d.datacl, E'\\n') AS \"Access privileges\"\n\tFROM pg_catalog.pg_database d\n\tORDER BY 1;", + "user": "postgres", + "timestamp": "2017-07-31 13:36:44.642" + } + }, + "beat": { + "hostname": "baldur", + "name": "baldur", + "version": "7.0.0-alpha1" + }, + "prospector": { + "type": "log" + }, + "source": "/home/n/go/src/github.com/elastic/beats/filebeat/module/postgresql/log/test/postgresql-9.6-debian.log", + "fileset": { + "module": "postgresql", + "name": "log" + }, + "message": "2017-07-31 13:36:44.642 CEST [4989] postgres@postgres LOG: duration: 2.809 ms statement: SELECT d.datname as \"Name\",\n\t pg_catalog.pg_get_userbyid(d.datdba) as \"Owner\",\n\t pg_catalog.pg_encoding_to_char(d.encoding) as \"Encoding\",\n\t d.datcollate as \"Collate\",\n\t d.datctype as \"Ctype\",\n\t pg_catalog.array_to_string(d.datacl, E'\\n') AS \"Access privileges\"\n\tFROM pg_catalog.pg_database d\n\tORDER BY 1;" + }, + "fields": { + "@timestamp": [ + "2017-07-31T13:36:44.642Z" + ] + }, + "sort": [ + 1501508204642 + ] + }, + + { + "_index": "filebeat-7.0.0-alpha1-2017.08.01", + "_type": "doc", + "_id": "vcS2ml0BVlT2mTAm4zST", + "_version": 1, + "_score": null, + "_source": { + "@timestamp": "2017-07-31T13:39:16.249Z", + "offset": 1818, + "postgresql": { + "log": { + "thread_id": "5407", + "database": "users", + "level": "FATAL", + "timezone": "CEST", + "message": "database \"users\" does not exist", + "user": "postgres", + "timestamp": "2017-07-31 13:39:16.249" + } + }, + "beat": { + "hostname": "baldur", + "name": "baldur", + "version": "7.0.0-alpha1" + }, + "prospector": { + "type": "log" + }, + "source": "/home/n/go/src/github.com/elastic/beats/filebeat/module/postgresql/log/test/postgresql-9.6-debian.log", + "message": "2017-07-31 13:39:16.249 CEST [5407] postgres@users FATAL: database \"users\" does not exist", + "fileset": { + "module": "postgresql", + "name": "log" + } + }, + "fields": { + "@timestamp": [ + "2017-07-31T13:39:16.249Z" + ] + }, + "sort": [ + 1501508356249 + ] + }, + + { + "_index": "filebeat-7.0.0-alpha1-2017.08.01", + "_type": "doc", + "_id": "vsS2ml0BVlT2mTAm4zST", + "_version": 1, + "_score": null, + "_source": { + "@timestamp": "2017-07-31T13:39:17.945Z", + "offset": 1907, + "postgresql": { + "log": { + "thread_id": "5500", + "database": "user", + "level": "FATAL", + "timezone": "CEST", + "message": "database \"user\" does not exist", + "user": "postgres", + "timestamp": "2017-07-31 13:39:17.945" + } + }, + "beat": { + "hostname": "baldur", + "name": "baldur", + "version": "7.0.0-alpha1" + }, + "prospector": { + "type": "log" + }, + "source": "/home/n/go/src/github.com/elastic/beats/filebeat/module/postgresql/log/test/postgresql-9.6-debian.log", + "fileset": { + "module": "postgresql", + "name": "log" + }, + "message": "2017-07-31 13:39:17.945 CEST [5500] postgres@user FATAL: database \"user\" does not exist" + }, + "fields": { + "@timestamp": [ + "2017-07-31T13:39:17.945Z" + ] + }, + "sort": [ + 1501508357945 + ] + }, + + { + "_index": "filebeat-7.0.0-alpha1-2017.08.01", + "_type": "doc", + "_id": "v8S2ml0BVlT2mTAm4zST", + "_version": 1, + "_score": null, + "_source": { + "@timestamp": "2017-07-31T13:39:21.025Z", + "offset": 2620, + "postgresql": { + "log": { + "duration": "37.598", + "thread_id": "5404", + "database": "postgres", + "level": "LOG", + "timezone": "CEST", + "query": "SELECT n.nspname as \"Schema\",\n\t c.relname as \"Name\",\n\t CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' END as \"Type\",\n\t pg_catalog.pg_get_userbyid(c.relowner) as \"Owner\"\n\tFROM pg_catalog.pg_class c\n\t LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace\n\tWHERE c.relkind IN ('r','')\n\t AND n.nspname <> 'pg_catalog'\n\t AND n.nspname <> 'information_schema'\n\t AND n.nspname !~ '^pg_toast'\n\t AND pg_catalog.pg_table_is_visible(c.oid)\n\tORDER BY 1,2;", + "user": "postgres", + "timestamp": "2017-07-31 13:39:21.025" + } + }, + "beat": { + "hostname": "baldur", + "name": "baldur", + "version": "7.0.0-alpha1" + }, + "prospector": { + "type": "log" + }, + "source": "/home/n/go/src/github.com/elastic/beats/filebeat/module/postgresql/log/test/postgresql-9.6-debian.log", + "message": "2017-07-31 13:39:21.025 CEST [5404] postgres@postgres LOG: duration: 37.598 ms statement: SELECT n.nspname as \"Schema\",\n\t c.relname as \"Name\",\n\t CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' END as \"Type\",\n\t pg_catalog.pg_get_userbyid(c.relowner) as \"Owner\"\n\tFROM pg_catalog.pg_class c\n\t LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace\n\tWHERE c.relkind IN ('r','')\n\t AND n.nspname <> 'pg_catalog'\n\t AND n.nspname <> 'information_schema'\n\t AND n.nspname !~ '^pg_toast'\n\t AND pg_catalog.pg_table_is_visible(c.oid)\n\tORDER BY 1,2;", + "fileset": { + "module": "postgresql", + "name": "log" + } + }, + "fields": { + "@timestamp": [ + "2017-07-31T13:39:21.025Z" + ] + }, + "sort": [ + 1501508361025 + ] + }, + + { + "_index": "filebeat-7.0.0-alpha1-2017.08.01", + "_type": "doc", + "_id": "wMS2ml0BVlT2mTAm4zST", + "_version": 1, + "_score": null, + "_source": { + "@timestamp": "2017-07-31T13:39:31.619Z", + "offset": 2733, + "postgresql": { + "log": { + "duration": "9.482", + "thread_id": "5502", + "database": "clients", + "level": "LOG", + "timezone": "CEST", + "query": "select * from clients;", + "user": "postgres", + "timestamp": "2017-07-31 13:39:31.619" + } + }, + "beat": { + "hostname": "baldur", + "name": "baldur", + "version": "7.0.0-alpha1" + }, + "prospector": { + "type": "log" + }, + "source": "/home/n/go/src/github.com/elastic/beats/filebeat/module/postgresql/log/test/postgresql-9.6-debian.log", + "message": "2017-07-31 13:39:31.619 CEST [5502] postgres@clients LOG: duration: 9.482 ms statement: select * from clients;", + "fileset": { + "module": "postgresql", + "name": "log" + } + }, + "fields": { + "@timestamp": [ + "2017-07-31T13:39:31.619Z" + ] + }, + "sort": [ + 1501508371619 + ] + }, + + { + "_index": "filebeat-7.0.0-alpha1-2017.08.01", + "_type": "doc", + "_id": "wcS2ml0BVlT2mTAm4zST", + "_version": 1, + "_score": null, + "_source": { + "@timestamp": "2017-07-31T13:39:40.147Z", + "offset": 2847, + "postgresql": { + "log": { + "duration": "0.765", + "thread_id": "5502", + "database": "clients", + "level": "LOG", + "timezone": "CEST", + "query": "select id from clients;", + "user": "postgres", + "timestamp": "2017-07-31 13:39:40.147" + } + }, + "beat": { + "hostname": "baldur", + "name": "baldur", + "version": "7.0.0-alpha1" + }, + "prospector": { + "type": "log" + }, + "source": "/home/n/go/src/github.com/elastic/beats/filebeat/module/postgresql/log/test/postgresql-9.6-debian.log", + "fileset": { + "module": "postgresql", + "name": "log" + }, + "message": "2017-07-31 13:39:40.147 CEST [5502] postgres@clients LOG: duration: 0.765 ms statement: select id from clients;" + }, + "fields": { + "@timestamp": [ + "2017-07-31T13:39:40.147Z" + ] + }, + "sort": [ + 1501508380147 + ] + }, + + { + "_index": "filebeat-7.0.0-alpha1-2017.08.01", + "_type": "doc", + "_id": "wsS2ml0BVlT2mTAm4zST", + "_version": 1, + "_score": null, + "_source": { + "@timestamp": "2017-07-31T13:40:54.310Z", + "offset": 3559, + "postgresql": { + "log": { + "duration": "26.082", + "thread_id": "5502", + "database": "clients", + "level": "LOG", + "timezone": "CEST", + "query": "SELECT n.nspname as \"Schema\",\n\t c.relname as \"Name\",\n\t CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' END as \"Type\",\n\t pg_catalog.pg_get_userbyid(c.relowner) as \"Owner\"\n\tFROM pg_catalog.pg_class c\n\t LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace\n\tWHERE c.relkind IN ('r','')\n\t AND n.nspname <> 'pg_catalog'\n\t AND n.nspname <> 'information_schema'\n\t AND n.nspname !~ '^pg_toast'\n\t AND pg_catalog.pg_table_is_visible(c.oid)\n\tORDER BY 1,2;", + "user": "postgres", + "timestamp": "2017-07-31 13:40:54.310" + } + }, + "beat": { + "hostname": "baldur", + "name": "baldur", + "version": "7.0.0-alpha1" + }, + "prospector": { + "type": "log" + }, + "source": "/home/n/go/src/github.com/elastic/beats/filebeat/module/postgresql/log/test/postgresql-9.6-debian.log", + "message": "2017-07-31 13:40:54.310 CEST [5502] postgres@clients LOG: duration: 26.082 ms statement: SELECT n.nspname as \"Schema\",\n\t c.relname as \"Name\",\n\t CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' END as \"Type\",\n\t pg_catalog.pg_get_userbyid(c.relowner) as \"Owner\"\n\tFROM pg_catalog.pg_class c\n\t LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace\n\tWHERE c.relkind IN ('r','')\n\t AND n.nspname <> 'pg_catalog'\n\t AND n.nspname <> 'information_schema'\n\t AND n.nspname !~ '^pg_toast'\n\t AND pg_catalog.pg_table_is_visible(c.oid)\n\tORDER BY 1,2;", + "fileset": { + "module": "postgresql", + "name": "log" + } + }, + "fields": { + "@timestamp": [ + "2017-07-31T13:40:54.310Z" + ] + }, + "sort": [ + 1501508454310 + ] + }, + + { + "_index": "filebeat-7.0.0-alpha1-2017.08.01", + "_type": "doc", + "_id": "w8S2ml0BVlT2mTAm4zST", + "_version": 1, + "_score": null, + "_source": { + "@timestamp": "2017-07-31T13:43:22.645Z", + "offset": 3751, + "postgresql": { + "log": { + "duration": "36.162", + "thread_id": "5502", + "database": "clients", + "level": "LOG", + "timezone": "CEST", + "query": "create table cats(name varchar(50) primary key, toy varchar (50) not null, born timestamp not null);", + "user": "postgres", + "timestamp": "2017-07-31 13:43:22.645" + } + }, + "beat": { + "hostname": "baldur", + "name": "baldur", + "version": "7.0.0-alpha1" + }, + "prospector": { + "type": "log" + }, + "source": "/home/n/go/src/github.com/elastic/beats/filebeat/module/postgresql/log/test/postgresql-9.6-debian.log", + "message": "2017-07-31 13:43:22.645 CEST [5502] postgres@clients LOG: duration: 36.162 ms statement: create table cats(name varchar(50) primary key, toy varchar (50) not null, born timestamp not null);", + "fileset": { + "module": "postgresql", + "name": "log" + } + }, + "fields": { + "@timestamp": [ + "2017-07-31T13:43:22.645Z" + ] + }, + "sort": [ + 1501508602645 + ] + }, + + { + "_index": "filebeat-7.0.0-alpha1-2017.08.01", + "_type": "doc", + "_id": "xMS2ml0BVlT2mTAm4zST", + "_version": 1, + "_score": null, + "_source": { + "@timestamp": "2017-07-31T13:46:02.670Z", + "offset": 3907, + "postgresql": { + "log": { + "duration": "10.540", + "thread_id": "5502", + "database": "clients", + "level": "LOG", + "timezone": "CEST", + "query": "insert into cats(name, toy, born) values('kate', 'ball', now());", + "user": "postgres", + "timestamp": "2017-07-31 13:46:02.670" + } + }, + "beat": { + "hostname": "baldur", + "name": "baldur", + "version": "7.0.0-alpha1" + }, + "prospector": { + "type": "log" + }, + "source": "/home/n/go/src/github.com/elastic/beats/filebeat/module/postgresql/log/test/postgresql-9.6-debian.log", + "fileset": { + "module": "postgresql", + "name": "log" + }, + "message": "2017-07-31 13:46:02.670 CEST [5502] postgres@clients LOG: duration: 10.540 ms statement: insert into cats(name, toy, born) values('kate', 'ball', now());" + }, + "fields": { + "@timestamp": [ + "2017-07-31T13:46:02.670Z" + ] + }, + "sort": [ + 1501508762670 + ] + }, + + { + "_index": "filebeat-7.0.0-alpha1-2017.08.01", + "_type": "doc", + "_id": "xcS2ml0BVlT2mTAm4zST", + "_version": 1, + "_score": null, + "_source": { + "@timestamp": "2017-07-31T13:46:23.016Z", + "offset": 4064, + "postgresql": { + "log": { + "duration": "5.156", + "thread_id": "5502", + "database": "clients", + "level": "LOG", + "timezone": "CEST", + "query": "insert into cats(name, toy, born) values('frida', 'horse', now());", + "user": "postgres", + "timestamp": "2017-07-31 13:46:23.016" + } + }, + "beat": { + "hostname": "baldur", + "name": "baldur", + "version": "7.0.0-alpha1" + }, + "prospector": { + "type": "log" + }, + "source": "/home/n/go/src/github.com/elastic/beats/filebeat/module/postgresql/log/test/postgresql-9.6-debian.log", + "message": "2017-07-31 13:46:23.016 CEST [5502] postgres@clients LOG: duration: 5.156 ms statement: insert into cats(name, toy, born) values('frida', 'horse', now());", + "fileset": { + "module": "postgresql", + "name": "log" + } + }, + "fields": { + "@timestamp": [ + "2017-07-31T13:46:23.016Z" + ] + }, + "sort": [ + 1501508783016 + ] + }, + + { + "_index": "filebeat-7.0.0-alpha1-2017.08.01", + "_type": "doc", + "_id": "x8S2ml0BVlT2mTAm8zQ2", + "_version": 1, + "_score": null, + "_source": { + "@timestamp": "2017-07-31T13:46:55.637Z", + "offset": 4258, + "postgresql": { + "log": { + "duration": "25.871", + "thread_id": "5502", + "database": "clients", + "level": "LOG", + "timezone": "CEST", + "query": "create table dogs(name varchar(50) primary key, owner varchar (50) not null, born timestamp not null);", + "user": "postgres", + "timestamp": "2017-07-31 13:46:55.637" + } + }, + "beat": { + "hostname": "baldur", + "name": "baldur", + "version": "7.0.0-alpha1" + }, + "prospector": { + "type": "log" + }, + "source": "/home/n/go/src/github.com/elastic/beats/filebeat/module/postgresql/log/test/postgresql-9.6-debian.log", + "message": "2017-07-31 13:46:55.637 CEST [5502] postgres@clients LOG: duration: 25.871 ms statement: create table dogs(name varchar(50) primary key, owner varchar (50) not null, born timestamp not null);", + "fileset": { + "module": "postgresql", + "name": "log" + } + }, + "fields": { + "@timestamp": [ + "2017-07-31T13:46:55.637Z" + ] + }, + "sort": [ + 1501508815637 + ] + } +] diff --git a/filebeat/module/postgresql/log/test/postgresql-ubuntu-9.5.log b/filebeat/module/postgresql/log/test/postgresql-ubuntu-9.5.log new file mode 100644 index 000000000000..5a3df37b16a6 --- /dev/null +++ b/filebeat/module/postgresql/log/test/postgresql-ubuntu-9.5.log @@ -0,0 +1,78 @@ +2017-04-03 22:32:14.322 CEST [31225] postgres@mydb LOG: could not receive data from client: Connection reset by peer +2017-04-03 22:32:14.322 CEST [31225] postgres@mydb LOG: unexpected EOF on client connection with an open transaction +2017-04-03 22:35:22.389 CEST [3474] postgres@mydb LOG: could not receive data from client: Connection reset by peer +2017-04-03 22:36:56.464 CEST [3525] postgres@mydb LOG: could not receive data from client: Connection reset by peer +2017-04-03 22:37:12.961 CEST [3570] postgres@mydb LOG: could not receive data from client: Connection reset by peer +2017-04-07 21:05:28.549 CEST [21483] postgres@mydb LOG: could not receive data from client: Connection reset by peer +2017-04-07 21:09:41.345 CEST [21597] postgres@mydb LOG: could not receive data from client: Connection reset by peer +2017-04-07 22:45:30.218 CEST [22603] postgres@mydb ERROR: operator does not exist: jsonb @> at character 49 +2017-04-07 22:45:30.218 CEST [22603] postgres@mydb HINT: No operator matches the given name and argument type(s). You might need to add explicit type casts. +2017-04-07 22:45:30.218 CEST [22603] postgres@mydb STATEMENT: SELECT id, user FROM users WHERE NOT user @> %s +2017-04-07 22:46:09.751 CEST [22608] postgres@mydb ERROR: column "%s" does not exist at character 52 +2017-04-07 22:46:09.751 CEST [22608] postgres@mydb STATEMENT: SELECT id, user FROM users WHERE NOT user @> "%s" +2017-04-07 23:02:51.199 CEST [24341] postgres@mydb LOG: could not receive data from client: Connection reset by peer +2017-04-07 23:02:51.199 CEST [24341] postgres@mydb LOG: unexpected EOF on client connection with an open transaction +2017-04-07 23:04:36.087 CEST [20730] postgres@mydb ERROR: syntax error at or near "{" at character 49 +2017-04-07 23:04:36.087 CEST [20730] postgres@mydb STATEMENT: INSERT INTO users (id, user) VALUES (1, {"attr": "yes"}); +2017-04-07 23:04:51.462 CEST [20730] postgres@mydb ERROR: syntax error at or near "{" at character 49 +2017-04-07 23:04:51.462 CEST [20730] postgres@mydb STATEMENT: INSERT INTO users (id, user) VALUES (1, {attr: "yes"}); +2017-04-07 23:05:06.217 CEST [20730] postgres@mydb ERROR: column "a" does not exist at character 42 +2017-04-07 23:05:06.217 CEST [20730] postgres@mydb STATEMENT: INSERT INTO users (id, user) VALUES (1, '{"attr": "yes"}'); +2017-04-07 23:05:18.295 CEST [20730] postgres@mydb ERROR: column "attr" does not exist at character 42 +2017-04-07 23:05:18.295 CEST [20730] postgres@mydb STATEMENT: INSERT INTO users (id, user) VALUES ("1", '{"attr": "no"}'); +2017-04-07 23:13:47.505 CEST [24489] postgres@mydb LOG: could not receive data from client: Connection reset by peer +2017-04-07 23:13:47.505 CEST [24489] postgres@mydb LOG: unexpected EOF on client connection with an open transaction +2017-04-08 12:32:51.056 CEST [20730] postgres@mydb ERROR: duplicate key value violates unique constraint "users_pkey" +2017-04-08 12:32:51.056 CEST [20730] postgres@mydb DETAIL: Key (id)=(1) already exists. +2017-04-08 12:32:51.056 CEST [20730] postgres@mydb STATEMENT: INSERT INTO users (id, user) VALUES ('1', '{"attr": "yes"}'); +2017-04-08 21:54:37.443 CEST [30630] postgres@mydb LOG: could not receive data from client: Connection reset by peer +2017-04-08 21:54:37.468 CEST [30502] postgres@mydb LOG: could not receive data from client: Connection reset by peer +2017-04-08 21:54:37.618 CEST [20769] LOG: received fast shutdown request +2017-04-08 21:54:37.618 CEST [20769] LOG: aborting any active transactions +2017-04-08 21:54:37.618 CEST [20774] LOG: autovacuum launcher shutting down +2017-04-08 21:54:37.622 CEST [20771] LOG: shutting down +2017-04-08 21:54:37.644 CEST [20769] LOG: database system is shut down +2017-04-08 21:56:02.932 CEST [797] LOG: database system was shut down at 2017-04-08 21:54:37 CEST +2017-04-08 21:56:02.944 CEST [797] LOG: MultiXact member wraparound protections are now enabled +2017-04-08 21:56:02.946 CEST [780] LOG: database system is ready to accept connections +2017-04-08 21:56:02.947 CEST [802] LOG: autovacuum launcher started +2017-04-08 21:56:03.362 CEST [891] [unknown]@[unknown] LOG: incomplete startup packet +2017-05-27 14:07:53.007 UTC [32567] LOG: database system was shut down at 2017-05-27 14:07:52 UTC +2017-05-27 14:07:53.010 UTC [32567] LOG: MultiXact member wraparound protections are now enabled +2017-05-27 14:07:53.015 UTC [32566] LOG: database system is ready to accept connections +2017-05-27 14:07:53.016 UTC [32571] LOG: autovacuum launcher started +2017-05-27 14:07:53.463 UTC [32573] [unknown]@[unknown] LOG: incomplete startup packet +2017-05-27 14:08:13.661 UTC [1308] postgres@mydb FATAL: database "mydb" does not exist +2017-05-27 14:59:26.553 UTC [1994] postgres@mydb LOG: could not receive data from client: Connection reset by peer +2017-05-27 14:59:26.555 UTC [1989] postgres@mydb LOG: could not receive data from client: Connection reset by peer +2017-06-06 07:54:13.753 CEST [9110] LOG: received fast shutdown request +2017-06-06 07:54:13.753 CEST [9110] LOG: aborting any active transactions +2017-06-06 07:54:13.753 CEST [9115] LOG: autovacuum launcher shutting down +2017-06-06 07:54:13.755 CEST [9112] LOG: shutting down +2017-06-06 07:54:13.816 CEST [9110] LOG: database system is shut down +2017-06-06 07:55:39.725 CEST [12969] LOG: database system was shut down at 2017-06-06 07:54:13 CEST +2017-06-06 07:55:39.736 CEST [12969] LOG: MultiXact member wraparound protections are now enabled +2017-06-06 07:55:39.739 CEST [12968] LOG: database system is ready to accept connections +2017-06-06 07:55:39.739 CEST [12973] LOG: autovacuum launcher started +2017-06-06 07:55:40.155 CEST [12975] [unknown]@[unknown] LOG: incomplete startup packet +2017-06-06 07:55:40.156 CEST [12975] [unknown]@[unknown] LOG: incomplete startup packet +2017-06-10 19:37:30.681 CEST [17398] LOG: database system was shut down at 2017-06-10 19:37:29 CEST +2017-06-10 19:37:30.695 CEST [17398] LOG: MultiXact member wraparound protections are now enabled +2017-06-10 19:37:30.702 CEST [17397] LOG: database system is ready to accept connections +2017-06-10 19:37:30.702 CEST [17402] LOG: autovacuum launcher started +2017-06-10 19:37:31.104 CEST [17404] [unknown]@[unknown] LOG: incomplete startup packet +2017-06-10 20:27:55.911 CEST [17397] LOG: received fast shutdown request +2017-06-10 20:27:55.911 CEST [17397] LOG: aborting any active transactions +2017-06-10 20:27:55.911 CEST [17402] LOG: autovacuum launcher shutting down +2017-06-10 20:27:55.914 CEST [17399] LOG: shutting down +2017-06-10 20:27:55.973 CEST [17397] LOG: database system is shut down +2017-06-10 20:27:57.022 CEST [24490] LOG: database system was shut down at 2017-06-10 20:27:55 CEST +2017-06-10 20:27:57.032 CEST [24490] LOG: MultiXact member wraparound protections are now enabled +2017-06-10 20:27:57.035 CEST [24494] LOG: autovacuum launcher started +2017-06-10 20:27:57.035 CEST [24489] LOG: database system is ready to accept connections +2017-06-10 20:27:57.475 CEST [24496] [unknown]@[unknown] LOG: incomplete startup packet +2017-06-17 16:58:03.937 CEST [24489] LOG: received fast shutdown request +2017-06-17 16:58:03.937 CEST [24489] LOG: aborting any active transactions +2017-06-17 16:58:03.938 CEST [24494] LOG: autovacuum launcher shutting down +2017-06-17 16:58:03.940 CEST [24491] LOG: shutting down +2017-06-17 16:58:04.040 CEST [24489] LOG: database system is shut down diff --git a/filebeat/modules.d/postgresql.yml.disabled b/filebeat/modules.d/postgresql.yml.disabled new file mode 100644 index 000000000000..c82734a9570f --- /dev/null +++ b/filebeat/modules.d/postgresql.yml.disabled @@ -0,0 +1,8 @@ +- module: postgresql + # All logs + log: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: diff --git a/filebeat/tests/system/test_modules.py b/filebeat/tests/system/test_modules.py index a07fafab13f1..4b0f79bc59d2 100644 --- a/filebeat/tests/system/test_modules.py +++ b/filebeat/tests/system/test_modules.py @@ -64,6 +64,26 @@ def test_modules(self): test_file=test_file, cfgfile=cfgfile) + def _test_expected_events(self, module, test_file, res, objects): + with open(test_file + "-expected.json", "r") as f: + expected = json.load(f) + + if len(expected) > len(objects): + res = self.es.search(index=self.index_name, + body={"query": {"match_all": {}}, + "size": len(expected)}) + objects = [o["_source"] for o in res["hits"]["hits"]] + + assert len(expected) == res['hits']['total'], "expected {} but got {}".format(len(expected), len(objects)) + + for ev in expected: + found = False + for obj in objects: + if ev["_source"][module] == obj[module]: + found = True + break + assert found, "The following expected object was not found: {}".format(obj) + def run_on_file(self, module, fileset, test_file, cfgfile): print("Testing {}/{} on {}".format(module, fileset, test_file)) @@ -116,18 +136,7 @@ def run_on_file(self, module, fileset, test_file, cfgfile): self.assert_fields_are_documented(obj) if os.path.exists(test_file + "-expected.json"): - with open(test_file + "-expected.json", "r") as f: - expected = json.load(f) - assert len(expected) == len(objects), "expected {} but got {}".format(len(expected), len(objects)) - for ev in expected: - found = False - for obj in objects: - if ev["_source"][module] == obj[module]: - found = True - break - if not found: - raise Exception("The following expected object was" + - " not found: {}".format(obj)) + self._test_expected_events(module, test_file, res, objects) @unittest.skipIf(not INTEGRATION_TESTS or os.getenv("TESTING_ENVIRONMENT") == "2x",