Add PostgresSQL module for Filebeat

CHANGELOG.asciidoc

@@ -66,6 +66,8 @@ https://github.com/elastic/beats/compare/v6.0.0-beta1...master[Check the HEAD di
 
 *Filebeat*
 
+- Add PostgreSQL module with slowlog support. {pull}4763[4763]
+
 *Heartbeat*
 
 *Metricbeat*

filebeat/docs/fields.asciidoc

@@ -22,6 +22,7 @@ grouped in the following categories:
 * <<exported-fields-log>>
 * <<exported-fields-mysql>>
 * <<exported-fields-nginx>>
+* <<exported-fields-postgresql>>
 * <<exported-fields-redis>>
 * <<exported-fields-system>>
 
@@ -1277,6 +1278,92 @@ type: text
 The error message
 
 
+[[exported-fields-postgresql]]
+== PostgreSQL fields
+
+Module for parsing the PostgreSQL log files.
+
+
+
+[float]
+== postgresql fields
+
+Fields from PostgreSQL logs.
+
+
+
+[float]
+== log fields
+
+Fields from the PostgreSQL log files.
+
+
+
+[float]
+=== `postgresql.log.timestamp`
+
+The timestamp from the log line.
+
+
+[float]
+=== `postgresql.log.timezone`
+
+The timezone of timestamp.
+
+
+[float]
+=== `postgresql.log.thread_id`
+
+type: long
+
+Processs id
+
+
+[float]
+=== `postgresql.log.user`
+
+example: admin
+
+Name of user
+
+[float]
+=== `postgresql.log.database`
+
+example: mydb
+
+Name of database
+
+[float]
+=== `postgresql.log.level`
+
+example: FATAL
+
+The log level.
+
+[float]
+=== `postgresql.log.duration`
+
+type: float
+
+example: 30.0
+
+Duration of a query.
+
+[float]
+=== `postgresql.log.query`
+
+example: SELECT * FROM users;
+
+Query statment.
+
+[float]
+=== `postgresql.log.message`
+
+type: text
+
+The logged message.
+
+
 [[exported-fields-redis]]
 == Redis fields
 

filebeat/docs/modules/postgresql.asciidoc

@@ -0,0 +1,43 @@
+////
+This file is generated! See scripts/docs_collector.py
+////
+
+[[filebeat-module-postgresql]]
+== PostgreSQL module
+
+This module collects and parses the logs created by https://www.postgresql.org/[PostgreSQL].
+
+[float]
+=== Compatibility
+
+The PostgreSQL module was tested with logs from versions 9.5 on Ubuntu and 9.6 on Debian.
+
+[float]
+=== Dashboard
+
+This module comes with two sample dashboards.
+
+The first dashboard is for regulars logs.
+
+image::./images/filebeat-postgresql-overview.png[]
+
+The second one shows the slowlogs of PostgreSQL.
+
+image::./images/filebeat-postgresql-postgresql-overview.png[]
+
+[float]
+=== Logs fileset settings
+
+[float]
+==== var.paths
+
+An array of paths where to look for the log files. If left empty, Filebeat
+will choose the paths depending on your operating systems.
+
+
+[float]
+=== Fields
+
+For a description of each field in the metricset, see the
+<<exported-fields-postgresql,exported fields>> section.
+

filebeat/docs/modules_list.asciidoc

@@ -8,6 +8,7 @@ This file is generated! See scripts/docs_collector.py
   * <<filebeat-module-icinga>>
   * <<filebeat-module-mysql>>
   * <<filebeat-module-nginx>>
+  * <<filebeat-module-postgresql>>
   * <<filebeat-module-redis>>
   * <<filebeat-module-system>>
 
@@ -20,5 +21,6 @@ include::modules/auditd.asciidoc[]
 include::modules/icinga.asciidoc[]
 include::modules/mysql.asciidoc[]
 include::modules/nginx.asciidoc[]
+include::modules/postgresql.asciidoc[]
 include::modules/redis.asciidoc[]
 include::modules/system.asciidoc[]

filebeat/filebeat.reference.yml

@@ -166,6 +166,20 @@ filebeat.modules:
     # can be added under this section.
     #prospector:
 
+#----------------------------- PostgreSQL Module -----------------------------
+#- module: postgresql
+  # Logs
+  #log:
+    #enabled: true
+
+    # Set custom paths for the log files. If left empty,
+    # Filebeat will choose the paths depending on your OS.
+    #var.paths:
+
+    # Prospector configuration (advanced). Any prospector configuration option
+    # can be added under this section.
+    #prospector:
+
 #-------------------------------- Redis Module -------------------------------
 #- module: redis
   # Main logs

filebeat/module/postgresql/_meta/config.reference.yml

@@ -0,0 +1,12 @@
+#- module: postgresql
+  # Logs
+  #log:
+    #enabled: true
+
+    # Set custom paths for the log files. If left empty,
+    # Filebeat will choose the paths depending on your OS.
+    #var.paths:
+
+    # Prospector configuration (advanced). Any prospector configuration option
+    # can be added under this section.
+    #prospector:

filebeat/module/postgresql/_meta/config.yml

@@ -0,0 +1,8 @@
+- module: postgresql
+  # All logs
+  log:
+    enabled: true
+
+    # Set custom paths for the log files. If left empty,
+    # Filebeat will choose the paths depending on your OS.
+    #var.paths:

filebeat/module/postgresql/_meta/docs.asciidoc

@@ -0,0 +1,30 @@
+== PostgreSQL module
+
+This module collects and parses the logs created by https://www.postgresql.org/[PostgreSQL].
+
+[float]
+=== Compatibility
+
+The PostgreSQL module was tested with logs from versions 9.5 on Ubuntu and 9.6 on Debian.
+
+[float]
+=== Dashboard
+
+This module comes with two sample dashboards.
+
+The first dashboard is for regulars logs.
+
+image::./images/filebeat-postgresql-overview.png[]
+
+The second one shows the slowlogs of PostgreSQL.
+
+image::./images/filebeat-postgresql-postgresql-overview.png[]
+
+[float]
+=== Logs fileset settings
+
+[float]
+==== var.paths
+
+An array of paths where to look for the log files. If left empty, Filebeat
+will choose the paths depending on your operating systems.

filebeat/module/postgresql/_meta/fields.yml

@@ -0,0 +1,11 @@
+- key: postgresql
+  title: "PostgreSQL"
+  description: >
+    Module for parsing the PostgreSQL log files.
+  short_config: true
+  fields:
+    - name: postgresql
+      type: group
+      description: >
+          Fields from PostgreSQL logs.
+      fields:

filebeat/module/postgresql/_meta/kibana/default/dashboard/Filebeat-Postgresql-overview.json

@@ -0,0 +1,64 @@
+{
+  "objects": [
+    {
+      "attributes": {
+        "description": "",
+        "kibanaSavedObjectMeta": {
+          "searchSourceJSON": "{\"filter\":[]}"
+        },
+        "savedSearchId": "ef9da9b0-776c-11e7-9e03-393c0d1aa325",
+        "title": "PostgreSQL Log Level Count",
+        "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}",
+        "version": 1,
+        "visState": "{\"title\":\"PostgreSQL Log Level Count\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"postgresql.log.level\",\"size\":12,\"order\":\"desc\",\"orderBy\":\"1\"}}]}"
+      },
+      "id": "28563b50-776d-11e7-9e03-393c0d1aa325",
+      "type": "visualization",
+      "version": 1
+    },
+    {
+      "attributes": {
+        "columns": [
+          "postgresql.log.user",
+          "postgresql.log.database",
+          "postgresql.log.level",
+          "postgresql.log.message",
+          "postgresql.log.query"
+        ],
+        "description": "",
+        "hits": 0,
+        "kibanaSavedObjectMeta": {
+          "searchSourceJSON": "{\"index\":\"a0b887a0-763d-11e7-9e03-393c0d1aa325\",\"highlightAll\":true,\"version\":true,\"query\":{\"query\":\"postgresql.log.level:*\",\"language\":\"lucene\"},\"filter\":[]}"
+        },
+        "sort": [
+          "@timestamp",
+          "desc"
+        ],
+        "title": "PostgreSQL All Logs",
+        "version": 1
+      },
+      "id": "ef9da9b0-776c-11e7-9e03-393c0d1aa325",
+      "type": "search",
+      "version": 1
+    },
+    {
+      "attributes": {
+        "description": "",
+        "hits": 0,
+        "kibanaSavedObjectMeta": {
+          "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[],\"highlightAll\":true,\"version\":true}"
+        },
+        "optionsJSON": "{\"darkTheme\":false}",
+        "panelsJSON": "[{\"size_x\":6,\"size_y\":3,\"panelIndex\":1,\"type\":\"visualization\",\"id\":\"28563b50-776d-11e7-9e03-393c0d1aa325\",\"col\":1,\"row\":4},{\"size_x\":12,\"size_y\":3,\"panelIndex\":2,\"type\":\"search\",\"id\":\"ef9da9b0-776c-11e7-9e03-393c0d1aa325\",\"col\":1,\"row\":1,\"columns\":[\"postgresql.log.user\",\"postgresql.log.database\",\"postgresql.log.level\",\"postgresql.log.message\",\"postgresql.log.query\"],\"sort\":[\"@timestamp\",\"desc\"]}]",
+        "timeRestore": false,
+        "title": "PostgreSQL All Logs",
+        "uiStateJSON": "{\"P-1\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}}",
+        "version": 1
+      },
+      "id": "a21284c0-776e-11e7-9e03-393c0d1aa325",
+      "type": "dashboard",
+      "version": 1
+    }
+  ],
+  "version": "6.0.0-beta1-SNAPSHOT"
+}