elastic · jsoriano · Jan 18, 2019 · Jan 18, 2019
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -42,6 +42,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add grok pattern to support redis 5.0.3 log timestamp. {issue}9819[9819] {pull}10033[10033]
 - Now save the 'first seen' timestamp in `event.created` (previously `read_timestamp`),
   instead of saving the parsed date. Now aligned with `event.created` semantics elsewhere. {pull}10139[10139]
+- Rename `mysql.error.thread_id` and `mysql.slowlog.id` to `mysql.thread_id`. {pull}10161[10161]
+- Remove `mysql.error.timestamp`  and `mysql.slowlog.timestamp`. {pull}10161[10161]
 
 *Heartbeat*
 

diff --git a/dev-tools/ecs-migration.yml b/dev-tools/ecs-migration.yml
@@ -551,7 +551,7 @@
   alias: true
 
 - from: mysql.error.thread_id
-  to: process.thread.id
+  to: mysql.thread_id
   alias: true
 
 - from: mysql.slowlog.ip
@@ -566,6 +566,10 @@
   to: user.name
   alias: true
 
+- from: mysql.slowlog.id
+  to: mysql.thread_id
+  alias: true
+
 ## NGINX module
 
 - from: nginx.access.user_name

diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc
@@ -6322,27 +6322,29 @@ Fields from the MySQL log files.
 
 
 
-[float]
-== error fields
-
-Contains fields from the MySQL error logs.
+*`mysql.thread_id`*::
++
+--
+type: long
 
+The connection or thread ID for the query.
 
 
-*`mysql.error.timestamp`*::
-+
 --
-The timestamp from the log line.
+
+[float]
+== error fields
+
+Contains fields from the MySQL error logs.
 
 
---
 
 *`mysql.error.thread_id`*::
 +
 --
 type: alias
 
-alias to: process.thread.id
+alias to: mysql.thread_id
 
 --
 
@@ -6419,16 +6421,6 @@ type: long
 The number of rows modified by the query.
 
 
---
-
-*`mysql.slowlog.timestamp`*::
-+
---
-type: long
-
-The unix timestamp taken from the `SET timestamp` query.
-
-
 --
 
 *`mysql.slowlog.query`*::
@@ -6442,10 +6434,9 @@ The slow query.
 *`mysql.slowlog.id`*::
 +
 --
-type: long
-
-The connection or thread ID for the query.
+type: alias
 
+alias to: mysql.thread_id
 
 --
 

diff --git a/filebeat/module/mysql/_meta/fields.yml b/filebeat/module/mysql/_meta/fields.yml
@@ -9,3 +9,7 @@
       description: >
         Fields from the MySQL log files.
       fields:
+        - name: thread_id
+          type: long
+          description: >
+            The connection or thread ID for the query.
diff --git a/filebeat/module/mysql/error/_meta/fields.yml b/filebeat/module/mysql/error/_meta/fields.yml
@@ -3,13 +3,9 @@
   description: >
     Contains fields from the MySQL error logs.
   fields:
-    - name: timestamp
-      description: >
-        The timestamp from the log line.
-
     - name: thread_id
       type: alias
-      path: process.thread.id
+      path: mysql.thread_id
       migration: true
     - name: level
       type: alias

diff --git a/filebeat/module/mysql/error/ingest/pipeline.json b/filebeat/module/mysql/error/ingest/pipeline.json
@@ -5,7 +5,7 @@
         "field": "message",
         "patterns": [
           "%{LOCALDATETIME:mysql.error.timestamp} (\\[%{DATA:log.level}\\] )?%{GREEDYDATA:message}",
-          "%{DATA:mysql.error.timestamp} %{NUMBER:process.thread.id:long} \\[%{DATA:log.level}\\] %{GREEDYDATA:message}",
+          "%{DATA:mysql.error.timestamp} %{NUMBER:mysql.thread_id:long} \\[%{DATA:log.level}\\] %{GREEDYDATA:message}",
           "%{GREEDYDATA:message}"
         ],
         "ignore_missing": true,
@@ -28,6 +28,11 @@
         ],
         "ignore_failure": true
       }
+    }, {
+      "remove":{
+        "field": "mysql.error.timestamp",
+        "ignore_missing": true
+      }
     }],
   "on_failure" : [{
     "set" : {

diff --git a/filebeat/module/mysql/error/test/error.log-expected.json b/filebeat/module/mysql/error/test/error.log-expected.json
@@ -8,7 +8,6 @@
         "input.type": "log",
         "log.offset": 0,
         "message": "mysqld_safe Starting mysqld daemon with databases from /usr/local/var/mysql",
-        "mysql.error.timestamp": "161209 13:08:33",
         "service.type": "mysql"
     },
     {
@@ -21,8 +20,7 @@
         "log.level": "Warning",
         "log.offset": 92,
         "message": "TIMESTAMP with implicit DEFAULT value is deprecated. Please use --explicit_defaults_for_timestamp server option (see documentation for more details).",
-        "mysql.error.timestamp": "2016-12-09T12:08:33.335060Z",
-        "process.thread.id": 0,
+        "mysql.thread_id": 0,
         "service.type": "mysql"
     },
     {
@@ -35,8 +33,7 @@
         "log.level": "Warning",
         "log.offset": 282,
         "message": "Insecure configuration for --secure-file-priv: Current value does not restrict location of generated files. Consider setting it to a valid, non-empty path.",
-        "mysql.error.timestamp": "2016-12-09T12:08:33.335892Z",
-        "process.thread.id": 0,
+        "mysql.thread_id": 0,
         "service.type": "mysql"
     },
     {
@@ -49,8 +46,7 @@
         "log.level": "Note",
         "log.offset": 478,
         "message": "/usr/local/Cellar/mysql/5.7.10/bin/mysqld (mysqld 5.7.10) starting as process 61571 ...",
-        "mysql.error.timestamp": "2016-12-09T12:08:33.336610Z",
-        "process.thread.id": 0,
+        "mysql.thread_id": 0,
         "service.type": "mysql"
     },
     {
@@ -63,8 +59,7 @@
         "log.level": "Warning",
         "log.offset": 603,
         "message": "Setting lower_case_table_names=2 because file system for /usr/local/var/mysql/ is case insensitive",
-        "mysql.error.timestamp": "2016-12-09T12:08:33.345527Z",
-        "process.thread.id": 0,
+        "mysql.thread_id": 0,
         "service.type": "mysql"
     },
     {
@@ -77,8 +72,7 @@
         "log.level": "Note",
         "log.offset": 742,
         "message": "InnoDB: Mutexes and rw_locks use GCC atomic builtins",
-        "mysql.error.timestamp": "2016-12-09T12:08:33.351596Z",
-        "process.thread.id": 0,
+        "mysql.thread_id": 0,
         "service.type": "mysql"
     },
     {
@@ -91,8 +85,7 @@
         "log.level": "Note",
         "log.offset": 832,
         "message": "/usr/local/Cellar/mysql/5.7.10/bin/mysqld: ready for connections.",
-        "mysql.error.timestamp": "2016-12-09T12:08:33.784722Z",
-        "process.thread.id": 0,
+        "mysql.thread_id": 0,
         "service.type": "mysql"
     },
     {
@@ -115,8 +108,7 @@
         "log.level": "Note",
         "log.offset": 1002,
         "message": "InnoDB: page_cleaner: 1000ms intended loop took 772568ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.)",
-        "mysql.error.timestamp": "2016-12-09T22:21:02.443689Z",
-        "process.thread.id": 0,
+        "mysql.thread_id": 0,
         "service.type": "mysql"
     },
     {
@@ -129,7 +121,6 @@
         "log.level": "Warning",
         "log.offset": 1176,
         "message": "Using unique option prefix myisam-recover instead of myisam-recover-options is deprecated and will be removed in a future release. Please use the full name instead.",
-        "mysql.error.timestamp": "161209 14:18:50",
         "service.type": "mysql"
     },
     {
@@ -142,7 +133,6 @@
         "log.level": "Note",
         "log.offset": 1367,
         "message": "Plugin 'FEDERATED' is disabled.",
-        "mysql.error.timestamp": "161209 14:18:50",
         "service.type": "mysql"
     },
     {
@@ -154,7 +144,6 @@
         "input.type": "log",
         "log.offset": 1422,
         "message": "InnoDB: The InnoDB memory heap is disabled",
-        "mysql.error.timestamp": "161209 14:18:50",
         "service.type": "mysql"
     }
 ]
diff --git a/filebeat/module/mysql/fields.go b/filebeat/module/mysql/fields.go
diff --git a/filebeat/module/mysql/slowlog/_meta/fields.yml b/filebeat/module/mysql/slowlog/_meta/fields.yml
@@ -24,17 +24,13 @@
       type: long
       description: >
         The number of rows modified by the query.
-    - name: timestamp
-      type: long
-      description: >
-        The unix timestamp taken from the `SET timestamp` query.
     - name: query
       description: >
         The slow query.
     - name: id
-      type: long
-      description: >
-        The connection or thread ID for the query.
+      type: alias
+      path: mysql.thread_id
+      migration: true
     - name: schema
       type: keyword
       description: >

diff --git a/filebeat/module/mysql/slowlog/ingest/pipeline.json b/filebeat/module/mysql/slowlog/ingest/pipeline.json
@@ -4,7 +4,7 @@
     "grok": {
       "field": "message",
       "patterns":[
-        "^# User@Host: %{USER:user.name}(\\[%{USER:mysql.slowlog.current_user}\\])? @ %{HOSTNAME:source.domain}? \\[%{IP:source.ip}?\\](\\s*Id:\\s* %{NUMBER:mysql.slowlog.id:long})?\n(# Thread_id:%{SPACE}%{NUMBER:mysql.slowlog.id}%{METRICSPACE}Schema: %{WORD:mysql.slowlog.schema}?%{METRICSPACE}QC_hit: %{WORD:mysql.slowlog.query_cache_hit}\n)?%{METRICSPACE}Query_time: %{NUMBER:mysql.slowlog.query_time.sec:float}%{METRICSPACE}Lock_time: %{NUMBER:mysql.slowlog.lock_time.sec:float}%{METRICSPACE}Rows_sent: %{NUMBER:mysql.slowlog.rows_sent:long}%{METRICSPACE}Rows_examined: %{NUMBER:mysql.slowlog.rows_examined:long}%{METRICSPACE}(Rows_affected: %{NUMBER:mysql.slowlog.rows_affected:long}%{METRICSPACE})?(Full_scan: %{WORD:mysql.slowlog.full_scan}%{METRICSPACE})?(Full_join: %{WORD:mysql.slowlog.full_join}%{METRICSPACE})?(Tmp_table: %{WORD:mysql.slowlog.tmp_table}%{METRICSPACE})?(Tmp_table_on_disk: %{WORD:mysql.slowlog.tmp_table_on_disk}%{METRICSPACE})?(Filesort: %{WORD:mysql.slowlog.filesort}%{METRICSPACE})?(Filesort_on_disk: %{WORD:mysql.slowlog.filesort_on_disk}%{METRICSPACE})?(Merge_passes: %{NUMBER:mysql.slowlog.merge_passes}%{METRICSPACE})?(Priority_queue: %{WORD:mysql.slowlog.priority_queue}%{METRICSPACE})?%{EXPLAIN}?(use %{WORD:mysql.slowlog.schema};\n)?SET timestamp=%{NUMBER:mysql.slowlog.timestamp:long};\n%{GREEDYMULTILINE:mysql.slowlog.query}"
+        "^# User@Host: %{USER:user.name}(\\[%{USER:mysql.slowlog.current_user}\\])? @ %{HOSTNAME:source.domain}? \\[%{IP:source.ip}?\\](\\s*Id:\\s* %{NUMBER:mysql.thread_id:long})?\n(# Thread_id:%{SPACE}%{NUMBER:mysql.thread_id}%{METRICSPACE}Schema: %{WORD:mysql.slowlog.schema}?%{METRICSPACE}QC_hit: %{WORD:mysql.slowlog.query_cache_hit}\n)?%{METRICSPACE}Query_time: %{NUMBER:mysql.slowlog.query_time.sec:float}%{METRICSPACE}Lock_time: %{NUMBER:mysql.slowlog.lock_time.sec:float}%{METRICSPACE}Rows_sent: %{NUMBER:mysql.slowlog.rows_sent:long}%{METRICSPACE}Rows_examined: %{NUMBER:mysql.slowlog.rows_examined:long}%{METRICSPACE}(Rows_affected: %{NUMBER:mysql.slowlog.rows_affected:long}%{METRICSPACE})?(Full_scan: %{WORD:mysql.slowlog.full_scan}%{METRICSPACE})?(Full_join: %{WORD:mysql.slowlog.full_join}%{METRICSPACE})?(Tmp_table: %{WORD:mysql.slowlog.tmp_table}%{METRICSPACE})?(Tmp_table_on_disk: %{WORD:mysql.slowlog.tmp_table_on_disk}%{METRICSPACE})?(Filesort: %{WORD:mysql.slowlog.filesort}%{METRICSPACE})?(Filesort_on_disk: %{WORD:mysql.slowlog.filesort_on_disk}%{METRICSPACE})?(Merge_passes: %{NUMBER:mysql.slowlog.merge_passes}%{METRICSPACE})?(Priority_queue: %{WORD:mysql.slowlog.priority_queue}%{METRICSPACE})?%{EXPLAIN}?(use %{WORD:mysql.slowlog.schema};\n)?SET timestamp=%{NUMBER:mysql.slowlog.timestamp:long};\n%{GREEDYMULTILINE:mysql.slowlog.query}"
         ],
       "pattern_definitions" : {
         "GREEDYMULTILINE": "(.|\n)*",
@@ -51,6 +51,11 @@
       "formats": ["UNIX"],
       "ignore_failure": true
     }
+  }, {
+    "remove":{
+      "field": "mysql.slowlog.timestamp",
+      "ignore_missing": true
+    }
   }, {
     "gsub": {
       "field": "mysql.slowlog.query",

diff --git a/filebeat/module/mysql/slowlog/test/mariadb-10.1.21.log-expected.json b/filebeat/module/mysql/slowlog/test/mariadb-10.1.21.log-expected.json
@@ -12,14 +12,13 @@
         ],
         "log.offset": 24,
         "mysql.slowlog.current_user": "root",
-        "mysql.slowlog.id": "5",
         "mysql.slowlog.lock_time.sec": 0.0,
         "mysql.slowlog.query": "select sleep(2);",
         "mysql.slowlog.query_cache_hit": false,
         "mysql.slowlog.query_time.sec": 2.000652,
         "mysql.slowlog.rows_examined": 0,
         "mysql.slowlog.rows_sent": 1,
-        "mysql.slowlog.timestamp": 1528898676,
+        "mysql.thread_id": "5",
         "service.type": "mysql",
         "source.domain": "localhost",
         "source.ip": "121.0.0.1",

diff --git a/filebeat/module/mysql/slowlog/test/mariadb-10.2.12.log-expected.json b/filebeat/module/mysql/slowlog/test/mariadb-10.2.12.log-expected.json
@@ -12,15 +12,14 @@
         ],
         "log.offset": 437,
         "mysql.slowlog.current_user": "root",
-        "mysql.slowlog.id": "8",
         "mysql.slowlog.lock_time.sec": 0.0,
         "mysql.slowlog.query": "select sleep(2)\nAS foo;",
         "mysql.slowlog.query_cache_hit": false,
         "mysql.slowlog.query_time.sec": 2.000227,
         "mysql.slowlog.rows_affected": 0,
         "mysql.slowlog.rows_examined": 0,
         "mysql.slowlog.rows_sent": 1,
-        "mysql.slowlog.timestamp": 1547741043,
+        "mysql.thread_id": "8",
         "service.type": "mysql",
         "source.domain": "localhost",
         "user.name": "root"
@@ -42,7 +41,6 @@
         "mysql.slowlog.filesort_on_disk": false,
         "mysql.slowlog.full_join": false,
         "mysql.slowlog.full_scan": true,
-        "mysql.slowlog.id": "25844",
         "mysql.slowlog.lock_time.sec": 0.0,
         "mysql.slowlog.merge_passes": "0",
         "mysql.slowlog.priority_queue": false,
@@ -53,9 +51,9 @@
         "mysql.slowlog.rows_examined": 53022772,
         "mysql.slowlog.rows_sent": 0,
         "mysql.slowlog.schema": "blah",
-        "mysql.slowlog.timestamp": 1547741058,
         "mysql.slowlog.tmp_table": true,
         "mysql.slowlog.tmp_table_on_disk": false,
+        "mysql.thread_id": "25844",
         "service.type": "mysql",
         "source.ip": "192.168.0.10",
         "user.name": "root"

diff --git a/filebeat/module/mysql/slowlog/test/mariadb-explain.log-expected.json b/filebeat/module/mysql/slowlog/test/mariadb-explain.log-expected.json
@@ -12,15 +12,14 @@
         ],
         "log.offset": 24,
         "mysql.slowlog.current_user": "root",
-        "mysql.slowlog.id": "2",
         "mysql.slowlog.lock_time.sec": 0.000337,
         "mysql.slowlog.query": "select count(*) from customer, orders, nation\n  where c_custkey=o_custkey\n    and c_nationkey=n_nationkey\n    and n_name='GERMANY';",
         "mysql.slowlog.query_cache_hit": false,
         "mysql.slowlog.query_time.sec": 5.524103,
         "mysql.slowlog.rows_examined": 65633,
         "mysql.slowlog.rows_sent": 1,
         "mysql.slowlog.schema": "dbt3sf1",
-        "mysql.slowlog.timestamp": 1384261412,
+        "mysql.thread_id": "2",
         "service.type": "mysql",
         "source.domain": "localhost",
         "user.name": "root"