From 27f7b158e6d24f3367fd2e0c81dbc1e0f78c53e0 Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Fri, 11 Jan 2019 13:50:28 -0500 Subject: [PATCH] Convert Filebeat nginx.error to ECS (#10007) - Convert many fields under `nginx.error.*` to ECS. Previous field names are field aliases towards the new corresponding ECS field: - nginx.error.level => log.level - nginx.error.pid => process.pid - nginx.error.tid => process.thread.id - nginx.error.message => message - read_timestamp => event.created (not aliased, still used elsewhere --- CHANGELOG.next.asciidoc | 1 + dev-tools/ecs-migration.yml | 16 ++++++++++ filebeat/docs/fields.asciidoc | 32 ++++++++----------- filebeat/module/nginx/error/_meta/fields.yml | 31 +++++++++--------- .../module/nginx/error/ingest/pipeline.json | 8 ++--- .../nginx/error/test/error.log-expected.json | 20 ++++++------ filebeat/module/nginx/fields.go | 2 +- 7 files changed, 60 insertions(+), 50 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 8fcdaa6011df..cc59e824b068 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -29,6 +29,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Rename many `traefik.access.*` fields to map to ECS. {pull}9005[9005] - Fix parsing of GC entries in elasticsearch server log. {issue}9513[9513] {pull}9810[9810] - Rename a few `logstash.*` fields to map to ECS, remove logstash.slowlog.message. {pull}9935[9935] +- Rename a few `nginx.error.*` fields to map to ECS. {pull}10007[10007] *Heartbeat* diff --git a/dev-tools/ecs-migration.yml b/dev-tools/ecs-migration.yml index 2222fcac62fa..72636ce29978 100644 --- a/dev-tools/ecs-migration.yml +++ b/dev-tools/ecs-migration.yml @@ -514,6 +514,22 @@ to: user_agent.original alias: true +- from: nginx.error.level + to: log.level + alias: true + +- from: nginx.error.pid + to: process.pid + alias: true + +- from: nginx.error.tid + to: process.thread.id + alias: true + +- from: nginx.error.message + to: message + alias: true + ## PostgreSQL module - from: postgresql.log.timezone diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index 719c52438f95..6d3685cd048f 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -9938,53 +9938,49 @@ Contains fields for the Nginx error logs. -*`nginx.error.level`*:: +*`nginx.error.connection_id`*:: + -- -type: keyword +type: long -Error level (e.g. error, critical). +Connection identifier. -- -*`nginx.error.pid`*:: +*`nginx.error.level`*:: + -- -type: long - -Process identifier (PID). +type: alias +alias to: log.level -- -*`nginx.error.tid`*:: +*`nginx.error.pid`*:: + -- -type: long - -Thread identifier. +type: alias +alias to: process.pid -- -*`nginx.error.connection_id`*:: +*`nginx.error.tid`*:: + -- -type: long - -Connection identifier. +type: alias +alias to: process.thread.id -- *`nginx.error.message`*:: + -- -type: text - -The error message +type: alias +alias to: message -- diff --git a/filebeat/module/nginx/error/_meta/fields.yml b/filebeat/module/nginx/error/_meta/fields.yml index bc908c10a08e..b61ec5dfb800 100644 --- a/filebeat/module/nginx/error/_meta/fields.yml +++ b/filebeat/module/nginx/error/_meta/fields.yml @@ -3,23 +3,24 @@ description: > Contains fields for the Nginx error logs. fields: - - name: level - type: keyword - description: > - Error level (e.g. error, critical). - - name: pid - type: long - description: > - Process identifier (PID). - - name: tid - type: long - description: > - Thread identifier. - name: connection_id type: long description: > Connection identifier. + + - name: level + type: alias + path: log.level + migration: true + - name: pid + type: alias + path: process.pid + migration: true + - name: tid + type: alias + path: process.thread.id + migration: true - name: message - type: text - description: > - The error message + type: alias + path: message + migration: true diff --git a/filebeat/module/nginx/error/ingest/pipeline.json b/filebeat/module/nginx/error/ingest/pipeline.json index f8231e046077..dd1dbc6098ff 100644 --- a/filebeat/module/nginx/error/ingest/pipeline.json +++ b/filebeat/module/nginx/error/ingest/pipeline.json @@ -4,18 +4,14 @@ "grok": { "field": "message", "patterns": [ - "%{DATA:nginx.error.time} \\[%{DATA:nginx.error.level}\\] %{NUMBER:nginx.error.pid}#%{NUMBER:nginx.error.tid}: (\\*%{NUMBER:nginx.error.connection_id} )?%{GREEDYDATA:nginx.error.message}" + "%{DATA:nginx.error.time} \\[%{DATA:log.level}\\] %{NUMBER:process.pid:long}#%{NUMBER:process.thread.id:long}: (\\*%{NUMBER:nginx.error.connection_id:long} )?%{GREEDYDATA:message}" ], "ignore_missing": true } - },{ - "remove":{ - "field": "message" - } }, { "rename": { "field": "@timestamp", - "target_field": "read_timestamp" + "target_field": "event.created" } }, { "date": { diff --git a/filebeat/module/nginx/error/test/error.log-expected.json b/filebeat/module/nginx/error/test/error.log-expected.json index bc04d11c8385..6ee80bcd578d 100644 --- a/filebeat/module/nginx/error/test/error.log-expected.json +++ b/filebeat/module/nginx/error/test/error.log-expected.json @@ -6,12 +6,12 @@ "event.module": "nginx", "fileset.name": "error", "input.type": "log", + "log.level": "error", "log.offset": 0, - "nginx.error.connection_id": "1", - "nginx.error.level": "error", - "nginx.error.message": "open() \"/usr/local/Cellar/nginx/1.10.2_1/html/favicon.ico\" failed (2: No such file or directory), client: 127.0.0.1, server: localhost, request: \"GET /favicon.ico HTTP/1.1\", host: \"localhost:8080\", referrer: \"http://localhost:8080/\"", - "nginx.error.pid": "54053", - "nginx.error.tid": "0" + "message": "open() \"/usr/local/Cellar/nginx/1.10.2_1/html/favicon.ico\" failed (2: No such file or directory), client: 127.0.0.1, server: localhost, request: \"GET /favicon.ico HTTP/1.1\", host: \"localhost:8080\", referrer: \"http://localhost:8080/\"", + "nginx.error.connection_id": 1, + "process.pid": 54053, + "process.thread.id": 0 }, { "@timestamp": "2016-10-25T14:50:44.000Z", @@ -20,11 +20,11 @@ "event.module": "nginx", "fileset.name": "error", "input.type": "log", + "log.level": "error", "log.offset": 273, - "nginx.error.connection_id": "3", - "nginx.error.level": "error", - "nginx.error.message": "open() \"/usr/local/Cellar/nginx/1.10.2_1/html/adsasd\" failed (2: No such file or directory), client: 127.0.0.1, server: localhost, request: \"GET /adsasd HTTP/1.1\", host: \"localhost:8080\"", - "nginx.error.pid": "54053", - "nginx.error.tid": "0" + "message": "open() \"/usr/local/Cellar/nginx/1.10.2_1/html/adsasd\" failed (2: No such file or directory), client: 127.0.0.1, server: localhost, request: \"GET /adsasd HTTP/1.1\", host: \"localhost:8080\"", + "nginx.error.connection_id": 3, + "process.pid": 54053, + "process.thread.id": 0 } ] \ No newline at end of file diff --git a/filebeat/module/nginx/fields.go b/filebeat/module/nginx/fields.go index ec1b8a22f757..3719f90c166d 100644 --- a/filebeat/module/nginx/fields.go +++ b/filebeat/module/nginx/fields.go @@ -31,5 +31,5 @@ func init() { // Asset returns asset data func Asset() string { - return "eJysmMFymzAQhu9+ip2c0pmGB+DQS9rO9NBODrkzCiygRmjpanHit+8InMRhJCxsdMIY/f+32kWw3MEzHnKwjbavOwDRYjCHmz/+980OoEJXsu5Fk83h2w4A4DdVg0GoiaFX7LRtQFqEcQoYaqDWBl22A3AtsRQl2Vo3OQgPuAOoNZrK5aPUHVjV4Ye9H3LoMYeGaeiPZwIMfvwchaBm6mIAfpz6nXqqskTn3k+HjBfM/bgnK0pbd7QYV+QDZNL3PO8oIZxTpCeqDoVDK9nTQdB9uuaNz5BtZn/UxJ2SHEKTFvD9eGwR7NA9IQPVk4A/8FE45D0yMLqerMMR7iOQID9jR4KF7gujnQTxldFqztgraXOwKC/Ez1lN/KK4wqrQ/ezCTjesplCOxbTAsNLe0cAlZhd5Dg658IcrPf28LDAvxbNDaalaadiK9BnjvwGdZEGFpHDZrA2UTUasG23VfGqKoccu9shOk70k4vDUtGqaqr8oqVqb3eNiTwKZEyWDC+mkcdTIjHxNviMaKfaqQbv2fh5vinHi2tTH77E4x3zbhshWeypZ4V6X82wshxYML6qztLSnIJ36S/O0XMARk0nG0HYTjIhMKkavpGyvx4jJpGIE9uULKCIqqRAUMlmJQC6rB2NCD6h1KMVGdUru6lL1MNtUq4e5smDJxdZ2Pcu1SQrvtJewxJVW7dsNUuRd7JItuyQr2qKVa1b8+KrXIGVn9VIXvqTBCh8K7Sj0jL8I7axiKpyhcrzseqgFpVQYxkaT3Sh/y2LJydNy2KqgFqRWrtB2pXReMIb2hoTMn7bIjXvlUX5Vq2xwj+Eu5BkPL8TzvuZMH/xjAvCicItZk01IX6FkLbpU5ksWxOh1uAULdOlnCB6Yxu8FukIrutbIcPvw63vEVzbzfWwZVXViGzYsyVosZSyjrazv3zXP2nfonGrCHZjg67wnSPjqMZXcm+7/AAAA///B08Vj" + return "eJysl8F2ozoMhvd5Cp3uywOwuJuec3f3rmbPcUGAp8ZiJJFO3n4OJG1TxiY2iVcpRf//yRIGPcMbnkrwnfW/DwBq1WEJT//Pfz8dABqUmu2olnwJ/xwAAP6jZnIILTGMhsX6DrRHWELAUQetdSjFAUB6Yq1q8q3tSlCe8ADQWnSNlIvUM3gz4Jf9vPQ0Ygkd0zRergQY5vXvIgQt0xADmNe137WnqWsU+bwcMt4wn9cLeTXWy8Vi2ZEvkLP+zPOJEsK5Rnql5lQJei1eT4ry7Z4PPke+W/2jJR6MlhAK2sCf148ewU/DKzJQexaYf8xZCPIRGRhlJC+4wH0lEuRnHEixsmPlrGgQ3zhr1oyj0b4Ej/pO/Fa0xO+GG2wqO65uHGzH5pzKpZk2GDLthSausdjlOQlyNf/M9JzjikBciueA2lOTadirjgXjrwlFi6BCUrrschNlVxDbznqzDk0xnLGrI7JY8nsyDoemddO5+6uamtzqXjb7LFCIGp0kpJPG0SIz8j31jmik2JsOfe7zvDwUS2Bu6ePPWJxjfWxD5Ki9lmzwaOt1NbZTC6YX1dna2muQwfykdVl2cMRkkjGsfwhGRCYVYzRa9/djxGRSMQLn8g6KiEoqBIVMMhFIinZyLvSCykOpHtSnJHe36gzzmG6dYe5sWJLY3uaz3Fuk8Em7hyWulHVud0iRb7E9R3ZNXq1Hr/fs+OVTr0MqbuqlbnxNk1c+VVYo9I7fhXZTMRXOUb3cdj/UhlIqDGNnyT+ofttiycWzenpUQ21IZe7Q41rptmAM7QMJmb8dkQ+elRf5rFG5Ju+x1iWt8BQUGJRvDMMvn5pgG/RqW4t8Y+R1eMTcachRV4TiUr7Ex0i2cbeRqUaR4u/IFD/d7ac9o2mKXa4Dipgud+AKR639/gQAAP//qeS/3Q==" }