diff --git a/dev-tools/ecs-migration.yml b/dev-tools/ecs-migration.yml index a748eb6e676..b64f2c70621 100644 --- a/dev-tools/ecs-migration.yml +++ b/dev-tools/ecs-migration.yml @@ -216,6 +216,10 @@ ## Apache module +- from: apache2.access.remote_ip + to: source.address + alias: true + - from: apache2.access.user_name to: user.name alias: true @@ -424,6 +428,10 @@ ## HAProxy module +- from: haproxy.client.ip + to: source.address + alias: true + - from: haproxy.client.port to: source.port alias: true diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index 4137809eb24..952a85c6e8a 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -63,7 +63,7 @@ Aliases for backward compatibility with old apache2 fields -- type: alias -alias to: apache.access.remote_ip +alias to: source.address -- @@ -363,16 +363,6 @@ Contains fields for the Apache HTTP Server access logs. -*`apache.access.remote_ip`*:: -+ --- -type: keyword - -Client IP address or hostname. - - --- - *`apache.access.ssl.protocol`*:: + -- @@ -4742,9 +4732,9 @@ Information about the client doing the request *`haproxy.client.ip`*:: + -- -IP address of the client which initiated the TCP connection to haproxy. -If connection is via unix socket, socket path is in this field. +type: alias +alias to: source.address -- diff --git a/filebeat/module/apache/_meta/fields.yml b/filebeat/module/apache/_meta/fields.yml index 2a4ba231b99..bf60c88f2f8 100644 --- a/filebeat/module/apache/_meta/fields.yml +++ b/filebeat/module/apache/_meta/fields.yml @@ -14,7 +14,7 @@ fields: - name: remote_ip type: alias - path: apache.access.remote_ip + path: source.address migration: true - name: ssl.protocol type: alias diff --git a/filebeat/module/apache/access/_meta/fields.yml b/filebeat/module/apache/access/_meta/fields.yml index 6489ce4c798..00d629cf853 100644 --- a/filebeat/module/apache/access/_meta/fields.yml +++ b/filebeat/module/apache/access/_meta/fields.yml @@ -3,11 +3,6 @@ description: > Contains fields for the Apache HTTP Server access logs. fields: - - name: remote_ip - type: keyword - description: > - Client IP address or hostname. - - name: ssl.protocol type: keyword description: > diff --git a/filebeat/module/apache/access/ingest/default.json b/filebeat/module/apache/access/ingest/default.json index 60beae8e989..ca4bc7c317f 100644 --- a/filebeat/module/apache/access/ingest/default.json +++ b/filebeat/module/apache/access/ingest/default.json @@ -4,9 +4,9 @@ "grok": { "field": "message", "patterns":[ - "%{IPORHOST:apache.access.remote_ip} - %{DATA:user.name} \\[%{HTTPDATE:apache.access.time}\\] \"%{WORD:http.request.method} %{DATA:url.original} HTTP/%{NUMBER:http.version}\" %{NUMBER:http.response.status_code:long} (?:%{NUMBER:apache.access.body_sent.bytes:long}|-)( \"%{DATA:http.request.referrer}\")?( \"%{DATA:apache.access.agent}\")?", - "%{IPORHOST:apache.access.remote_ip} - %{DATA:user.name} \\[%{HTTPDATE:apache.access.time}\\] \"-\" %{NUMBER:http.response.status_code:long} -", - "\\[%{HTTPDATE:apache.access.time}\\] %{IPORHOST:apache.access.remote_ip} %{DATA:apache.access.ssl.protocol} %{DATA:apache.access.ssl.cipher} \"%{WORD:http.request.method} %{DATA:url.original} HTTP/%{NUMBER:http.version}\" %{NUMBER:apache.access.body_sent.bytes}" + "%{IPORHOST:source.address} - %{DATA:user.name} \\[%{HTTPDATE:apache.access.time}\\] \"%{WORD:http.request.method} %{DATA:url.original} HTTP/%{NUMBER:http.version}\" %{NUMBER:http.response.status_code:long} (?:%{NUMBER:apache.access.body_sent.bytes:long}|-)( \"%{DATA:http.request.referrer}\")?( \"%{DATA:apache.access.agent}\")?", + "%{IPORHOST:source.address} - %{DATA:user.name} \\[%{HTTPDATE:apache.access.time}\\] \"-\" %{NUMBER:http.response.status_code:long} -", + "\\[%{HTTPDATE:apache.access.time}\\] %{IPORHOST:source.address} %{DATA:apache.access.ssl.protocol} %{DATA:apache.access.ssl.cipher} \"%{WORD:http.request.method} %{DATA:url.original} HTTP/%{NUMBER:http.version}\" %{NUMBER:apache.access.body_sent.bytes}" ], "ignore_missing": true } @@ -16,7 +16,7 @@ } }, { "grok": { - "field": "apache.access.remote_ip", + "field": "source.address", "ignore_missing": true, "patterns": [ "^(%{IP:source.ip}|%{HOSTNAME:source.domain})$" diff --git a/filebeat/module/apache/access/test/ssl-request.log-expected.json b/filebeat/module/apache/access/test/ssl-request.log-expected.json index 07bb38b970a..c99db08e596 100644 --- a/filebeat/module/apache/access/test/ssl-request.log-expected.json +++ b/filebeat/module/apache/access/test/ssl-request.log-expected.json @@ -2,7 +2,6 @@ { "@timestamp": "2018-08-10T07:45:56.000Z", "apache.access.body_sent.bytes": "1375", - "apache.access.remote_ip": "172.30.0.119", "apache.access.ssl.cipher": "ECDHE-RSA-AES128-GCM-SHA256", "apache.access.ssl.protocol": "TLSv1.2", "ecs.version": "1.0.0-beta2", @@ -14,6 +13,7 @@ "input.type": "log", "log.offset": 0, "service.type": "apache", + "source.address": "172.30.0.119", "source.ip": "172.30.0.119", "url.original": "/nagiosxi/ajaxhelper.php?cmd=getxicoreajax&opts=%7B%22func%22%3A%22get_admin_tasks_html%22%2C%22args%22%3A%22%22%7D&nsp=b5c7d5d4b6f7d0cf0c92f9cbdf737f6a5c838218425e6ae21" } diff --git a/filebeat/module/apache/access/test/test.log-expected.json b/filebeat/module/apache/access/test/test.log-expected.json index 71afc41cec5..215a1aad978 100644 --- a/filebeat/module/apache/access/test/test.log-expected.json +++ b/filebeat/module/apache/access/test/test.log-expected.json @@ -2,7 +2,6 @@ { "@timestamp": "2016-12-26T14:16:29.000Z", "apache.access.body_sent.bytes": 209, - "apache.access.remote_ip": "::1", "ecs.version": "1.0.0-beta2", "event.dataset": "apache.access", "event.module": "apache", @@ -13,6 +12,7 @@ "input.type": "log", "log.offset": 0, "service.type": "apache", + "source.address": "::1", "source.ip": "::1", "url.original": "/favicon.ico", "user.name": "-" @@ -20,7 +20,6 @@ { "@timestamp": "2016-12-26T16:22:13.000Z", "apache.access.body_sent.bytes": 499, - "apache.access.remote_ip": "192.168.33.1", "ecs.version": "1.0.0-beta2", "event.dataset": "apache.access", "event.module": "apache", @@ -32,6 +31,7 @@ "input.type": "log", "log.offset": 73, "service.type": "apache", + "source.address": "192.168.33.1", "source.ip": "192.168.33.1", "url.original": "/hello", "user.name": "-", @@ -47,7 +47,6 @@ }, { "@timestamp": "2016-12-26T14:16:48.000Z", - "apache.access.remote_ip": "::1", "ecs.version": "1.0.0-beta2", "event.dataset": "apache.access", "event.module": "apache", @@ -56,13 +55,13 @@ "input.type": "log", "log.offset": 238, "service.type": "apache", + "source.address": "::1", "source.ip": "::1", "user.name": "-" }, { "@timestamp": "2017-05-29T19:02:48.000Z", "apache.access.body_sent.bytes": 612, - "apache.access.remote_ip": "172.17.0.1", "ecs.version": "1.0.0-beta2", "event.dataset": "apache.access", "event.module": "apache", @@ -74,6 +73,7 @@ "input.type": "log", "log.offset": 285, "service.type": "apache", + "source.address": "172.17.0.1", "source.ip": "172.17.0.1", "url.original": "/stringpatch", "user.name": "-", @@ -89,7 +89,6 @@ { "@timestamp": "2017-05-29T19:02:48.000Z", "apache.access.body_sent.bytes": 612, - "apache.access.remote_ip": "monitoring-server", "ecs.version": "1.0.0-beta2", "event.dataset": "apache.access", "event.module": "apache", @@ -101,6 +100,7 @@ "input.type": "log", "log.offset": 443, "service.type": "apache", + "source.address": "monitoring-server", "source.domain": "monitoring-server", "url.original": "/status", "user.name": "-", diff --git a/filebeat/module/apache/fields.go b/filebeat/module/apache/fields.go index 4eb3cdf339d..346716117fe 100644 --- a/filebeat/module/apache/fields.go +++ b/filebeat/module/apache/fields.go @@ -31,5 +31,5 @@ func init() { // Asset returns asset data func Asset() string { - return "eJysmM+O4zYMxu95CmLvq0OPORQoChQt0AILzNwNRWZsdWTRpegZ5O0L/5vNJLIj2ZlToAy/7yeSkS1+hze8HEG32tR4ABArDo/w7bdh4dsBoMRg2LZiyR/h1wMAwPgl/ENl5/qgUBNLYcifbXUE4a5fPFt0ZTgOAd/B6wZnm1+GNQC5tHiEiqlrp5WI1+DnrA4Y4EwMJ23ePjSXYKhptdiTdVYu8GGlBnLlbDHZTxLXLF94jMEQPpdjSLHwawnGhgQL2375dlbSPfnNN62Wek6FGgnUkkpjK9ZjOqa03hOE4FTLJGTI7YJYEUrlMLatkXdTRGVSGE5UXoqAXtTpInhrlweyrpVC0wXkov+YydHHqUhcimeDUlOZaViLtIrxvw6DqKhC0nY5t/86dorYVtbrLR3XYxfvyMGS37LjeGiKM2NoyQcsDJW51Z2SPQqoIFq6ENNJ4zgjc/Yv7ku9FzRS7HWFXjY0dzEEppZ++be17H97hMPCMX4tWeK7NbdVWN9SdFuLOmspvQZp9L90W44NHEsyyRjWPwVjQSYVo9Vi6v0YSzKpGJHzeAPFgkoqBMVMMhEoqHPnXOzBlIdSPKlPKexu1R7mOd3aw+xsWApLuc1n2Vuk+Am7hWVZKeVJUSEtvCJvOakNebEevexJdKCODaoKST3US823oc4LXwobKPZI34T2UDEVzpEZ/m0/1IpSKgxjZck/qX7rYsnFs3J5VkOtSGVm6Hmt9FhwCW1GQuYvJ2P+ddnhO+beEhxVKhaXdh0KQVe5r+jxqBS/1ubevVqm4bp5H5niJ5v9pGbUpdrk2syTnxzj6XY99JCKKty9+B/gbnh0uDZLmR2Ns6qxJ9W0vHMetGDV//1OXrT1YbIYRlZS44zx5+vrD3hBfkeezPru/uSKsUHysOkNLx/Et/VcgR2AnUUv8NcP0GXJPRAx1BSkd/zJFcV5OHnaRPTy8jfMqjBd0RNAVkZPmzFGTVhIRRQlZQLlyFe3bzzEjZYjxIIegL7WCL5rTshA51Gg/9B3XRg7bR42DHCRjeQd7ys4f0xNz9Rcd/2gm9XoKwfMpmL2ORo1+9YqO2N9NRA6qios5wNfHf4PAAD//0mO0CA=" + return "eJysmE2PpDYQhu/9K0p7Xx9y5BApihTlkEiRZu7IbarBGeMiZTMj/n1kPnb7w9A2NKeW6Xrfx1WFwf4OHzgUIDupGjwBeO0NFvDtt3Hg2wmgQqdYd16TLeDXEwDAdBP+pqo3Icg1xL5UZC+6LsBzHwYvGk3lijHgO1jZ4mLzyzgG4IcOC6iZ+m4eiXiNfkZLhw4uxHCW6uNLcgWK2k56fdZG+wG+tG+ATLVYzPazxDXLDY9S6NyP4RhSLPxagrElj6Xubu4uSjKQ393ppG8KcNSzQiGrim8RwtXqmuWUhTmbj8bOGdExeVJkMr2nHIlp9mJDKJVD6a5BPkwRlUlhOFM1lA6tF+fB471dHsi2VgpN75DL8DOTI8SJSFyKZ4u+oSrTsPG+E4z/9ei8iCokTZdz+69nI4h1ra3c03EBu/xEdprsnhnHQ1OcGV1H1mGpqMqt7pzsSUA4L33vYjppHBdkzn7ibuq9opFiL2u0fkdzl2NgaunXn611//uVG1ZW72vJCj+1uq/C9pSi01rV2UrpNUgr/6X7cuzgWJNJxtD2JRgrMqkYnfSqOY6xJpOKEVmPd1CsqKRCUMwkE4GcuPTGxF5MeSjli/qU3OFWDTCv6dYAc7Bhya3lNp/laJHiK+welnWllDdFjbTyZbxnpVZkvbZo/ZFEzx/eNZJ4qpeab0W99TyU2lHslb4L7aliKpwhNf7tONSGUioMY63Jvqh+22LJxdN+eFVDbUhlZuh1rfRccA1tQULmm5Uxf5ds8BNzdwmGahGLS9sOOSfr3E/0eFSKX6dz914d07jdfIxM8fO7/XzDKCuxy7VdDnxyjOfd9dhDIqrw8OF/goczo9O1WcqR0XRENfWkmIcPHgOtWIXrd7Jeautmi/Gkyje4YPz5/v4PvCF/Is9mobt/cMXYIOeo5wOHL+L7km7whuvt7S9YVGHeE/8kWgXZOOvZjTFpjiYRgihKypGPIVvff2IQt9IXEAt6AvreINi+PSMDXSaB8COU2U2lXXb3I1xkInnr6QbOH3OXMbXXbTbqZnXWxhO9q5ghR5NmaK2qV9rWI6GhusZqWWHF6f8AAAD//9lEqWo=" } diff --git a/filebeat/module/haproxy/_meta/fields.yml b/filebeat/module/haproxy/_meta/fields.yml index 23e4a2177d4..674d20f29be 100644 --- a/filebeat/module/haproxy/_meta/fields.yml +++ b/filebeat/module/haproxy/_meta/fields.yml @@ -92,10 +92,9 @@ type: group fields: - name: ip - description: > - IP address of the client which initiated the TCP connection to haproxy. - - If connection is via unix socket, socket path is in this field. + type: alias + path: source.address + migration: true - name: port type: alias path: source.port diff --git a/filebeat/module/haproxy/fields.go b/filebeat/module/haproxy/fields.go index ae8d6ff1209..1797a39e9d9 100644 --- a/filebeat/module/haproxy/fields.go +++ b/filebeat/module/haproxy/fields.go @@ -31,5 +31,5 @@ func init() { // Asset returns asset data func Asset() string { - return "eJzMWd1u20oOvs9TELnZFsjxAwTYAyyyZ7cFdttc5N6gNbRFZDSjzlBO/PYH8yNZkiXbSlK0uQksDcmP/+ToD3imwz2UWDv7ergBEBZN93Cbn9zeACjyheNa2Jp7+PMGANrz8H+rGk03AFsmrfx9fPkHGKyozzT8yaGme9g529T5yQTfI6P888ht66wRMmodfnZvR1y+YUVgtyAldQTwyTrQ7IUMuc/wUnJRgqOCeE8K0CionS3Ie1KRrrDGUBH4rU5RbLB4XgIin5/E8IIePGkqJEi2UKHBHY0whBfhiSe3JzeBKL24GpBGL5kmsE5IRiITMCMT0sQK6vULsrDZrYUrWld+Tu5TOAzhELCBirVmT4U1yoOvyQhkPuFtgLBHx7bx8KOhhvpcU+xoa3aniI64I6wPw7S1bsIV5AU3mn3Z+mXLBnU26JWINwchv3aE6jxI01QbcsFxkQLEofEVSw6XCE5zxFxSsqC2O2APlE6trgQUTRaN/nM8uQhHm2HZ7r/Cj3fAptCNCuSOxPH1KuRsvMKYR+86+tGQF9+WBXLUK0kb2loXqgJ7sIZaC+cUjoKuBdea9uejy5L+4WGn7Qb1Qpy8oMKmqhpchUo58n5c4S/WdHLOunVF3uNuVuRf4RDkQyHPdkH7A3z512PshGygQB9RRX6nugq9ykTA2MYVNH14zkMldWITeWcLO5VV5Co2GCukF5RZFR+sUZwSJAaY920zYHMsMe1zMipUmBNxlVXL1AkEICXKsGuHgKrJYUziT08Pj2AdfHl6evx8rgfM1v0HawTZ+K40FbYx4oPletSAhfC+C+Mc56e+7I8xAMPxZwwusRy8upByhTVF41yoYn1sdgDq1CPBUykwVyNpU8kGE7PVx4Hs3PiBKHNR+QCQJRqlU/72h7QPxHoyELwVqhfWug1LO6j9Hwg3d7lzeL/1kbadNNMBvdbkmEzRGpX9EVWE6Q4hj8W21OO59jLeLtnjzDOX51/N1roqVjvAjW2kPycpG0GU1Ha1NyR2C4Prm1lb/TlS5+tj15xype4Gt9Cq2LAwSu5VodQNh5W8SK1uxmwHvmAPe0ZoDL+Ct8UzyV3+DzVKGd7HssY+abWaUKq2Tm5OHYGacRgcgWPbvFYnVBXvHCZTiGvo1Ie5ho2b/LS0JKutxSOSy6JYLZQwpLgoQJGX3F/ngvLfxyPAxwB9R/QtdFQP42VvnUobxfr1skaEF225IzsgmTbLmVzr2vx/yX597BsbdigluVCgMEyuuZamRFxxfZoUAN+NPkDtKCzDwClzE+O/wh7NhSd0RQm1bnYcMxD3yBo3mgBHnarxw7J8bmQorBE2ZGScIOesP0rKHdnVWT7zbu8DaYy4w5q9XRfDwW4xlLOcrgGjbTFOm4UgZjhcI9zRLgzQ7/PHPJOrnMFyeG9AzLBYYIH3h8J5RnNQultFkdla8Kgp7GCoVP/54jI7UQEd+doaT/M9/z+RCTjS2LugCQvLNPE0lAt1AWtpHKl1Ye0zn90pxiMIwPf4BjXcBmb/3KNu6BYoZCWwUVykVavbxPKIUqJKQ0OS2a5GrU6rKatNIC4JFbmzE+Yp5P+xlzAxZeKO2xgEqIZai6dyfdyJbzPR8XDidgtxGa5iaR8uLdPT6GCHnR76BaXxb8mOENSrzqTzfOaSYzJixxPukoA9pf3t4zVvRSUGnaRxhhRgG7UvHCZg8a1qvzhsI4Yrozad/WlB6/BlnYWsNZuzTnqwVa1JhlECgap/ZRpQVSSlVXfdGTQqEe3Jxa3Qi2OzeyvoeFOcluLhXf8J5CeuaGB0RxVyjI3uqifvV3ftOuYjgczfMJPGOt1+ygvlJXzLzku8qW992Ep8iRHZ+97UfYdpT3el6fjlZcoucR++bJf4GSQEvG1krVDwkomWXqUfPx95MiHJto3Ww1Z3B8ZKGh8DZUDxfp0+UI0h5uSnrbNVv/N9GqiwserwGXAr5Mb+7jt4iZadhsXsRPP08Bg/6qQ15j0LY1pup7/aTcJcYuCphMCioFqGAV9o66cWzNFHvN8C5d8BAAD//xXwP9M=" + return "eJzMWdtu2zwSvs9TDHKzLZD6AQJsgUW2uy3w/20ucm/Q5FgalCJVcpTUb/+DB8k62lbiovWdRc7MxzkP+QG+4+EeSlE7+/NwA8DEGu/hNn+5vQFQ6KWjmsmae/h4AwDtfvjbqkbjDcCeUCt/Hxc/gBEV9pmGHx9qvIfC2abOX2b4Hhnlv0due2cNo1Hb8LdbHXH5KioEuwcusSOAd9aBJs9o0L2Hl5JkCQ4l0jMqEEZB7axE71FFOmmNQRn4baYodkJ+XwMi75/F8CI8eNQoOUi2UAkjChxhCAvhi0f3jG4GUVq4GJAWnjNNYJ2QjEQmYIZnpLFlobcvgphMsWWqcFv5JblPYTOETUAGKtKaPEprlAdfo2HIfMJqgPAsHNnGw48GG+xzTb6jrSmmiI64I6yrYdpbN2MK9Cx2mnzZ2mVPRuis0AsR7w6MfutQqNMgTVPt0AXDRQpgJ4yviLO7RHCaIuYSkwa1LYA8YNq1uRBQVFlU+q+x5CocbYRlvf8OO94BGakbFcgdsqPLj5Cj8QJlHq3r8EeDnn2bFtBhLyXtcG9dyArkwRpsNZxDOAq6FFyr2l+PLkv6l4dC253QK3HSigybsmowlVDKoffjDH82p6Nz1m0r9F4UiyI/hU2QN4U4K8LpD/D5P4+xEpIBKXxEFflNz8r4k2ccxjZO4vzmJQuV2IlN5J0u7FxUoavIiJghPQtePOKDNYpSgEQH874tBmSOKab9jkaFDDMRV1m17jiBALgUPKzawaFqdCIG8bunh0ewDj4/PT2+P1UDFvP+gzUsyPguNUnbGPZBcz1qEJLpuXPj7OdTW/bbGIBh+zMGl1gOls6EnLRGNs6FLNbHZgegphYJlkqOuRlJmws2mOmtrgeyM+MVUeakcgWQpTBKp/jtN2lXxDppCF4L1TNp3bqlHeT+K8LNVe4U3q99pG0lzXSAP2t0hEa2SiV/RBVhukOIY7Yt9bivPY+3C/bY8yzF+Rezt66K2Q7Ezjbc75OUjSBKbKvaKwK7hUH1zRSx0CSGWqwFl22W3+QKNdhQUeFEws5uUJZbSbV1vF7WhGosaKLYnFjGlXdeWpLVJsgRyXlRpFZKGFKcFaDQcy56S57y3+MWoKPXvMElVhqqh/G8tabSVjlgX9aI8KwuC7QDknm1zE7z6dfV3v+j/fLYVzYUgkt0IWuI0E7mBJfCdUN1UvYwO3wz+gC1wzChAqXGJzH+FIZbkh6FkyXUuinIhD5CPAvSYqcxTPsDXo0f5spTdVxaw2TQ8DhATml/FJQF2s1JPstm7wNpDLvDlrzdymG3tRrKSU6XgNFWjsNmJYgFDpcId1iErvZt9lhmcpExiA9vdYgFFis08HZXOM1oCUp31ce8mAseNYbBSCjV/746zc5kQIe+tsYPwQ5k/y8yAYda9G5NwhQxTzwP5UxeEDU3DtVWWvudTjb6H0eLAN/iitBwG5j9+1noBm8BQ1QCGUUyzT/deJQbmVKo1GQlme280p5pM6e1GcQlCoXuZNs3hfwXeQ5tYCbuuI1BgGqw1XhK18dB9TYTHTcnbrcQJ9QqpvbhJDHfIg4Gy/lOnAU3/jXREZx606l0mc9ScMx67LjtXOOwU9o/3l/zqFKKcCZunEEFovXaF+ISiH17tN/sthHDhV6b9v4yp3XiZZuFbDWZk0Z6sFWtkYdeAoGqf48ZUFXIpVV33R5hVCJ6RhdHNc+OTPFa0PH6Nk2qwwv4CeQnqnCgdIeVoOgb3f1Lfj26y9d55CMBL1/7ohZ1upLkF8yT8Z6c53h93tqwlfgSPbL3CNQ9jrS7u9R0fA6Z00scUs/rJb5NBIe3DW+VYHFORWvvt49vOh5NCLJ9o/Ww1N2BsZzax0AZULz9TFc8xhBzstPe2apf+d4NjrCz6vAexJ7Rje3dN/CaU3YnlIsdzdPDY3xpSWPMWwbGNNzOP6XNwlyj4LmAEFJizUOHl9r6uQFz9LL2R6D8JwAA///iRBtm" } diff --git a/filebeat/module/haproxy/log/ingest/pipeline.json b/filebeat/module/haproxy/log/ingest/pipeline.json index d8027be9a44..08f4315c88e 100644 --- a/filebeat/module/haproxy/log/ingest/pipeline.json +++ b/filebeat/module/haproxy/log/ingest/pipeline.json @@ -5,13 +5,13 @@ "grok": { "field": "message", "patterns": [ - "%{HAPROXY_DATE:haproxy.request_date} %{IPORHOST:haproxy.source} %{PROG:process.name}(?:\\[%{POSINT:process.pid:long}\\])?: %{GREEDYDATA} %{IPORHOST:haproxy.client.ip}:%{POSINT:source.port:long} %{WORD} %{IPORHOST:destination.ip}:%{POSINT:destination.port:long} \\(%{WORD:haproxy.frontend_name}/%{WORD:haproxy.mode}\\)", + "%{HAPROXY_DATE:haproxy.request_date} %{IPORHOST:haproxy.source} %{PROG:process.name}(?:\\[%{POSINT:process.pid:long}\\])?: %{GREEDYDATA} %{IPORHOST:source.address}:%{POSINT:source.port:long} %{WORD} %{IPORHOST:destination.ip}:%{POSINT:destination.port:long} \\(%{WORD:haproxy.frontend_name}/%{WORD:haproxy.mode}\\)", - "(%{NOTSPACE:process.name}\\[%{NUMBER:process.pid:long}\\]: )?%{IP:haproxy.client.ip}:%{NUMBER:source.port:long} \\[%{NOTSPACE:haproxy.request_date}\\] %{NOTSPACE:haproxy.frontend_name} %{NOTSPACE:haproxy.backend_name}/%{NOTSPACE:haproxy.server_name} %{NUMBER:haproxy.http.request.time_wait_ms:long}/%{NUMBER:haproxy.total_waiting_time_ms:long}/%{NUMBER:haproxy.connection_wait_time_ms:long}/%{NUMBER:haproxy.http.request.time_wait_without_data_ms:long}/%{NUMBER:haproxy.http.request.time_active_ms:long} %{NUMBER:http.response.status_code:long} %{NUMBER:haproxy.bytes_read:long} %{NOTSPACE:haproxy.http.request.captured_cookie} %{NOTSPACE:haproxy.http.response.captured_cookie} %{NOTSPACE:haproxy.termination_state} %{NUMBER:haproxy.connections.active:long}/%{NUMBER:haproxy.connections.frontend:long}/%{NUMBER:haproxy.connections.backend:long}/%{NUMBER:haproxy.connections.server:long}/%{NUMBER:haproxy.connections.retries:long} %{NUMBER:haproxy.server_queue:long}/%{NUMBER:haproxy.backend_queue:long} (\\{%{DATA:haproxy.http.request.captured_headers}\\} \\{%{DATA:haproxy.http.response.captured_headers}\\} |\\{%{DATA}\\} )?\"%{GREEDYDATA:haproxy.http.request.raw_request_line}\"", + "(%{NOTSPACE:process.name}\\[%{NUMBER:process.pid:long}\\]: )?%{IP:source.address}:%{NUMBER:source.port:long} \\[%{NOTSPACE:haproxy.request_date}\\] %{NOTSPACE:haproxy.frontend_name} %{NOTSPACE:haproxy.backend_name}/%{NOTSPACE:haproxy.server_name} %{NUMBER:haproxy.http.request.time_wait_ms:long}/%{NUMBER:haproxy.total_waiting_time_ms:long}/%{NUMBER:haproxy.connection_wait_time_ms:long}/%{NUMBER:haproxy.http.request.time_wait_without_data_ms:long}/%{NUMBER:haproxy.http.request.time_active_ms:long} %{NUMBER:http.response.status_code:long} %{NUMBER:haproxy.bytes_read:long} %{NOTSPACE:haproxy.http.request.captured_cookie} %{NOTSPACE:haproxy.http.response.captured_cookie} %{NOTSPACE:haproxy.termination_state} %{NUMBER:haproxy.connections.active:long}/%{NUMBER:haproxy.connections.frontend:long}/%{NUMBER:haproxy.connections.backend:long}/%{NUMBER:haproxy.connections.server:long}/%{NUMBER:haproxy.connections.retries:long} %{NUMBER:haproxy.server_queue:long}/%{NUMBER:haproxy.backend_queue:long} (\\{%{DATA:haproxy.http.request.captured_headers}\\} \\{%{DATA:haproxy.http.response.captured_headers}\\} |\\{%{DATA}\\} )?\"%{GREEDYDATA:haproxy.http.request.raw_request_line}\"", - "(%{NOTSPACE:process.name}\\[%{NUMBER:process.pid:long}\\]: )?%{IP:haproxy.client.ip}:%{NUMBER:source.port:long} \\[%{NOTSPACE:haproxy.request_date}\\] %{NOTSPACE:haproxy.frontend_name}/%{NOTSPACE:haproxy.bind_name} %{GREEDYDATA:haproxy.error_message}", + "(%{NOTSPACE:process.name}\\[%{NUMBER:process.pid:long}\\]: )?%{IP:source.address}:%{NUMBER:source.port:long} \\[%{NOTSPACE:haproxy.request_date}\\] %{NOTSPACE:haproxy.frontend_name}/%{NOTSPACE:haproxy.bind_name} %{GREEDYDATA:haproxy.error_message}", - "%{HAPROXY_DATE} %{IPORHOST:haproxy.source} (%{NOTSPACE:process.name}\\[%{NUMBER:process.pid:long}\\]: )?%{IP:haproxy.client.ip}:%{NUMBER:source.port:long} \\[%{NOTSPACE:haproxy.request_date}\\] %{NOTSPACE:haproxy.frontend_name} %{NOTSPACE:haproxy.backend_name}/%{NOTSPACE:haproxy.server_name} %{NUMBER:haproxy.total_waiting_time_ms:long}/%{NUMBER:haproxy.connection_wait_time_ms:long}/%{NUMBER:haproxy.tcp.processing_time_ms:long} %{NUMBER:haproxy.bytes_read:long} %{NOTSPACE:haproxy.termination_state} %{NUMBER:haproxy.connections.active:long}/%{NUMBER:haproxy.connections.frontend:long}/%{NUMBER:haproxy.connections.backend:long}/%{NUMBER:haproxy.connections.server:long}/%{NUMBER:haproxy.connections.retries:long} %{NUMBER:haproxy.server_queue:long}/%{NUMBER:haproxy.backend_queue:long}" + "%{HAPROXY_DATE} %{IPORHOST:haproxy.source} (%{NOTSPACE:process.name}\\[%{NUMBER:process.pid:long}\\]: )?%{IP:source.address}:%{NUMBER:source.port:long} \\[%{NOTSPACE:haproxy.request_date}\\] %{NOTSPACE:haproxy.frontend_name} %{NOTSPACE:haproxy.backend_name}/%{NOTSPACE:haproxy.server_name} %{NUMBER:haproxy.total_waiting_time_ms:long}/%{NUMBER:haproxy.connection_wait_time_ms:long}/%{NUMBER:haproxy.tcp.processing_time_ms:long} %{NUMBER:haproxy.bytes_read:long} %{NOTSPACE:haproxy.termination_state} %{NUMBER:haproxy.connections.active:long}/%{NUMBER:haproxy.connections.frontend:long}/%{NUMBER:haproxy.connections.backend:long}/%{NUMBER:haproxy.connections.server:long}/%{NUMBER:haproxy.connections.retries:long} %{NUMBER:haproxy.server_queue:long}/%{NUMBER:haproxy.backend_queue:long}" ], "ignore_missing": false, "pattern_definitions": { @@ -41,10 +41,10 @@ }, { "grok": { - "field": "haproxy.client.ip", - "ignore_missing": true, + "field": "source.address", + "ignore_failure": true, "patterns": [ - "^(%{IP:source.ip}|%{HOSTNAME:source.domain})$" + "^%{IP:source.ip}$" ] } }, diff --git a/filebeat/module/haproxy/log/test/default.log-expected.json b/filebeat/module/haproxy/log/test/default.log-expected.json index 6b35d2a9b13..efedfabf69c 100644 --- a/filebeat/module/haproxy/log/test/default.log-expected.json +++ b/filebeat/module/haproxy/log/test/default.log-expected.json @@ -6,7 +6,6 @@ "event.dataset": "haproxy.log", "event.module": "haproxy", "fileset.name": "log", - "haproxy.client.ip": "1.2.3.4", "haproxy.frontend_name": "main", "haproxy.mode": "HTTP", "haproxy.source": "1.2.3.4", @@ -15,6 +14,7 @@ "process.name": "haproxy", "process.pid": 24551, "service.type": "haproxy", + "source.address": "1.2.3.4", "source.geo.continent_name": "North America", "source.geo.country_iso_code": "US", "source.geo.location.lat": 37.751, diff --git a/filebeat/module/haproxy/log/test/haproxy.log-expected.json b/filebeat/module/haproxy/log/test/haproxy.log-expected.json index 0eabd5720cb..b269eb0e09f 100644 --- a/filebeat/module/haproxy/log/test/haproxy.log-expected.json +++ b/filebeat/module/haproxy/log/test/haproxy.log-expected.json @@ -8,7 +8,6 @@ "haproxy.backend_name": "docs_microservice", "haproxy.backend_queue": 0, "haproxy.bytes_read": 168, - "haproxy.client.ip": "1.2.3.4", "haproxy.connection_wait_time_ms": 1, "haproxy.connections.active": 6, "haproxy.connections.backend": 0, @@ -37,6 +36,7 @@ "process.name": "haproxy", "process.pid": 32450, "service.type": "haproxy", + "source.address": "1.2.3.4", "source.geo.continent_name": "North America", "source.geo.country_iso_code": "US", "source.geo.location.lat": 37.751, diff --git a/filebeat/module/haproxy/log/test/httplog-no-headers.log-expected.json b/filebeat/module/haproxy/log/test/httplog-no-headers.log-expected.json index 9b3489a9b30..5f80d850cfc 100644 --- a/filebeat/module/haproxy/log/test/httplog-no-headers.log-expected.json +++ b/filebeat/module/haproxy/log/test/httplog-no-headers.log-expected.json @@ -8,7 +8,6 @@ "haproxy.backend_name": "http-webservices", "haproxy.backend_queue": 0, "haproxy.bytes_read": 213, - "haproxy.client.ip": "127.0.0.1", "haproxy.connection_wait_time_ms": -1, "haproxy.connections.active": 1, "haproxy.connections.backend": 0, @@ -33,6 +32,7 @@ "process.name": "haproxy", "process.pid": 19312, "service.type": "haproxy", + "source.address": "127.0.0.1", "source.ip": "127.0.0.1", "source.port": 35982 }, @@ -45,7 +45,6 @@ "haproxy.backend_name": "http-webservices", "haproxy.backend_queue": 0, "haproxy.bytes_read": 213, - "haproxy.client.ip": "127.0.0.1", "haproxy.connection_wait_time_ms": -1, "haproxy.connections.active": 1, "haproxy.connections.backend": 0, @@ -70,6 +69,7 @@ "process.name": "haproxy", "process.pid": 29785, "service.type": "haproxy", + "source.address": "127.0.0.1", "source.ip": "127.0.0.1", "source.port": 43738 }, @@ -82,7 +82,6 @@ "haproxy.backend_name": "http-webservices", "haproxy.backend_queue": 0, "haproxy.bytes_read": 213, - "haproxy.client.ip": "127.0.0.1", "haproxy.connection_wait_time_ms": -1, "haproxy.connections.active": 1, "haproxy.connections.backend": 0, @@ -111,6 +110,7 @@ "process.name": "haproxy", "process.pid": 7873, "service.type": "haproxy", + "source.address": "127.0.0.1", "source.ip": "127.0.0.1", "source.port": 44542 } diff --git a/filebeat/module/haproxy/log/test/tcplog.log-expected.json b/filebeat/module/haproxy/log/test/tcplog.log-expected.json index 3d4bbe1205d..d5ffb2e3f13 100644 --- a/filebeat/module/haproxy/log/test/tcplog.log-expected.json +++ b/filebeat/module/haproxy/log/test/tcplog.log-expected.json @@ -8,7 +8,6 @@ "haproxy.backend_name": "app", "haproxy.backend_queue": 0, "haproxy.bytes_read": 212, - "haproxy.client.ip": "127.0.0.1", "haproxy.connection_wait_time_ms": -1, "haproxy.connections.active": 1, "haproxy.connections.backend": 0, @@ -27,6 +26,7 @@ "process.name": "haproxy", "process.pid": 25457, "service.type": "haproxy", + "source.address": "127.0.0.1", "source.ip": "127.0.0.1", "source.port": 40962 }