From 8e29ab772dddb6df40ee9457729b1a621af0e200 Mon Sep 17 00:00:00 2001 From: Chris Mark Date: Fri, 15 May 2020 12:33:17 +0300 Subject: [PATCH] Fix source.address not being set for nginx ingress_controller (#18511) Signed-off-by: chrismark Co-authored-by: chendo --- CHANGELOG.next.asciidoc | 1 + .../ingress_controller/ingest/pipeline.yml | 2 +- .../test/test.log-expected.json | 132 +++++++++++++++--- 3 files changed, 112 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index ce7034d05c8c..12a1c6199694 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -140,6 +140,7 @@ field. You can revert this change by configuring tags for the module and omittin - Fix Cisco ASA ASA 3020** and 106023 messages {pull}17964[17964] - Unescape file name from SQS message. {pull}18370[18370] - Improve cisco asa and ftd pipelines' failure handler to avoid mapping temporary fields. {issue}18391[18391] {pull}18392[18392] +- Fix source.address not being set for nginx ingress_controller {pull}18511[18511] - Fix PANW module wrong mappings for bytes and packets counters. {issue}18522[18522] {pull}18525[18525] - Fixed ingestion of some Cisco ASA and FTD messages when a hostname was used instead of an IP for NAT fields. {issue}14034[14034] {pull}18376[18376] - Fix a rate limit related issue in httpjson input for Okta module. {issue}18530[18530] {pull}18534[18534] diff --git a/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml b/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml index 9721be136e31..e1a2aab119bc 100644 --- a/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml +++ b/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml @@ -39,7 +39,7 @@ processors: if: ctx.source?.address == null value: "" - script: - if: ctx.nginx?.access?.remote_ip_list != null && ctx.nginx.ingress_controller.remote_ip_list.length > 0 + if: ctx.nginx?.ingress_controller?.remote_ip_list != null && ctx.nginx.ingress_controller.remote_ip_list.length > 0 lang: painless source: >- boolean isPrivate(def dot, def ip) { diff --git a/filebeat/module/nginx/ingress_controller/test/test.log-expected.json b/filebeat/module/nginx/ingress_controller/test/test.log-expected.json index a2bf0f6c6e08..566713644159 100644 --- a/filebeat/module/nginx/ingress_controller/test/test.log-expected.json +++ b/filebeat/module/nginx/ingress_controller/test/test.log-expected.json @@ -32,8 +32,12 @@ "nginx.ingress_controller.upstream.response.length": 59, "nginx.ingress_controller.upstream.response.status_code": 200, "nginx.ingress_controller.upstream.response.time": 0.0, + "related.ip": [ + "192.168.64.1" + ], "service.type": "nginx", - "source.address": "", + "source.address": "192.168.64.1", + "source.ip": "192.168.64.1", "url.original": "/products", "user_agent.device.name": "Other", "user_agent.name": "curl", @@ -73,8 +77,12 @@ "nginx.ingress_controller.upstream.response.length": 59, "nginx.ingress_controller.upstream.response.status_code": 200, "nginx.ingress_controller.upstream.response.time": 0.0, + "related.ip": [ + "192.168.64.1" + ], "service.type": "nginx", - "source.address": "", + "source.address": "192.168.64.1", + "source.ip": "192.168.64.1", "url.original": "/products/42", "user_agent.device.name": "Other", "user_agent.name": "curl", @@ -114,8 +122,12 @@ "nginx.ingress_controller.upstream.response.length": 59, "nginx.ingress_controller.upstream.response.status_code": 200, "nginx.ingress_controller.upstream.response.time": 0.001, + "related.ip": [ + "192.168.64.1" + ], "service.type": "nginx", - "source.address": "", + "source.address": "192.168.64.1", + "source.ip": "192.168.64.1", "url.original": "/products/42", "user_agent.device.name": "Other", "user_agent.name": "curl", @@ -155,8 +167,12 @@ "nginx.ingress_controller.upstream.response.length": 59, "nginx.ingress_controller.upstream.response.status_code": 200, "nginx.ingress_controller.upstream.response.time": 0.0, + "related.ip": [ + "192.168.64.1" + ], "service.type": "nginx", - "source.address": "", + "source.address": "192.168.64.1", + "source.ip": "192.168.64.1", "url.original": "/products/42", "user_agent.device.name": "Other", "user_agent.name": "curl", @@ -191,8 +207,12 @@ ], "nginx.ingress_controller.upstream.alternative_name": "", "nginx.ingress_controller.upstream.name": "", + "related.ip": [ + "192.168.64.1" + ], "service.type": "nginx", - "source.address": "", + "source.address": "192.168.64.1", + "source.ip": "192.168.64.1", "url.original": "/products/42" }, { @@ -223,8 +243,12 @@ ], "nginx.ingress_controller.upstream.alternative_name": "", "nginx.ingress_controller.upstream.name": "", + "related.ip": [ + "192.168.64.1" + ], "service.type": "nginx", - "source.address": "", + "source.address": "192.168.64.1", + "source.ip": "192.168.64.1", "url.original": "/products/42" }, { @@ -260,8 +284,12 @@ "nginx.ingress_controller.upstream.response.length": 59, "nginx.ingress_controller.upstream.response.status_code": 200, "nginx.ingress_controller.upstream.response.time": 0.0, + "related.ip": [ + "192.168.64.1" + ], "service.type": "nginx", - "source.address": "", + "source.address": "192.168.64.1", + "source.ip": "192.168.64.1", "url.original": "/products/42", "user_agent.device.name": "Other", "user_agent.name": "Wget", @@ -301,8 +329,12 @@ "nginx.ingress_controller.upstream.response.length": 59, "nginx.ingress_controller.upstream.response.status_code": 200, "nginx.ingress_controller.upstream.response.time": 0.0, + "related.ip": [ + "192.168.64.1" + ], "service.type": "nginx", - "source.address": "", + "source.address": "192.168.64.1", + "source.ip": "192.168.64.1", "url.original": "/products/42", "user_agent.device.name": "Other", "user_agent.name": "Chrome", @@ -346,8 +378,12 @@ "nginx.ingress_controller.upstream.response.length": 59, "nginx.ingress_controller.upstream.response.status_code": 200, "nginx.ingress_controller.upstream.response.time": 0.0, + "related.ip": [ + "192.168.64.1" + ], "service.type": "nginx", - "source.address": "", + "source.address": "192.168.64.1", + "source.ip": "192.168.64.1", "url.original": "/favicon.ico", "user_agent.device.name": "Other", "user_agent.name": "Chrome", @@ -390,8 +426,12 @@ "nginx.ingress_controller.upstream.response.length": 61, "nginx.ingress_controller.upstream.response.status_code": 200, "nginx.ingress_controller.upstream.response.time": 0.001, + "related.ip": [ + "192.168.64.1" + ], "service.type": "nginx", - "source.address": "", + "source.address": "192.168.64.1", + "source.ip": "192.168.64.1", "url.original": "/v2", "user_agent.device.name": "Other", "user_agent.name": "Chrome", @@ -435,8 +475,12 @@ "nginx.ingress_controller.upstream.response.length": 59, "nginx.ingress_controller.upstream.response.status_code": 200, "nginx.ingress_controller.upstream.response.time": 0.002, + "related.ip": [ + "192.168.64.1" + ], "service.type": "nginx", - "source.address": "", + "source.address": "192.168.64.1", + "source.ip": "192.168.64.1", "url.original": "/favicon.ico", "user_agent.device.name": "Other", "user_agent.name": "Chrome", @@ -479,8 +523,12 @@ "nginx.ingress_controller.upstream.response.length": 59, "nginx.ingress_controller.upstream.response.status_code": 200, "nginx.ingress_controller.upstream.response.time": 0.001, + "related.ip": [ + "192.168.64.1" + ], "service.type": "nginx", - "source.address": "", + "source.address": "192.168.64.1", + "source.ip": "192.168.64.1", "url.original": "/products/42", "user_agent.device.name": "Other", "user_agent.name": "Safari", @@ -524,8 +572,12 @@ "nginx.ingress_controller.upstream.response.length": 59, "nginx.ingress_controller.upstream.response.status_code": 200, "nginx.ingress_controller.upstream.response.time": 0.001, + "related.ip": [ + "192.168.64.1" + ], "service.type": "nginx", - "source.address": "", + "source.address": "192.168.64.1", + "source.ip": "192.168.64.1", "url.original": "/favicon.ico", "user_agent.device.name": "Other", "user_agent.name": "Safari", @@ -568,8 +620,12 @@ "nginx.ingress_controller.upstream.response.length": 59, "nginx.ingress_controller.upstream.response.status_code": 200, "nginx.ingress_controller.upstream.response.time": 0.002, + "related.ip": [ + "192.168.64.1" + ], "service.type": "nginx", - "source.address": "", + "source.address": "192.168.64.1", + "source.ip": "192.168.64.1", "url.original": "/products/42", "user_agent.device.name": "Other", "user_agent.name": "Safari", @@ -612,8 +668,12 @@ "nginx.ingress_controller.upstream.response.length": 59, "nginx.ingress_controller.upstream.response.status_code": 200, "nginx.ingress_controller.upstream.response.time": 0.001, + "related.ip": [ + "192.168.64.1" + ], "service.type": "nginx", - "source.address": "", + "source.address": "192.168.64.1", + "source.ip": "192.168.64.1", "url.original": "/", "user_agent.device.name": "Other", "user_agent.name": "Safari", @@ -657,8 +717,12 @@ "nginx.ingress_controller.upstream.response.length": 59, "nginx.ingress_controller.upstream.response.status_code": 200, "nginx.ingress_controller.upstream.response.time": 0.002, + "related.ip": [ + "192.168.64.1" + ], "service.type": "nginx", - "source.address": "", + "source.address": "192.168.64.1", + "source.ip": "192.168.64.1", "url.original": "/favicon.ico", "user_agent.device.name": "Other", "user_agent.name": "Safari", @@ -701,8 +765,12 @@ "nginx.ingress_controller.upstream.response.length": 61, "nginx.ingress_controller.upstream.response.status_code": 200, "nginx.ingress_controller.upstream.response.time": 0.002, + "related.ip": [ + "192.168.64.1" + ], "service.type": "nginx", - "source.address": "", + "source.address": "192.168.64.1", + "source.ip": "192.168.64.1", "url.original": "/v2", "user_agent.device.name": "Other", "user_agent.name": "Safari", @@ -746,8 +814,12 @@ "nginx.ingress_controller.upstream.response.length": 59, "nginx.ingress_controller.upstream.response.status_code": 200, "nginx.ingress_controller.upstream.response.time": 0.0, + "related.ip": [ + "192.168.64.1" + ], "service.type": "nginx", - "source.address": "", + "source.address": "192.168.64.1", + "source.ip": "192.168.64.1", "url.original": "/favicon.ico", "user_agent.device.name": "Other", "user_agent.name": "Safari", @@ -790,8 +862,12 @@ "nginx.ingress_controller.upstream.response.length": 59, "nginx.ingress_controller.upstream.response.status_code": 200, "nginx.ingress_controller.upstream.response.time": 0.001, + "related.ip": [ + "192.168.64.1" + ], "service.type": "nginx", - "source.address": "", + "source.address": "192.168.64.1", + "source.ip": "192.168.64.1", "url.original": "/products/42?address=delhi+technological+university", "user_agent.device.name": "Other", "user_agent.name": "Python Requests", @@ -831,8 +907,12 @@ "nginx.ingress_controller.upstream.response.length": 61, "nginx.ingress_controller.upstream.response.status_code": 200, "nginx.ingress_controller.upstream.response.time": 0.001, + "related.ip": [ + "192.168.64.1" + ], "service.type": "nginx", - "source.address": "", + "source.address": "192.168.64.1", + "source.ip": "192.168.64.1", "url.original": "/v2", "user_agent.device.name": "Other", "user_agent.name": "Firefox", @@ -875,8 +955,12 @@ "nginx.ingress_controller.upstream.response.length": 59, "nginx.ingress_controller.upstream.response.status_code": 200, "nginx.ingress_controller.upstream.response.time": 0.0, + "related.ip": [ + "192.168.64.1" + ], "service.type": "nginx", - "source.address": "", + "source.address": "192.168.64.1", + "source.ip": "192.168.64.1", "url.original": "/favicon.ico", "user_agent.device.name": "Other", "user_agent.name": "Firefox", @@ -919,8 +1003,12 @@ "nginx.ingress_controller.upstream.response.length": 61, "nginx.ingress_controller.upstream.response.status_code": 200, "nginx.ingress_controller.upstream.response.time": 0.0, + "related.ip": [ + "192.168.64.1" + ], "service.type": "nginx", - "source.address": "", + "source.address": "192.168.64.1", + "source.ip": "192.168.64.1", "url.original": "/v2/some", "user_agent.device.name": "Other", "user_agent.name": "Firefox",