Cherry-pick elastic#18511 to 7.7: Fix source.address not being set fo…

…r nginx ingress_controller (elastic#18569) * Fix source.address not being set for nginx ingress_controller Signed-off-by: chrismark <[email protected]> Co-authored-by: chendo <[email protected]> (cherry picked from commit 49c8888)
leweafan · May 18, 2020 · fc3eae8 · fc3eae8
1 parent 5c5ec09
commit fc3eae8
Show file tree

Hide file tree

Showing 3 changed files with 46 additions and 23 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -128,6 +128,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fixed typo in log message. {pull}17897[17897]
 - Unescape file name from SQS message. {pull}18370[18370]
 - Improve cisco asa and ftd pipelines' failure handler to avoid mapping temporary fields. {issue}18391[18391] {pull}18392[18392]
+- Fix source.address not being set for nginx ingress_controller {pull}18511[18511]
 - Fix `googlecloud.audit` pipeline to only take in fields that are explicitly defined by the dataset. {issue}18465[18465] {pull}18472[18472]
 - Fix a rate limit related issue in httpjson input for Okta module. {issue}18530[18530] {pull}18534[18534]
 - Fixed ingestion of some Cisco ASA and FTD messages when a hostname was used instead of an IP for NAT fields. {issue}14034[14034] {pull}18376[18376]

diff --git a/filebeat/module/nginx/ingress_controller/ingest/pipeline.json b/filebeat/module/nginx/ingress_controller/ingest/pipeline.json
@@ -54,7 +54,7 @@
         },
         {
             "script": {
-                "if": "ctx.nginx?.access?.remote_ip_list != null && ctx.nginx.ingress_controller.remote_ip_list.length > 0",
+                "if": "ctx.nginx?.ingress_controller?.remote_ip_list != null && ctx.nginx.ingress_controller.remote_ip_list.length > 0",
                 "lang": "painless",
                 "source": "boolean isPrivate(def dot, def ip) { try { StringTokenizer tok = new StringTokenizer(ip, dot); int firstByte = Integer.parseInt(tok.nextToken());       int secondByte = Integer.parseInt(tok.nextToken());       if (firstByte == 10) {         return true;       }       if (firstByte == 192 && secondByte == 168) {         return true;       }       if (firstByte == 172 && secondByte >= 16 && secondByte <= 31) {         return true;       }       if (firstByte == 127) {         return true;       }       return false;     } catch (Exception e) {       return false;     }   }   try {    ctx.source.address = null;    if (ctx.nginx.ingress_controller.remote_ip_list == null) { return; }    def found = false;    for (def item : ctx.nginx.ingress_controller.remote_ip_list) {        if (!isPrivate(params.dot, item)) {            ctx.source.address = item;            found = true;            break;        }    }    if (!found) {     ctx.source.address = ctx.nginx.ingress_controller.remote_ip_list[0];   }} catch (Exception e) { ctx.source.address = null; }",
                 "params": {

diff --git a/filebeat/module/nginx/ingress_controller/test/test.log-expected.json b/filebeat/module/nginx/ingress_controller/test/test.log-expected.json
@@ -26,7 +26,8 @@
         "nginx.ingress_controller.upstream.response.status_code": 200,
         "nginx.ingress_controller.upstream.response.time": 0.0,
         "service.type": "nginx",
-        "source.address": "",
+        "source.address": "192.168.64.1",
+        "source.ip": "192.168.64.1",
         "url.original": "/products",
         "user.name": "-",
         "user_agent.device.name": "Other",
@@ -61,7 +62,8 @@
         "nginx.ingress_controller.upstream.response.status_code": 200,
         "nginx.ingress_controller.upstream.response.time": 0.0,
         "service.type": "nginx",
-        "source.address": "",
+        "source.address": "192.168.64.1",
+        "source.ip": "192.168.64.1",
         "url.original": "/products/42",
         "user.name": "-",
         "user_agent.device.name": "Other",
@@ -96,7 +98,8 @@
         "nginx.ingress_controller.upstream.response.status_code": 200,
         "nginx.ingress_controller.upstream.response.time": 0.001,
         "service.type": "nginx",
-        "source.address": "",
+        "source.address": "192.168.64.1",
+        "source.ip": "192.168.64.1",
         "url.original": "/products/42",
         "user.name": "-",
         "user_agent.device.name": "Other",
@@ -131,7 +134,8 @@
         "nginx.ingress_controller.upstream.response.status_code": 200,
         "nginx.ingress_controller.upstream.response.time": 0.0,
         "service.type": "nginx",
-        "source.address": "",
+        "source.address": "192.168.64.1",
+        "source.ip": "192.168.64.1",
         "url.original": "/products/42",
         "user.name": "-",
         "user_agent.device.name": "Other",
@@ -161,7 +165,8 @@
         "nginx.ingress_controller.upstream.alternative_name": "",
         "nginx.ingress_controller.upstream.name": "",
         "service.type": "nginx",
-        "source.address": "",
+        "source.address": "192.168.64.1",
+        "source.ip": "192.168.64.1",
         "url.original": "/products/42",
         "user.name": "-",
         "user_agent.device.name": "Other",
@@ -190,7 +195,8 @@
         "nginx.ingress_controller.upstream.alternative_name": "",
         "nginx.ingress_controller.upstream.name": "",
         "service.type": "nginx",
-        "source.address": "",
+        "source.address": "192.168.64.1",
+        "source.ip": "192.168.64.1",
         "url.original": "/products/42",
         "user.name": "-",
         "user_agent.device.name": "Other",
@@ -224,7 +230,8 @@
         "nginx.ingress_controller.upstream.response.status_code": 200,
         "nginx.ingress_controller.upstream.response.time": 0.0,
         "service.type": "nginx",
-        "source.address": "",
+        "source.address": "192.168.64.1",
+        "source.ip": "192.168.64.1",
         "url.original": "/products/42",
         "user.name": "-",
         "user_agent.device.name": "Other",
@@ -259,7 +266,8 @@
         "nginx.ingress_controller.upstream.response.status_code": 200,
         "nginx.ingress_controller.upstream.response.time": 0.0,
         "service.type": "nginx",
-        "source.address": "",
+        "source.address": "192.168.64.1",
+        "source.ip": "192.168.64.1",
         "url.original": "/products/42",
         "user.name": "-",
         "user_agent.device.name": "Other",
@@ -297,7 +305,8 @@
         "nginx.ingress_controller.upstream.response.status_code": 200,
         "nginx.ingress_controller.upstream.response.time": 0.0,
         "service.type": "nginx",
-        "source.address": "",
+        "source.address": "192.168.64.1",
+        "source.ip": "192.168.64.1",
         "url.original": "/favicon.ico",
         "user.name": "-",
         "user_agent.device.name": "Other",
@@ -335,7 +344,8 @@
         "nginx.ingress_controller.upstream.response.status_code": 200,
         "nginx.ingress_controller.upstream.response.time": 0.001,
         "service.type": "nginx",
-        "source.address": "",
+        "source.address": "192.168.64.1",
+        "source.ip": "192.168.64.1",
         "url.original": "/v2",
         "user.name": "-",
         "user_agent.device.name": "Other",
@@ -373,7 +383,8 @@
         "nginx.ingress_controller.upstream.response.status_code": 200,
         "nginx.ingress_controller.upstream.response.time": 0.002,
         "service.type": "nginx",
-        "source.address": "",
+        "source.address": "192.168.64.1",
+        "source.ip": "192.168.64.1",
         "url.original": "/favicon.ico",
         "user.name": "-",
         "user_agent.device.name": "Other",
@@ -411,7 +422,8 @@
         "nginx.ingress_controller.upstream.response.status_code": 200,
         "nginx.ingress_controller.upstream.response.time": 0.001,
         "service.type": "nginx",
-        "source.address": "",
+        "source.address": "192.168.64.1",
+        "source.ip": "192.168.64.1",
         "url.original": "/products/42",
         "user.name": "-",
         "user_agent.device.name": "Other",
@@ -449,7 +461,8 @@
         "nginx.ingress_controller.upstream.response.status_code": 200,
         "nginx.ingress_controller.upstream.response.time": 0.001,
         "service.type": "nginx",
-        "source.address": "",
+        "source.address": "192.168.64.1",
+        "source.ip": "192.168.64.1",
         "url.original": "/favicon.ico",
         "user.name": "-",
         "user_agent.device.name": "Other",
@@ -487,7 +500,8 @@
         "nginx.ingress_controller.upstream.response.status_code": 200,
         "nginx.ingress_controller.upstream.response.time": 0.002,
         "service.type": "nginx",
-        "source.address": "",
+        "source.address": "192.168.64.1",
+        "source.ip": "192.168.64.1",
         "url.original": "/products/42",
         "user.name": "-",
         "user_agent.device.name": "Other",
@@ -525,7 +539,8 @@
         "nginx.ingress_controller.upstream.response.status_code": 200,
         "nginx.ingress_controller.upstream.response.time": 0.001,
         "service.type": "nginx",
-        "source.address": "",
+        "source.address": "192.168.64.1",
+        "source.ip": "192.168.64.1",
         "url.original": "/",
         "user.name": "-",
         "user_agent.device.name": "Other",
@@ -563,7 +578,8 @@
         "nginx.ingress_controller.upstream.response.status_code": 200,
         "nginx.ingress_controller.upstream.response.time": 0.002,
         "service.type": "nginx",
-        "source.address": "",
+        "source.address": "192.168.64.1",
+        "source.ip": "192.168.64.1",
         "url.original": "/favicon.ico",
         "user.name": "-",
         "user_agent.device.name": "Other",
@@ -601,7 +617,8 @@
         "nginx.ingress_controller.upstream.response.status_code": 200,
         "nginx.ingress_controller.upstream.response.time": 0.002,
         "service.type": "nginx",
-        "source.address": "",
+        "source.address": "192.168.64.1",
+        "source.ip": "192.168.64.1",
         "url.original": "/v2",
         "user.name": "-",
         "user_agent.device.name": "Other",
@@ -639,7 +656,8 @@
         "nginx.ingress_controller.upstream.response.status_code": 200,
         "nginx.ingress_controller.upstream.response.time": 0.0,
         "service.type": "nginx",
-        "source.address": "",
+        "source.address": "192.168.64.1",
+        "source.ip": "192.168.64.1",
         "url.original": "/favicon.ico",
         "user.name": "-",
         "user_agent.device.name": "Other",
@@ -677,7 +695,8 @@
         "nginx.ingress_controller.upstream.response.status_code": 200,
         "nginx.ingress_controller.upstream.response.time": 0.001,
         "service.type": "nginx",
-        "source.address": "",
+        "source.address": "192.168.64.1",
+        "source.ip": "192.168.64.1",
         "url.original": "/products/42?address=delhi+technological+university",
         "user.name": "-",
         "user_agent.device.name": "Other",
@@ -712,7 +731,8 @@
         "nginx.ingress_controller.upstream.response.status_code": 200,
         "nginx.ingress_controller.upstream.response.time": 0.001,
         "service.type": "nginx",
-        "source.address": "",
+        "source.address": "192.168.64.1",
+        "source.ip": "192.168.64.1",
         "url.original": "/v2",
         "user.name": "-",
         "user_agent.device.name": "Other",
@@ -750,7 +770,8 @@
         "nginx.ingress_controller.upstream.response.status_code": 200,
         "nginx.ingress_controller.upstream.response.time": 0.0,
         "service.type": "nginx",
-        "source.address": "",
+        "source.address": "192.168.64.1",
+        "source.ip": "192.168.64.1",
         "url.original": "/favicon.ico",
         "user.name": "-",
         "user_agent.device.name": "Other",
@@ -788,7 +809,8 @@
         "nginx.ingress_controller.upstream.response.status_code": 200,
         "nginx.ingress_controller.upstream.response.time": 0.0,
         "service.type": "nginx",
-        "source.address": "",
+        "source.address": "192.168.64.1",
+        "source.ip": "192.168.64.1",
         "url.original": "/v2/some",
         "user.name": "-",
         "user_agent.device.name": "Other",