Drop fields in aws vpcflow data stream when value equals to "-" (elas…

…tic#435) * Drop fields in aws vpcflow data stream when value equals to "-" * bump aws package version to 0.3.14
build-security · Dec 4, 2020 · f950811 · f950811
1 parent 947c83f
commit f950811
Show file tree

Hide file tree

Showing 4 changed files with 91 additions and 1 deletion.
diff --git a/packages/aws/data_stream/vpcflow/_dev/test/pipeline/test-tcp-flag-sequence.log b/packages/aws/data_stream/vpcflow/_dev/test/pipeline/test-tcp-flag-sequence.log
@@ -0,0 +1,6 @@
+version vpc-id subnet-id instance-id interface-id account-id type srcaddr dstaddr srcport dstport pkt-srcaddr pkt-dstaddr protocol bytes packets start end action tcp-flags log-status
+3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 IPv4 52.213.180.42 10.0.0.62 43416 5001 52.213.180.42 10.0.0.62 6 568 8 1566848875 1566848933 ACCEPT 2 OK
+version vpc-id subnet-id instance-id interface-id account-id type srcaddr dstaddr srcport dstport pkt-srcaddr pkt-dstaddr protocol bytes packets start end action tcp-flags log-status
+3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 - - - - - - - - - - 1566848875 1566848933 - - SKIPDATA
+version vpc-id subnet-id instance-id interface-id account-id type srcaddr dstaddr srcport dstport pkt-srcaddr pkt-dstaddr protocol bytes packets start end action tcp-flags log-status
+3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 - - - - - - - - - - 1566848875 1566848933 - - NODATA
diff --git a/packages/aws/data_stream/vpcflow/_dev/test/pipeline/test-tcp-flag-sequence.log-expected.json b/packages/aws/data_stream/vpcflow/_dev/test/pipeline/test-tcp-flag-sequence.log-expected.json
@@ -0,0 +1,58 @@
+{
+    "expected": [
+        {
+            "cloud": {
+                "provider": "aws"
+            },
+            "event": {
+                "kind": "event",
+                "original": "version vpc-id subnet-id instance-id interface-id account-id type srcaddr dstaddr srcport dstport pkt-srcaddr pkt-dstaddr protocol bytes packets start end action tcp-flags log-status"
+            }
+        },
+        {
+            "cloud": {
+                "provider": "aws"
+            },
+            "event": {
+                "kind": "event",
+                "original": "3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 IPv4 52.213.180.42 10.0.0.62 43416 5001 52.213.180.42 10.0.0.62 6 568 8 1566848875 1566848933 ACCEPT 2 OK"
+            }
+        },
+        {
+            "cloud": {
+                "provider": "aws"
+            },
+            "event": {
+                "kind": "event",
+                "original": "version vpc-id subnet-id instance-id interface-id account-id type srcaddr dstaddr srcport dstport pkt-srcaddr pkt-dstaddr protocol bytes packets start end action tcp-flags log-status"
+            }
+        },
+        {
+            "cloud": {
+                "provider": "aws"
+            },
+            "event": {
+                "kind": "event",
+                "original": "3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 - - - - - - - - - - 1566848875 1566848933 - - SKIPDATA"
+            }
+        },
+        {
+            "cloud": {
+                "provider": "aws"
+            },
+            "event": {
+                "kind": "event",
+                "original": "version vpc-id subnet-id instance-id interface-id account-id type srcaddr dstaddr srcport dstport pkt-srcaddr pkt-dstaddr protocol bytes packets start end action tcp-flags log-status"
+            }
+        },
+        {
+            "cloud": {
+                "provider": "aws"
+            },
+            "event": {
+                "kind": "event",
+                "original": "3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 - - - - - - - - - - 1566848875 1566848933 - - NODATA"
+            }
+        }
+    ]
+}
diff --git a/packages/aws/data_stream/vpcflow/elasticsearch/ingest_pipeline/default.yml b/packages/aws/data_stream/vpcflow/elasticsearch/ingest_pipeline/default.yml
@@ -25,6 +25,32 @@ processors:
       field: ["aws.vpcflow.start", "aws.vpcflow.end"]
       ignore_missing: true
 
+  - script:
+      lang: painless
+      ignore_failure: true
+      if: ctx?.aws != null
+      source: >-
+        void handleMap(Map map) {
+          for (def x : map.values()) {
+            if (x instanceof Map) {
+                handleMap(x);
+            } else if (x instanceof List) {
+                handleList(x);
+            }
+          }
+          map.values().removeIf(v -> v instanceof String && v == "-");
+        }
+        void handleList(List list) {
+          for (def x : list) {
+              if (x instanceof Map) {
+                  handleMap(x);
+              } else if (x instanceof List) {
+                  handleList(x);
+              }
+          }
+        }
+        handleMap(ctx.aws);
+
   # IP Geolocation Lookup
   - geoip:
       field: source.ip

diff --git a/packages/aws/manifest.yml b/packages/aws/manifest.yml
@@ -1,7 +1,7 @@
 format_version: 1.0.0
 name: aws
 title: AWS
-version: 0.3.13
+version: 0.3.14
 license: basic
 description: AWS Integration
 type: integration