[Filebeat][Google workspace] Backport integration fixes (#33666)

* Backport integration improvements: - Use event for google workspace fingerprint - Fix pagination and cursor for google workspace data streams * Add changelog entry * Add processing for user_accounts event parameters * Add PR number * Ensure only scalars are added to generate the id (cherry picked from commit 311b137) # Conflicts: # x-pack/filebeat/module/google_workspace/config/common.js
elastic · Nov 14, 2022 · 2ddbd0c · 2ddbd0c
1 parent 1fb0297
commit 2ddbd0c
Show file tree

Hide file tree

Showing 9 changed files with 217 additions and 20 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -37,6 +37,33 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 
 *Filebeat*
 
+- Fix `httpjson` input page number initialization and documentation. {pull}33400[33400]
+- Add handling of AAA operations for Cisco ASA module. {issue}32257[32257] {pull}32789[32789]
+- Fix gc.log always shipped even if gc fileset is disabled {issue}30995[30995]
+- Fix handling of empty array in httpjson input. {pull}32001[32001]
+- Fix reporting of `filebeat.events.active` in log events such that the current value is always reported instead of the difference from the last value. {pull}33597[33597]
+- Fix splitting array of strings/arrays in httpjson input {issue}30345[30345] {pull}33609[33609]
+- Fix Google workspace pagination and document ID generation. {pull}33666[33666]
+
+*Heartbeat*
+- Fix bug affecting let's encrypt and other users of cross-signed certs, where cert expiration was incorrectly calculated. {issue}33215[33215]
+- Fix broken disable feature for kibana configured monitors. {pull}33293[33293]
+- Fix states client support for output options. {pull}33405[33405]
+- Fix states client reloader under managed mode. {pull}33405[33405]
+- Fix bug where states.duration_ms was incorrect type. {pull}33563[33563]
+
+*Auditbeat*
+
+
+*Filebeat*
+
+
+*Auditbeat*
+
+
+*Filebeat*
+
+
 *Heartbeat*
 
 

diff --git a/x-pack/filebeat/module/google_workspace/admin/config/config.yml b/x-pack/filebeat/module/google_workspace/admin/config/config.yml
@@ -17,21 +17,42 @@ request.proxy_url: {{ .proxy_url }}
 request.transforms:
   - set:
       target: url.params.startTime
-      value: '[[if eq .last_response.page 0]][[.cursor.last_execution_datetime]][[else]][[.last_response.url.params.Get "startTime"]][[end]]'
+      value: '[[.cursor.last_execution_datetime]]'
       default: '[[formatDate (now (parseDuration "-{{.initial_interval}}"))]]'
 response.split:
   target: body.items
   split:
     target: body.events
     keep_parent: true
 response.pagination:
+  - set:
+      target: url.params.startTime
+      value: '[[.last_response.url.params.Get "startTime"]]'
+      fail_on_template_error: true
   - set:
       target: url.params.pageToken
-      value: "[[.last_response.body.nextPageToken]]"
+      value: >-
+        [[- if index .last_response.body "nextPageToken" -]]
+          [[- .last_response.body.nextPageToken -]]
+        [[- end -]]
       fail_on_template_error: true
 cursor:
   last_execution_datetime:
-    value: "[[formatDate now]]"
+    value: >-
+      [[- $time := .last_event.id.time -]]
+      [[- if not (parseDate $time "RFC3339").IsZero -]]
+        [[- $time -]]
+      [[- else if not (parseDate $time "2006-01-02T15:04:05").IsZero -]]
+        [[- formatDate (parseDate $time "2006-01-02T15:04:05") -]]
+      [[- else if not (parseDate $time "2006-01-02T15:04:05Z").IsZero -]]
+        [[- formatDate (parseDate $time "2006-01-02T15:04:05Z") -]]
+      [[- else if not (parseDate $time "2006-01-02T15:04:05.999Z").IsZero -]]
+        [[- formatDate (parseDate $time "2006-01-02T15:04:05.999Z") -]]
+      [[- else if not (parseDate $time "2006-01-02T15:04:05 MST").IsZero -]]
+        [[- formatDate (parseDate $time "2006-01-02T15:04:05 MST") -]]
+      [[- else -]]
+        [[- formatDate now -]]
+      [[- end -]]
 
 {{ else if eq .input "file" }}
 type: log

diff --git a/x-pack/filebeat/module/google_workspace/config/common.js b/x-pack/filebeat/module/google_workspace/config/common.js
@@ -18,17 +18,35 @@ var googleWorkspace = (function () {
         ignore_missing: true,
     });
 
-    var addID = new processor.Fingerprint({
-        fields: [
+    var addID = function(evt) {
+        var keys = [
             "json.id.time",
             "json.id.uniqueQualifier",
             "json.id.applicationName",
             "json.id.customerId",
+<<<<<<< HEAD
         ],
         target_field: "@metadata.id",
         ignore_missing: true,
         fail_on_error: false,
     });
+=======
+        ];
+        Object.keys(evt.Get("json.events")).forEach(function(evtsKey) {
+            var key = "json.events."+evtsKey;
+            var value = evt.Get(key);
+            if (!Array.isArray(value) && !(typeof value === "object")) {
+                keys.push(key);
+            }
+        });
+        new processor.Fingerprint({
+            fields: keys,
+            target_field: "@metadata._id",
+            ignore_missing: true,
+            fail_on_error: false,
+        }).Run(evt);
+    };
+>>>>>>> 311b1371e1 ([Filebeat][Google workspace] Backport integration fixes (#33666))
 
     var convertFields = new processor.Convert({
         fields: [

diff --git a/x-pack/filebeat/module/google_workspace/drive/config/config.yml b/x-pack/filebeat/module/google_workspace/drive/config/config.yml
@@ -17,21 +17,42 @@ request.proxy_url: {{ .proxy_url }}
 request.transforms:
   - set:
       target: url.params.startTime
-      value: '[[if eq .last_response.page 0]][[.cursor.last_execution_datetime]][[else]][[.last_response.url.params.Get "startTime"]][[end]]'
+      value: '[[.cursor.last_execution_datetime]]'
       default: '[[formatDate (now (parseDuration "-{{.initial_interval}}"))]]'
 response.split:
   target: body.items
   split:
     target: body.events
     keep_parent: true
 response.pagination:
+  - set:
+      target: url.params.startTime
+      value: '[[.last_response.url.params.Get "startTime"]]'
+      fail_on_template_error: true
   - set:
       target: url.params.pageToken
-      value: "[[.last_response.body.nextPageToken]]"
+      value: >-
+        [[- if index .last_response.body "nextPageToken" -]]
+          [[- .last_response.body.nextPageToken -]]
+        [[- end -]]
       fail_on_template_error: true
 cursor:
   last_execution_datetime:
-    value: "[[formatDate now]]"
+    value: >-
+      [[- $time := .last_event.id.time -]]
+      [[- if not (parseDate $time "RFC3339").IsZero -]]
+        [[- $time -]]
+      [[- else if not (parseDate $time "2006-01-02T15:04:05").IsZero -]]
+        [[- formatDate (parseDate $time "2006-01-02T15:04:05") -]]
+      [[- else if not (parseDate $time "2006-01-02T15:04:05Z").IsZero -]]
+        [[- formatDate (parseDate $time "2006-01-02T15:04:05Z") -]]
+      [[- else if not (parseDate $time "2006-01-02T15:04:05.999Z").IsZero -]]
+        [[- formatDate (parseDate $time "2006-01-02T15:04:05.999Z") -]]
+      [[- else if not (parseDate $time "2006-01-02T15:04:05 MST").IsZero -]]
+        [[- formatDate (parseDate $time "2006-01-02T15:04:05 MST") -]]
+      [[- else -]]
+        [[- formatDate now -]]
+      [[- end -]]
 
 {{ else if eq .input "file" }}
 type: log

diff --git a/x-pack/filebeat/module/google_workspace/groups/config/config.yml b/x-pack/filebeat/module/google_workspace/groups/config/config.yml
@@ -17,21 +17,42 @@ request.proxy_url: {{ .proxy_url }}
 request.transforms:
   - set:
       target: url.params.startTime
-      value: '[[if eq .last_response.page 0]][[.cursor.last_execution_datetime]][[else]][[.last_response.url.params.Get "startTime"]][[end]]'
+      value: '[[.cursor.last_execution_datetime]]'
       default: '[[formatDate (now (parseDuration "-{{.initial_interval}}"))]]'
 response.split:
   target: body.items
   split:
     target: body.events
     keep_parent: true
 response.pagination:
+  - set:
+      target: url.params.startTime
+      value: '[[.last_response.url.params.Get "startTime"]]'
+      fail_on_template_error: true
   - set:
       target: url.params.pageToken
-      value: "[[.last_response.body.nextPageToken]]"
+      value: >-
+        [[- if index .last_response.body "nextPageToken" -]]
+          [[- .last_response.body.nextPageToken -]]
+        [[- end -]]
       fail_on_template_error: true
 cursor:
   last_execution_datetime:
-    value: "[[formatDate now]]"
+    value: >-
+      [[- $time := .last_event.id.time -]]
+      [[- if not (parseDate $time "RFC3339").IsZero -]]
+        [[- $time -]]
+      [[- else if not (parseDate $time "2006-01-02T15:04:05").IsZero -]]
+        [[- formatDate (parseDate $time "2006-01-02T15:04:05") -]]
+      [[- else if not (parseDate $time "2006-01-02T15:04:05Z").IsZero -]]
+        [[- formatDate (parseDate $time "2006-01-02T15:04:05Z") -]]
+      [[- else if not (parseDate $time "2006-01-02T15:04:05.999Z").IsZero -]]
+        [[- formatDate (parseDate $time "2006-01-02T15:04:05.999Z") -]]
+      [[- else if not (parseDate $time "2006-01-02T15:04:05 MST").IsZero -]]
+        [[- formatDate (parseDate $time "2006-01-02T15:04:05 MST") -]]
+      [[- else -]]
+        [[- formatDate now -]]
+      [[- end -]]
 
 {{ else if eq .input "file" }}
 type: log

diff --git a/x-pack/filebeat/module/google_workspace/login/config/config.yml b/x-pack/filebeat/module/google_workspace/login/config/config.yml
@@ -17,21 +17,42 @@ request.proxy_url: {{ .proxy_url }}
 request.transforms:
   - set:
       target: url.params.startTime
-      value: '[[if eq .last_response.page 0]][[.cursor.last_execution_datetime]][[else]][[.last_response.url.params.Get "startTime"]][[end]]'
+      value: '[[.cursor.last_execution_datetime]]'
       default: '[[formatDate (now (parseDuration "-{{.initial_interval}}"))]]'
 response.split:
   target: body.items
   split:
     target: body.events
     keep_parent: true
 response.pagination:
+  - set:
+      target: url.params.startTime
+      value: '[[.last_response.url.params.Get "startTime"]]'
+      fail_on_template_error: true
   - set:
       target: url.params.pageToken
-      value: "[[.last_response.body.nextPageToken]]"
+      value: >-
+        [[- if index .last_response.body "nextPageToken" -]]
+          [[- .last_response.body.nextPageToken -]]
+        [[- end -]]
       fail_on_template_error: true
 cursor:
   last_execution_datetime:
-    value: "[[formatDate now]]"
+    value: >-
+      [[- $time := .last_event.id.time -]]
+      [[- if not (parseDate $time "RFC3339").IsZero -]]
+        [[- $time -]]
+      [[- else if not (parseDate $time "2006-01-02T15:04:05").IsZero -]]
+        [[- formatDate (parseDate $time "2006-01-02T15:04:05") -]]
+      [[- else if not (parseDate $time "2006-01-02T15:04:05Z").IsZero -]]
+        [[- formatDate (parseDate $time "2006-01-02T15:04:05Z") -]]
+      [[- else if not (parseDate $time "2006-01-02T15:04:05.999Z").IsZero -]]
+        [[- formatDate (parseDate $time "2006-01-02T15:04:05.999Z") -]]
+      [[- else if not (parseDate $time "2006-01-02T15:04:05 MST").IsZero -]]
+        [[- formatDate (parseDate $time "2006-01-02T15:04:05 MST") -]]
+      [[- else -]]
+        [[- formatDate now -]]
+      [[- end -]]
 
 {{ else if eq .input "file" }}
 type: log

diff --git a/x-pack/filebeat/module/google_workspace/saml/config/config.yml b/x-pack/filebeat/module/google_workspace/saml/config/config.yml
@@ -17,21 +17,42 @@ request.proxy_url: {{ .proxy_url }}
 request.transforms:
   - set:
       target: url.params.startTime
-      value: '[[if eq .last_response.page 0]][[.cursor.last_execution_datetime]][[else]][[.last_response.url.params.Get "startTime"]][[end]]'
+      value: '[[.cursor.last_execution_datetime]]'
       default: '[[formatDate (now (parseDuration "-{{.initial_interval}}"))]]'
 response.split:
   target: body.items
   split:
     target: body.events
     keep_parent: true
 response.pagination:
+  - set:
+      target: url.params.startTime
+      value: '[[.last_response.url.params.Get "startTime"]]'
+      fail_on_template_error: true
   - set:
       target: url.params.pageToken
-      value: "[[.last_response.body.nextPageToken]]"
+      value: >-
+        [[- if index .last_response.body "nextPageToken" -]]
+          [[- .last_response.body.nextPageToken -]]
+        [[- end -]]
       fail_on_template_error: true
 cursor:
   last_execution_datetime:
-    value: "[[formatDate now]]"
+    value: >-
+      [[- $time := .last_event.id.time -]]
+      [[- if not (parseDate $time "RFC3339").IsZero -]]
+        [[- $time -]]
+      [[- else if not (parseDate $time "2006-01-02T15:04:05").IsZero -]]
+        [[- formatDate (parseDate $time "2006-01-02T15:04:05") -]]
+      [[- else if not (parseDate $time "2006-01-02T15:04:05Z").IsZero -]]
+        [[- formatDate (parseDate $time "2006-01-02T15:04:05Z") -]]
+      [[- else if not (parseDate $time "2006-01-02T15:04:05.999Z").IsZero -]]
+        [[- formatDate (parseDate $time "2006-01-02T15:04:05.999Z") -]]
+      [[- else if not (parseDate $time "2006-01-02T15:04:05 MST").IsZero -]]
+        [[- formatDate (parseDate $time "2006-01-02T15:04:05 MST") -]]
+      [[- else -]]
+        [[- formatDate now -]]
+      [[- end -]]
 
 {{ else if eq .input "file" }}
 type: log

diff --git a/x-pack/filebeat/module/google_workspace/user_accounts/config/config.yml b/x-pack/filebeat/module/google_workspace/user_accounts/config/config.yml
@@ -17,21 +17,42 @@ request.proxy_url: {{ .proxy_url }}
 request.transforms:
   - set:
       target: url.params.startTime
-      value: '[[if eq .last_response.page 0]][[.cursor.last_execution_datetime]][[else]][[.last_response.url.params.Get "startTime"]][[end]]'
+      value: '[[.cursor.last_execution_datetime]]'
       default: '[[formatDate (now (parseDuration "-{{.initial_interval}}"))]]'
 response.split:
   target: body.items
   split:
     target: body.events
     keep_parent: true
 response.pagination:
+  - set:
+      target: url.params.startTime
+      value: '[[.last_response.url.params.Get "startTime"]]'
+      fail_on_template_error: true
   - set:
       target: url.params.pageToken
-      value: "[[.last_response.body.nextPageToken]]"
+      value: >-
+        [[- if index .last_response.body "nextPageToken" -]]
+          [[- .last_response.body.nextPageToken -]]
+        [[- end -]]
       fail_on_template_error: true
 cursor:
   last_execution_datetime:
-    value: "[[formatDate now]]"
+    value: >-
+      [[- $time := .last_event.id.time -]]
+      [[- if not (parseDate $time "RFC3339").IsZero -]]
+        [[- $time -]]
+      [[- else if not (parseDate $time "2006-01-02T15:04:05").IsZero -]]
+        [[- formatDate (parseDate $time "2006-01-02T15:04:05") -]]
+      [[- else if not (parseDate $time "2006-01-02T15:04:05Z").IsZero -]]
+        [[- formatDate (parseDate $time "2006-01-02T15:04:05Z") -]]
+      [[- else if not (parseDate $time "2006-01-02T15:04:05.999Z").IsZero -]]
+        [[- formatDate (parseDate $time "2006-01-02T15:04:05.999Z") -]]
+      [[- else if not (parseDate $time "2006-01-02T15:04:05 MST").IsZero -]]
+        [[- formatDate (parseDate $time "2006-01-02T15:04:05 MST") -]]
+      [[- else -]]
+        [[- formatDate now -]]
+      [[- end -]]
 
 {{ else if eq .input "file" }}
 type: log

diff --git a/x-pack/filebeat/module/google_workspace/user_accounts/config/pipeline.js b/x-pack/filebeat/module/google_workspace/user_accounts/config/pipeline.js
@@ -10,8 +10,34 @@ var userAccounts = (function () {
         evt.Put("event.category", ["iam"]);
     };
 
+    var getParamValue = function(param) {
+        if (param.value) {
+            return param.value;
+        }
+        if (param.multiValue) {
+            return param.multiValue;
+        }
+        if (param.intValue !== null) {
+            return param.intValue;
+        }
+    };
+
+    var flattenParams = function(evt) {
+        var params = evt.Get("json.events.parameters");
+        if (!params || !Array.isArray(params)) {
+            return;
+        }
+
+        params.forEach(function(p){
+            evt.Put("google_workspace.user_accounts."+p.name, getParamValue(p));
+        });
+
+        evt.Delete("json.events.parameters");
+    };
+
     var pipeline = new processor.Chain()
         .Add(categorizeEvent)
+        .Add(flattenParams)
         .Build();
 
     return {