[Filebeat] Parse additonal debug data fields for Okta module (#25818)

* #25689: Parse additonal debug data fields for Okta module * update generated data * update changelog * added additional test data & `uri_parts` processor * update fields * fix changelog * update fields Co-authored-by: Marius Iversen <[email protected]>
elastic · Jun 24, 2021 · 4aff295 · 4aff295
1 parent 1ce38f4
commit 4aff295
Show file tree

Hide file tree

Showing 7 changed files with 498 additions and 22 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -827,8 +827,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add `include_s3_metadata` config option to the `aws-s3` input for including object metadata in events. {pull}26267[26267]
 - RFC 5424 and UNIX socket support in the Syslog input are now GA {pull}26293[26293]
 - Update grok patterns for HA Proxy module {issue}25827[25827] {pull}25835[25835]
+- Update Okta module to parse additional fields to `okta.debug_context.debug_data`. {issue}25689[25689] {pull}25818[25818]
 - Added dataset `anomalithreatstream` to the `threatintel` module to ingest indicators from Anomali ThreatStream {pull}26350[26350]
-
 - Add support for `copytruncate` method when rotating input logs with an external tool in `filestream` input. {pull}23457[23457]
 
 *Heartbeat*

diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc
@@ -113109,6 +113109,133 @@ type: keyword
 
 --
 
+[float]
+=== suspicious_activity
+
+The suspicious activity fields from the debug data.
+
+
+
+*`okta.debug_context.debug_data.suspicious_activity.browser`*::
++
+--
+The browser used.
+
+
+type: keyword
+
+--
+
+*`okta.debug_context.debug_data.suspicious_activity.event_city`*::
++
+--
+The city where the suspicious activity took place.
+
+
+type: keyword
+
+--
+
+*`okta.debug_context.debug_data.suspicious_activity.event_country`*::
++
+--
+The country where the suspicious activity took place.
+
+
+type: keyword
+
+--
+
+*`okta.debug_context.debug_data.suspicious_activity.event_id`*::
++
+--
+The event ID.
+
+
+type: keyword
+
+--
+
+*`okta.debug_context.debug_data.suspicious_activity.event_ip`*::
++
+--
+The IP of the suspicious event.
+
+
+type: ip
+
+--
+
+*`okta.debug_context.debug_data.suspicious_activity.event_latitude`*::
++
+--
+The latitude where the suspicious activity took place.
+
+
+type: float
+
+--
+
+*`okta.debug_context.debug_data.suspicious_activity.event_longitude`*::
++
+--
+The longitude where the suspicious activity took place.
+
+
+type: float
+
+--
+
+*`okta.debug_context.debug_data.suspicious_activity.event_state`*::
++
+--
+The state where the suspicious activity took place.
+
+
+type: keyword
+
+--
+
+*`okta.debug_context.debug_data.suspicious_activity.event_transaction_id`*::
++
+--
+The event transaction ID.
+
+
+type: keyword
+
+--
+
+*`okta.debug_context.debug_data.suspicious_activity.event_type`*::
++
+--
+The event type.
+
+
+type: keyword
+
+--
+
+*`okta.debug_context.debug_data.suspicious_activity.os`*::
++
+--
+The OS of the system from where the suspicious activity occured.
+
+
+type: keyword
+
+--
+
+*`okta.debug_context.debug_data.suspicious_activity.timestamp`*::
++
+--
+The timestamp of when the activity occurred.
+
+
+type: date
+
+--
+
 [float]
 === authentication_context
 

diff --git a/x-pack/filebeat/module/okta/fields.go b/x-pack/filebeat/module/okta/fields.go
diff --git a/x-pack/filebeat/module/okta/system/_meta/fields.yml b/x-pack/filebeat/module/okta/system/_meta/fields.yml
@@ -213,6 +213,72 @@
       description: >
         The URL.
 
+    - name: suspicious_activity
+      description: >
+        The suspicious activity fields from the debug data.
+      type: group
+      fields:
+
+        - name: browser
+          type: keyword
+          description: >
+            The browser used.
+
+        - name: event_city
+          type: keyword
+          description: >
+            The city where the suspicious activity took place.
+
+        - name: event_country
+          type: keyword
+          description: >
+            The country where the suspicious activity took place.
+
+        - name: event_id
+          type: keyword
+          description: >
+            The event ID.
+
+        - name: event_ip
+          type: ip
+          description: >
+            The IP of the suspicious event.
+
+        - name: event_latitude
+          type: float
+          description: >
+            The latitude where the suspicious activity took place.
+
+        - name: event_longitude
+          type: float
+          description: >
+            The longitude where the suspicious activity took place.
+
+        - name: event_state
+          type: keyword
+          description: >
+            The state where the suspicious activity took place.
+
+        - name: event_transaction_id
+          type: keyword
+          description: >
+            The event transaction ID.
+
+        - name: event_type
+          type: keyword
+          description: >
+            The event type.
+
+        - name: os
+          type: keyword
+          description: >
+            The OS of the system from where the suspicious activity occured.
+
+        - name: timestamp
+          type: date
+          description: >
+            The timestamp of when the activity occurred.
+
 - name: authentication_context
   title: Authentication Context
   short: Fields that let you store information about authentication context.

diff --git a/x-pack/filebeat/module/okta/system/ingest/pipeline.yml b/x-pack/filebeat/module/okta/system/ingest/pipeline.yml
@@ -4,23 +4,6 @@ processors:
   - set:
       field: event.ingested
       value: "{{_ingest.timestamp}}"
-  - script:
-      description: Drops null/empty values recursively
-      lang: painless
-      source: |
-        boolean drop(Object o) {
-          if (o == null || o == "") {
-            return true;
-          } else if (o instanceof Map) {
-            ((Map) o).values().removeIf(v -> drop(v));
-            return (((Map) o).size() == 0);
-          } else if (o instanceof List) {
-            ((List) o).removeIf(v -> drop(v));
-            return (((List) o).length == 0);
-          }
-          return false;
-        }
-        drop(ctx);
   - remove:
       field: message
       ignore_missing: true
@@ -265,6 +248,72 @@ processors:
       target_field: okta.debug_context.debug_data.url
       ignore_missing: true
       ignore_failure: true
+  - uri_parts:
+      field: okta.debug_context.debug_data.url
+      ignore_failure: true
+      if: ctx?.okta?.debug_context?.debug_data?.url != null
+  - rename:
+      field: json.debugContext.debugData.suspiciousActivityBrowser
+      target_field: okta.debug_context.debug_data.suspicious_activity.browser
+      ignore_missing: true
+      ignore_failure: true
+  - rename:
+      ignore_failure: true
+      field: json.debugContext.debugData.suspiciousActivityEventCity
+      target_field: okta.debug_context.debug_data.suspicious_activity.event_city
+      ignore_missing: true
+  - rename:
+      field: json.debugContext.debugData.suspiciousActivityEventCountry
+      target_field: okta.debug_context.debug_data.suspicious_activity.event_country
+      ignore_missing: true
+      ignore_failure: true
+  - rename:
+      field: json.debugContext.debugData.suspiciousActivityEventId
+      target_field: okta.debug_context.debug_data.suspicious_activity.event_id
+      ignore_missing: true
+      ignore_failure: true
+  - rename:
+      field: json.debugContext.debugData.suspiciousActivityEventIp
+      target_field: okta.debug_context.debug_data.suspicious_activity.event_ip
+      ignore_missing: true
+      ignore_failure: true
+  - rename:
+      field: json.debugContext.debugData.suspiciousActivityEventLatitude
+      target_field: okta.debug_context.debug_data.suspicious_activity.event_latitude
+      ignore_missing: true
+      ignore_failure: true
+  - rename:
+      field: json.debugContext.debugData.suspiciousActivityEventLongitude
+      target_field: okta.debug_context.debug_data.suspicious_activity.event_longitude
+      ignore_missing: true
+      ignore_failure: true
+  - rename:
+      field: json.debugContext.debugData.suspiciousActivityEventState
+      target_field: okta.debug_context.debug_data.suspicious_activity.event_state
+      ignore_missing: true
+      ignore_failure: true
+  - rename:
+      field: json.debugContext.debugData.suspiciousActivityEventTransactionId
+      target_field: okta.debug_context.debug_data.suspicious_activity.event_transaction_id
+      ignore_missing: true
+      ignore_failure: true
+  - rename:
+      field: json.debugContext.debugData.suspiciousActivityEventType
+      target_field: okta.debug_context.debug_data.suspicious_activity.event_type
+      ignore_missing: true
+      ignore_failure: true
+  - rename:
+      field: json.debugContext.debugData.suspiciousActivityOs
+      target_field: okta.debug_context.debug_data.suspicious_activity.os
+      ignore_missing: true
+      ignore_failure: true
+  - date:
+      field: json.debugContext.debugData.suspiciousActivityTimestamp
+      target_field: okta.debug_context.debug_data.suspicious_activity.timestamp
+      ignore_failure: true
+      formats:
+        - ISO8601
+      if: ctx?.json?.debugContext?.debugData?.suspiciousActivityTimestamp != null
   - rename:
       field: json.authenticationContext.authenticationProvider
       target_field: okta.authentication_context.authentication_provider
@@ -452,6 +501,7 @@ processors:
       field:
         - okta_target_user
         - okta_target_group
+        - json
       ignore_missing: true
   - set:
       field: client.user.id
@@ -498,9 +548,6 @@ processors:
       value: "{{destination.ip}}"
       allow_duplicates: false
       if: ctx?.destination?.ip != null
-  - remove:
-      field: json
-      ignore_missing: true
   - user_agent:
       field: user_agent.original
       ignore_missing: true
@@ -544,6 +591,23 @@ processors:
       field: destination.as.organization_name
       target_field: destination.as.organization.name
       ignore_missing: true
+  - script:
+      description: Drops null/empty values recursively
+      lang: painless
+      source: |
+        boolean drop(Object o) {
+          if (o == null || o == "") {
+            return true;
+          } else if (o instanceof Map) {
+            ((Map) o).values().removeIf(v -> drop(v));
+            return (((Map) o).size() == 0);
+          } else if (o instanceof List) {
+            ((List) o).removeIf(v -> drop(v));
+            return (((List) o).length == 0);
+          }
+          return false;
+        }
+        drop(ctx);
 
 on_failure:
   - set: