[o365] Could not index event to Elasticsearch

[kowalczyk-p]

Filebeat 8.9.1 is fetching logs from Office365/Microsoft365 management activity API and sending them to Elasticsearch via Logstash. Recetly I observe raise in Could not index event to Elasticsearch errors:

{"create"=>{"_index"=>".ds-logs-filebeat.8.9.2-o365-2024.01.23-000004", "_id"=>"-NreWo0BSez6H-H_YS06", "status"=>400, "error"=>{"type"=>"document_parsing_exception", "reason"=>"[1:1128] object mapping for [o365.audit.OperationProperties] tried to parse field [OperationProperties] as object, but found a concrete value"}}

Sample event:

{
  "event": {
    "action": "CreateArtifact",
    "category": "web",
    "dataset": "o365.audit",
    "code": "PowerBIAudit",
    "module": "o365",
    "kind": "event",
    "outcome": "success",
    "id": "a4420e70-b7a1-4d5b-8175-11e3364acd22",
    "provider": "PowerBI",
    "type": "info"
  },
  "o365": {
    "audit": {
      "Activity": "CreateArtifact",
      "WorkspaceName": "obszar_robaczy",
      "OrganizationId": "53d83e1d-xxx-xxx-84e9-01ec5045dd81",
      "Operation": "CreateArtifact",
      "Id": "a4420e70-b7a1-xxx-xxx-11e3364acd22",
      "CreationTime": "2024-01-30T14:23:40",
      "Timestamp": "2024-01-30T14:22:50",
      "UserId": "[email protected]",
      "ClientIP": "31.135.18.197",
      "RecordType": 20,
      "ResultStatus": "InProgress",
      "ObjectDisplayName": "test_lakehouse",
      "OperationId": "a84f7f73-xxxx-xxxx-8cf3-094f69c23756",
      "Experience": "Lakehouse",
      "WorkspaceId": "91dad513-xxxx-xxxx-94bb-f5cbf305691c",
      "ObjectId": "0e00d1cf-825a-4d78-98ff-8a8199357669",
      "UserAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
      "Workload": "PowerBI",
      "RequestId": "fcbbe282-xxx-xxxx-xxxx-dc1e6d9b090b",
      "OperationProperties": "[{\"Name\":\"SystemArtifactType\",​\"Value\":\"None\"}]",
      "ObjectType": "Lakehouse",
      "UserType": 0,
      "UserKey": "xxxxxxxx"
    }
  },
  "@version": "1",
  "input": {
    "type": "o365audit"
  },
  "ABS_SIEM_OUTPUT": {
    "index_name": "logs-filebeat.8.9.2-o365",
    "pipeline": "filebeat-8.9.2-o365-audit-pipeline"
  },
  "source": {
    "ip": "31.135.11.111"
  },
  "network": {
    "type": "ipv4"
  },
  "host": {
    "id": "53d83e1d-82ae-4273-84e9-01ec5045dd81",
    "ip": "31.135.11.111",
    "name": "assecobusinesssolutions.onmicrosoft.com"
  },
  "related": {
    "user": "username",
    "ip": "31.135.11.111"
  },
  "@timestamp": "2024-01-30T14:23:40.000Z",
  "service": {
    "type": "o365"
  },
  "client": {
    "ip": "31.135.11.111",
    "address": "31.135.11.111"
  },
  "organization": {
    "id": "53d83e1d-1111-111-84e9-01ec5045dd81",
    "name": "53d83e1d-1111-111-84e9-01ec5045dd81"
  },
  "user": {
    "email": "[email protected]",
    "id": "[email protected]",
    "domain": "domain.pl",
    "name": "username"
  },
  "user_agent": {
    "original": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
  },
  "ecs": {
    "version": "1.12.0"
  },
  "data_stream": {
    "dataset": "filebeat.8.9.2",
    "type": "logs",
    "namespace": "o365"
  },
  "agent": {
    "version": "8.9.2",
    "ephemeral_id": "567e8840-1111-1111-1111-671d671de964",
    "id": "02258696-1111-1111-1111-85404ad17474",
    "name": "filebeat-downloader-1",
    "type": "filebeat"
  },
  "tags": [
    "forwarded",
    "beats",
    "logstash_pipeline_beats",
    "beats_input_raw_event"
  ],
  "fileset": {
    "name": "audit"
  }
}

In other events OperationProperties is list of objects with "Name" and "Value" keys.

[elasticmachine]

Pinging @elastic/security-service-integrations (Team:Security-Service Integrations)

[efd6]

Does this happen for all documents that include that field or only some? Can you conditionally unmarshal the JSON if that field is a string type?

[efd6]

@kowalczyk-p I've sent changes to handle un-parsed JSON in that field. This adds a processor to the ingest pipeline

  - json:
      tag: json-extract-stringly-OperationProperties
      field: o365.audit.OperationProperties
      if: ctx.o365?.audit?.OperationProperties instanceof String
      on_failure:
        - remove:
            field: o365.audit.OperationProperties
        - append:
            field: error.message
            value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'

You can add a custom pipeline that has this processor to address the issue before the fix is released.

However, I think that the issue may be due to logstash (I don't have your config to check this), and in the quoted event, there is a ZWSP ("[{\"Name\":\"SystemArtifactType\",<ZWSP>\"Value\":\"None\"}]") that causes the JSON parser to fail unless that is removed (the test case has had this codepoint removed from the input). The processor above will fail on this, but will note the failure and remove the syntactically invalid JSON string, allowing the event to be ingested. The presence of the ZWSP further suggests there is an upstream issue; possibly in the logstash pipeline?