ROX-27688: Add JSON aggregator lua function to the audit log aggregat…

…or (#2160)

dp-terraform/helm/rhacs-terraform/charts/audit-logs/templates/02-configmap.yaml

@@ -14,3 +14,42 @@ data:
   vector.yaml: |
 {{ tpl (toYaml .Values.customConfig) . | indent 4 }}
   {{- end }}
+  json-aggregator.lua: |
+    -- Copied from https://github.com/vectordotdev/vector/issues/4952
+    -- This lua function will aggregate valid multiline JSON events
+    -- see: https://vector.dev/guides/advanced/merge-multiline-logs-with-lua/
+    function process(event, emit)
+      if merged_event == nil then -- a global variable containing the merged event
+        merged_event = event -- if it is empty, set it to the current event
+      else -- otherwise, concatenate the line in the stored merged event with the next line
+        merged_event = safe_merge(merged_event, event)
+        if not merged_event then
+          return
+        end
+      end
+
+      -- Count brackets to handle top level JSON arrays
+      local _, b1 = merged_event.log.message:gsub("%[","")
+      local _, b2 = merged_event.log.message:gsub("%]","")
+      if b1 > b2 then
+        return -- continue to merge events until JSON document is complete
+      end
+      -- Count curly braces to handle JSON document
+      local _, c1 = merged_event.log.message:gsub("{","")
+      local _, c2 = merged_event.log.message:gsub("}","")
+      if c1 > c2 then
+        return -- continue to merge events until JSON document is complete
+      end
+
+      emit(merged_event) -- emit the resulting event
+      merged_event = nil -- clear the merged event
+    end
+
+    function safe_merge(merged_event, event)
+      if #merged_event.log.message + #event.log.message > 10240 then
+        return nil
+      else
+        merged_event.log.message = merged_event.log.message .. event.log.message
+        return merged_event
+      end
+    end

dp-terraform/helm/rhacs-terraform/charts/audit-logs/values.yaml

@@ -56,9 +56,16 @@ customConfig:
       encoding:
         codec: "json"
   transforms:
+    aggregator:
+      type: "lua"
+      version: "2"
+      inputs: [ "http_server" ]
+      hooks:
+        process: "process"
+      source: "require('json-aggregator')"
     conversion:
       type: "remap"
-      inputs: ["http_server"]
+      inputs: ["aggregator"]
       source: ". = parse_json!(string!(.message))"
 
 # Secrets used to set environment variables for Vector pod.