for supporting JSON logs from Zeek (idaholab#65); getting closer

arkime/etc/config.ini

@@ -1628,6 +1628,11 @@ zeek.smb_files.times_changed=db:zeek.smb_files.times_changed;group:zeek_smb;kind
 zeek.smb_files.data_offset_req=db:zeek.smb_files.data_offset_req;group:zeek_smb;kind:integer;viewerOnly:true;friendly:Data Offset Requested;help:Data Offset Requested
 zeek.smb_files.data_len_req=db:zeek.smb_files.data_len_req;group:zeek_smb;kind:integer;viewerOnly:true;friendly:Data Length Requested;help:Data Length Requested
 zeek.smb_files.data_len_rsp=db:zeek.smb_files.data_len_rsp;group:zeek_smb;kind:integer;viewerOnly:true;friendly:Data Length In Response;help:Data Length In Response
+zeek.smb_files.ts=db:zeek.smb_files.ts;group:zeek_smb;kind:termfield;viewerOnly:true;friendly:Zeek Timestamp for Referenced File;help:Zeek Timestamp for Referenced File
+zeek.smb_files.orig_h=db:zeek.smb_files.orig_h;group:zeek_smb;kind:termfield;viewerOnly:true;friendly:Originating IP for Referenced File;help:Originating IP for Referenced File
+zeek.smb_files.orig_p=db:zeek.smb_files.orig_p;group:zeek_smb;kind:integer;viewerOnly:true;friendly:Originating Port for Referenced File;help:Originating Port for Referenced File
+zeek.smb_files.resp_h=db:zeek.smb_files.resp_h;group:zeek_smb;kind:termfield;viewerOnly:true;friendly:Responding IP for Referenced File;help:Responding IP for Referenced File
+zeek.smb_files.resp_p=db:zeek.smb_files.resp_p;group:zeek_smb;kind:integer;viewerOnly:true;friendly:Responding Port for Referenced File;help:Responding Port for Referenced File
 
 # smb_mapping.log
 # https://docs.zeek.org/en/stable/scripts/base/protocols/smb/main.zeek.html#type-SMB::TreeInfo
@@ -2658,7 +2663,7 @@ o_zeek_s7comm_upload_download=require:zeek.s7comm_upload_download;title:Zeek s7c
 o_zeek_signatures=require:zeek.signatures;title:Zeek signatures.log;fields:event.module,rule.category,rule.name,vulnerability.category,vulnerability.enumeration,vulnerability.id,zeek.signatures.sub_message,zeek.signatures.signature_count,zeek.signatures.host_count
 o_zeek_sip=require:zeek.sip;title:Zeek sip.log;fields:zeek.sip.trans_depth,zeek.sip.method,zeek.sip.uri,zeek.sip.date,zeek.sip.request_from,zeek.sip.request_to,zeek.sip.response_from,zeek.sip.response_to,zeek.sip.reply_to,zeek.sip.call_id,zeek.sip.seq,zeek.sip.subject,zeek.sip.request_path,zeek.sip.response_path,zeek.sip.user_agent,zeek.sip.status_code,zeek.sip.status_msg,zeek.sip.warning,zeek.sip.request_body_len,zeek.sip.response_body_len,zeek.sip.content_type,zeek.sip.version
 o_zeek_smb_cmd=require:zeek.smb_cmd;title:Zeek smb_cmd.log;fields:zeek.smb_cmd.command,zeek.smb_cmd.sub_command,zeek.smb_cmd.argument,zeek.smb_cmd.status,zeek.smb_cmd.rtt,zeek.smb_cmd.version,zeek.smb_cmd.user,zeek.smb_cmd.tree,zeek.smb_cmd.tree_service
-o_zeek_smb_files=require:zeek.smb_files;title:Zeek smb_files.log;fields:zeek.smb_files.action,zeek.smb_files.path,zeek.smb_files.name,zeek.smb_files.size,zeek.smb_files.prev_name,zeek.smb_files.times_modified,zeek.smb_files.times_accessed,zeek.smb_files.times_created,zeek.smb_files.times_changed,zeek.smb_files.data_offset_req,zeek.smb_files.data_len_req,zeek.smb_files.data_len_rsp
+o_zeek_smb_files=require:zeek.smb_files;title:Zeek smb_files.log;fields:zeek.smb_files.action,zeek.smb_files.path,zeek.smb_files.name,zeek.smb_files.size,zeek.smb_files.prev_name,zeek.smb_files.times_modified,zeek.smb_files.times_accessed,zeek.smb_files.times_created,zeek.smb_files.times_changed,zeek.smb_files.data_offset_req,zeek.smb_files.data_len_req,zeek.smb_files.data_len_rsp,zeek.smb_files.ts,zeek.smb_files.orig_h,zeek.smb_files.orig_p,zeek.smb_files.resp_h,zeek.smb_files.resp_p
 o_zeek_smb_mapping=require:zeek.smb_mapping;title:Zeek smb_mapping.log;fields:zeek.smb_mapping.path,zeek.smb_mapping.resource_type,zeek.smb_mapping.native_file_system,zeek.smb_mapping.share_type
 o_zeek_smtp=require:zeek.smtp;title:Zeek smtp.log;fields:zeek.smtp.trans_depth,zeek.smtp.helo,zeek.smtp.mailfrom,zeek.smtp.rcptto,zeek.smtp.date,zeek.smtp.from,zeek.smtp.to,zeek.smtp.cc,zeek.smtp.reply_to,zeek.smtp.msg_id,zeek.smtp.in_reply_to,zeek.smtp.subject,zeek.smtp.x_originating_ip,zeek.smtp.first_received,zeek.smtp.second_received,zeek.smtp.last_reply,zeek.smtp.last_reply_code,zeek.smtp.last_reply_msg,zeek.smtp.path,zeek.smtp.user_agent,zeek.smtp.tls,zeek.smtp.is_webmail
 o_zeek_snmp=require:zeek.snmp;title:Zeek snmp.log;fields:zeek.snmp.duration,zeek.snmp.version,zeek.snmp.community,zeek.snmp.get_requests,zeek.snmp.get_bulk_requests,zeek.snmp.get_responses,zeek.snmp.set_requests,zeek.snmp.display_string,zeek.snmp.up_since

arkime/wise/source.zeeklogs.js

@@ -749,15 +749,15 @@ class MalcolmSource extends WISESource {
       "vulnerability.reference",
       "vulnerability.scanner.vendor",
       "zeek.bacnet.bvlc_function",
-      "zeek.bacnet.invoke_id",
       "zeek.bacnet.instance_number",
+      "zeek.bacnet.invoke_id",
       "zeek.bacnet.pdu_service",
       "zeek.bacnet.pdu_type",
       "zeek.bacnet.result_code",
-      "zeek.bacnet_device_control.time_duration",
       "zeek.bacnet_device_control.device_state",
       "zeek.bacnet_device_control.result",
       "zeek.bacnet_device_control.result_code",
+      "zeek.bacnet_device_control.time_duration",
       "zeek.bacnet_discovery.object_name",
       "zeek.bacnet_discovery.object_type",
       "zeek.bacnet_discovery.range",
@@ -1114,20 +1114,20 @@ class MalcolmSource extends WISESource {
       "zeek.login.success",
       "zeek.modbus.exception",
       "zeek.modbus.func",
+      "zeek.modbus.mei_type",
       "zeek.modbus.network_direction",
       "zeek.modbus.trans_id",
       "zeek.modbus.unit_id",
-      "zeek.modbus.mei_type",
       "zeek.modbus_detailed.address",
       "zeek.modbus_detailed.quantity",
       "zeek.modbus_detailed.values",
       "zeek.modbus_mask_write_register.and_mask",
       "zeek.modbus_mask_write_register.or_mask",
-      "zeek.modbus_read_device_identification.conformity_level_code",
       "zeek.modbus_read_device_identification.conformity_level",
+      "zeek.modbus_read_device_identification.conformity_level_code",
       "zeek.modbus_read_device_identification.device_id_code",
-      "zeek.modbus_read_device_identification.object_id_code",
       "zeek.modbus_read_device_identification.object_id",
+      "zeek.modbus_read_device_identification.object_id_code",
       "zeek.modbus_read_device_identification.object_value",
       "zeek.modbus_read_write_multiple_registers.read_quantity",
       "zeek.modbus_read_write_multiple_registers.read_registers",
@@ -1826,13 +1826,18 @@ class MalcolmSource extends WISESource {
       "zeek.smb_files.data_len_rsp",
       "zeek.smb_files.data_offset_req",
       "zeek.smb_files.name",
+      "zeek.smb_files.orig_h",
+      "zeek.smb_files.orig_p",
       "zeek.smb_files.path",
       "zeek.smb_files.prev_name",
+      "zeek.smb_files.resp_h",
+      "zeek.smb_files.resp_p",
       "zeek.smb_files.size",
       "zeek.smb_files.times_accessed",
       "zeek.smb_files.times_changed",
       "zeek.smb_files.times_created",
       "zeek.smb_files.times_modified",
+      "zeek.smb_files.ts",
       "zeek.smb_mapping.native_file_system",
       "zeek.smb_mapping.path",
       "zeek.smb_mapping.resource_type",

dashboards/templates/composable/component/zeek.json

@@ -407,6 +407,10 @@
         "zeek.smb_files.times_created": { "type": "date" },
         "zeek.smb_files.times_modified": { "type": "date" },
         "zeek.smb_files.ts": { "type": "date" },
+        "zeek.smb_files.orig_h": { "type": "ip" },
+        "zeek.smb_files.orig_p": { "type": "integer" },
+        "zeek.smb_files.resp_h": { "type": "ip" },
+        "zeek.smb_files.resp_p": { "type": "integer" },
         "zeek.smb_mapping.native_file_system": { "type": "keyword" },
         "zeek.smb_mapping.path": { "type": "keyword", "ignore_above": 1024, "fields": { "text": { "type": "text" } } },
         "zeek.smb_mapping.resource_type": { "type": "keyword" },

docs/contributing-logstash.md

@@ -38,8 +38,8 @@ The following modifications must be made in order for Malcolm to parse new Zeek
     * Follow patterns for existing log files as an example
     * For common Zeek fields such as the `id` four-tuple, timestamp, etc., use the same convention used by existing Zeek logs in that file (e.g., `ts`, `uid`, `orig_h`, `orig_p`, `resp_h`, `resp_p`)
     * Take care, especially when copy-pasting filter code, the Zeek delimiter isn't modified from a tab character to a space character (see "*zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP*" warnings in that file)
-1. If necessary, perform log normalization in [`logstash/pipelines/zeek/12_zeek_normalize.conf`]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/logstash/pipelines/zeek/12_zeek_normalize.conf) for values such as action (`event.action`), result (`event.result`), application protocol version (`network.protocol_version`), etc.
-1. If necessary, define conversions for floating point or integer values in [`logstash/pipelines/zeek/11_zeek_parse.conf`]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/logstash/pipelines/zeek/14_zeek_convert.conf)
+1. If necessary, perform log normalization in [`logstash/pipelines/zeek/13_zeek_normalize.conf`]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/logstash/pipelines/zeek/13_zeek_normalize.conf) for values such as action (`event.action`), result (`event.result`), application protocol version (`network.protocol_version`), etc.
+1. If necessary, define conversions for floating point or integer values in [`logstash/pipelines/zeek/14_zeek_convert.conf`]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/logstash/pipelines/zeek/14_zeek_convert.conf)
 1. Identify the new fields and add them as described in [Adding new log fields](contributing-new-log-fields.md#NewFields)
 
 The script [`scripts/zeek_script_to_malcolm_boilerplate.py`]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/scripts/zeek_script_to_malcolm_boilerplate.py) may help by autogenerating these filters.

logstash/pipelines/zeek/12_zeek_mutate.conf

@@ -1644,9 +1644,15 @@ filter {
           }
         }
 
-        # collect referenced file FUID(s) at parent level (here rather than in 12_zeek_normalize.conf because
+        # collect referenced file UIDs(s)/FUID(s) at parent level (here rather than in 13_zeek_normalize.conf because
         # this would have already been done as a root-level fuid array in the main "rename" above if we
         # had not had to move it up a level just now)
+        if ([zeek][smb_files][uid]) {
+          mutate { id => "mutate_merge_zeek_smb_files_uid"
+                   merge => { "[zeek][uid]" => "[zeek][smb_files][uid]" } }
+          mutate { id => "mutate_remove_zeek_smb_files_uid"
+                   remove_field => [ "[zeek][smb_files][uid]" ] }
+        }
         if ([zeek][smb_files][fuid]) {
           mutate { id => "mutate_merge_zeek_smb_files_fuid"
                    merge => { "[zeek][fuid]" => "[zeek][smb_files][fuid]" } }

logstash/pipelines/zeek/13_zeek_normalize.conf

@@ -1450,6 +1450,10 @@ filter {
                                                          merge => { "[related][ip]" => "[zeek][ospf][fwd_addrs]" } } }
   if ([zeek][ospf][dest_router_id]) {           mutate { id => "mutate_merge_field_related_ip_zeek_ospf_dest_router_id"
                                                          merge => { "[related][ip]" => "[zeek][ospf][dest_router_id]" } } }
+  if ([zeek][smb_files][orig_h]) {              mutate { id => "mutate_merge_field_related_ip_zeek_smb_files_orig_h"
+                                                         merge => { "[related][ip]" => "[zeek][smb_files][orig_h]" } } }
+  if ([zeek][smb_files][resp_h]) {              mutate { id => "mutate_merge_field_related_ip_zeek_smb_files_resp_h"
+                                                         merge => { "[related][ip]" => "[zeek][smb_files][resp_h]" } } }
   if ([zeek][radius][framed_addr]) {            mutate { id => "mutate_merge_field_related_ip_zeek_radius_framed_addr"
                                                          merge => { "[related][ip]" => "[zeek][radius][framed_addr]" } } }
   if ([zeek][smtp][path]) {                     mutate { id => "mutate_merge_field_related_ip_zeek_smtp_path"