for idaholab#74, auto-gen Zeek intel files based on STIX

Dockerfiles/zeek.Dockerfile

@@ -126,7 +126,7 @@ RUN apt-get -q update && \
     mkdir -p "${ZEEK_DIR}"/var/lib/zkg/clones/package/spicy-plugin/plugin/lib/ && \
       ln -s -r "${ZEEK_DIR}"/lib/zeek/plugins/packages/spicy-plugin/lib/bif \
                "${ZEEK_DIR}"/var/lib/zkg/clones/package/spicy-plugin/plugin/lib/bif && \
-    mkdir -p "${ZEEK_DIR}"/share/zeek/site/intel && \
+    mkdir -p "${ZEEK_DIR}"/share/zeek/site/intel/STIX && \
         touch "${ZEEK_DIR}"/share/zeek/site/intel/__load__.zeek && \
     cd /usr/lib/locale && \
       ( ls | grep -Piv "^(en|en_US|en_US\.utf-?8|C\.utf-?8)$" | xargs -l -r rm -rf ) && \
@@ -138,7 +138,7 @@ RUN apt-get -q update && \
 ADD shared/bin/docker-uid-gid-setup.sh /usr/local/bin/
 ADD shared/bin/pcap_arkime_and_zeek_processor.py /usr/local/bin/
 ADD shared/bin/pcap_utils.py /usr/local/bin/
-ADD docs/stix/*.py /usr/local/bin/
+ADD shared/bin/stix*.py ${ZEEK_DIR}/bin/
 ADD shared/pcaps /tmp/pcaps
 ADD zeek/supervisord.conf /etc/supervisord.conf
 ADD zeek/config/*.zeek ${ZEEK_DIR}/share/zeek/site/

README.md

@@ -82,6 +82,7 @@ In short, Malcolm provides an easily deployable network analysis tool suite for
     - [Event severity scoring](#Severity)
         + [Customizing event severity scoring](#SeverityConfig)
     - [Zeek Intelligence Framework](#ZeekIntel)
+        + [STIX™](#ZeekIntelSTIX)
     - [Alerting](#Alerting)
     - ["Best Guess" Fingerprinting for ICS Protocols](#ICSBestGuess)
     - [API](#API)
@@ -265,7 +266,7 @@ Malcolm leverages the following excellent open source tools, among others.
     * J-Gras' [Zeek::AF_Packet](https://github.com/J-Gras/zeek-af_packet-plugin) plugin
     * Johanna Amann's [CVE-2020-0601](https://github.com/0xxon/cve-2020-0601) ECC certificate validation plugin and [CVE-2020-13777](https://github.com/0xxon/cve-2020-13777) GnuTLS unencrypted session ticket detection plugin
     * Lexi Brent's [EternalSafety](https://github.com/0xl3x1/zeek-EternalSafety) plugin
-    * MITRE Cyber Analytics Repository's [Bro/Zeek ATT&CK-Based Analytics (BZAR)](https://github.com/mitre-attack/car/tree/master/implementations) script
+    * MITRE Cyber Analytics Repository's [Bro/Zeek ATT&CK®-Based Analytics (BZAR)](https://github.com/mitre-attack/car/tree/master/implementations) script
     * Salesforce's [gQUIC](https://github.com/salesforce/GQUIC_Protocol_Analyzer) analyzer
     * Salesforce's [HASSH](https://github.com/salesforce/hassh) SSH fingerprinting plugin
     * Salesforce's [JA3](https://github.com/salesforce/ja3) TLS fingerprinting plugin
@@ -566,7 +567,7 @@ Various other environment variables inside of `docker-compose.yml` can be tweake
 
 * `EXTRACTED_FILE_ENABLE_CAPA` – if set to `true`, [Zeek-extracted files](#ZeekFileExtraction) that are determined to be PE (portable executable) files will be scanned with [Capa](https://github.com/fireeye/capa)
 
-* `EXTRACTED_FILE_CAPA_VERBOSE` – if set to `true`, all Capa rule hits will be logged; otherwise (`false`) only [MITRE ATT&CK technique](https://attack.mitre.org/techniques) classifications will be logged
+* `EXTRACTED_FILE_CAPA_VERBOSE` – if set to `true`, all Capa rule hits will be logged; otherwise (`false`) only [MITRE ATT&CK® technique](https://attack.mitre.org/techniques) classifications will be logged
 
 * `EXTRACTED_FILE_ENABLE_CLAMAV` – if set to `true`, [Zeek-extracted files](#ZeekFileExtraction) will be scanned with [ClamAV](https://www.clamav.net/)
 
@@ -1482,6 +1483,12 @@ Note that Malcolm does not manage updates for these intelligence files. You shou
 docker-compose exec --user $(id -u) zeek /usr/local/bin/entrypoint.sh true
 ```
 
+#### <a name="ZeekIntelSTIX"></a>STIX™
+
+In addition to loading Zeek intelligence files, Malcolm will automatically generate Zeek intelligence files for all [Structured Threat Information Expression (STIX™)](https://oasis-open.github.io/cti-documentation/stix/intro.html) v2.0/v2.1 JSON files found under `./zeek/intel/STIX`.
+
+Note that only indicators of [cyber-observable objects](https://docs.oasis-open.org/cti/stix/v2.1/cs01/stix-v2.1-cs01.html#_mlbmudhl16lr) matched with the equals (`=`) [comparison operator](https://docs.oasis-open.org/cti/stix/v2.1/cs01/stix-v2.1-cs01.html#_t11hn314cr7w) against a single value can be expressed as Zeek intelligence items. More complex STIX indicators will be silently ignored.
+
 ### <a name="Alerting"></a>Alerting
 
 See [Alerting](https://opensearch.org/docs/latest/monitoring-plugins/alerting/index/) in the OpenSearch documentation.

malcolm-iso/build.sh

@@ -90,7 +90,7 @@ if [ -d "$WORKDIR" ]; then
   mkdir -p "$MALCOLM_DEST_DIR/pcap/processed/"
   mkdir -p "$MALCOLM_DEST_DIR/scripts/"
   mkdir -p "$MALCOLM_DEST_DIR/yara/rules/"
-  mkdir -p "$MALCOLM_DEST_DIR/zeek/intel/"
+  mkdir -p "$MALCOLM_DEST_DIR/zeek/intel/STIX"
   mkdir -p "$MALCOLM_DEST_DIR/zeek-logs/current/"
   mkdir -p "$MALCOLM_DEST_DIR/zeek-logs/upload/"
   mkdir -p "$MALCOLM_DEST_DIR/zeek-logs/processed/"

scripts/malcolm_appliance_packager.sh

@@ -74,7 +74,7 @@ if mkdir "$DESTDIR"; then
   mkdir $VERBOSE -p "$DESTDIR/pcap/upload/"
   mkdir $VERBOSE -p "$DESTDIR/pcap/processed/"
   mkdir $VERBOSE -p "$DESTDIR/yara/rules/"
-  mkdir $VERBOSE -p "$DESTDIR/zeek/intel/"
+  mkdir $VERBOSE -p "$DESTDIR/zeek/intel/STIX"
   mkdir $VERBOSE -p "$DESTDIR/zeek-logs/current/"
   mkdir $VERBOSE -p "$DESTDIR/zeek-logs/upload/"
   mkdir $VERBOSE -p "$DESTDIR/zeek-logs/processed/"

sensor-iso/config/hooks/normal/0910-sensor-build.hook.chroot

@@ -104,9 +104,10 @@ EOF
 
 # set up default zeek local policy
 cp -f /usr/local/etc/zeek/*.zeek /usr/local/etc/zeek/*.txt "${ZEEK_DIR}"/share/zeek/site/
-mkdir -p /opt/sensor/sensor_ctl/zeek/intel
+mkdir -p /opt/sensor/sensor_ctl/zeek/intel/STIX
 touch /opt/sensor/sensor_ctl/zeek/intel/__load__.zeek
 [[ -f /usr/local/bin/zeek_intel_setup.sh ]] && mv /usr/local/bin/zeek_intel_setup.sh "${ZEEK_DIR}"/bin/
+[[ -f /usr/local/bin/stix_to_zeek_intel.py ]] && mv /usr/local/bin/stix*.py "${ZEEK_DIR}"/bin/
 
 # cleanup
 cd /usr/local/src

sensor-iso/config/includes.chroot/opt/zeek/bin/zeekdeploy.sh

@@ -77,7 +77,7 @@ ZEEK_EXTRACTOR_SCRIPT="$ZEEK_INSTALL_PATH"/share/zeek/site/"$EXTRACTOR_ZEEK_SCRI
 
 # make sure "intel" directory exists, even if empty
 export INTEL_DIR=/opt/sensor/sensor_ctl/zeek/intel
-mkdir -p "$INTEL_DIR"
+mkdir -p "$INTEL_DIR"/STIX
 touch "$INTEL_DIR"/__load__.zeek
 # autoconfigure load directives for intel files
 [[ -x "$ZEEK_INSTALL_PATH"/bin/zeek_intel_setup.sh ]] && "$ZEEK_INSTALL_PATH"/bin/zeek_intel_setup.sh /bin/true

shared/bin/zeek_intel_setup.sh

@@ -13,6 +13,7 @@ ENCODING="utf-8"
 
 ZEEK_DIR=${ZEEK_DIR:-"/opt/zeek"}
 INTEL_DIR=${INTEL_DIR:-"${ZEEK_DIR}/share/zeek/site/intel"}
+STIX_TO_ZEEK_SCRIPT=${STIX_TO_ZEEK_SCRIPT:-"${ZEEK_DIR}/bin/stix_to_zeek_intel.py"}
 
 # create directive to @load every subdirectory in /opt/zeek/share/zeek/site/intel
 if [[ -d "${INTEL_DIR}" ]] && (( $(find "${INTEL_DIR}" -mindepth 1 -maxdepth 1 -type d 2>/dev/null | wc -l) > 0 )); then
@@ -27,10 +28,17 @@ if [[ -d "${INTEL_DIR}" ]] && (( $(find "${INTEL_DIR}" -mindepth 1 -maxdepth 1 -
 
 EOF
     LOOSE_INTEL_FILES=()
+    STIX_JSON_FILES=()
 
     # process subdirectories under INTEL_DIR
     for DIR in $(find . -mindepth 1 -maxdepth 1 -type d 2>/dev/null); do
-        if [[ -f "${DIR}"/__load__.zeek ]]; then
+        if [[ "${DIR}" == "./STIX" ]]; then
+            # this directory contains STIX JSON files we'll need to convert to zeek intel files then load
+            while IFS= read -r line; do
+                STIX_JSON_FILES+=( "$line" )
+            done < <( find "${INTEL_DIR}/${DIR}" -type f ! -name ".*" 2>/dev/null )
+
+        elif [[ -f "${DIR}"/__load__.zeek ]]; then
             # this intel feed has its own load directive and should take care of itself
             echo "@load ${DIR}" >> __load__.zeek
         else
@@ -41,6 +49,14 @@ EOF
         fi
     done
 
+    # process STIX JSON files by converting them to Zeek intel format
+    if (( ${#STIX_JSON_FILES[@]} )) && [[ -f "${STIX_TO_ZEEK_SCRIPT}" ]]; then
+        "${STIX_TO_ZEEK_SCRIPT}" -i "${STIX_JSON_FILES[@]}" >./stix_autogen.zeek 2>/dev/null
+        LOOSE_INTEL_FILES+=( "${INTEL_DIR}/stix_autogen.zeek" )
+    else
+        rm -f ./stix_autogen.zeek
+    fi
+
     # explicitly load all of the "loose" intel files in other subdirectories that didn't __load__.zeek themselves
     if (( ${#LOOSE_INTEL_FILES[@]} )); then
         echo >> __load__.zeek