Style Automation

.flake8

@@ -0,0 +1,3 @@
+[flake8]
+max-line-length = 88
+extend-ignore = E203, E501, W503

.github/workflows/style_automation.yml

@@ -0,0 +1,27 @@
+---
+name: Style Automation
+on:
+  push:
+  pull_request:
+    branches:
+      - master
+jobs:
+  style:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install black mypy flake8 isort
+
+      - name: Code quality checks
+        run: |
+          python -m black --extend-exclude .*_pb2.*\.py --check --diff src/python
+          python -m flake8 --exclude src/python/build/,src/python/strelka/proto/ src/python
+          python -m isort --extend-skip-glob *_pb2*.py --profile black --check-only src/python

.pre-commit-config.yaml

@@ -0,0 +1,33 @@
+---
+repos:
+  - repo: https://github.com/psf/black
+    rev: "22.6.0"
+    hooks:
+      - id: black
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.3.0
+    hooks:
+      - id: check-yaml
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+      - id: check-added-large-files
+      - id: check-merge-conflict
+      - id: mixed-line-ending
+      - id: no-commit-to-branch
+        args:
+          - -b main
+  - repo: https://github.com/PyCQA/flake8
+    rev: "4.0.1"
+    hooks:
+      - id: flake8
+  - repo: https://github.com/PyCQA/isort
+    rev: "5.10.1"
+    hooks:
+      - id: isort
+        args: ["--profile", "black", "--filter-files"]
+#  - repo: https://github.com/pre-commit/mirrors-mypy
+#    rev: v0.961
+#    hooks:
+#      - id: mypy
+#        additional_dependencies:
+#          - types-requests

CONTRIBUTING.md

@@ -14,6 +14,46 @@ Bugs should be submitted as issues using the bug report template.
 Enhancements should be submitted as issues using the feature request template.
 
 ## Development Environment
+
+Clone the repo
+
+```
+git clone https://github.com/target/strelka.git
+```
+
+Python should be set up to use a virtualenv.
+
+```
+cd strelka/
+python -m venv env
+```
+
+Activate the virtualenv
+
+```
+source env/bin/activate
+```
+
+Install build requirements
+
+```
+cd src/python/
+pip install -r requirements.txt
+```
+
+Install strelka
+
+```
+python setup.py install
+```
+
+Install pre-commit hooks
+
+```
+pre-commit install
+```
+
+
 Development builds can tested using the default docker-compose.yaml file (`build/docker-compose.yaml`). To bring the project up with docker-compse, use the following command as a template:
     ```bash
     docker-compose -f build/docker-compose.yaml --project-name strelka up
@@ -76,8 +116,11 @@ tests_configuration/test_taste.py ..............................................
 ```
 
 ## Style Guides
+
 ### Python
-Python code should attempt to adhere as closely to [PEP8](https://www.python.org/dev/peps/pep-0008/) as possible. We may ask authors to refactor code for better PEP8 compliance, but we do not enforce 100% compliance.
+Python code should attempt to adhere as closely to [PEP8](https://www.python.org/dev/peps/pep-0008/) as possible.
+
+Conformance is ensured using `black`, `isort`, `flake8`, and `mypy` in pre-commit hooks and CI/CD actions. Pushes and PRs may be automatically rejected due to non-conformance. Review errors/warnings from the style modules for tips.
 
 ### Scanners
 * Write event data in snake_case JSON format

build/python/backend/requirements.txt

@@ -28,6 +28,7 @@ opencv-python==4.6.0.66
 openpyxl==3.0.9
 pefile==2022.5.30
 pgpdump3==1.5.2
+pre-commit==3.0.1
 py-tlsh==4.7.2
 pycdlib==1.13.0
 pyelftools==0.28

src/python/bin/strelka-backend

@@ -8,10 +8,10 @@ import argparse
 import logging.config
 import os
 import sys
+
 import redis
 import yaml
 
-
 from strelka import strelka
 
 

src/python/bin/strelka-mmrpc

@@ -1,14 +1,12 @@
 #! /usr/bin/env python3
 import argparse
-from concurrent import futures
 import json
 import time
+from concurrent import futures
 
 import grpc
 from mmbot import mmbot
-
-from strelka.proto import mmbot_pb2
-from strelka.proto import mmbot_pb2_grpc
+from strelka.proto import mmbot_pb2, mmbot_pb2_grpc
 
 
 class MmbotServicer(mmbot_pb2_grpc.MmbotServicer):

src/python/requirements.txt

@@ -28,6 +28,7 @@ opencv-python==4.6.0.66
 openpyxl==3.0.9
 pefile==2022.5.30
 pgpdump3==1.5.2
+pre-commit==3.0.1
 py-tlsh==4.7.2
 pycdlib==1.13.0
 pyelftools==0.28

src/python/setup.py

@@ -2,16 +2,16 @@
 import setuptools
 
 setuptools.setup(
-    name='strelka',
-    author='Target Brands, Inc.',
-    description='strelka: container-based file analysis at scale',
-    license='Apache 2.0',
+    name="strelka",
+    author="Target Brands, Inc.",
+    description="strelka: container-based file analysis at scale",
+    license="Apache 2.0",
     packages=setuptools.find_packages(),
-    scripts=['bin/strelka-backend', 'bin/strelka-mmrpc'],
+    scripts=["bin/strelka-backend", "bin/strelka-mmrpc"],
     zip_safe=False,
     entry_points={
-        'console_scripts': [
-            'strelka = strelka.__main__:main',
+        "console_scripts": [
+            "strelka = strelka.__main__:main",
         ]
-    }
+    },
 )

src/python/strelka/__main__.py

@@ -3,10 +3,10 @@
 import os
 import sys
 import time
+from importlib.resources import files
 
 import yaml
 
-from importlib.resources import files
 from strelka import strelka
 
 
@@ -24,24 +24,24 @@ def main():
 
     if os.path.exists("/etc/strelka/backend.yaml"):
         backend_cfg_path = "/etc/strelka/backend.yaml"
-    elif os.path.exists(files('strelka.config').joinpath('backend.yaml')):
-        backend_cfg_path = files('strelka.config').joinpath('backend.yaml')
+    elif os.path.exists(files("strelka.config").joinpath("backend.yaml")):
+        backend_cfg_path = files("strelka.config").joinpath("backend.yaml")
     else:
         logging.exception("no backend configuration found")
         sys.exit(1)
 
     if os.path.exists("/etc/strelka/taste/taste.yara"):
         taste_path = "/etc/strelka/taste/taste.yara"
-    elif os.path.exists(files('strelka.config').joinpath('taste.yara')):
-        taste_path = str(files('strelka.config').joinpath('taste.yara'))
+    elif os.path.exists(files("strelka.config").joinpath("taste.yara")):
+        taste_path = str(files("strelka.config").joinpath("taste.yara"))
     else:
         logging.exception("no taste path found")
         sys.exit(1)
 
     if os.path.exists("/etc/strelka/yara/rules.yara"):
         yara_rules_path = "/etc/strelka/yara/rules.yara"
-    elif os.path.exists(files('strelka.config').joinpath('rules.yara')):
-        yara_rules_path = str(files('strelka.config').joinpath('rules.yara'))
+    elif os.path.exists(files("strelka.config").joinpath("rules.yara")):
+        yara_rules_path = str(files("strelka.config").joinpath("rules.yara"))
     else:
         logging.exception("no yara rules path found")
         sys.exit(1)

src/python/strelka/auxiliary/xl4ma/analyzer.py

@@ -5,11 +5,12 @@
 import os
 import tempfile
 from pathlib import Path
+
 from strelka.auxiliary.xl4ma.extract import iocs
+from strelka.auxiliary.xl4ma.xl4decoder import decode
 from strelka.auxiliary.xl4ma.xls_wrapper import XLSWrapper
 from strelka.auxiliary.xl4ma.xlsb_wrapper import XLSBWrapper
 from strelka.auxiliary.xl4ma.xlsm_wrapper import XLSMWrapper
-from strelka.auxiliary.xl4ma.xl4decoder import decode
 
 
 def _make_temp_file(data, file_type):

src/python/strelka/auxiliary/xl4ma/extract.py

@@ -9,11 +9,15 @@
 def iocs(excel_doc_decoded):
     extracted = set()
     for decoded in excel_doc_decoded:
-        if url := re.findall('(https?://[A-Za-z0-9-._]+/[A-Za-z0-9-._~:/?#\[\]@!$&\'\(\)*+,;%=]+[^,\s\)])', decoded, flags=re.IGNORECASE):
+        if url := re.findall(
+            r"(https?://[A-Za-z0-9-._]+/[A-Za-z0-9-._~:/?#\[\]@!$&'\(\)*+,;%=]+[^,\s\)])",
+            decoded,
+            flags=re.IGNORECASE,
+        ):
             scheme, netloc, path, params, query, fragment = urlparse(url[0])
-            if netloc.startswith('0x'):
+            if netloc.startswith("0x"):
                 netloc = socket.inet_ntoa(struct.pack(">L", int(netloc, 16)))
-            if netloc.startswith('0o'):
+            if netloc.startswith("0o"):
                 netloc = socket.inet_ntoa(struct.pack(">L", int(netloc, 8)))
             extracted.add(f"{scheme}://{netloc}{path}")
 

src/python/strelka/auxiliary/xl4ma/xl4decoder.py

@@ -1,12 +1,13 @@
 # Authors: Ryan Borre
 
 import tempfile
+from os import devnull
+
 import xlrd2
 from openpyxl.workbook import Workbook
 from pyxlsb2 import open_workbook
 from pyxlsb2.formula import Formula
 from pyxlsb2.records import ErrorValue
-from os import devnull
 from strelka.auxiliary.xl4ma.xl4interpreter import Interpreter
 
 
@@ -47,7 +48,7 @@ def _decode_xls(file_path, defined_names):
                         book_sheet.cell(
                             row + 1, col + 1, wb[sheet_name].cell(row, col).value
                         )
-                except:
+                except Exception:
                     pass
     temp_file = tempfile.NamedTemporaryFile(suffix=".xlsx", delete=False)
     book.save(temp_file.name)
@@ -96,7 +97,7 @@ def _decode_xlsb(file_path, defined_names):
                                 cell.col + 1,
                                 str(cell.value).rstrip("\x00"),
                             )
-        except:
+        except Exception:
             pass
     temp_file = tempfile.NamedTemporaryFile(suffix=".xlsx", delete=False)
     book.save(temp_file.name)
@@ -107,7 +108,7 @@ def _decode_xlsb(file_path, defined_names):
 
 # XLSM
 def _decode_xlsm(file_path, defined_names):
-    with tempfile.NamedTemporaryFile(suffix=f".xlsm", delete=False) as temp_file, open(
+    with tempfile.NamedTemporaryFile(suffix=".xlsm", delete=False) as temp_file, open(
         file_path, "rb"
     ) as fp:
         temp_file.write(fp.read())

src/python/strelka/auxiliary/xl4ma/xl4interpreter.py

@@ -1,9 +1,10 @@
 # Authors: Ryan Borre
 
 import logging
-import formulas
 import os
 
+import formulas
+
 
 class Interpreter:
     def __init__(self, defined_names):
@@ -159,7 +160,7 @@ def calculate(self, temp_file):
                 if isinstance(result, str):
                     self.results.add(str(result))
 
-        except:
+        except Exception:
             logging.info("formula error")
 
         temp_file.close()

src/python/strelka/auxiliary/xl4ma/xls_wrapper.py

@@ -1,10 +1,11 @@
 # Authors: Ryan Borre
 
-import xlrd
-import xlrd2
 from enum import Enum
 from os import devnull
 
+import xlrd
+import xlrd2
+
 
 class VISIBILITY(Enum):
     VISIBLE = 0
@@ -73,7 +74,7 @@ def parse_sheets(self, file_path):
                                     .value,
                                 }
                             )
-                    except:
+                    except Exception:
                         pass
 
             results["sheets"].append(

src/python/strelka/auxiliary/xl4ma/xlsb_wrapper.py

@@ -9,7 +9,7 @@ class XLSBWrapper:
     def __init__(self, file_path):
         try:
             self.workbook = open_workbook(file_path)
-        except Exception as e:
+        except Exception:
             return
 
     def get_defined_names(self):
@@ -54,7 +54,7 @@ def parse_sheets(self, file_path):
                                     "value": cell.value,
                                 }
                             )
-            except:
+            except Exception:
                 pass
             results["sheets"].append(
                 {