Merge pull request #885 from MaibornWolff/dev

chore: merge for release 1.2.0

.github/workflows/build_push_release.yml

@@ -3,6 +3,7 @@ name: Build and push release images
 on:
   workflow_dispatch:
     inputs:
+      #checkov:skip=CKV_GHA_7:This is a false positive
       release:
         description: 'SecObserve release (without the v)'
         required: true

.github/workflows/check_backend.yml

@@ -4,6 +4,10 @@ on: [push]
 
 permissions: read-all
 
+
+env:
+  POETRY_NO_INTERACTION: 1
+
 jobs:
   build:
 
@@ -12,30 +16,30 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: Set up Python 3.11
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: 3.11
 
       - name: Install dependencies
         working-directory: ./backend
         run: |
-          python -m pip install --upgrade pip
-          python -m pip install --upgrade -r requirements/dev.txt
+          python -m pip install --upgrade poetry
+          python -m poetry install  --without prod,unittests --no-root
 
       - name: Flake8
         working-directory: ./backend
         run: |
-          flake8 . --count --show-source --statistics
+          poetry run flake8 . --count --show-source --statistics
 
       - name: Black
         working-directory: ./backend
         run: |
-          black . --check
+          poetry run black . --check
 
       - name: isort
         working-directory: ./backend
         run: |
-          isort . -c
+          poetry run isort . -c
 
       - name: MyPy
         working-directory: ./backend

.github/workflows/publish_docs.yml

@@ -14,7 +14,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: 3.x
       - uses: actions/cache@v3

backend/application/__init__.py

@@ -1 +1 @@
-__version__ = "1.1.1"
+__version__ = "1.2.0"

backend/application/core/api/views.py

@@ -216,9 +216,8 @@ def apply_rules(self, request, pk=None):
         product = self.__get_product(pk)
         user_has_permission_or_403(product, Permissions.Product_Rule_Apply)
 
-        for parser in Parser.objects.all():
-            rule_engine = Rule_Engine(product, parser)
-            rule_engine.apply_all_rules_for_product_and_parser()
+        rule_engine = Rule_Engine(product)
+        rule_engine.apply_all_rules_for_product()
 
         return Response(status=HTTP_204_NO_CONTENT)
 

backend/application/import_observations/parsers/cyclone_dx/parser.py

@@ -375,16 +375,6 @@ def _get_dependencies(
     #         ):
     #             i += 1
 
-    #             if list_pointer == 0:
-    #                 print("---------------------------------")
-    #                 print(f"i: {i}")
-    #                 print(f"list_pointer: {list_pointer}")
-    #                 print(f"ref: {ref}")
-    #                 print(f"component_version: {component_version}")
-    #                 print(f"depends_on: {depends_on}")
-    #                 print(f"dependencies: {dependencies}")
-    #                 print(f"current_dependency: {current_dependency}")
-
     #             if len(dependencies) <= i:
     #                 if not dependencies or not current_dependency:
     #                     dependencies.append(f"{ref} --> {component_version}")

backend/application/import_observations/services/import_observations.py

@@ -196,9 +196,7 @@ def process_data(import_parameters: ImportParameters) -> Tuple[int, int, int, st
 
     scanner = ""
 
-    rule_engine = Rule_Engine(
-        product=import_parameters.product, parser=import_parameters.parser
-    )
+    rule_engine = Rule_Engine(product=import_parameters.product)
 
     # Read current observations for the same vulnerability check, to find updated and resolved observations
     observations_before: dict[str, Observation] = {}

backend/application/rules/api/serializers.py

@@ -9,6 +9,12 @@ class Meta:
         model = Rule
         exclude = ["product"]
 
+    def validate(self, attrs):
+        if not attrs.get("parser") and not attrs.get("scanner_prefix"):
+            raise ValidationError("Either Parser or Scanner Prefix must be set")
+
+        return super().validate(attrs)
+
 
 class ProductRuleSerializer(ModelSerializer):
     product_data = NestedProductSerializer(source="product", read_only=True)
@@ -23,3 +29,9 @@ def validate_product(self, value):
             raise ValidationError("Product cannot be changed")
 
         return value
+
+    def validate(self, attrs):
+        if not attrs.get("parser") and not attrs.get("scanner_prefix"):
+            raise ValidationError("Either Parser or Scanner Prefix must be set")
+
+        return super().validate(attrs)

backend/application/rules/migrations/0008_rule_description_observation_alter_rule_parser.py

@@ -0,0 +1,26 @@
+# Generated by Django 4.2.8 on 2023-12-17 14:56
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("core", "0025_observation_origin_component_dependencies"),
+        ("rules", "0007_indices"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="rule",
+            name="description_observation",
+            field=models.CharField(blank=True, max_length=255),
+        ),
+        migrations.AlterField(
+            model_name="rule",
+            name="parser",
+            field=models.ForeignKey(
+                null=True, on_delete=django.db.models.deletion.CASCADE, to="core.parser"
+            ),
+        ),
+    ]

backend/application/rules/models.py

@@ -15,9 +15,10 @@ class Rule(Model):
     name = CharField(max_length=255)
     description = TextField(max_length=2048, blank=True)
     product = ForeignKey(Product, blank=True, null=True, on_delete=CASCADE)
-    parser = ForeignKey(Parser, on_delete=CASCADE)
+    parser = ForeignKey(Parser, null=True, on_delete=CASCADE)
     scanner_prefix = CharField(max_length=255, blank=True)
     title = CharField(max_length=255, blank=True)
+    description_observation = CharField(max_length=255, blank=True)
     origin_component_name_version = CharField(max_length=513, blank=True)
     origin_docker_image_name_tag = CharField(max_length=513, blank=True)
     origin_endpoint_url = TextField(max_length=2048, blank=True)

backend/application/rules/services/rule_engine.py

@@ -2,7 +2,7 @@
 from typing import Optional
 
 from application.commons.services.global_request import get_current_user
-from application.core.models import Observation, Parser, Product
+from application.core.models import Observation, Product
 from application.core.services.observation import (
     get_current_severity,
     get_current_status,
@@ -15,28 +15,22 @@
 
 
 class Rule_Engine:
-    def __init__(self, product: Product, parser: Parser):
-        product_parser_rules = Rule.objects.filter(
-            product=product, parser=parser, enabled=True
-        )
+    def __init__(self, product: Product):
+        product_parser_rules = Rule.objects.filter(product=product, enabled=True)
         self.rules: list[Rule] = list(product_parser_rules)
 
         if product.product_group:
             product_group_parser_rules = Rule.objects.filter(
                 product=product.product_group,
-                parser=parser,
                 enabled=True,
             )
             self.rules += list(product_group_parser_rules)
 
         if product.apply_general_rules:
-            parser_rules = Rule.objects.filter(
-                product__isnull=True, parser=parser, enabled=True
-            )
-            self.rules += list(parser_rules)
+            general_rules = Rule.objects.filter(product__isnull=True, enabled=True)
+            self.rules += list(general_rules)
 
         self.product = product
-        self.parser = parser
 
     def apply_rules_for_observation(self, observation: Observation) -> None:
         previous_product_rule = None
@@ -51,14 +45,17 @@ def apply_rules_for_observation(self, observation: Observation) -> None:
         rule_found = False
         for rule in self.rules:
             if (  # pylint: disable=too-many-boolean-expressions
-                observation.parser == rule.parser
+                (not rule.parser or observation.parser == rule.parser)
                 and (
                     not rule.scanner_prefix
                     or observation.scanner.lower().startswith(
                         rule.scanner_prefix.lower()
                     )
                 )
                 and self._check_regex(rule.title, observation.title)
+                and self._check_regex(
+                    rule.description_observation, observation.description
+                )
                 and self._check_regex(
                     rule.origin_component_name_version,
                     observation.origin_component_name_version,
@@ -119,10 +116,14 @@ def apply_rules_for_observation(self, observation: Observation) -> None:
                 observation, previous_product_rule, previous_general_rule
             )
 
-    def apply_all_rules_for_product_and_parser(self) -> None:
-        for observation in Observation.objects.filter(
-            product=self.product, parser=self.parser
-        ):
+    def apply_all_rules_for_product(self) -> None:
+        if self.product.is_product_group:
+            products = Product.objects.filter(product_group=self.product)
+            observations = Observation.objects.filter(product__in=products)
+        else:
+            observations = Observation.objects.filter(product=self.product)
+
+        for observation in observations:
             self.apply_rules_for_observation(observation)
 
     def _check_regex(self, pattern: str, value: str) -> bool:

backend/bin/run_mypy.sh

@@ -27,4 +27,8 @@ export OIDC_EMAIL=dummy
 # --- Huey ---
 export HUEY_FILENAME=./huey.db
 
-mypy application
+if [ -z "$GITHUB_WORKSPACE" ]; then 
+    mypy application
+else 
+    poetry run mypy application
+fi

backend/bin/run_pylint.sh

@@ -27,4 +27,8 @@ export OIDC_EMAIL=dummy
 # --- Huey ---
 export HUEY_FILENAME=./huey.db
 
-pylint application
+if [ -z "$GITHUB_WORKSPACE" ]; then 
+    pylint application
+else 
+    poetry run pylint application
+fi

backend/config/settings/dist.py

@@ -11,7 +11,10 @@
     default="NxYPEF5lNGgk3yonndjSbwP77uNJxOvfKTjF5aVBqsHktNlf1wfJHHvJ8iifk32r",
 )
 # https://docs.djangoproject.com/en/dev/ref/settings/#allowed-hosts
-ALLOWED_HOSTS = env("ALLOWED_HOSTS", default=["localhost", "0.0.0.0", "127.0.0.1"])
+ALLOWED_HOSTS = env(
+    "ALLOWED_HOSTS", default=["localhost", "0.0.0.0", "127.0.0.1"]  # nosec B104
+)
+# This file is not used for production
 
 # CACHES
 # ------------------------------------------------------------------------------