Fix/reuse report ooi entities

boefjes/boefjes/job_handler.py

@@ -1,4 +1,3 @@
-import json
 import os
 import traceback
 from collections.abc import Callable
@@ -132,40 +131,22 @@ def handle(self, boefje_meta: BoefjeMeta) -> None:
 
             boefje_meta.arguments["input"] = ooi.serialize()
 
-        boefje_results: list[tuple[set, bytes | str]] = []
         boefje_meta.runnable_hash = plugin.runnable_hash
+        boefje_meta.environment = get_environment_settings(boefje_meta, plugin.boefje_schema)
 
-        try:
-            logger.info("Getting environment settings for boefje %s[%s]", boefje_meta.boefje.id, str(boefje_meta.id))
-            boefje_meta.environment = get_environment_settings(boefje_meta, plugin.boefje_schema)
-
-            logger.info("Starting boefje %s[%s]", boefje_meta.boefje.id, str(boefje_meta.id))
-            boefje_meta.started_at = datetime.now(timezone.utc)
-            boefje_results = self.job_runner.run(boefje_meta, boefje_meta.environment)
+        logger.info("Starting boefje %s[%s]", boefje_meta.boefje.id, str(boefje_meta.id))
 
-        except SettingsNotConformingToSchema:
-            logger.exception(
-                "Error running boefje due to settings/schema mismatch %s[%s]",
-                boefje_meta.boefje.id,
-                str(boefje_meta.id),
-            )
-            boefje_results = [
-                (
-                    {"error/boefje"},
-                    f"Error running boefje ({boefje_meta.id}) due to settings/schema mismatch: {0}".format(
-                        json.dumps(plugin.boefje_schema)
-                    ),
-                )
-            ]
+        boefje_meta.started_at = datetime.now(timezone.utc)
 
-            raise
+        boefje_results: list[tuple[set, bytes | str]] = []
 
+        try:
+            boefje_results = self.job_runner.run(boefje_meta, boefje_meta.environment)
         except:
             logger.exception("Error running boefje %s[%s]", boefje_meta.boefje.id, str(boefje_meta.id))
             boefje_results = [({"error/boefje"}, traceback.format_exc())]
 
             raise
-
         finally:
             boefje_meta.ended_at = datetime.now(timezone.utc)
             logger.info("Saving to Bytes for boefje %s[%s]", boefje_meta.boefje.id, str(boefje_meta.id))

boefjes/boefjes/local_repository.py

@@ -132,7 +132,7 @@ def _cached_resolve_normalizers(path: Path) -> dict[str, NormalizerResource]:
 
     for path, package in paths_and_packages:
         try:
-            normalizer_resources.append(get_normalizer_resource(path, package, hash(path)))
+            normalizer_resources.append(get_normalizer_resource(path, package, hash_path(path)))
         except ModuleException as exc:
             logger.exception(exc)
 

bytes/bytes/api/router.py

@@ -1,4 +1,4 @@
-from base64 import b64decode
+from base64 import b64decode, b64encode
 from uuid import UUID
 
 import structlog
@@ -8,7 +8,7 @@
 from httpx import codes
 from starlette.responses import JSONResponse
 
-from bytes.api.models import BoefjeOutput
+from bytes.api.models import BoefjeOutput, File
 from bytes.auth import authenticate_token
 from bytes.config import get_settings
 from bytes.database.sql_meta_repository import MetaIntegrityError, ObjectNotFoundException, create_meta_data_repository
@@ -226,6 +226,7 @@ def get_raw(
     organization: str | None = None,
     boefje_meta_id: UUID | None = None,
     normalized: bool | None = None,
+    raw_ids: list[UUID] | None = Query(None),
     limit: int = 1,
     mime_types: list[str] | None = Query(None),
     meta_repository: MetaDataRepository = Depends(create_meta_data_repository),
@@ -237,6 +238,7 @@ def get_raw(
     query_filter = RawDataFilter(
         organization=organization,
         boefje_meta_id=boefje_meta_id,
+        raw_ids=raw_ids,
         normalized=normalized,
         mime_types=parsed_mime_types,
         limit=limit,
@@ -247,6 +249,38 @@ def get_raw(
     return meta_repository.get_raw(query_filter)
 
 
+@router.get("/raws", response_model=BoefjeOutput, tags=[RAW_TAG])
+def get_raws(
+    organization: str | None = None,
+    boefje_meta_id: UUID | None = None,
+    raw_ids: list[UUID] | None = Query(None),
+    normalized: bool | None = None,
+    limit: int = 1,
+    mime_types: list[str] | None = Query(None),
+    meta_repository: MetaDataRepository = Depends(create_meta_data_repository),
+) -> BoefjeOutput:
+    """Get a filtered list of RawData"""
+
+    parsed_mime_types = [] if mime_types is None else [MimeType(value=mime_type) for mime_type in mime_types]
+
+    query_filter = RawDataFilter(
+        organization=organization,
+        boefje_meta_id=boefje_meta_id,
+        raw_ids=raw_ids,
+        normalized=normalized,
+        mime_types=parsed_mime_types,
+        limit=limit,
+    )
+
+    logger.info("mime_types: %s", parsed_mime_types)
+
+    raws = meta_repository.get_raws(query_filter)
+
+    return BoefjeOutput(
+        files=[File(name=raw_id, content=b64encode(raw.value), tags=raw.mime_types) for raw_id, raw in raws]
+    )
+
+
 @router.get("/mime_types", response_model=dict[str, int], tags=[RAW_TAG])
 def get_raw_count_per_mime_type(
     organization: str | None = None,

bytes/bytes/database/sql_meta_repository.py

@@ -140,6 +140,15 @@ def get_raw(self, query_filter: RawDataFilter) -> list[RawDataMeta]:
 
         return [to_raw_meta(raw_file_in_db) for raw_file_in_db in query]
 
+    def get_raws(self, query_filter: RawDataFilter) -> list[tuple[uuid.UUID, RawData]]:
+        logger.debug("Querying raw data: %s", query_filter.model_dump_json())
+        query = self.session.query(RawFileInDB)
+        query = query_filter.apply(query)
+
+        raw_metas_pairs = [(raw_meta.id, to_boefje_meta(raw_meta.boefje_meta)) for raw_meta in query]
+
+        return self.raw_repository.get_raws(raw_metas_pairs)
+
     def get_raw_by_id(self, raw_id: uuid.UUID) -> RawData:
         raw_in_db: RawFileInDB | None = self.session.get(RawFileInDB, str(raw_id))
 

bytes/bytes/raw/file_raw_repository.py

@@ -75,6 +75,20 @@ def get_raw(self, raw_id: UUID, boefje_meta: BoefjeMeta) -> RawData:
         contents = file_path.read_bytes()
         return RawData(value=self.file_middleware.decode(contents), boefje_meta=boefje_meta)
 
+    def get_raws(self, raw_metas_pairs: list[tuple[UUID, BoefjeMeta]]) -> list[tuple[UUID, RawData]]:
+        try:
+            raws = [
+                (raw_id, self._raw_file_path(raw_id, boefje_meta).read_bytes(), boefje_meta)
+                for raw_id, boefje_meta in raw_metas_pairs
+            ]
+        except FileNotFoundError:
+            raise BytesFileNotFoundException()
+
+        return [
+            (raw_id, RawData(value=self.file_middleware.decode(raw), boefje_meta=boefje_meta))
+            for raw_id, raw, boefje_meta in raws
+        ]
+
     def _raw_file_path(self, raw_id: UUID, boefje_meta: BoefjeMeta) -> Path:
         return self.base_path / boefje_meta.organization / self._index(raw_id) / str(raw_id)
 

bytes/bytes/repositories/meta_repository.py

@@ -29,6 +29,7 @@ class NormalizerMetaFilter(BaseModel):
 class RawDataFilter(BaseModel):
     organization: str | None = None
     boefje_meta_id: UUID | None = None
+    raw_ids: list[UUID] | None = None
     normalized: bool | None = None
     mime_types: list[MimeType] = Field(default_factory=list)
     limit: int | None = 1
@@ -50,6 +51,9 @@ def apply(self, query: Query) -> Query:
         if self.mime_types:
             query = query.filter(RawFileInDB.mime_types.contains([m.value for m in self.mime_types]))
 
+        if self.raw_ids:
+            query = query.filter(RawFileInDB.id.in_([x.hex for x in self.raw_ids]))
+
         return query.offset(self.offset).limit(self.limit)
 
 
@@ -87,6 +91,9 @@ def get_raw_by_id(self, raw_id: UUID) -> RawData:
     def get_raw(self, query_filter: RawDataFilter) -> list[RawDataMeta]:
         raise NotImplementedError()
 
+    def get_raws(self, query_filter: RawDataFilter) -> list[tuple[UUID, RawData]]:
+        raise NotImplementedError()
+
     def has_raw(self, boefje_meta: BoefjeMeta, mime_types: list[MimeType]) -> bool:
         raise NotImplementedError()
 

bytes/bytes/repositories/raw_repository.py

@@ -13,3 +13,6 @@ def save_raw(self, raw_id: UUID, raw: RawData) -> None:
 
     def get_raw(self, raw_id: UUID, boefje_meta: BoefjeMeta) -> RawData:
         raise NotImplementedError()
+
+    def get_raws(self, raw_metas_pairs: list[tuple[UUID, BoefjeMeta]]) -> list[tuple[UUID, RawData]]:
+        raise NotImplementedError()

bytes/tests/client.py

@@ -8,7 +8,7 @@
 import httpx
 from httpx import HTTPError, HTTPStatusError
 
-from bytes.api.models import BoefjeOutput
+from bytes.api.models import BoefjeOutput, File
 from bytes.models import BoefjeMeta, NormalizerMeta, RawDataMeta
 from bytes.repositories.meta_repository import BoefjeMetaFilter, NormalizerMetaFilter, RawDataFilter
 
@@ -157,11 +157,21 @@ def get_raw_meta(self, raw_id: UUID) -> RawDataMeta:
         return RawDataMeta.model_validate(response.json())
 
     @retry_with_login
-    def get_raws(self, query_filter: RawDataFilter) -> dict[str, str]:
+    def get_raw_metas(self, query_filter: RawDataFilter) -> dict[str, str]:
         params = query_filter.model_dump(exclude_none=True)
         params["mime_types"] = [m.value for m in query_filter.mime_types]
 
         response = self.client.get("/bytes/raw", params=params)
         self._verify_response(response)
 
         return response.json()  # type: ignore
+
+    @retry_with_login
+    def get_raws(self, query_filter: RawDataFilter) -> list[File]:
+        params = query_filter.model_dump(exclude_none=True)
+        params["mime_types"] = [m.value for m in query_filter.mime_types]
+
+        response = self.client.get("/bytes/raws", params=params)
+        self._verify_response(response)
+
+        return response.json().get("files", [])

bytes/tests/integration/test_bytes_api.py

@@ -1,5 +1,5 @@
 import uuid
-from base64 import b64encode
+from base64 import b64decode, b64encode
 
 import httpx
 import pytest
@@ -262,7 +262,7 @@ def test_save_raw_with_one_mime_type(bytes_api_client: BytesAPIClient) -> None:
     assert retrieved_raw == raw
     assert (
         len(
-            bytes_api_client.get_raws(
+            bytes_api_client.get_raw_metas(
                 RawDataFilter(boefje_meta_id=boefje_meta.id, normalized=False, mime_types=[MimeType(value="bad/mime")])
             )
         )
@@ -296,6 +296,27 @@ def test_save_raw_no_mime_types(bytes_api_client: BytesAPIClient) -> None:
     assert get_raw_without_mime_type_response.content == raw
 
 
+def test_get_many_actual_raw_files(bytes_api_client: BytesAPIClient) -> None:
+    boefje_meta = get_boefje_meta(meta_id=uuid.uuid4())
+    bytes_api_client.save_boefje_meta(boefje_meta)
+    mime_types = ["text/kat-test", "text/html"]
+    second_mime_types = ["text/kat-test", "text/status-code"]
+
+    raw = b"test 123456"
+    second_raw = b"second test 200"
+    first_id = bytes_api_client.save_raw(boefje_meta.id, raw, mime_types)
+    second_id = bytes_api_client.save_raw(boefje_meta.id, second_raw, second_mime_types)
+
+    result = bytes_api_client.get_raws(RawDataFilter(raw_ids=[first_id], limit=10))
+    assert len(result) == 1
+    assert b64decode(result[0]["content"]) == raw
+
+    result = bytes_api_client.get_raws(RawDataFilter(raw_ids=[first_id, second_id], limit=10))
+    assert len(result) == 2
+    assert b64decode(result[0]["content"]) == raw
+    assert b64decode(result[1]["content"]) == second_raw
+
+
 def test_raw_mimes(bytes_api_client: BytesAPIClient) -> None:
     boefje_meta = get_boefje_meta(meta_id=uuid.uuid4())
     bytes_api_client.save_boefje_meta(boefje_meta)
@@ -317,21 +338,21 @@ def test_raw_mimes(bytes_api_client: BytesAPIClient) -> None:
     assert bytes_api_client.get_raw(first_id) == raw
     assert bytes_api_client.get_raw(second_id) == second_raw
 
-    retrieved_raws = bytes_api_client.get_raws(
+    retrieved_raws = bytes_api_client.get_raw_metas(
         RawDataFilter(
             boefje_meta_id=boefje_meta.id, normalized=False, mime_types=[MimeType(value=x) for x in mime_types]
         )
     )
     assert len(retrieved_raws) == 1
     assert {x["value"] for x in retrieved_raws[0]["mime_types"]} == set(mime_types)
 
-    retrieved_raws = bytes_api_client.get_raws(
+    retrieved_raws = bytes_api_client.get_raw_metas(
         RawDataFilter(boefje_meta_id=boefje_meta.id, normalized=False, mime_types=[MimeType(value="text/html")])
     )
     assert len(retrieved_raws) == 1
     assert {x["value"] for x in retrieved_raws[0]["mime_types"]} == set(mime_types)
 
-    retrieved_raws = bytes_api_client.get_raws(
+    retrieved_raws = bytes_api_client.get_raw_metas(
         RawDataFilter(boefje_meta_id=boefje_meta.id, normalized=False, mime_types=[MimeType(value="bad/mime")])
     )
     assert len(retrieved_raws) == 0
@@ -342,7 +363,7 @@ def test_raw_mimes(bytes_api_client: BytesAPIClient) -> None:
         mime_types=[MimeType(value="text/kat-test")],
         limit=3,
     )
-    retrieved_raws = bytes_api_client.get_raws(query_filter)
+    retrieved_raws = bytes_api_client.get_raw_metas(query_filter)
 
     assert len(retrieved_raws) == 2
     assert (

docs/source/developer-documentation/development-tutorial/creating-boefje.md

@@ -2,8 +2,6 @@
 
 There are two ways to create a Boefje. The first is to create a Boefje with the User Interface (UI). The second option is to create it in the backend.
 
-.. \_boefje-variants:
-
 ## Boefje variants
 
 Boefje variants are Boefjes that use the same container image. In OpenKAT, all Boefjes with the same container image will be seen as 'variants' of each other and will be shown together on those Boefje detail pages.

docs/source/index.rst

@@ -8,7 +8,6 @@ Welcome to the OpenKAT documentation!
    :caption: Contents
 
    introduction/index
-   katalogus/index
    manual/index
    modules/index
    installation-and-deployment/index