Merge branch 'main' into feature/mula/refactor-queue

* main: (51 commits) Fix static files for container images/Debian packages when DEBUG is on (#2742) OOI selection at Aggregate report does not remember changed selection (#2619) fix schema errors on empty / missing schemas (#2744) Updated `phonenumbers` and `django-phonenumber-field` (#2757) Remove octopoes coverage workflow (#2755) Bump actions/configure-pages from 4 to 5 (#2745) Add xtdb-cli tool to Octopoes (#2733) Dont report vulnerabilities without version info of the software for snyk (#2730) Feature/boefjes to oci images (#2709) Query non-reference fields and subclass-specific fields through path queries (#2662) Fix in System Specific (#2732) Plugins overview in appendix not showing any plugins (#2694) Feat stepper design v2 (#2704) Undo project-directory in Rocky (#2734) Remove Docker Compose: "version" (#2718) Upgrade `pre-commit` hooks (#2729) Fix #1739 (#2705) Improve generate report (#2633) Fix critical vulnerability counter (#2712) Fix pdf alignment (#2674) ...
minvws · Apr 4, 2024 · 652fc2c · 652fc2c
2 parents 2fa57b6 + 6762666
commit 652fc2c
Show file tree

Hide file tree

Showing 469 changed files with 35,310 additions and 19,353 deletions.
diff --git a/.env-defaults b/.env-defaults
@@ -13,7 +13,7 @@ SCHEDULER_API=http://scheduler:8000
 KEIKO_API=http://keiko:8000
 KATALOGUS_API=http://katalogus:8000
 XTDB_URI=http://crux:3000
-BOEFJE_API=http://boefje:8000
+BOEFJES_API=http://boefje:8000
 
 # Bytes uses JWT for authentication
 BYTES_API=http://bytes:8000
@@ -30,3 +30,7 @@ DJANGO_CSRF_TRUSTED_ORIGINS=http://localhost,http://127.0.0.1
 
 # This allows running pytest inside the container
 ROCKY_DB_USER_CREATEDB=CREATEDB
+
+# This is normally False when DEBUG is true, but we override that in settings.py
+# so it possible to set DEBUG to True in production like environments.
+COMPRESS_ENABLED=False
diff --git a/.github/workflows/build-rdo-package.yml b/.github/workflows/build-rdo-package.yml
@@ -162,7 +162,7 @@ jobs:
         run: python3.10 -m venv /var/www/html/.venv
 
       - name: Create scheduler release archive
-        run: tar -cvzf ${{ env.PKGDIR }}/scheduler_${{ env.RELEASE_VERSION }}.tar.gz --exclude=./.git* --exclude=Makefile --exclude=Dockerfile --exclude=base.yml --exclude=requirements* --exclude=tests .
+        run: tar -cvzf ${{ env.PKGDIR }}/scheduler_${{ env.RELEASE_VERSION }}.tar.gz --exclude=./.git* --exclude=Makefile --exclude=Dockerfile --exclude=requirements* --exclude=tests .
         working-directory: ./mula
 
       - name: Create virtual env

diff --git a/.github/workflows/deploy_docs.yml b/.github/workflows/deploy_docs.yml
@@ -42,7 +42,7 @@ jobs:
         run: make docs
 
       - name: Setup Pages
-        uses: actions/configure-pages@v4
+        uses: actions/configure-pages@v5
 
       - name: Upload artifact
         uses: actions/upload-pages-artifact@v3

diff --git a/.github/workflows/keiko_test.yml b/.github/workflows/keiko_test.yml
@@ -9,9 +9,11 @@ on:
       - '*'
     paths:
       - keiko/**
+      - .github/workflows/keiko_test.yml
   pull_request:
     paths:
       - keiko/**
+      - .github/workflows/keiko_test.yml
 
 jobs:
   test:
@@ -34,7 +36,7 @@ jobs:
 
       - name: Install Latex
         run: |
-          sudo apt-get install -y --no-install-recommends \
+          sudo apt-get update && sudo apt-get install -y --no-install-recommends \
             chktex \
             fonts-recommended \
             latexmk \

diff --git a/.github/workflows/mula_tests.yml b/.github/workflows/mula_tests.yml
@@ -30,7 +30,7 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Build the images
-        run: DOCKER_BUILDKIT=1 COMPOSE_DOCKER_CLI_BUILD=1 docker compose -f base.yml -f .ci/docker-compose.yml build --build-arg PYTHON_VERSION=${{ matrix.version }}
+        run: DOCKER_BUILDKIT=1 COMPOSE_DOCKER_CLI_BUILD=1 docker compose --project-directory . -f .ci/docker-compose.yml build --build-arg PYTHON_VERSION=${{ matrix.version }}
         working-directory: ./mula
 
       - name: Run unit tests

diff --git a/.github/workflows/octopoes_coverage.yml b/.github/workflows/octopoes_coverage.yml
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -21,12 +21,16 @@ repos:
     exclude: |
       (?x)(
       \.svcg$ |
-      ^boefjes/tests/examples/rdns-nxdomain.txt$
+      ^boefjes/tests/examples/rdns-nxdomain.txt$ |
+      ^boefjes/tests/examples/raw/
       )
+  - id: fix-byte-order-marker
+  - id: pretty-format-json
+    args: ["--autofix", "--no-ensure-ascii", "--no-sort-keys"]
 
 
 - repo: https://github.com/abravalheri/validate-pyproject
-  rev: v0.15
+  rev: v0.16
   hooks:
   - id: validate-pyproject
     files: pyproject.toml$
@@ -40,24 +44,24 @@ repos:
     additional_dependencies: ["rstcheck[sphinx]"]
 
 - repo: https://github.com/MarketSquare/robotframework-tidy
-  rev: "4.8.1"
+  rev: "4.11.0"
   hooks:
   - id: robotidy
 
 - repo: https://github.com/jendrikseipp/vulture
-  rev: v2.10
+  rev: v2.11
   hooks:
   - id: vulture
     exclude: |
       /tests/
 
 - repo: https://github.com/charliermarsh/ruff-pre-commit
-  rev: 'v0.1.13'
+  rev: 'v0.3.4'
   hooks:
   - id: ruff
 
 - repo: https://github.com/asottile/pyupgrade
-  rev: v3.15.0
+  rev: v3.15.2
   hooks:
   - id: pyupgrade
     args: [--py310-plus]
@@ -69,15 +73,15 @@ repos:
     args: [--target-version, "4.2"]
 
 - repo: https://github.com/psf/black
-  rev: "23.12.1"
+  rev: "24.3.0"
   hooks:
   - id: black
 
 - repo: https://github.com/pre-commit/mirrors-mypy
-  rev: v1.8.0
+  rev: v1.9.0
   hooks:
   - id: mypy
-    additional_dependencies: ['types-PyYAML', 'types-requests', 'types-cachetools', 'types-retry', 'pydantic', 'pynacl']
+    additional_dependencies: ['types-PyYAML', 'types-cachetools', 'types-retry', 'pydantic', 'pynacl', 'httpx']
     exclude: |
       (?x)(
       ^boefjes/boefjes/plugins |
@@ -141,7 +145,7 @@ repos:
     exclude: '^rocky/rocky/templates/admin/.*\.html$'
 
 - repo: https://github.com/thibaudcolas/pre-commit-stylelint
-  rev: v16.1.0
+  rev: v16.3.0
   hooks:
     - id: stylelint
       args: [ --fix ]
@@ -151,13 +155,13 @@ repos:
       files: "^(rocky\/assets\/css\/|docs\/source\/).*.(css|scss|sass)$"
 
 - repo: https://github.com/shellcheck-py/shellcheck-py
-  rev: v0.9.0.6
+  rev: v0.10.0.1
   hooks:
   - id: shellcheck
     args: ["-e", "SC1091"]
 
 - repo: https://github.com/scop/pre-commit-shfmt
-  rev: v3.7.0-4
+  rev: v3.8.0-1
   hooks:
   - id: shfmt
     args: ["-w", "-s", "-i", "4", "-sr"]
diff --git a/.stylelintrc.json b/.stylelintrc.json
@@ -1,6 +1,8 @@
 {
   "extends": "stylelint-config-standard-scss",
-  "ignoreFiles": ["rocky/assets/css/themes/soft/fonts/tabler-icons/tabler-icons.scss"],
+  "ignoreFiles": [
+    "rocky/assets/css/themes/soft/fonts/tabler-icons/tabler-icons.scss"
+  ],
   "rules": {
     "number-max-precision": 5,
     "color-hex-length": "long",

diff --git a/Makefile b/Makefile
@@ -27,7 +27,7 @@ kat: env-if-empty build up
 	@echo "The KAT frontend is running at http://localhost:8000,"
 	@echo "An initial superuser has been created"
 	@echo "The username is stored in DJANGO_SUPERUSER_EMAIL in the .env-default file."
-	@echo "run 'grep 'DJANGO_SUPERUSER_EMAIL' .env-default' to find it."
+	@echo "run 'grep 'DJANGO_SUPERUSER_EMAIL' .env-defaults' to find it."
 	@echo "The related password can be found as DJANGO_SUPERUSER_PASSWORD in the .env file."
 	@echo "run 'grep 'DJANGO_SUPERUSER_PASSWORD' .env' to find it."
 	@echo

diff --git a/boefjes/.ci/docker-compose.yml b/boefjes/.ci/docker-compose.yml
@@ -1,5 +1,3 @@
-version: "3.9"
-
 services:
   katalogus_integration:
     build:

diff --git a/boefjes/.dockerignore b/boefjes/.dockerignore
@@ -33,4 +33,3 @@ sql_migrations
 .dockerignore
 export_migrations
 docs
-base.yml
diff --git a/boefjes/Makefile b/boefjes/Makefile
@@ -6,7 +6,7 @@ MAKEFLAGS += --warn-undefined-variables
 MAKEFLAGS += --no-builtin-rules
 # Makefile Reference: https://tech.davis-hansson.com/p/make/
 
-.PHONY: help sql migrate migrations debian ubuntu clean
+.PHONY: help sql migrate migrations debian ubuntu clean images
 
 # use HIDE to run commands invisibly, unless VERBOSE defined
 HIDE:=$(if $(VERBOSE),,@)
@@ -16,6 +16,10 @@ export revid	# Revision id to generate raw sql for
 export rev1		# Previous revision id for generating migrations
 export rev2		# New revision id for the new migration file
 
+# We set this to build images with the right target platform in the `images` target. This prevents arm systems such as
+# systems with an Apple silicon chip to build images they cannot use.
+export DOCKER_DEFAULT_PLATFORM=$(shell docker system info --format '{{.OSType}}/{{.Architecture}}')
+
 ##
 ##|------------------------------------------------------------------------|
 ##			Help
@@ -28,11 +32,18 @@ help: ## Show this help.
 ##|------------------------------------------------------------------------|
 ##			Development
 ##|------------------------------------------------------------------------|
-build: seed
+
+build: seed images
 
 seed:  # Seed the katalogus database
 	-docker compose run --rm katalogus python -m boefjes.seed
 
+images:  # Build the images for the containerized boefjes
+  # Dns-records is disabled for now, see the discussion in https://github.com/minvws/nl-kat-coordination/pull/2709
+	# docker build -f images/base.Dockerfile -t openkat/dns-records --build-arg BOEFJE_PATH=./boefjes/plugins/kat_dns .
+	docker build -f ./boefjes/plugins/kat_dnssec/boefje.Dockerfile -t openkat/dns-sec --build-arg BOEFJE_PATH=./boefjes/plugins/kat_dnssec .
+
+
 ##
 ##|------------------------------------------------------------------------|
 ##			Migrations
@@ -57,7 +68,7 @@ check:
 ##			Tests
 ##|------------------------------------------------------------------------|
 
-ci-docker-compose := docker compose -f base.yml  -f .ci/docker-compose.yml
+ci-docker-compose := docker compose --project-directory . -f .ci/docker-compose.yml
 
 
 test: itest ## Run all tests.

diff --git a/boefjes/base.yml b/boefjes/base.yml
diff --git a/boefjes/boefjes/api.py b/boefjes/boefjes/api.py
@@ -6,8 +6,8 @@
 from uuid import UUID
 
 from fastapi import Depends, FastAPI, HTTPException, Response
+from httpx import HTTPError, HTTPStatusError
 from pydantic import BaseModel, ConfigDict, Field
-from requests import HTTPError
 from uvicorn import Config, Server
 
 from boefjes.clients.bytes_client import BytesAPIClient
@@ -73,7 +73,7 @@ def get_scheduler_client():
 
 def get_bytes_client():
     return BytesAPIClient(
-        settings.bytes_api,
+        str(settings.bytes_api),
         username=settings.bytes_username,
         password=settings.bytes_password,
     )
@@ -140,7 +140,7 @@ def get_task(task_id, scheduler_client):
     try:
         task = scheduler_client.get_task(task_id)
     except HTTPError as e:
-        if e.response.status_code == 404:
+        if isinstance(e, HTTPStatusError) and e.response.status_code == 404:
             raise HTTPException(status_code=404, detail="Task not found")
         else:
             logger.exception("Failed to get task from scheduler")

diff --git a/boefjes/boefjes/app.py b/boefjes/boefjes/app.py
@@ -6,8 +6,8 @@
 import time
 from queue import Queue
 
+from httpx import HTTPError
 from pydantic import ValidationError
-from requests import HTTPError
 
 from boefjes.clients.scheduler_client import (
     QueuePrioritizedItem,