refactor settings to the __init__ and base file and no longer make us…

…e of production/development/github files

.github/workflows/ci.yml

@@ -13,6 +13,7 @@ jobs:
           POSTGRES_USER: signals
           POSTGRES_PASSWORD: insecure
           POSTGRES_DB: signals
+          IS_GITHUB_WORKFLOW: True
         ports: ['5432:5432']
         options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
 
@@ -41,7 +42,7 @@ jobs:
       - name: Run tests
         run: |
           cd api/app
-          tox -- --ds signals.settings.github
+          tox -- --ds signals.settings
 
       - name: Upload coverage report
         uses: actions/upload-artifact@v1

api/Dockerfile

@@ -20,7 +20,7 @@ RUN apt-get update \
 
 RUN mkdir -p /media && mkdir -p /static && chown datapunt /media && chown datapunt /static
 
-ENV DJANGO_SETTINGS_MODULE=signals.settings.production
+ENV DJANGO_SETTINGS_MODULE=signals.settings
 ARG DJANGO_SECRET_KEY=insecure_docker_build_key
 
 COPY app /app/

api/app/logs/__init__.py

@@ -0,0 +1,35 @@
+# SPDX-License-Identifier: MPL-2.0
+# Copyright (C) 2022 Gemeente Amsterdam
+import copy
+
+from typing import List, Union
+
+from .filters import DebugQueryFilter
+from .handlers import ColoredHandler
+from .config import BASE_LOGGING
+
+
+__all__ = ["ColoredHandler", "DebugQueryFilter", "get_configuration"]
+
+
+def get_configuration(local_apps: List[str], logging_level: Union[str, int]):
+    """
+    This function returns a dictionary config object that can be used as the
+    LOGGING environment variable.
+
+    It will construct a logger for every app passed via the local_apps
+    list parameter with the given level in the logging_level.
+
+    :param local_apps:
+    :param logging_level:
+    :return:
+    """
+
+    config = copy.deepcopy(BASE_LOGGING)
+
+    for app_name in local_apps:
+        config["loggers"].update(
+            {app_name: {"level": logging_level, "handlers": ["colorize"]}}
+        )
+
+    return config

api/app/logs/config.py

@@ -0,0 +1,50 @@
+# SPDX-License-Identifier: MPL-2.0
+# Copyright (C) 2022 Gemeente Amsterdam
+BASE_LOGGING = {
+    'version': 1,
+    'disable_existing_loggers': False,
+    'formatters': {
+        'elaborate': {
+            'format': '{levelname} {name} {module}.{filename} {message}',
+            'style': '{'
+        }
+    },
+    'filters': {
+        'require_debug_queries': {
+            '()': 'logs.filters.DebugQueryFilter'
+        },
+        'static_fields': {
+            '()': 'logs.filters.StaticFieldFilter',
+            'fields': {
+                'project': 'SignalsAPI',
+                'environment': 'Any',
+                'hideme': 'True'
+            },
+        },
+    },
+    'handlers': {
+        'sentry': {
+            'level': 'WARNING',
+            'class': 'raven.contrib.django.raven_compat.handlers.SentryHandler',
+        },
+        'colorize': {
+            'class': 'logs.handlers.ColoredHandler',
+            'formatter': 'elaborate'
+        },
+        'colorless': {
+            'class': 'logging.StreamHandler',
+        },
+    },
+    'loggers': {
+        'django': {
+            'level': 'DEBUG',
+            'handlers': ['colorless'],
+        },
+        'django.db.backends': {
+            'level': 'DEBUG',
+            'handlers': ['colorless'],
+            'filters': ['require_debug_queries'],
+            'propagate': False,
+        },
+    },
+}

api/app/logs/filters.py

@@ -0,0 +1,28 @@
+# SPDX-License-Identifier: MPL-2.0
+# Copyright (C) 2022 Gemeente Amsterdam
+from logging import LogRecord, Filter
+
+from django.conf import settings
+
+
+class DebugQueryFilter(Filter):
+    """
+    Makes the query debug information dependant on the settings.
+    """
+    def filter(self, record: LogRecord) -> bool:
+        return settings.LOG_QUERIES and settings.DEBUG
+
+
+class StaticFieldFilter(Filter):
+    """
+    Python logging filter that adds the given static contextual information
+    in the ``fields`` dictionary to all logging records.
+    """
+
+    def __init__(self, fields):
+        self.static_fields = fields
+
+    def filter(self, record):
+        for k, v in self.static_fields.items():
+            setattr(record, k, v)
+        return True

api/app/logs/handlers.py

@@ -0,0 +1,85 @@
+# SPDX-License-Identifier: MPL-2.0
+# Copyright (C) 2022 Gemeente Amsterdam
+import sys
+
+from logging import LogRecord, Handler
+
+
+class ColoredHandler(Handler):
+    terminator = "\n"
+    output = None
+
+    def __init__(self) -> None:
+        """
+        Obligatory call to the Handler init.
+        Uses system output, not prints!
+        """
+        Handler.__init__(self)
+        self.output = sys.stderr
+
+    def yellow(self, message: str) -> str:
+        return "\033[93m{}\033[0m".format(message)
+
+    def red(self, message: str) -> str:
+        return "\033[91m{}\033[0m".format(message)
+
+    def blue(self, message: str) -> str:
+        return "\033[96m{}\033[0m".format(message)
+
+    def green(self, message: str) -> str:
+        return "\033[92m{}\033[0m".format(message)
+
+    def colorize(self, record: LogRecord) -> str:
+        """
+        Add the correct colour depending on the level of the LogRecord.
+        :param record:
+        :return:
+        """
+        message = self.format(record=record)
+
+        if record.levelname in ["ERROR", "WARNING"]:
+            return self.yellow(message=message)
+        elif record.levelname == "CRITICAL":
+            return self.red(message=message)
+        elif record.levelname == "INFO":
+            return self.blue(message=message)
+        else:
+            return self.green(message=message)
+
+    def flush(self):
+        """
+        Flushes the stream. This forces the output to print the buffered
+        content to the console.
+        """
+        self.acquire()
+
+        try:
+            if self.output and hasattr(self.output, "flush"):
+                self.output.flush()
+        finally:
+            self.release()
+
+    def emit(self, record: LogRecord):
+        """
+        Adds colour to the message output to infer priority. If this fails for
+        some reason it will fallback to a regular message.
+
+        If that fails there is still the built-in error handler of the emit.
+
+        :param record:
+        :return:
+        """
+        try:
+            try:
+                parsed_message = self.colorize(record=record)
+            except Exception:
+                print('error called')
+                parsed_message = self.format(record=record)
+
+            stream = self.output
+            stream.write(parsed_message)
+            stream.write(self.terminator)
+            self.flush()
+        except Exception:
+            print('error2 called')
+            self.handleError(record)

api/app/manage.py

@@ -5,7 +5,7 @@
 import sys
 
 if __name__ == '__main__':
-    os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'signals.settings.development')
+    os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'signals.settings')
 
     from django.core.management import execute_from_command_line
 

api/app/pytest.ini

@@ -1,6 +1,6 @@
 [pytest]
 #addopts=-n2
-DJANGO_SETTINGS_MODULE = signals.settings.testing
+DJANGO_SETTINGS_MODULE = signals.settings
 python_files =
     test.py
     tests.py

api/app/signals/celery.py

@@ -4,7 +4,7 @@
 
 from celery import Celery
 
-os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'signals.settings.production')
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'signals.settings')
 app = Celery('signals')
 app.config_from_object('django.conf:settings', namespace='CELERY')
 app.autodiscover_tasks()

api/app/signals/settings/__init__.py

@@ -2,4 +2,31 @@
 # Copyright (C) 2018 - 2021 Gemeente Amsterdam
 # Temporary fix to make the settings restructure work with current deployment settings. Can be
 # removed after the Ansible playbooks for deployment are updated with the production settings.
-from signals.settings.base import *  # noqa
+import sys
+from .base import *  # noqa
+from .feature_flags import *
+from logs import get_configuration
+
+try:
+    from .local import *
+except ImportError:
+    pass
+
+if "test" in sys.argv or "pytest" in sys.argv[0]:
+    try:
+        from .testing import *
+    except ImportError:
+        pass
+
+LOGGING = get_configuration(local_apps=SIGNAL_APPS, logging_level=LOGGING_LEVEL)
+
+DATABASES = {
+    'default': {
+        'ENGINE': 'django.contrib.gis.db.backends.postgis',
+        'NAME': DATABASE_NAME,
+        'USER': DATABASE_USER,
+        'PASSWORD': DATABASE_PASSWORD,
+        'HOST': DATABASE_HOST,
+        'PORT': DATABASE_PORT
+    },
+}