Squash: Refactor settings and logging

.github/workflows/ci.yml

@@ -13,7 +13,7 @@ jobs:
           POSTGRES_USER: signals
           POSTGRES_PASSWORD: insecure
           POSTGRES_DB: signals
-          IS_GITHUB_WORKFLOW: True
+
         ports: ['5432:5432']
         options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
 
@@ -40,9 +40,11 @@ jobs:
           pip install -r api/requirements.txt
 
       - name: Run tests
+        env:
+          DATABASE_HOST: localhost
         run: |
           cd api/app
-          tox -- --ds signals.settings
+          tox
 
       - name: Upload coverage report
         uses: actions/upload-artifact@v1

.gitignore

@@ -39,7 +39,7 @@ htmlcov/
 .cache
 nosetests.xml
 coverage.xml
-
+docker-compose.override.yml
 # Translations
 *.mo
 *.pot

README.md

@@ -67,6 +67,9 @@ request, as any failed checks will abort the build pipeline.
 
 ### Running and developing locally using Docker and docker-compose
 
+Starting the project for the first time requires you to copy the local.py.dist file 
+in the signals.settings folder and rename this copied file to local.py.
+
 Assuming no services are running, start the database and queue:
 
 ```console

api/README.md

@@ -31,15 +31,14 @@ Django settings
 
 Settings for all different environments are defined in `app/signals/settings/...`. The file
 `base.py` contains all the default settings. Defaults should be production ready. You can
-override this, if needed, in one of the other specific settings files (e.g. `testing`).
+override this, if needed, in the other specific settings files (e.g. `testing`, `local`).
 
 Current available settings:
 
-- `production`      -> Used for Acceptance/Production instances
-- `testing`         -> Used for Jenkins test pipeline and testing with `tox`
-- `development`     -> Used for Local Docker instances
-- `local`           -> Override settings in `development` for local usage (not tracked in Git)
-
+- `base`      -> Used for Acceptance/Production instances
+- `testing`   -> Used for Jenkins test pipeline and testing with `tox`
+- `local`     -> Used for Local Docker instances.
+                 Override settings in `local` for local usage (not tracked in Git)
 
 Tox usage
 =========

api/app/logs/config.py

@@ -5,7 +5,7 @@
     'disable_existing_loggers': False,
     'formatters': {
         'elaborate': {
-            'format': '{levelname} {name} {module}.{filename} {message}',
+            'format': '{levelname} {module}.{filename} {message}',
             'style': '{'
         }
     },

api/app/logs/handlers.py

@@ -73,13 +73,11 @@ def emit(self, record: LogRecord):
             try:
                 parsed_message = self.colorize(record=record)
             except Exception:
-                print('error called')
                 parsed_message = self.format(record=record)
 
             stream = self.output
             stream.write(parsed_message)
             stream.write(self.terminator)
             self.flush()
         except Exception:
-            print('error2 called')
             self.handleError(record)

api/app/manage.py

@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: MPL-2.0
-# Copyright (C) 2018 - 2021 Gemeente Amsterdam
+# Copyright (C) 2018 - 2022 Gemeente Amsterdam
 #!/usr/bin/env python
 import os
 import sys

api/app/signals/celery.py

@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: MPL-2.0
-# Copyright (C) 2018 - 2021 Gemeente Amsterdam
+# Copyright (C) 2018 - 2022 Gemeente Amsterdam
 import os
 
 from celery import Celery

api/app/signals/settings/__init__.py

@@ -2,31 +2,34 @@
 # Copyright (C) 2018 - 2021 Gemeente Amsterdam
 # Temporary fix to make the settings restructure work with current deployment settings. Can be
 # removed after the Ansible playbooks for deployment are updated with the production settings.
-import sys
-from .base import *  # noqa
-from .feature_flags import *
+import os, sys # noqa
+
 from logs import get_configuration
 
-try:
-    from .local import *
-except ImportError:
-    pass
+from .base import *  # noqa
+from .feature_flags import * # noqa
+
+if "test" in sys.argv or "pytest" in sys.argv[0] or os.getenv('TESTING') == 'true':
+    try:
+        from .testing import * # noqa
+    except ImportError:
+        pass
 
-if "test" in sys.argv or "pytest" in sys.argv[0]:
+else:
     try:
-        from .testing import *
+        from .local import *  # noqa
     except ImportError:
         pass
 
-LOGGING = get_configuration(local_apps=SIGNAL_APPS, logging_level=LOGGING_LEVEL)
+LOGGING = get_configuration(local_apps=SIGNAL_APPS, logging_level=LOGGING_LEVEL) # noqa
 
 DATABASES = {
     'default': {
         'ENGINE': 'django.contrib.gis.db.backends.postgis',
-        'NAME': DATABASE_NAME,
-        'USER': DATABASE_USER,
-        'PASSWORD': DATABASE_PASSWORD,
-        'HOST': DATABASE_HOST,
-        'PORT': DATABASE_PORT
+        'NAME': DATABASE_NAME, # noqa:
+        'USER': DATABASE_USER, # noqa
+        'PASSWORD': DATABASE_PASSWORD, # noqa
+        'HOST': DATABASE_HOST, # noqa
+        'PORT': DATABASE_PORT # noqa
     },
-}
+} # noqa

api/app/signals/settings/base.py

@@ -125,7 +125,7 @@
     }
 ]
 
-#DATABASE
+# Database settings
 DATABASE_NAME = os.getenv('DATABASE_NAME', 'signals')
 DATABASE_USER = os.getenv('DATABASE_USER', 'signals')
 DATABASE_PASSWORD = os.getenv('DATABASE_PASSWORD', 'insecure')
@@ -259,111 +259,6 @@
     'dsn': os.getenv('SENTRY_RAVEN_DSN'),
 }
 
-# Django Logging settings
-# LOGGING = {
-#     'version': 1,
-#     'disable_existing_loggers': False,
-#     'root': {
-#         'level': 'INFO',
-#         'handlers': ['console', 'sentry'],
-#     },
-#     'formatters': {
-#         'console': {
-#             'format': '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
-#         }
-#     },
-#     'filters': {
-#
-#     },
-#     'handlers': {
-#         'console': {
-#             'level': 'INFO',
-#             'class': 'logs.handlers.ColoredHandler',
-#             'formatter': 'console',
-#         },
-#         'sentry': {
-#             'level': 'WARNING',
-#             'class': 'raven.contrib.django.raven_compat.handlers.SentryHandler',
-#         },
-#     },
-#     'loggers': {
-#         'signals': {
-#             'level': 'WARNING',
-#             'handlers': ['console'],
-#             'propagate': True,
-#         },
-#         'django': {
-#             'level': 'DEBUG',
-#             'handlers': ['console'],
-#             'propagate': False,
-#         },
-#         'raven': {
-#             'level': 'DEBUG',
-#             'handlers': ['console'],
-#             'propagate': False,
-#         },
-#         'sentry.errors': {
-#             'level': 'DEBUG',
-#             'handlers': ['console'],
-#             'propagate': False,
-#         },
-#
-#         # Debug all batch jobs
-#         'doc': {
-#             'level': 'INFO',
-#             'handlers': ['console'],
-#             'propagate': False,
-#         },
-#         'index': {
-#             'level': 'INFO',
-#             'handlers': ['console'],
-#             'propagate': False,
-#         },
-#         'search': {
-#             'level': 'ERROR',
-#             'handlers': ['console'],
-#             'propagate': False,
-#         },
-#         'elasticsearch': {
-#             'level': 'ERROR',
-#             'handlers': ['console'],
-#             'propagate': False,
-#         },
-#         'urllib3': {
-#             'level': 'ERROR',
-#             'handlers': ['console'],
-#             'propagate': False,
-#         },
-#         'factory.containers': {
-#             'level': 'INFO',
-#             'handlers': ['console'],
-#             'propagate': False,
-#         },
-#         'factory.generate': {
-#             'handlers': ['console'],
-#             'level': 'INFO',
-#             'propagate': False,
-#         },
-#         'requests.packages.urllib3.connectionpool': {
-#             'level': 'ERROR',
-#             'handlers': ['console'],
-#             'propagate': False,
-#         },
-#         'signals.apps.reporting.management.commands.dump_horeca_csv_files': {
-#             'level': 'DEBUG',
-#             'handlers': ['console'],
-#             'propagate': False,
-#         },
-#
-#         # Log all unhandled exceptions
-#         'django.request': {
-#             'level': 'ERROR',
-#             'handlers': ['console'],
-#             'propagate': False,
-#         },
-#     },
-# }
-
 # Django REST framework settings
 REST_FRAMEWORK = dict(
     PAGE_SIZE=100,

api/app/signals/settings/feature_flags.py

@@ -1,6 +1,7 @@
 # SPDX-License-Identifier: MPL-2.0
 # Copyright (C) 2022 Gemeente Amsterdam
 import os
+
 TRUE_VALUES = [True, 'True', 'true', '1']
 FEATURE_FLAGS = {
     'API_DETERMINE_STADSDEEL_ENABLED': os.getenv('API_DETERMINE_STADSDEEL_ENABLED', True) in TRUE_VALUES,

api/app/signals/settings/local.py.dist

@@ -19,7 +19,6 @@ FRONTEND_URL = os.getenv('FRONTEND_URL', 'http://dummy_link')
 
 SECURE_SSL_REDIRECT = False
 SECURE_REDIRECT_EXEMPT = [r'^status/', ]  # Allow health checks on localhost.
-SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https')
 SESSION_COOKIE_SECURE = False
 CSRF_COOKIE_SECURE = False
 
@@ -29,4 +28,4 @@ DATABASE_NAME = 'signals'
 DATABASE_USER = 'signals'
 DATABASE_PASSWORD = 'insecure'
 DATABASE_HOST = 'database'
-DATABASE_POST = '5432'
+DATABASE_POST = '5432'

api/app/signals/settings/testing.py

@@ -1,11 +1,10 @@
 # SPDX-License-Identifier: MPL-2.0
 # Copyright (C) 2018 - 2022 Gemeente Amsterdam
 import os
+
 from .base import INSTALLED_APPS
 from .feature_flags import FEATURE_FLAGS
 
-DEBUG = False
-LOG_QUERIES = False
 LOGGING_LEVEL = "INFO"
 
 ALLOWED_HOSTS = ['*']
@@ -18,8 +17,8 @@
 ]
 CORS_ALLOW_ALL_ORIGINS = False
 
-DATABASE_HOST = 'database'  # noqa: F405
-DATABASE_PORT = '5432'  # noqa: F405
+DATABASE_HOST = os.getenv('DATABASE_HOST', 'database')
+DATABASE_PORT = '5432'
 
 SECRET_KEY = 'insecure'
 CELERY_TASK_ALWAYS_EAGER = True
@@ -28,8 +27,6 @@
 SITE_DOMAIN = 'localhost:8000'
 
 SECURE_SSL_REDIRECT = False
-SECURE_REDIRECT_EXEMPT = [r'^status/', ]  # Allow health checks on localhost.
-SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https')
 SESSION_COOKIE_SECURE = False
 CSRF_COOKIE_SECURE = False
 
@@ -61,24 +58,10 @@
 
 FRONTEND_URL = 'http://dummy_link'
 
-
-if os.getenv('IS_GITHUB_WORKFLOW', False):
-    # Have database host to localhost for the github workflow
-    DATABASE_HOST = 'localhost'
-
-
 INSTALLED_APPS += [  # noqa
     'change_log.tests',  # Added so that we can test the chane_log with a "test only" model
 ]
 
-FEATURE_FLAGS['SYSTEM_MAIL_FEEDBACK_RECEIVED_ENABLED'] = True  # noqa
-FEATURE_FLAGS['REPORTER_MAIL_HANDLED_NEGATIVE_CONTACT_ENABLED'] = True  # noqa
-FEATURE_FLAGS['SIGNAL_HISTORY_LOG_ENABLED'] = True  # noqa
-
-def overwrite_test_settings(apps, flags):
-    apps += [  # noqa
-        'change_log.tests',  # Added so that we can test the chane_log with a "test only" model
-    ]
-    flags['SYSTEM_MAIL_FEEDBACK_RECEIVED_ENABLED'] = True  # noqa
-    flags['REPORTER_MAIL_HANDLED_NEGATIVE_CONTACT_ENABLED'] = True  # noqa
-    flags['SIGNAL_HISTORY_LOG_ENABLED'] = True  # noqa
+FEATURE_FLAGS['SYSTEM_MAIL_FEEDBACK_RECEIVED_ENABLED'] = True
+FEATURE_FLAGS['REPORTER_MAIL_HANDLED_NEGATIVE_CONTACT_ENABLED'] = True
+FEATURE_FLAGS['SIGNAL_HISTORY_LOG_ENABLED'] = True

api/app/signals/wsgi.py

@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: MPL-2.0
-# Copyright (C) 2018 - 2021 Gemeente Amsterdam
+# Copyright (C) 2018 - 2022 Gemeente Amsterdam
 import os
 
 from django.core.wsgi import get_wsgi_application

api/app/tox.ini

@@ -6,6 +6,10 @@ toxworkdir = /tmp/
 [testenv]
 envdir = {toxworkdir}/test
 sitepackages = True
+setenv =
+    TESTING = true
+passenv =
+    DATABASE_HOST
 deps =
     -r../requirements.txt
 basepython = python3
@@ -14,12 +18,10 @@ whitelist_externals =
     flake8
     isort
 commands =
-    pytest: pytest --cov=signals --cov-report=term --cov-report=html:{toxworkdir}/test/htmlcov --no-cov-on-fail {posargs:} --tb=short
-;    flake8: flake8 signals
-;    isort: isort --recursive --diff --check-only signals tests
-;    spdx: python check_spdx.py .
-;commands =
-;    pytest: pytest {posargs:}
+    pytest: pytest -n 4 --cov=signals --cov-report=term --cov-report=html:{toxworkdir}/test/htmlcov --no-cov-on-fail {posargs:} --tb=short
+    flake8: flake8 signals
+    isort: isort --recursive --diff --check-only signals tests
+    spdx: python check_spdx.py .
 
 [pytest]
 DJANGO_SETTINGS_MODULE = signals.settings