diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7a904f90c..bacca1e2a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,7 +13,7 @@ jobs: POSTGRES_USER: signals POSTGRES_PASSWORD: insecure POSTGRES_DB: signals - IS_GITHUB_WORKFLOW: True + ports: ['5432:5432'] options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 @@ -40,9 +40,11 @@ jobs: pip install -r api/requirements.txt - name: Run tests + env: + DATABASE_HOST: localhost run: | cd api/app - tox -- --ds signals.settings + tox - name: Upload coverage report uses: actions/upload-artifact@v1 diff --git a/.gitignore b/.gitignore index 7f605a870..b63336494 100755 --- a/.gitignore +++ b/.gitignore @@ -39,7 +39,7 @@ htmlcov/ .cache nosetests.xml coverage.xml - +docker-compose.override.yml # Translations *.mo *.pot diff --git a/README.md b/README.md index fea80ead3..6eb30a7e7 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,9 @@ request, as any failed checks will abort the build pipeline. ### Running and developing locally using Docker and docker-compose +Starting the project for the first time requires you to copy the local.py.dist file +in the signals.settings folder and rename this copied file to local.py. + Assuming no services are running, start the database and queue: ```console diff --git a/api/README.md b/api/README.md index 121a7c72e..8e925a5cc 100644 --- a/api/README.md +++ b/api/README.md @@ -31,15 +31,14 @@ Django settings Settings for all different environments are defined in `app/signals/settings/...`. The file `base.py` contains all the default settings. Defaults should be production ready. You can -override this, if needed, in one of the other specific settings files (e.g. `testing`). +override this, if needed, in the other specific settings files (e.g. `testing`, `local`). Current available settings: -- `production` -> Used for Acceptance/Production instances -- `testing` -> Used for Jenkins test pipeline and testing with `tox` -- `development` -> Used for Local Docker instances -- `local` -> Override settings in `development` for local usage (not tracked in Git) - +- `base` -> Used for Acceptance/Production instances +- `testing` -> Used for Jenkins test pipeline and testing with `tox` +- `local` -> Used for Local Docker instances. + Override settings in `local` for local usage (not tracked in Git) Tox usage ========= diff --git a/api/app/logs/config.py b/api/app/logs/config.py index 3761c9c0e..5b31a5548 100644 --- a/api/app/logs/config.py +++ b/api/app/logs/config.py @@ -5,7 +5,7 @@ 'disable_existing_loggers': False, 'formatters': { 'elaborate': { - 'format': '{levelname} {name} {module}.{filename} {message}', + 'format': '{levelname} {module}.{filename} {message}', 'style': '{' } }, diff --git a/api/app/logs/handlers.py b/api/app/logs/handlers.py index 3c5ca20f1..1803e19f0 100644 --- a/api/app/logs/handlers.py +++ b/api/app/logs/handlers.py @@ -73,7 +73,6 @@ def emit(self, record: LogRecord): try: parsed_message = self.colorize(record=record) except Exception: - print('error called') parsed_message = self.format(record=record) stream = self.output @@ -81,5 +80,4 @@ def emit(self, record: LogRecord): stream.write(self.terminator) self.flush() except Exception: - print('error2 called') self.handleError(record) diff --git a/api/app/manage.py b/api/app/manage.py index 6401c67b7..700377282 100755 --- a/api/app/manage.py +++ b/api/app/manage.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: MPL-2.0 -# Copyright (C) 2018 - 2021 Gemeente Amsterdam +# Copyright (C) 2018 - 2022 Gemeente Amsterdam #!/usr/bin/env python import os import sys diff --git a/api/app/signals/celery.py b/api/app/signals/celery.py index e1cd3c392..fbece2e7b 100644 --- a/api/app/signals/celery.py +++ b/api/app/signals/celery.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: MPL-2.0 -# Copyright (C) 2018 - 2021 Gemeente Amsterdam +# Copyright (C) 2018 - 2022 Gemeente Amsterdam import os from celery import Celery diff --git a/api/app/signals/settings/__init__.py b/api/app/signals/settings/__init__.py index 74300337b..43ab1b157 100644 --- a/api/app/signals/settings/__init__.py +++ b/api/app/signals/settings/__init__.py @@ -2,31 +2,34 @@ # Copyright (C) 2018 - 2021 Gemeente Amsterdam # Temporary fix to make the settings restructure work with current deployment settings. Can be # removed after the Ansible playbooks for deployment are updated with the production settings. -import sys -from .base import * # noqa -from .feature_flags import * +import os, sys # noqa + from logs import get_configuration -try: - from .local import * -except ImportError: - pass +from .base import * # noqa +from .feature_flags import * # noqa + +if "test" in sys.argv or "pytest" in sys.argv[0] or os.getenv('TESTING') == 'true': + try: + from .testing import * # noqa + except ImportError: + pass -if "test" in sys.argv or "pytest" in sys.argv[0]: +else: try: - from .testing import * + from .local import * # noqa except ImportError: pass -LOGGING = get_configuration(local_apps=SIGNAL_APPS, logging_level=LOGGING_LEVEL) +LOGGING = get_configuration(local_apps=SIGNAL_APPS, logging_level=LOGGING_LEVEL) # noqa DATABASES = { 'default': { 'ENGINE': 'django.contrib.gis.db.backends.postgis', - 'NAME': DATABASE_NAME, - 'USER': DATABASE_USER, - 'PASSWORD': DATABASE_PASSWORD, - 'HOST': DATABASE_HOST, - 'PORT': DATABASE_PORT + 'NAME': DATABASE_NAME, # noqa: + 'USER': DATABASE_USER, # noqa + 'PASSWORD': DATABASE_PASSWORD, # noqa + 'HOST': DATABASE_HOST, # noqa + 'PORT': DATABASE_PORT # noqa }, -} +} # noqa diff --git a/api/app/signals/settings/base.py b/api/app/signals/settings/base.py index 7924945fa..dae24ab65 100644 --- a/api/app/signals/settings/base.py +++ b/api/app/signals/settings/base.py @@ -125,7 +125,7 @@ } ] -#DATABASE +# Database settings DATABASE_NAME = os.getenv('DATABASE_NAME', 'signals') DATABASE_USER = os.getenv('DATABASE_USER', 'signals') DATABASE_PASSWORD = os.getenv('DATABASE_PASSWORD', 'insecure') @@ -259,111 +259,6 @@ 'dsn': os.getenv('SENTRY_RAVEN_DSN'), } -# Django Logging settings -# LOGGING = { -# 'version': 1, -# 'disable_existing_loggers': False, -# 'root': { -# 'level': 'INFO', -# 'handlers': ['console', 'sentry'], -# }, -# 'formatters': { -# 'console': { -# 'format': '%(asctime)s - %(name)s - %(levelname)s - %(message)s' -# } -# }, -# 'filters': { -# -# }, -# 'handlers': { -# 'console': { -# 'level': 'INFO', -# 'class': 'logs.handlers.ColoredHandler', -# 'formatter': 'console', -# }, -# 'sentry': { -# 'level': 'WARNING', -# 'class': 'raven.contrib.django.raven_compat.handlers.SentryHandler', -# }, -# }, -# 'loggers': { -# 'signals': { -# 'level': 'WARNING', -# 'handlers': ['console'], -# 'propagate': True, -# }, -# 'django': { -# 'level': 'DEBUG', -# 'handlers': ['console'], -# 'propagate': False, -# }, -# 'raven': { -# 'level': 'DEBUG', -# 'handlers': ['console'], -# 'propagate': False, -# }, -# 'sentry.errors': { -# 'level': 'DEBUG', -# 'handlers': ['console'], -# 'propagate': False, -# }, -# -# # Debug all batch jobs -# 'doc': { -# 'level': 'INFO', -# 'handlers': ['console'], -# 'propagate': False, -# }, -# 'index': { -# 'level': 'INFO', -# 'handlers': ['console'], -# 'propagate': False, -# }, -# 'search': { -# 'level': 'ERROR', -# 'handlers': ['console'], -# 'propagate': False, -# }, -# 'elasticsearch': { -# 'level': 'ERROR', -# 'handlers': ['console'], -# 'propagate': False, -# }, -# 'urllib3': { -# 'level': 'ERROR', -# 'handlers': ['console'], -# 'propagate': False, -# }, -# 'factory.containers': { -# 'level': 'INFO', -# 'handlers': ['console'], -# 'propagate': False, -# }, -# 'factory.generate': { -# 'handlers': ['console'], -# 'level': 'INFO', -# 'propagate': False, -# }, -# 'requests.packages.urllib3.connectionpool': { -# 'level': 'ERROR', -# 'handlers': ['console'], -# 'propagate': False, -# }, -# 'signals.apps.reporting.management.commands.dump_horeca_csv_files': { -# 'level': 'DEBUG', -# 'handlers': ['console'], -# 'propagate': False, -# }, -# -# # Log all unhandled exceptions -# 'django.request': { -# 'level': 'ERROR', -# 'handlers': ['console'], -# 'propagate': False, -# }, -# }, -# } - # Django REST framework settings REST_FRAMEWORK = dict( PAGE_SIZE=100, diff --git a/api/app/signals/settings/feature_flags.py b/api/app/signals/settings/feature_flags.py index e32ec8fd3..9e44cd04d 100644 --- a/api/app/signals/settings/feature_flags.py +++ b/api/app/signals/settings/feature_flags.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: MPL-2.0 # Copyright (C) 2022 Gemeente Amsterdam import os + TRUE_VALUES = [True, 'True', 'true', '1'] FEATURE_FLAGS = { 'API_DETERMINE_STADSDEEL_ENABLED': os.getenv('API_DETERMINE_STADSDEEL_ENABLED', True) in TRUE_VALUES, diff --git a/api/app/signals/settings/local.py.dist b/api/app/signals/settings/local.py.dist index fbcba769f..b9ee3bcea 100644 --- a/api/app/signals/settings/local.py.dist +++ b/api/app/signals/settings/local.py.dist @@ -19,7 +19,6 @@ FRONTEND_URL = os.getenv('FRONTEND_URL', 'http://dummy_link') SECURE_SSL_REDIRECT = False SECURE_REDIRECT_EXEMPT = [r'^status/', ] # Allow health checks on localhost. -SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https') SESSION_COOKIE_SECURE = False CSRF_COOKIE_SECURE = False @@ -29,4 +28,4 @@ DATABASE_NAME = 'signals' DATABASE_USER = 'signals' DATABASE_PASSWORD = 'insecure' DATABASE_HOST = 'database' -DATABASE_POST = '5432' \ No newline at end of file +DATABASE_POST = '5432' diff --git a/api/app/signals/settings/testing.py b/api/app/signals/settings/testing.py index 8e7abef6b..4c7729d6a 100644 --- a/api/app/signals/settings/testing.py +++ b/api/app/signals/settings/testing.py @@ -1,11 +1,10 @@ # SPDX-License-Identifier: MPL-2.0 # Copyright (C) 2018 - 2022 Gemeente Amsterdam import os + from .base import INSTALLED_APPS from .feature_flags import FEATURE_FLAGS -DEBUG = False -LOG_QUERIES = False LOGGING_LEVEL = "INFO" ALLOWED_HOSTS = ['*'] @@ -18,8 +17,8 @@ ] CORS_ALLOW_ALL_ORIGINS = False -DATABASE_HOST = 'database' # noqa: F405 -DATABASE_PORT = '5432' # noqa: F405 +DATABASE_HOST = os.getenv('DATABASE_HOST', 'database') +DATABASE_PORT = '5432' SECRET_KEY = 'insecure' CELERY_TASK_ALWAYS_EAGER = True @@ -28,8 +27,6 @@ SITE_DOMAIN = 'localhost:8000' SECURE_SSL_REDIRECT = False -SECURE_REDIRECT_EXEMPT = [r'^status/', ] # Allow health checks on localhost. -SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https') SESSION_COOKIE_SECURE = False CSRF_COOKIE_SECURE = False @@ -61,24 +58,10 @@ FRONTEND_URL = 'http://dummy_link' - -if os.getenv('IS_GITHUB_WORKFLOW', False): - # Have database host to localhost for the github workflow - DATABASE_HOST = 'localhost' - - INSTALLED_APPS += [ # noqa 'change_log.tests', # Added so that we can test the chane_log with a "test only" model ] -FEATURE_FLAGS['SYSTEM_MAIL_FEEDBACK_RECEIVED_ENABLED'] = True # noqa -FEATURE_FLAGS['REPORTER_MAIL_HANDLED_NEGATIVE_CONTACT_ENABLED'] = True # noqa -FEATURE_FLAGS['SIGNAL_HISTORY_LOG_ENABLED'] = True # noqa - -def overwrite_test_settings(apps, flags): - apps += [ # noqa - 'change_log.tests', # Added so that we can test the chane_log with a "test only" model - ] - flags['SYSTEM_MAIL_FEEDBACK_RECEIVED_ENABLED'] = True # noqa - flags['REPORTER_MAIL_HANDLED_NEGATIVE_CONTACT_ENABLED'] = True # noqa - flags['SIGNAL_HISTORY_LOG_ENABLED'] = True # noqa \ No newline at end of file +FEATURE_FLAGS['SYSTEM_MAIL_FEEDBACK_RECEIVED_ENABLED'] = True +FEATURE_FLAGS['REPORTER_MAIL_HANDLED_NEGATIVE_CONTACT_ENABLED'] = True +FEATURE_FLAGS['SIGNAL_HISTORY_LOG_ENABLED'] = True diff --git a/api/app/signals/utils/staticfieldfilter.py b/api/app/signals/utils/staticfieldfilter.py deleted file mode 100644 index 4da74222e..000000000 --- a/api/app/signals/utils/staticfieldfilter.py +++ /dev/null @@ -1,5 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 -# Copyright (C) 2018 - 2021 Gemeente Amsterdam -import logging - - diff --git a/api/app/signals/wsgi.py b/api/app/signals/wsgi.py index e2f3a14a9..aaea4bcf5 100644 --- a/api/app/signals/wsgi.py +++ b/api/app/signals/wsgi.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: MPL-2.0 -# Copyright (C) 2018 - 2021 Gemeente Amsterdam +# Copyright (C) 2018 - 2022 Gemeente Amsterdam import os from django.core.wsgi import get_wsgi_application diff --git a/api/app/tox.ini b/api/app/tox.ini index b2f2e94ed..6b8427583 100644 --- a/api/app/tox.ini +++ b/api/app/tox.ini @@ -6,6 +6,10 @@ toxworkdir = /tmp/ [testenv] envdir = {toxworkdir}/test sitepackages = True +setenv = + TESTING = true +passenv = + DATABASE_HOST deps = -r../requirements.txt basepython = python3 @@ -14,12 +18,10 @@ whitelist_externals = flake8 isort commands = - pytest: pytest --cov=signals --cov-report=term --cov-report=html:{toxworkdir}/test/htmlcov --no-cov-on-fail {posargs:} --tb=short -; flake8: flake8 signals -; isort: isort --recursive --diff --check-only signals tests -; spdx: python check_spdx.py . -;commands = -; pytest: pytest {posargs:} + pytest: pytest -n 4 --cov=signals --cov-report=term --cov-report=html:{toxworkdir}/test/htmlcov --no-cov-on-fail {posargs:} --tb=short + flake8: flake8 signals + isort: isort --recursive --diff --check-only signals tests + spdx: python check_spdx.py . [pytest] DJANGO_SETTINGS_MODULE = signals.settings diff --git a/api/deploy/test/docker-compose.yml b/api/deploy/test/docker-compose.yml index f263be40a..87136c44f 100644 --- a/api/deploy/test/docker-compose.yml +++ b/api/deploy/test/docker-compose.yml @@ -27,7 +27,6 @@ services: DATABASE_PASSWORD: insecure ENVIRONMENT: test DJANGO_SETTINGS_MODULE: signals.settings - IS_DOCKER: True volumes: - ./run_tests.sh:/app/run_tests.sh user: root diff --git a/docker-compose.yml b/docker-compose.yml index 414458fab..b74443c02 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -69,7 +69,6 @@ services: - AUTOMATICALLY_CREATE_CHILD_SIGNALS_PER_CONTAINER=True - RABBITMQ_HOST=rabbit - ELASTICSEARCH_HOST=elasticsearch:9200 - - IS_DOCKER=True volumes: - ./api/app:/app - ./api/deploy:/deploy @@ -89,7 +88,6 @@ services: - SWIFT_ENABLED=False - AUTOMATICALLY_CREATE_CHILD_SIGNALS_PER_CONTAINER=True - RABBITMQ_HOST=rabbit - - IS_DOCKER=True volumes: - ./api/app:/app - ./api/deploy:/deploy @@ -117,7 +115,6 @@ services: - INITIALIZE_WITH_DUMMY_DATA=0 - AUTOMATICALLY_CREATE_CHILD_SIGNALS_PER_CONTAINER=True - RABBITMQ_HOST=rabbit - - IS_DOCKER=True volumes: - ./api/app:/app - ./api/deploy:/deploy