From e434e8f90bf48860d6a976629494a979868a0c85 Mon Sep 17 00:00:00 2001 From: admire Date: Sat, 19 Dec 2020 15:26:24 +0200 Subject: [PATCH 1/4] Refactor dockerfile and install front end app by default --- .env.example | 17 ++++++++++++++++- Dockerfile.g3wsuite.dockerfile | 20 +++++++++----------- config/g3w-suite/settings_docker.py | 6 +++++- config/nginx/django.conf | 11 +++++++++-- docker-compose.yml | 10 +++++++--- cmd.sh => scripts/docker-entrypoint.sh | 19 ++++++++++++++----- scripts/setup.sh | 21 +++++++++++++++++++++ 7 files changed, 81 insertions(+), 23 deletions(-) rename cmd.sh => scripts/docker-entrypoint.sh (62%) mode change 100755 => 100644 create mode 100644 scripts/setup.sh diff --git a/.env.example b/.env.example index a9834704c..ff35b990c 100644 --- a/.env.example +++ b/.env.example @@ -19,4 +19,19 @@ TILESTACHE_CACHE_BUFFER_SIZE=256 TILESTACHE_CACHE_TOKEN=374h5g96831hsgetvmkdel # Gunicorn workers (default to 8) -G3WSUITE_GUNICORN_NUM_WORKERS=8 \ No newline at end of file +G3WSUITE_GUNICORN_NUM_WORKERS=8 + +# QGIS Server env variables +PGSERVICEFILE=/pg_service/pg_service.conf +QGIS_SERVER_PARALLEL_RENDERING=1 +QGIS_SERVER_LOG_FILE=/qgis_server/error.log +MAX_CACHE_LAYERS=100 +QGIS_SERVER_LOG_LEVEL=2 +QGIS_SERVER_MAX_THREADS=-1 +# Mounth the /qgis_server folder if you need these +QGIS_SERVER_CACHE_DIRECTORY=/qgis_server/ +QGIS_SERVER_IGNORE_BAD_LAYERS=false +QGIS_SERVER_WMS_MAX_HEIGHT=-1 +QGIS_SERVER_WMS_MAX_WIDTH=-1 +QGIS_PLUGINPATH=/qgis_server/plugin +QGIS_SERVER_TRUST_LAYER_METADATA=1 diff --git a/Dockerfile.g3wsuite.dockerfile b/Dockerfile.g3wsuite.dockerfile index ea19fc73a..0a857e737 100644 --- a/Dockerfile.g3wsuite.dockerfile +++ b/Dockerfile.g3wsuite.dockerfile @@ -3,19 +3,17 @@ FROM g3wsuite/g3w-suite-deps-ltr:dev LABEL maintainer="Gis3W" Description="This image is used to install python requirements and code for g3w-suite deployment" Vendor="Gis3W" Version="1.0" # Based on main CI Docker from g3w-suite, checkout code + caching, # custom settings file -RUN apt install git -y && \ - git clone https://github.com/g3w-suite/g3w-admin.git --single-branch --branch dev /code && \ - cd /code && \ - git checkout dev +RUN apt install git -y # Override settings -COPY requirements_rl.txt . -RUN pip3 install -r requirements_rl.txt +ADD requirements_rl.txt /requirements_rl.txt -# Caching -RUN pip3 install -r /code/g3w-admin/caching/requirements.txt +ADD scripts /scripts +RUN chmod +x /scripts/*.sh -# Filemanager -RUN pip3 install -r /code/g3w-admin/filemanager/requirements.txt +RUN /scripts/setup.sh \ + && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* -CMD echo "Base image for g3w-suite-dev" && tail -f /dev/null \ No newline at end of file +CMD echo "Base image for g3w-suite-dev" && tail -f /dev/null + +ENTRYPOINT /scripts/docker-entrypoint.sh \ No newline at end of file diff --git a/config/g3w-suite/settings_docker.py b/config/g3w-suite/settings_docker.py index 3a440bf79..38891e28d 100644 --- a/config/g3w-suite/settings_docker.py +++ b/config/g3w-suite/settings_docker.py @@ -9,9 +9,11 @@ G3WADMIN_LOCAL_MORE_APPS = [ 'caching', 'editing', - 'filemanager' + 'filemanager', + 'frontend' ] + DATABASES = { 'default': { 'ENGINE': 'django.contrib.gis.db.backends.postgis', @@ -133,6 +135,8 @@ } SESSION_COOKIE_NAME = 'gi3w-suite-dev-iehtgdb264t5gr' +FRONTEND = True +FRONTEND_APP = 'frontend' diff --git a/config/nginx/django.conf b/config/nginx/django.conf index 07ed50698..03e07d861 100644 --- a/config/nginx/django.conf +++ b/config/nginx/django.conf @@ -1,5 +1,7 @@ -client_max_body_size 20M; +client_max_body_size 200M; +client_body_timeout 600; + upstream web { ip_hash; @@ -38,7 +40,12 @@ server { } location / { - proxy_read_timeout 120; + keepalive_timeout 500; + proxy_connect_timeout 600; + proxy_send_timeout 600; + send_timeout 600; + fastcgi_read_timeout 300; + proxy_read_timeout 600; proxy_set_header Host $host:$server_port; proxy_set_header X-Forwarded-For $remote_addr; proxy_pass http://web/; diff --git a/docker-compose.yml b/docker-compose.yml index 6fddfa6f3..772b17663 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,7 +16,12 @@ services: max-size: "200k" max-file: "10" volumes: - - ${WEBGIS_DOCKER_SHARED_VOLUME}:/var/lib/postgresql + - g3wsuite-pg-data:/var/lib/postgresql + healthcheck: + interval: 60s + timeout: 30s + retries: 3 + test: "pg_isready" networks: internal: aliases: @@ -37,7 +42,6 @@ services: - TILESTACHE_CACHE_TOKEN expose: - "8000" - command: /cmd.sh restart: always logging: driver: "json-file" @@ -48,7 +52,6 @@ services: depends_on: - postgis volumes: - - ./cmd.sh:/cmd.sh:ro - ${WEBGIS_DOCKER_SHARED_VOLUME}:/shared-volume - ${WEBGIS_DOCKER_SHARED_VOLUME}/node_modules:/code/node_modules - ./config/g3w-suite/overrides/templates:/code/templates:ro @@ -81,6 +84,7 @@ services: volumes: shared-volume: + g3wsuite-pg-data: networks: internal: diff --git a/cmd.sh b/scripts/docker-entrypoint.sh old mode 100755 new mode 100644 similarity index 62% rename from cmd.sh rename to scripts/docker-entrypoint.sh index 3971734e1..3c87cc42d --- a/cmd.sh +++ b/scripts/docker-entrypoint.sh @@ -1,17 +1,26 @@ #!/bin/bash -# CMD to be run inside the g3w-suite docker, this script is -# volume-mounted (ro) in the docker compose as /cmd.sh +# Entrypoint script # Start XVfb -rm /tmp/.X99-lock +if [[ -f /tmp/.X99-lock ]]; then + rm /tmp/.X99-lock +fi Xvfb :99 -screen 0 640x480x24 -nolisten tcp & export DISPLAY=:99 export QGIS_SERVER_PARALLEL_RENDERING=1 +# Start +cd /code/g3w-admin + +rm -rf /shared-volume/build_done + # Build the suite /code/ci_scripts/build_suite.sh # Setup once /code/ci_scripts/setup_suite.sh -# Start -cd /code/g3w-admin +# Run migrations to activate the front end app +python3 manage.py makemigrations +python3 manage.py migrate +python3 manage.py collectstatic --noinput + gunicorn base.wsgi:application --limit-request-fields 0 --error-logfile - \ --log-level=debug --timeout 120 --workers=${G3WSUITE_GUNICORN_NUM_WORKERS:-8} -b 0.0.0.0:8000 diff --git a/scripts/setup.sh b/scripts/setup.sh new file mode 100644 index 000000000..b1ed5413a --- /dev/null +++ b/scripts/setup.sh @@ -0,0 +1,21 @@ + +if [ -z "$(ls -A /code)" ]; then + git clone https://github.com/g3w-suite/g3w-admin.git --single-branch --branch dev /code && \ + cd /code && \ + git checkout dev +fi + +cp /requirements_rl.txt . + +# Override settings +pip3 install -r requirements_rl.txt + +# Front end +git submodule add -f https://github.com/g3w-suite/g3w-admin-frontend.git g3w-admin/frontend + +# Caching +pip3 install -r /code/g3w-admin/caching/requirements.txt + +# File manager +pip3 install -r /code/g3w-admin/filemanager/requirements.txt + From 0c120b4b60037dd24197799b9a4064ca3d5da698 Mon Sep 17 00:00:00 2001 From: admire Date: Sun, 27 Dec 2020 17:37:17 +0200 Subject: [PATCH 2/4] Activate frontend app using an env variable --- .env.example | 2 ++ config/g3w-suite/settings_docker.py | 16 +++++----------- docker-compose.yml | 3 ++- scripts/docker-entrypoint.sh | 23 +++++++++++++++++++---- scripts/setup.sh | 2 ++ 5 files changed, 30 insertions(+), 16 deletions(-) diff --git a/.env.example b/.env.example index ff35b990c..d6ab8c92c 100644 --- a/.env.example +++ b/.env.example @@ -12,6 +12,8 @@ G3WSUITE_POSTGRES_HOST=postgis G3WSUITE_POSTGRES_PORT=5432 G3WSUITE_QDJANGO_SERVER_URL=http://qgisserver/ows/ +# set this to true to activate the frontend module +FRONTEND_APP=False # Caching G3WSUITE_TILECACHE_PATH=/shared-volume/tile_cache/ diff --git a/config/g3w-suite/settings_docker.py b/config/g3w-suite/settings_docker.py index 38891e28d..7bfc5c64d 100644 --- a/config/g3w-suite/settings_docker.py +++ b/config/g3w-suite/settings_docker.py @@ -1,7 +1,6 @@ # Override settings for G3W-SUITE docker # Destination: /code/g3w-admin/base/settings/local_settings.py # Read connection parameters from environment - import os G3WADMIN_PROJECT_APPS = [] @@ -13,19 +12,18 @@ 'frontend' ] - DATABASES = { 'default': { 'ENGINE': 'django.contrib.gis.db.backends.postgis', 'NAME': os.getenv('G3WSUITE_POSTGRES_DBNAME'), - 'USER': os.getenv('G3WSUITE_POSTGRES_USER_LOCAL') if os.getenv('G3WSUITE_POSTGRES_USER_LOCAL') else "%s@%s" % (os.getenv('G3WSUITE_POSTGRES_USER'), os.getenv('G3WSUITE_POSTGRES_HOST')), + 'USER': os.getenv('G3WSUITE_POSTGRES_USER_LOCAL') if os.getenv('G3WSUITE_POSTGRES_USER_LOCAL') else "%s@%s" % ( + os.getenv('G3WSUITE_POSTGRES_USER'), os.getenv('G3WSUITE_POSTGRES_HOST')), 'PASSWORD': os.getenv('G3WSUITE_POSTGRES_PASS'), 'HOST': os.getenv('G3WSUITE_POSTGRES_HOST'), 'PORT': os.getenv('G3WSUITE_POSTGRES_PORT'), } } - MEDIA_ROOT = '/shared-volume/media/' MEDIA_URL = '/media/' STATIC_ROOT = '/shared-volume/static/' @@ -35,10 +33,10 @@ DATASOURCE_PATH = '/shared-volume/project_data/' -#CACHING SETTINGS -#======================================= +# CACHING SETTINGS +# ======================================= TILESTACHE_CACHE_NAME = 'default' -TILESTACHE_CACHE_TYPE = 'Disk' # or 'Memcache' +TILESTACHE_CACHE_TYPE = 'Disk' # or 'Memcache' TILESTACHE_CACHE_DISK_PATH = os.getenv('G3WSUITE_TILECACHE_PATH') TILESTACHE_CACHE_BUFFER_SIZE = os.getenv('TILESTACHE_CACHE_BUFFER_SIZE') TILESTACHE_CACHE_TOKEN = os.getenv('TILESTACHE_CACHE_TOKEN') @@ -135,8 +133,4 @@ } SESSION_COOKIE_NAME = 'gi3w-suite-dev-iehtgdb264t5gr' -FRONTEND = True -FRONTEND_APP = 'frontend' - - diff --git a/docker-compose.yml b/docker-compose.yml index 772b17663..148c937d9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -40,6 +40,7 @@ services: - G3WSUITE_POSTGRES_PORT - TILESTACHE_CACHE_BUFFER_SIZE - TILESTACHE_CACHE_TOKEN + - FRONTEND_APP expose: - "8000" restart: always @@ -55,7 +56,7 @@ services: - ${WEBGIS_DOCKER_SHARED_VOLUME}:/shared-volume - ${WEBGIS_DOCKER_SHARED_VOLUME}/node_modules:/code/node_modules - ./config/g3w-suite/overrides/templates:/code/templates:ro - - ./config/g3w-suite/settings_docker.py:/code/g3w-admin/base/settings/local_settings.py:ro + - ./config/g3w-suite/settings_docker.py:/code/g3w-admin/base/settings/local_settings.py networks: internal: diff --git a/scripts/docker-entrypoint.sh b/scripts/docker-entrypoint.sh index 3c87cc42d..4c137e72b 100644 --- a/scripts/docker-entrypoint.sh +++ b/scripts/docker-entrypoint.sh @@ -13,14 +13,29 @@ cd /code/g3w-admin rm -rf /shared-volume/build_done +# Activate the front end app settings + +if [[ ${FRONTEND_APP} =~ [Tt][Rr][Uu][Ee] ]] ; then + SETTINGS_LOCKFILE=/shared-volume/.settings.lockfile + if [[ ! -f ${SETTINGS_LOCKFILE} ]]; then + echo "FRONTEND = True" >> /code/g3w-admin/base/settings/local_settings.py + echo "FRONTEND_APP = 'frontend'" >> /code/g3w-admin/base/settings/local_settings.py + touch ${SETTINGS_LOCKFILE} + fi +fi + # Build the suite /code/ci_scripts/build_suite.sh # Setup once /code/ci_scripts/setup_suite.sh -# Run migrations to activate the front end app -python3 manage.py makemigrations -python3 manage.py migrate -python3 manage.py collectstatic --noinput + + +# Run migrations to activate the front end app based on env variable +if [[ ${FRONTEND_APP} =~ [Tt][Rr][Uu][Ee] ]]; then + python3 manage.py makemigrations + python3 manage.py migrate + python3 manage.py collectstatic --noinput +fi gunicorn base.wsgi:application --limit-request-fields 0 --error-logfile - \ --log-level=debug --timeout 120 --workers=${G3WSUITE_GUNICORN_NUM_WORKERS:-8} -b 0.0.0.0:8000 diff --git a/scripts/setup.sh b/scripts/setup.sh index b1ed5413a..1be7835d6 100644 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -11,8 +11,10 @@ cp /requirements_rl.txt . pip3 install -r requirements_rl.txt # Front end +#TODO make this as generic so that we can install as many plugins as possible git submodule add -f https://github.com/g3w-suite/g3w-admin-frontend.git g3w-admin/frontend + # Caching pip3 install -r /code/g3w-admin/caching/requirements.txt From 9709e298f9ba097a196a72ad16c3fa4671eb55ba Mon Sep 17 00:00:00 2001 From: admire Date: Sun, 27 Dec 2020 17:41:33 +0200 Subject: [PATCH 3/4] update readme for front end app --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index 88276d3a0..bbba97da4 100644 --- a/README.md +++ b/README.md @@ -116,6 +116,14 @@ This locking system can remain active if users do not exit the editing state cor 0 */1 * * * docker exec -e DISPLAY=:99 g3w-suite-docker_g3w-suite_1 python3 /code/g3w-admin/manage.py check_features_locked ``` +## Front-end App + +Set the environment variable +``` +FRONTEND_APP=True +``` +This will set the front end app as the default app + ## Style customization Templates can now be overridden by placing the overrides in the `config/g3w-suite/overrides/templates`, a Docker service restart is required to make the changes effective. From a65929474632e373d896eda754b737eae5785148 Mon Sep 17 00:00:00 2001 From: admire Date: Tue, 29 Dec 2020 10:46:35 +0200 Subject: [PATCH 4/4] PR feedback --- README.md | 4 ++ docker-compose-dev.yml | 91 ++++++++++++++++++++++++++++++++++++ docker-compose.yml | 2 +- scripts/docker-entrypoint.sh | 6 ++- 4 files changed, 101 insertions(+), 2 deletions(-) create mode 100644 docker-compose-dev.yml diff --git a/README.md b/README.md index bbba97da4..d922c1717 100644 --- a/README.md +++ b/README.md @@ -96,6 +96,10 @@ docker-compose up -d Data, projects, uploads and the database are stored in a shared mounted volume `shared-volume`, the volume should be on a persistent storage device and a backup policy must be enforced. +Currently, the volume is mounted in `/tmp/shared-volume-g3wsuite-dev`. In production +environments it is encouraged to change this to a permanent location. +This can be done by modifying the `.env` file. + ## First time setup - log into the application web administation panel using default credentials (_admin/admin_) diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml new file mode 100644 index 000000000..148c937d9 --- /dev/null +++ b/docker-compose-dev.yml @@ -0,0 +1,91 @@ +version: "3" +services: + postgis: + image: g3wsuite/postgis:11.0-2.5 + ports: + - "5439:5432" + environment: + - POSTGRES_DBNAME=${G3WSUITE_POSTGRES_DBNAME},data_testing,data_production + - POSTGRES_USER=${G3WSUITE_POSTGRES_USER_LOCAL} + - POSTGRES_PASS=${G3WSUITE_POSTGRES_PASS} + - ALLOW_IP_RANGE=0.0.0.0/0 + restart: on-failure + logging: + driver: "json-file" + options: + max-size: "200k" + max-file: "10" + volumes: + - g3wsuite-pg-data:/var/lib/postgresql + healthcheck: + interval: 60s + timeout: 30s + retries: 3 + test: "pg_isready" + networks: + internal: + aliases: + - ${WEBGIS_PUBLIC_HOSTNAME} + + g3w-suite: + image: g3wsuite/g3w-suite:dev + environment: + - G3WSUITE_TILECACHE_PATH + - G3WSUITE_QDJANGO_SERVER_URL + - G3WSUITE_POSTGRES_DBNAME + - G3WSUITE_POSTGRES_USER + - G3WSUITE_POSTGRES_USER_LOCAL + - G3WSUITE_POSTGRES_PASS + - G3WSUITE_POSTGRES_HOST + - G3WSUITE_POSTGRES_PORT + - TILESTACHE_CACHE_BUFFER_SIZE + - TILESTACHE_CACHE_TOKEN + - FRONTEND_APP + expose: + - "8000" + restart: always + logging: + driver: "json-file" + options: + max-size: "200k" + max-file: "10" + + depends_on: + - postgis + volumes: + - ${WEBGIS_DOCKER_SHARED_VOLUME}:/shared-volume + - ${WEBGIS_DOCKER_SHARED_VOLUME}/node_modules:/code/node_modules + - ./config/g3w-suite/overrides/templates:/code/templates:ro + - ./config/g3w-suite/settings_docker.py:/code/g3w-admin/base/settings/local_settings.py + + networks: + internal: + + nginx: + image: nginx:1.19.5 + ports: + - "8080:8080" + - "443:443" + expose: + - "8080" + volumes: + - ${WEBGIS_DOCKER_SHARED_VOLUME}:/shared-volume + - ${WEBGIS_DOCKER_SHARED_VOLUME}/var/www/.well-known:/var/www/.well-known + - ${WEBGIS_DOCKER_SHARED_VOLUME}/certs/letsencrypt:/etc/letsencrypt:ro + - ./config/g3w-suite/overrides/static:/shared-volume/static/overrides:ro + - ./config/nginx:/etc/nginx/conf.d:ro + logging: + driver: "json-file" + options: + max-size: "200k" + max-file: "10" + restart: always + networks: + internal: + +volumes: + shared-volume: + g3wsuite-pg-data: + +networks: + internal: diff --git a/docker-compose.yml b/docker-compose.yml index 148c937d9..ea48d7969 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,7 +16,7 @@ services: max-size: "200k" max-file: "10" volumes: - - g3wsuite-pg-data:/var/lib/postgresql + - ${WEBGIS_DOCKER_SHARED_VOLUME}:/var/lib/postgresql healthcheck: interval: 60s timeout: 30s diff --git a/scripts/docker-entrypoint.sh b/scripts/docker-entrypoint.sh index 4c137e72b..c9d44b1f6 100644 --- a/scripts/docker-entrypoint.sh +++ b/scripts/docker-entrypoint.sh @@ -11,7 +11,11 @@ export QGIS_SERVER_PARALLEL_RENDERING=1 # Start cd /code/g3w-admin -rm -rf /shared-volume/build_done +# When building in dev env you might want a clean build each time. +if [[ -z "${DEV}" ]]; then + rm -rf /shared-volume/build_done +fi + # Activate the front end app settings