Optimize docker and install front end by default

.env.example

@@ -12,11 +12,28 @@ G3WSUITE_POSTGRES_HOST=postgis
 G3WSUITE_POSTGRES_PORT=5432
 
 G3WSUITE_QDJANGO_SERVER_URL=http://qgisserver/ows/
+# set this to true to activate the frontend module
+FRONTEND_APP=False
 
 # Caching
 G3WSUITE_TILECACHE_PATH=/shared-volume/tile_cache/
 TILESTACHE_CACHE_BUFFER_SIZE=256
 TILESTACHE_CACHE_TOKEN=374h5g96831hsgetvmkdel
 
 # Gunicorn workers (default to 8)
-G3WSUITE_GUNICORN_NUM_WORKERS=8
+G3WSUITE_GUNICORN_NUM_WORKERS=8
+
+# QGIS Server env variables
+PGSERVICEFILE=/pg_service/pg_service.conf
+QGIS_SERVER_PARALLEL_RENDERING=1
+QGIS_SERVER_LOG_FILE=/qgis_server/error.log
+MAX_CACHE_LAYERS=100
+QGIS_SERVER_LOG_LEVEL=2
+QGIS_SERVER_MAX_THREADS=-1
+# Mounth the /qgis_server folder if you need these
+QGIS_SERVER_CACHE_DIRECTORY=/qgis_server/
+QGIS_SERVER_IGNORE_BAD_LAYERS=false
+QGIS_SERVER_WMS_MAX_HEIGHT=-1
+QGIS_SERVER_WMS_MAX_WIDTH=-1
+QGIS_PLUGINPATH=/qgis_server/plugin
+QGIS_SERVER_TRUST_LAYER_METADATA=1

Dockerfile.g3wsuite.dockerfile

@@ -3,19 +3,17 @@ FROM g3wsuite/g3w-suite-deps-ltr:dev
 LABEL maintainer="Gis3W" Description="This image is used to install python requirements and code for g3w-suite deployment" Vendor="Gis3W" Version="1.0"
 # Based on main CI Docker from  g3w-suite, checkout code + caching,
 # custom settings file
-RUN apt install git -y && \
-    git clone https://github.com/g3w-suite/g3w-admin.git --single-branch --branch dev /code && \
-    cd /code && \
-    git checkout dev
+RUN apt install git -y
 
 # Override settings
-COPY requirements_rl.txt .
-RUN pip3 install -r requirements_rl.txt
+ADD requirements_rl.txt /requirements_rl.txt
 
-# Caching
-RUN pip3 install -r /code/g3w-admin/caching/requirements.txt
+ADD scripts /scripts
+RUN chmod +x /scripts/*.sh
 
-# Filemanager
-RUN pip3 install -r /code/g3w-admin/filemanager/requirements.txt
+RUN /scripts/setup.sh \
+    && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
-CMD echo "Base image for g3w-suite-dev" && tail -f /dev/null
+CMD echo "Base image for g3w-suite-dev" && tail -f /dev/null
+
+ENTRYPOINT /scripts/docker-entrypoint.sh

README.md

@@ -96,6 +96,10 @@ docker-compose up -d
 Data, projects, uploads and the database are stored in a shared mounted volume `shared-volume`, the volume should be on a persistent storage device and a backup
 policy must be enforced.
 
+Currently, the volume is mounted in `/tmp/shared-volume-g3wsuite-dev`. In production
+environments it is encouraged to change this to a permanent location. 
+This can be done by modifying the `.env` file.
+
 ## First time setup
 
 - log into the application web administation panel using default credentials (_admin/admin_)
@@ -116,6 +120,14 @@ This locking system can remain active if users do not exit the editing state cor
 0 */1 * * * docker exec -e DISPLAY=:99 g3w-suite-docker_g3w-suite_1 python3 /code/g3w-admin/manage.py check_features_locked
 ```
 
+## Front-end App
+
+Set the environment variable 
+```
+FRONTEND_APP=True
+```
+This will set the front end app as the default app
+
 ## Style customization
 
 Templates can now be overridden by placing the overrides in the `config/g3w-suite/overrides/templates`, a Docker service restart is required to make the changes effective.

config/g3w-suite/settings_docker.py

@@ -1,29 +1,29 @@
 # Override settings for G3W-SUITE docker
 # Destination: /code/g3w-admin/base/settings/local_settings.py
 # Read connection parameters from environment
-
 import os
 
 G3WADMIN_PROJECT_APPS = []
 
 G3WADMIN_LOCAL_MORE_APPS = [
     'caching',
     'editing',
-    'filemanager'
+    'filemanager',
+    'frontend'
 ]
 
 DATABASES = {
     'default': {
         'ENGINE': 'django.contrib.gis.db.backends.postgis',
         'NAME': os.getenv('G3WSUITE_POSTGRES_DBNAME'),
-        'USER': os.getenv('G3WSUITE_POSTGRES_USER_LOCAL') if os.getenv('G3WSUITE_POSTGRES_USER_LOCAL') else "%s@%s" % (os.getenv('G3WSUITE_POSTGRES_USER'), os.getenv('G3WSUITE_POSTGRES_HOST')),
+        'USER': os.getenv('G3WSUITE_POSTGRES_USER_LOCAL') if os.getenv('G3WSUITE_POSTGRES_USER_LOCAL') else "%s@%s" % (
+        os.getenv('G3WSUITE_POSTGRES_USER'), os.getenv('G3WSUITE_POSTGRES_HOST')),
         'PASSWORD': os.getenv('G3WSUITE_POSTGRES_PASS'),
         'HOST': os.getenv('G3WSUITE_POSTGRES_HOST'),
         'PORT': os.getenv('G3WSUITE_POSTGRES_PORT'),
     }
 }
 
-
 MEDIA_ROOT = '/shared-volume/media/'
 MEDIA_URL = '/media/'
 STATIC_ROOT = '/shared-volume/static/'
@@ -33,10 +33,10 @@
 
 DATASOURCE_PATH = '/shared-volume/project_data/'
 
-#CACHING SETTINGS
-#=======================================
+# CACHING SETTINGS
+# =======================================
 TILESTACHE_CACHE_NAME = 'default'
-TILESTACHE_CACHE_TYPE = 'Disk' # or 'Memcache'
+TILESTACHE_CACHE_TYPE = 'Disk'  # or 'Memcache'
 TILESTACHE_CACHE_DISK_PATH = os.getenv('G3WSUITE_TILECACHE_PATH')
 TILESTACHE_CACHE_BUFFER_SIZE = os.getenv('TILESTACHE_CACHE_BUFFER_SIZE')
 TILESTACHE_CACHE_TOKEN = os.getenv('TILESTACHE_CACHE_TOKEN')
@@ -134,5 +134,3 @@
 
 SESSION_COOKIE_NAME = 'gi3w-suite-dev-iehtgdb264t5gr'
 
-
-

config/nginx/django.conf

@@ -1,5 +1,7 @@
 
-client_max_body_size 20M;
+client_max_body_size 200M;
+client_body_timeout  600;
+
 
 upstream web {
   ip_hash;
@@ -38,7 +40,12 @@ server {
   }
 
   location / {
-        proxy_read_timeout 120;
+        keepalive_timeout           500;
+        proxy_connect_timeout       600;
+        proxy_send_timeout          600;
+        send_timeout                600;
+        fastcgi_read_timeout        300;
+        proxy_read_timeout          600;
         proxy_set_header Host            $host:$server_port;
         proxy_set_header X-Forwarded-For $remote_addr;
         proxy_pass http://web/;

docker-compose-dev.yml

@@ -0,0 +1,91 @@
+version: "3"
+services:
+  postgis:
+    image: g3wsuite/postgis:11.0-2.5
+    ports:
+      - "5439:5432"
+    environment:
+      - POSTGRES_DBNAME=${G3WSUITE_POSTGRES_DBNAME},data_testing,data_production
+      - POSTGRES_USER=${G3WSUITE_POSTGRES_USER_LOCAL}
+      - POSTGRES_PASS=${G3WSUITE_POSTGRES_PASS}
+      - ALLOW_IP_RANGE=0.0.0.0/0
+    restart: on-failure
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "200k"
+        max-file: "10"
+    volumes:
+      - g3wsuite-pg-data:/var/lib/postgresql
+    healthcheck:
+      interval: 60s
+      timeout: 30s
+      retries: 3
+      test: "pg_isready"
+    networks:
+      internal:
+        aliases:
+          - ${WEBGIS_PUBLIC_HOSTNAME}
+
+  g3w-suite:
+    image: g3wsuite/g3w-suite:dev
+    environment:
+      - G3WSUITE_TILECACHE_PATH
+      - G3WSUITE_QDJANGO_SERVER_URL
+      - G3WSUITE_POSTGRES_DBNAME
+      - G3WSUITE_POSTGRES_USER
+      - G3WSUITE_POSTGRES_USER_LOCAL
+      - G3WSUITE_POSTGRES_PASS
+      - G3WSUITE_POSTGRES_HOST
+      - G3WSUITE_POSTGRES_PORT
+      - TILESTACHE_CACHE_BUFFER_SIZE
+      - TILESTACHE_CACHE_TOKEN
+      - FRONTEND_APP
+    expose:
+      - "8000"
+    restart: always
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "200k"
+        max-file: "10"
+
+    depends_on:
+      - postgis
+    volumes:
+      - ${WEBGIS_DOCKER_SHARED_VOLUME}:/shared-volume
+      - ${WEBGIS_DOCKER_SHARED_VOLUME}/node_modules:/code/node_modules
+      - ./config/g3w-suite/overrides/templates:/code/templates:ro
+      - ./config/g3w-suite/settings_docker.py:/code/g3w-admin/base/settings/local_settings.py
+
+    networks:
+      internal:
+
+  nginx:
+    image: nginx:1.19.5
+    ports:
+      - "8080:8080"
+      - "443:443"
+    expose:
+      - "8080"
+    volumes:
+      - ${WEBGIS_DOCKER_SHARED_VOLUME}:/shared-volume
+      - ${WEBGIS_DOCKER_SHARED_VOLUME}/var/www/.well-known:/var/www/.well-known
+      - ${WEBGIS_DOCKER_SHARED_VOLUME}/certs/letsencrypt:/etc/letsencrypt:ro
+      - ./config/g3w-suite/overrides/static:/shared-volume/static/overrides:ro
+      - ./config/nginx:/etc/nginx/conf.d:ro
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "200k"
+        max-file: "10"
+    restart: always
+    networks:
+      internal:
+
+volumes:
+  shared-volume:
+  g3wsuite-pg-data:
+
+networks:
+  internal:

docker-compose.yml

@@ -17,6 +17,11 @@ services:
         max-file: "10"
     volumes:
       - ${WEBGIS_DOCKER_SHARED_VOLUME}:/var/lib/postgresql
+    healthcheck:
+      interval: 60s
+      timeout: 30s
+      retries: 3
+      test: "pg_isready"
     networks:
       internal:
         aliases:
@@ -35,9 +40,9 @@ services:
       - G3WSUITE_POSTGRES_PORT
       - TILESTACHE_CACHE_BUFFER_SIZE
       - TILESTACHE_CACHE_TOKEN
+      - FRONTEND_APP
     expose:
       - "8000"
-    command: /cmd.sh
     restart: always
     logging:
       driver: "json-file"
@@ -48,11 +53,10 @@ services:
     depends_on:
       - postgis
     volumes:
-      - ./cmd.sh:/cmd.sh:ro
       - ${WEBGIS_DOCKER_SHARED_VOLUME}:/shared-volume
       - ${WEBGIS_DOCKER_SHARED_VOLUME}/node_modules:/code/node_modules
       - ./config/g3w-suite/overrides/templates:/code/templates:ro
-      - ./config/g3w-suite/settings_docker.py:/code/g3w-admin/base/settings/local_settings.py:ro
+      - ./config/g3w-suite/settings_docker.py:/code/g3w-admin/base/settings/local_settings.py
 
     networks:
       internal:
@@ -81,6 +85,7 @@ services:
 
 volumes:
   shared-volume:
+  g3wsuite-pg-data:
 
 networks:
   internal:

scripts/docker-entrypoint.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+# Entrypoint script
+
+# Start XVfb
+if [[  -f /tmp/.X99-lock ]]; then
+  rm /tmp/.X99-lock
+fi
+Xvfb :99 -screen 0 640x480x24 -nolisten tcp &
+export DISPLAY=:99
+export QGIS_SERVER_PARALLEL_RENDERING=1
+# Start
+cd /code/g3w-admin
+
+# When building in dev env you might want a clean build each time.
+if [[ -z "${DEV}" ]]; then
+  rm -rf /shared-volume/build_done
+fi
+
+
+# Activate the front end app settings
+
+if [[ ${FRONTEND_APP} =~ [Tt][Rr][Uu][Ee] ]] ; then
+  SETTINGS_LOCKFILE=/shared-volume/.settings.lockfile
+  if [[ ! -f ${SETTINGS_LOCKFILE} ]]; then
+    echo "FRONTEND = True"  >> /code/g3w-admin/base/settings/local_settings.py
+    echo "FRONTEND_APP = 'frontend'" >> /code/g3w-admin/base/settings/local_settings.py
+    touch ${SETTINGS_LOCKFILE}
+  fi
+fi
+
+# Build the suite
+/code/ci_scripts/build_suite.sh
+# Setup once
+/code/ci_scripts/setup_suite.sh
+
+
+# Run migrations to activate the front end app based on env variable
+if [[ ${FRONTEND_APP} =~ [Tt][Rr][Uu][Ee] ]]; then
+  python3 manage.py makemigrations
+  python3 manage.py migrate
+  python3 manage.py collectstatic --noinput
+fi
+
+gunicorn base.wsgi:application --limit-request-fields 0 --error-logfile - \
+    --log-level=debug --timeout 120 --workers=${G3WSUITE_GUNICORN_NUM_WORKERS:-8} -b 0.0.0.0:8000

scripts/setup.sh

@@ -0,0 +1,23 @@
+
+if [ -z "$(ls -A /code)" ]; then
+   git clone https://github.com/g3w-suite/g3w-admin.git --single-branch --branch dev /code && \
+   cd /code && \
+   git checkout dev
+fi
+
+cp /requirements_rl.txt .
+
+# Override settings
+pip3 install -r requirements_rl.txt
+
+# Front end
+#TODO make this as generic so that we can install as many plugins as possible
+git submodule add -f https://github.com/g3w-suite/g3w-admin-frontend.git  g3w-admin/frontend
+
+
+# Caching
+pip3 install -r /code/g3w-admin/caching/requirements.txt
+
+# File manager
+pip3 install -r /code/g3w-admin/filemanager/requirements.txt
+