Merging Release 1.2 changes into production environment

.env_template

@@ -51,6 +51,8 @@ USERS_OPEN_REGISTRATION=False
 SENTRY_DSN=
 INSTALL_DEV=True
 INSTALL_JUPYTER=True
+DISABLE_NUMBA_STARTUP_CALL=True
+R5_HOST=172.17.0.1
 
 # FRONTENDENV
 VUE_APP_I18N_LOCALE=en

.github/workflows/PR.yaml

@@ -22,7 +22,7 @@ jobs:
         run: echo ${{steps.get_file_changes.outputs.files}}
       - name: Build changed client
         if: contains(steps.get_file_changes.outputs.files, 'app/client')
-        run: make build-client-docker-image -e COMPONENT=client
+        run: make build-docker-image-pr -e COMPONENT=client
       - name: Build changed api
         if: contains(steps.get_file_changes.outputs.files, 'app/api')
-        run: make build-docker-image -e COMPONENT=api
+        run: make build-docker-image-pr -e COMPONENT=api

.github/workflows/Push.yaml

@@ -58,6 +58,7 @@ jobs:
       POSTGRES_USER: ${{ secrets.POSTGRES_USER }}
       POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }}
       POSTGRES_HOST: ${{ secrets.POSTGRES_HOST }}
+      R5_HOST: ${{ secrets.R5_HOST }}
       API_SECRET_KEY: ${{ secrets.API_SECRET_KEY }}
       SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
       EMAILS_FROM_EMAIL: ${{ secrets.EMAILS_FROM_EMAIL }}
@@ -67,6 +68,7 @@ jobs:
       FIRST_SUPERUSER_NAME: ${{ secrets.FIRST_SUPERUSER_NAME }}
       FIRST_SUPERUSER_SURNAME: ${{ secrets.FIRST_SUPERUSER_SURNAME }}
       SMTP_PASSWORD: ${{ secrets.SMTP_PASSWORD }}
+      BACKEND_CORS_ORIGIN: ${{ secrets.BACKEND_CORS_ORIGIN }}
     steps:
       - uses: actions/checkout@v2
       - uses: azure/[email protected]

.gitignore

@@ -12,8 +12,19 @@ ubuntu-bionic-18.04-cloudimg-console.log
 node_modules
 .env
 /.vscode
-
+r5-cache
 /app/api/app/app/exts/cpp/build
 __pycache__
 /app/api/src/data/*
-!/app/api/src/data/.gitkeep
+!/app/api/src/data/.gitkeep
+r5-cache
+
+# Unit test / coverage reports 
+.coverage
+coverage.xml
+
+
+# Temporary files
+NearestNDInterpolate.png
+costs.bin
+.DS_Store

Makefile

@@ -44,6 +44,7 @@ K8S_OBJ:=$(patsubst %.tpl.yaml,%.yaml,$(K8S_SRC))
 	DOMAIN=$(DOMAIN) \
 	VERSION=$(VERSION) \
 	POSTGRES_HOST=$(POSTGRES_HOST) \
+	R5_HOST=$(R5_HOST) \
 	POSTGRES_DB=$(POSTGRES_DB) \
 	POSTGRES_USER=$(POSTGRES_USER) \
 	POSTGRES_PASSWORD=$(POSTGRES_PASSWORD) \
@@ -74,18 +75,24 @@ setup-kube-config:
 docker-login:
 	$(DOCKER) login -u $(DOCKER_USERNAME) -p $(DOCKER_PASSWORD) $(REGISTRY)
 
-# target: make build-docker-image -e VERSION=some_git_sha_comit -e COMPONENT=api|client|geoserver|print|mapproxy
+# make docker-build for pr workflow (without tag)
+# target: make build-docker-image-pr -e COMPONENT=api|client
+.PHONY: build-docker-image-pr
+build-docker-image-pr: app/$(COMPONENT)/Dockerfile
+	$(DOCKER) build -f app/$(COMPONENT)/Dockerfile app/$(COMPONENT) --build-arg FONTAWESOME_NPM_AUTH_TOKEN=$(FONTAWESOME_NPM_AUTH_TOKEN)
+
+# target: make build-docker-image -e VERSION=some_git_sha_comit -e COMPONENT=api|client
 .PHONY: build-docker-image
 build-docker-image: app/$(COMPONENT)/Dockerfile
 	$(DOCKER) build -f app/$(COMPONENT)/Dockerfile --pull -t $(DOCKER_IMAGE) app/$(COMPONENT)
 
-# target: build-client-docker-image -e VERSION=some_git_sha_comit -e COMPONENT=api|client|geoserver|print|mapproxy
+# target: build-client-docker-image -e VERSION=some_git_sha_comit -e COMPONENT=api|client
 .PHONY: build-client-docker-image
 build-client-docker-image: app/$(COMPONENT)/Dockerfile
 	$(DOCKER) build -f app/$(COMPONENT)/Dockerfile --pull -t $(DOCKER_IMAGE) app/$(COMPONENT) --build-arg FONTAWESOME_NPM_AUTH_TOKEN=$(FONTAWESOME_NPM_AUTH_TOKEN)
 
 
-# target: make release-docker-image -e VERSION=some_git_sha_comit -e COMPONENT=api|client|geoserver|print|mapproxy
+# target: make release-docker-image -e VERSION=some_git_sha_comit -e COMPONENT=api|client
 .PHONY: release-docker-image
 release-docker-image: docker-login build-docker-image
 	$(DOCKER) push $(DOCKER_IMAGE)

app/api/.devcontainer.json

@@ -19,5 +19,9 @@
     "settings": {
         "python.defaultInterpreterPath": "/usr/local/bin/python",
         "python.formatting.provider": "black"
-    }
-}
+    },
+    "postCreateCommand": "bash scripts/post-create.sh",
+    "runServices": [
+        "api"
+    ]
+}

app/api/.vscode/settings.json

@@ -17,7 +17,8 @@
     "titleBar.inactiveForeground": "#e7e7e799",
     "sash.hoverBorder": "#1f6fd0",
     "statusBarItem.remoteBackground": "#1857a4",
-    "statusBarItem.remoteForeground": "#e7e7e7"
+    "statusBarItem.remoteForeground": "#e7e7e7",
+    "commandCenter.border": "#e7e7e799"
   },
   "editor.formatOnSave": true,
   "peacock.remoteColor": "#1857a4",

app/api/.vscode/tasks.json

@@ -6,11 +6,13 @@
             "command": "/usr/bin/g++",
             "args": [
                 "-fdiagnostics-color=always",
+                "-std=c++0x",
                 "-g",
                 "${file}",
                 "-o",
                 "${fileDirname}/${fileBasenameNoExtension}",
-                "-DDEBUG"
+                "-DDEBUG",
+                "-lpthread"
             ],
             "options": {
                 "cwd": "${fileDirname}"

app/api/Dockerfile

@@ -19,7 +19,7 @@ RUN apt install curl ca-certificates gnupg -y && \
     sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt/ bullseye-pgdg main" > /etc/apt/sources.list.d/postgresql.list' && \
     apt update -y && apt-get install postgresql-client-14 -y
 # Install Poetry
-RUN curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | POETRY_HOME=/opt/poetry python && \
+RUN curl -sSL https://install.python-poetry.org | POETRY_HOME=/opt/poetry python && \
     cd /usr/local/bin && \
     ln -s /opt/poetry/bin/poetry && \
     poetry config virtualenvs.create false
@@ -30,9 +30,4 @@ COPY ./pyproject.toml ./poetry.lock* /app/
 ARG INSTALL_DEV=false
 RUN bash -c "if [ $INSTALL_DEV == 'True' ] ; then poetry install --no-root ; else poetry install --no-root --no-dev ; fi"
 COPY . /app
-# chown all the files to the app user
-#RUN chown -R app:app /app
-# change to the app user
-#USER app
-# run gunicorn
-CMD gunicorn --bind 0.0.0.0:5000 src.main:app -k uvicorn.workers.UvicornWorker
+CMD ["uvicorn", "src.main:app", "--host", "0.0.0.0", "--port", "5000"]

app/api/README.md

@@ -481,6 +481,12 @@ Traefik UI: http://localhost.tiangolo.com:8090
 
 ### Dump empty table schema using pgDump
 
-To create an empty data dump used for the data preparation the following command can be executed.
+To create an empty schema-only dump the following command can be executed.
+`pg_dump -d goat -U postgres -h URL -F t -s -n basic -n customer -x > dump_schema.tar`
 
-`pg_dump -d db-name -U user-name -h host-name -s -n basic -n customer -x > dump_schema.sql`
+To create an empty data-only dump the following command can be executed.
+`pg_dump -d goat -U postgres -h URL -t YOUR-TABLE --data-only > dump_schema.sql`
+
+To create a full dump you can do the following.
+`pg_dump -U postgres -h URL -Fc -Z 9  --file=file.dump goat`
+`pg_restore -Fc -j 8  file.dump`

app/api/alembic/env.py

@@ -3,20 +3,13 @@
 import asyncio
 import os
 from logging.config import fileConfig
-
-from alembic_utils.replaceable_entity import register_entities
 from sqlalchemy import engine_from_config, pool
 from sqlalchemy.ext.asyncio import AsyncEngine
 from sqlmodel import SQLModel
 
 from alembic import context
 from src.core.config import settings
 from src.db.models import *
-from src.db.sql.init_sql import (
-    sql_function_entities,
-    sql_trigger_entities,
-    sql_view_entities,
-)
 
 # this is the Alembic Config object, which provides
 # access to the values within the .ini file in use.
@@ -34,24 +27,17 @@
 # my_important_option = config.get_main_option("my_important_option")
 # ... etc.
 
-
-# run migration for database custom functions, views, triggers
-sql_entities = sql_function_entities() + sql_view_entities() + sql_trigger_entities()
-register_entities(sql_entities)
-
-
 def include_object(object, name, type_, reflected, compare_to):
     print(type_)
     if (
-        type_ in ["table", "function", "extension", "trigger", "view"]
+        type_ in ["table"]
         and reflected
         and compare_to is None
     ):
         return False
     else:
         return True
 
-
 def run_migrations_offline():
     """Run migrations in 'offline' mode.
 

app/api/alembic/versions/786395e0b1f4_added_further_attributes_to_geostore_.py

@@ -0,0 +1,45 @@
+"""added further attributes to geostore table
+
+Revision ID: 786395e0b1f4
+Revises: 9d4e51a9535c
+Create Date: 2022-07-28 13:02:05.366050
+
+"""
+from alembic import op
+import sqlalchemy as sa
+import geoalchemy2
+import sqlmodel  
+
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = '786395e0b1f4'
+down_revision = '9d4e51a9535c'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('geostore', sa.Column('type', sa.Text(), nullable=True), schema='customer')
+    op.add_column('geostore', sa.Column('configuration', postgresql.JSONB(astext_type=sa.Text()), nullable=True), schema='customer')
+    op.add_column('geostore', sa.Column('attribution', sa.Text(), nullable=True), schema='customer')
+    op.alter_column('geostore', 'name',
+               existing_type=sa.TEXT(),
+               nullable=True,
+               schema='customer')
+    op.drop_index('ix_customer_geostore_name', table_name='geostore', schema='customer')
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_index('ix_customer_geostore_name', 'geostore', ['name'], unique=False, schema='customer')
+    op.alter_column('geostore', 'name',
+               existing_type=sa.TEXT(),
+               nullable=False,
+               schema='customer')
+    op.drop_column('geostore', 'attribution', schema='customer')
+    op.drop_column('geostore', 'configuration', schema='customer')
+    op.drop_column('geostore', 'type', schema='customer')
+    # ### end Alembic commands ###

app/api/alembic/versions/7e07e654204b_create_staticlayer_model.py

@@ -0,0 +1,40 @@
+"""Create StaticLayer Model
+
+Revision ID: 7e07e654204b
+Revises: 70814d19d9d2
+Create Date: 2022-07-05 12:01:46.007154
+
+"""
+from alembic import op
+import sqlalchemy as sa
+import geoalchemy2
+import sqlmodel  
+
+
+
+# revision identifiers, used by Alembic.
+revision = '7e07e654204b'
+down_revision = '70814d19d9d2'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('static_layer',
+    sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
+    sa.Column('creation_date', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP'), nullable=True),
+    sa.Column('user_id', sa.Integer(), nullable=False),
+    sa.Column('table_name', sa.String(length=63), nullable=False),
+    sa.ForeignKeyConstraint(['user_id'], ['customer.user.id'], ondelete='CASCADE'),
+    sa.PrimaryKeyConstraint('id'),
+    sa.UniqueConstraint('table_name'),
+    schema='customer'
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('static_layer', schema='customer')
+    # ### end Alembic commands ###

app/api/alembic/versions/803ea689f6b8_added_on_delete_cascade_for_user_.py

@@ -0,0 +1,45 @@
+"""added on delete cascade for user foreign keys
+
+Revision ID: 803ea689f6b8
+Revises: f5256a2f45c8
+Create Date: 2022-08-11 12:11:44.522079
+
+"""
+from alembic import op
+import sqlalchemy as sa
+import geoalchemy2
+import sqlmodel  
+
+
+
+# revision identifiers, used by Alembic.
+revision = '803ea689f6b8'
+down_revision = 'f5256a2f45c8'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_constraint('study_area_grid_visualization_grid_visualization_id_fkey', 'study_area_grid_visualization', schema='basic', type_='foreignkey')
+    op.drop_constraint('study_area_grid_visualization_study_area_id_fkey', 'study_area_grid_visualization', schema='basic', type_='foreignkey')
+    op.create_foreign_key(None, 'study_area_grid_visualization', 'study_area', ['study_area_id'], ['id'], source_schema='basic', referent_schema='basic', ondelete='CASCADE')
+    op.create_foreign_key(None, 'study_area_grid_visualization', 'grid_visualization', ['grid_visualization_id'], ['id'], source_schema='basic', referent_schema='basic', ondelete='CASCADE')
+    op.drop_constraint('user_role_user_id_fkey', 'user_role', schema='customer', type_='foreignkey')
+    op.drop_constraint('user_role_role_id_fkey', 'user_role', schema='customer', type_='foreignkey')
+    op.create_foreign_key(None, 'user_role', 'user', ['user_id'], ['id'], source_schema='customer', referent_schema='customer', ondelete='CASCADE')
+    op.create_foreign_key(None, 'user_role', 'role', ['role_id'], ['id'], source_schema='customer', referent_schema='customer', ondelete='CASCADE')
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_constraint(None, 'user_role', schema='customer', type_='foreignkey')
+    op.drop_constraint(None, 'user_role', schema='customer', type_='foreignkey')
+    op.create_foreign_key('user_role_role_id_fkey', 'user_role', 'role', ['role_id'], ['id'], source_schema='customer', referent_schema='customer')
+    op.create_foreign_key('user_role_user_id_fkey', 'user_role', 'user', ['user_id'], ['id'], source_schema='customer', referent_schema='customer')
+    op.drop_constraint(None, 'study_area_grid_visualization', schema='basic', type_='foreignkey')
+    op.drop_constraint(None, 'study_area_grid_visualization', schema='basic', type_='foreignkey')
+    op.create_foreign_key('study_area_grid_visualization_study_area_id_fkey', 'study_area_grid_visualization', 'study_area', ['study_area_id'], ['id'], source_schema='basic', referent_schema='basic')
+    op.create_foreign_key('study_area_grid_visualization_grid_visualization_id_fkey', 'study_area_grid_visualization', 'grid_visualization', ['grid_visualization_id'], ['id'], source_schema='basic', referent_schema='basic')
+    # ### end Alembic commands ###

app/api/alembic/versions/8c1f155cadc8_removed_not_needed_columns_from_node_.py

@@ -0,0 +1,45 @@
+"""removed not needed columns from node table
+
+Revision ID: 8c1f155cadc8
+Revises: fe15a9867004
+Create Date: 2022-09-10 12:50:42.700816
+
+"""
+from alembic import op
+import sqlalchemy as sa
+import geoalchemy2
+import sqlmodel  
+
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = '8c1f155cadc8'
+down_revision = 'fe15a9867004'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index('ix_basic_node_death_end', table_name='node', schema='basic')
+    op.drop_column('node', 'class_ids', schema='basic')
+    op.drop_column('node', 'death_end', schema='basic')
+    op.drop_column('node', 'cnt', schema='basic')
+    op.drop_column('node', 'bicycle', schema='basic')
+    op.drop_column('node', 'lit_classified', schema='basic')
+    op.drop_column('node', 'wheelchair_classified', schema='basic')
+    op.drop_column('node', 'foot', schema='basic')
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('node', sa.Column('foot', postgresql.ARRAY(sa.TEXT()), autoincrement=False, nullable=True), schema='basic')
+    op.add_column('node', sa.Column('wheelchair_classified', postgresql.ARRAY(sa.TEXT()), autoincrement=False, nullable=True), schema='basic')
+    op.add_column('node', sa.Column('lit_classified', postgresql.ARRAY(sa.TEXT()), autoincrement=False, nullable=True), schema='basic')
+    op.add_column('node', sa.Column('bicycle', postgresql.ARRAY(sa.TEXT()), autoincrement=False, nullable=True), schema='basic')
+    op.add_column('node', sa.Column('cnt', sa.INTEGER(), autoincrement=False, nullable=True), schema='basic')
+    op.add_column('node', sa.Column('death_end', sa.BOOLEAN(), autoincrement=False, nullable=True), schema='basic')
+    op.add_column('node', sa.Column('class_ids', postgresql.ARRAY(sa.INTEGER()), autoincrement=False, nullable=True), schema='basic')
+    op.create_index('ix_basic_node_death_end', 'node', ['death_end'], unique=False, schema='basic')
+    # ### end Alembic commands ###