BHBC-1284: Create a N8N webhook workflow to call the validation end-points

.docker/nginx/n8n/dev.conf

@@ -0,0 +1,19 @@
+server {
+  listen 5100;
+
+  location / {
+    proxy_pass http://n8n:5678;
+    proxy_redirect default;
+    proxy_set_header Host $host;
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header X-Forwarded-Host $server_name;
+
+    if ($request_method = 'OPTIONS') {
+      add_header "Access-Control-Allow-Origin" "*";
+      add_header "Access-Control-Allow-Methods" "GET, POST, OPTIONS, HEAD";
+      add_header "Access-Control-Allow-Headers" "Authorization, Origin, X-Requested-With, Content-Type, Accept, sessionid";
+      return 204;
+    }
+  }
+}

.gitignore

@@ -108,3 +108,6 @@ dist
 
 # Apple macOS folder attributes file
 **/.DS_Store
+
+# n8N - ignore root level temp config folder
+.n8n

Makefile

@@ -40,6 +40,9 @@ db-setup: | build-db-setup run-db-setup ## Performs all commands necessary to ru
 db-migrate: | build-db-migrate run-db-migrate ## Performs all commands necessary to run the database migrations
 db-rollback: | build-db-rollback run-db-rollback ## Performs all commands necessary to rollback the latest database migrations
 
+n8n-setup: | build-n8n-setup run-n8n-setup ## Performs all commands necessary to run the n8n setup
+n8n-export: | build-n8n-export run-n8n-export ## Performs all commands necessary to export the latest n8n credentials and workflows
+
 ## ------------------------------------------------------------------------------
 ## Setup/Cleanup Commands
 ## ------------------------------------------------------------------------------
@@ -133,26 +136,26 @@ run-postgres-debug: ## Runs the postgres db containers in debug mode, where all
 
 ## ------------------------------------------------------------------------------
 ## Build/Run Backend+Web Commands (backend + web frontend)
-## - Builds all of the biohub backend+web projects (db, db_setup, api, app)
+## - Builds all of the biohub backend+web projects (db, db_setup, api, app, n8n, n8n_nginx, n8n_setup)
 ## ------------------------------------------------------------------------------
 
 build-web: ## Builds all backend+web containers
 	@echo "==============================================="
 	@echo "Make: build-web - building web images"
 	@echo "==============================================="
-	@docker-compose -f docker-compose.yml build db db_setup api app
+	@docker-compose -f docker-compose.yml build db db_setup api app n8n n8n_nginx n8n_setup
 
 run-web: ## Runs all backend+web containers
 	@echo "==============================================="
 	@echo "Make: run-web - running web images"
 	@echo "==============================================="
-	@docker-compose -f docker-compose.yml up -d db db_setup api app
+	@docker-compose -f docker-compose.yml up -d db db_setup api app n8n n8n_nginx n8n_setup
 
 run-web-debug: ## Runs all backend+web containers in debug mode, where all container output is printed to the console
 	@echo "==============================================="
 	@echo "Make: run-web-debug - running web images in debug mode"
 	@echo "==============================================="
-	@docker-compose -f docker-compose.yml up db db_setup api app
+	@docker-compose -f docker-compose.yml up db db_setup api app n8n n8n_nginx n8n_setup
 
 ## ------------------------------------------------------------------------------
 ## Commands to shell into the target container
@@ -177,6 +180,12 @@ api: ## Executes into the workspace container.
 	@echo "==============================================="
 	@docker-compose exec api bash
 
+n8n: ## Executes into the n8n container.
+	@echo "==============================================="
+	@echo "Shelling into n8n container"
+	@echo "==============================================="
+	@docker-compose exec n8n bash
+
 ## ------------------------------------------------------------------------------
 ## Database migration commands
 ## ------------------------------------------------------------------------------
@@ -217,6 +226,34 @@ run-db-rollback: ## Rollback the latest database migrations
 	@echo "==============================================="
 	@docker-compose -f docker-compose.yml up db_rollback
 
+## ------------------------------------------------------------------------------
+## n8n commands
+## ------------------------------------------------------------------------------
+
+build-n8n-setup: ## Build the n8n setup image
+	@echo "==============================================="
+	@echo "Make: build-n8n-setup - building n8n setup image"
+	@echo "==============================================="
+	@docker-compose -f docker-compose.yml build n8n_setup
+
+run-n8n-setup: ## Run the n8n setup
+	@echo "==============================================="
+	@echo "Make: run-n8n-setup - running n8n setup"
+	@echo "==============================================="
+	@docker-compose -f docker-compose.yml up n8n_setup
+
+build-n8n-export: ## Build the n8n export image
+	@echo "==============================================="
+	@echo "Make: build-n8n-export - building n8n export image"
+	@echo "==============================================="
+	@docker-compose -f docker-compose.yml build n8n_export
+
+run-n8n-export: ## Run the n8n export
+	@echo "==============================================="
+	@echo "Make: run-n8n-export - exporting the n8n credentials and workflows"
+	@echo "==============================================="
+	@docker-compose -f docker-compose.yml up n8n_export
+
 ## ------------------------------------------------------------------------------
 ## Run `npm` commands for all projects
 ## ------------------------------------------------------------------------------

api/src/paths/project/{projectId}/survey/{surveyId}/observation/submission/upload.ts

@@ -138,7 +138,6 @@ export function uploadMedia(): RequestHandler {
       });
 
       //query to update the record with the key before uploading the file
-
       await updateSurveyOccurrenceSubmissionWithKey(submissionId, key, connection);
 
       await connection.commit();

app/src/features/surveys/view/SurveyObservations.tsx

@@ -57,14 +57,21 @@ const SurveyObservations = () => {
   const classes = useStyles();
 
   const importObservations = (): IUploadHandler => {
-    return (files, cancelToken, handleFileUploadProgress) => {
-      return biohubApi.observation.uploadObservationSubmission(
+    return async (files, cancelToken, handleFileUploadProgress) => {
+      const uploadResult = biohubApi.observation.uploadObservationSubmission(
         projectId,
         surveyId,
         files[0],
         cancelToken,
         handleFileUploadProgress
       );
+      const awaitedUploadResult = await uploadResult;
+
+      if (!awaitedUploadResult || !awaitedUploadResult.submissionId) {
+        return uploadResult;
+      }
+
+      return biohubApi.observation.initiateSubmissionValidation(awaitedUploadResult.submissionId, files[0].type);
     };
   };
 
@@ -241,8 +248,8 @@ const SurveyObservations = () => {
           </>
         )}
         {!isValidating &&
-          (submissionStatus?.status === 'Darwin Core Validated' ||
-            submissionStatus?.status === 'Template Validated') && (
+          submissionStatus &&
+          (submissionStatus.status === 'Darwin Core Validated' || submissionStatus.status === 'Template Validated') && (
             <>
               <Alert
                 icon={<Icon path={mdiFileOutline} size={1} />}
@@ -256,7 +263,7 @@ const SurveyObservations = () => {
               </Box>
             </>
           )}
-        {isValidating && (
+        {isValidating && submissionStatus && (
           <>
             <Alert
               icon={<Icon path={mdiClockOutline} size={1} />}

app/src/hooks/api/useObservationApi.ts

@@ -42,14 +42,6 @@ const useObservationApi = (axios: AxiosInstance) => {
       }
     );
 
-    if (data.submissionId) {
-      if (file.type === 'application/x-zip-compressed' || file.type === 'application/zip') {
-        initiateDwCSubmissionValidation(data.submissionId);
-      } else {
-        initiateXLSXSubmissionValidation(data.submissionId);
-      }
-    }
-
     return data;
   };
 
@@ -108,24 +100,17 @@ const useObservationApi = (axios: AxiosInstance) => {
     return data;
   };
 
-  /**
-   * Initiate the validation process for the submitted observations
-   * @param {number} submissionId
-   */
-  const initiateDwCSubmissionValidation = async (submissionId: number) => {
-    axios.post(`/api/dwc/validate`, {
-      occurrence_submission_id: submissionId
-    });
-  };
-
-  const initiateXLSXSubmissionValidation = async (submissionId: number) => {
-    axios.post(`/api/xlsx/validate`, {
-      occurrence_submission_id: submissionId
+  // Initiate the validation process for the submitted observations using n8n webhook
+  const initiateSubmissionValidation = async (submissionId: number, fileType: string) => {
+    await axios.post('http://localhost:5100/webhook/validate', {
+      occurrence_submission_id: submissionId,
+      file_type: fileType
     });
   };
 
   return {
     uploadObservationSubmission,
+    initiateSubmissionValidation,
     getSubmissionCSVForView,
     getObservationSubmission,
     deleteObservationSubmission

database/src/migrations/20210825170006_add_n8n_schema.ts

@@ -0,0 +1,15 @@
+import * as Knex from 'knex';
+
+export async function up(knex: Knex): Promise<void> {
+  await knex.raw(`
+    create schema if not exists n8n;
+
+    GRANT ALL ON SCHEMA n8n TO postgres;
+
+    set search_path = n8n, public;
+  `);
+}
+
+export async function down(knex: Knex): Promise<void> {
+  await knex.raw(``);
+}

docker-compose.yml

@@ -68,20 +68,102 @@ services:
       - db
       - db_setup
 
-  # Build the nginx web-server docker image
-  nginx:
-    image: ${DOCKER_PROJECT_NAME}-nginx-${DOCKER_NAMESPACE}-img
-    container_name: ${DOCKER_PROJECT_NAME}-nginx-${DOCKER_NAMESPACE}-container
+  # Build the n8n nginx proxy docker image
+  n8n_nginx:
+    image: ${DOCKER_PROJECT_NAME}-n8n-nginx-${DOCKER_NAMESPACE}-img
+    container_name: ${DOCKER_PROJECT_NAME}-n8n-nginx-${DOCKER_NAMESPACE}-container
     build:
-      context: ./.docker/nginx
+      context: ./.docker/nginx/n8n
       dockerfile: Dockerfile
     restart: always
     ports:
-      - 80:80
+      - 5100:5100
     networks:
       - local-network
+    depends_on:
+      - n8n
+
+  ## Build the n8n workflow automation image
+  n8n:
+    image: n8nio/n8n
+    container_name: ${DOCKER_PROJECT_NAME}-n8n-${DOCKER_NAMESPACE}-container
+    ports:
+      - ${N8N_PORT}:${N8N_PORT}
+    environment:
+      - NODE_ENV=${NODE_ENV}
+      - N8N_HOST=${N8N_HOST}
+      - N8N_PROTOCOL=${N8N_PROTOCOL}
+      - N8N_PORT=${N8N_PORT}
+      - WEBHOOK_TUNNEL_URL=${WEBHOOK_TUNNEL_URL}
+      - DB_TYPE=${N8N_DB_TYPE}
+      - DB_POSTGRESDB_HOST=${DB_HOST}
+      - DB_POSTGRESDB_USER=${DB_ADMIN}
+      - DB_POSTGRESDB_PASSWORD=${DB_ADMIN_PASS}
+      - DB_POSTGRESDB_PORT=${DB_PORT}
+      - DB_POSTGRESDB_DATABASE=${DB_DATABASE}
+      - DB_POSTGRESDB_SCHEMA=${N8N_DB_SCHEMA}
+      - N8N_BASIC_AUTH_ACTIVE=${N8N_BASIC_AUTH_ACTIVE}
+      - N8N_BASIC_AUTH_USER=${N8N_BASIC_AUTH_USER}
+      - N8N_BASIC_AUTH_PASSWORD=${N8N_BASIC_AUTH_PASSWORD}
+      - N8N_ENCRYPTION_KEY=${N8N_ENCRYPTION_KEY}
+    networks:
+      - local-network
+    volumes:
+      - ./.n8n:/home/node/.n8n
     depends_on:
       - api
+      - db
+    command: ["n8n", "start"]
+
+  ## Run the n8n setup
+  n8n_setup:
+    image: ${DOCKER_PROJECT_NAME}-n8n-setup-${DOCKER_NAMESPACE}-img
+    container_name: ${DOCKER_PROJECT_NAME}-n8n-setup-${DOCKER_NAMESPACE}-container
+    build:
+      context: ./n8n
+      dockerfile: ./.docker/n8n/Dockerfile.setup
+    environment:
+      - NODE_ENV=${NODE_ENV}
+      - DB_TYPE=${N8N_DB_TYPE}
+      - DB_POSTGRESDB_HOST=${DB_HOST}
+      - DB_POSTGRESDB_USER=${DB_ADMIN}
+      - DB_POSTGRESDB_PASSWORD=${DB_ADMIN_PASS}
+      - DB_POSTGRESDB_PORT=${DB_PORT}
+      - DB_POSTGRESDB_DATABASE=${DB_DATABASE}
+      - DB_POSTGRESDB_SCHEMA=${N8N_DB_SCHEMA}
+      - N8N_ENCRYPTION_KEY=${N8N_ENCRYPTION_KEY}
+    volumes:
+      - /opt/app-root/src/node_modules # prevents local node_modules overriding container node_modules
+      - ./n8n:/opt/app-root/src
+    networks:
+      - local-network
+    depends_on:
+      - n8n
+
+  ## Export the n8n credentials and workflows
+  n8n_export:
+    image: ${DOCKER_PROJECT_NAME}-n8n-export-${DOCKER_NAMESPACE}-img
+    container_name: ${DOCKER_PROJECT_NAME}-n8n-export-${DOCKER_NAMESPACE}-container
+    build:
+      context: ./n8n
+      dockerfile: ./.docker/n8n/Dockerfile.export
+    environment:
+      - NODE_ENV=${NODE_ENV}
+      - DB_TYPE=${N8N_DB_TYPE}
+      - DB_POSTGRESDB_HOST=${DB_HOST}
+      - DB_POSTGRESDB_USER=${DB_ADMIN}
+      - DB_POSTGRESDB_PASSWORD=${DB_ADMIN_PASS}
+      - DB_POSTGRESDB_PORT=${DB_PORT}
+      - DB_POSTGRESDB_DATABASE=${DB_DATABASE}
+      - DB_POSTGRESDB_SCHEMA=${N8N_DB_SCHEMA}
+      - N8N_ENCRYPTION_KEY=${N8N_ENCRYPTION_KEY}
+    volumes:
+      - /opt/app-root/src/node_modules # prevents local node_modules overriding container node_modules
+      - ./n8n:/opt/app-root/src
+    networks:
+      - local-network
+    depends_on:
+      - n8n
 
   ## Build the app docker image
   app:
@@ -109,6 +191,7 @@ services:
       - local-network
     depends_on:
       - api
+      - n8n
 
   ## Run the database migrations and seeding
   db_setup:
@@ -137,7 +220,7 @@ services:
         condition: service_healthy
     command: ["npm", "run", "setup"]
 
-  ## Run the  database migrations
+  ## Run the database migrations
   db_migrate:
     image: ${DOCKER_PROJECT_NAME}-db-migrate-${DOCKER_NAMESPACE}-img
     container_name: ${DOCKER_PROJECT_NAME}-db-migrate-${DOCKER_NAMESPACE}-container

env_config/env.docker

@@ -66,3 +66,26 @@ OBJECT_STORE_BUCKET_NAME=
 # ------------------------------------------------------------------------------
 DOCKER_PROJECT_NAME=biohubbc
 DOCKER_NAMESPACE=all
+
+# ------------------------------------------------------------------------------
+# n8n - https://docs.n8n.io/reference/environment-variables.html
+# ------------------------------------------------------------------------------
+N8N_HOST=localhost
+N8N_PROTOCOL=http
+N8N_PORT=5678
+# N8N_PROXY_PORT=5100
+# N8N_NGINX_PORT=5101
+WEBHOOK_TUNNEL_URL=http://localhost:5100/
+
+N8N_DB_TYPE=postgresdb
+N8N_DB_HOST=localhost
+N8N_DB_USER=n8n
+N8N_DB_PASS=n8n
+N8N_DB_PORT=5433
+N8N_DB_DATABASE=n8n
+N8N_DB_SCHEMA=n8n
+
+N8N_BASIC_AUTH_ACTIVE=true
+N8N_BASIC_AUTH_USER=n8n
+N8N_BASIC_AUTH_PASSWORD=n8n
+N8N_ENCRYPTION_KEY=secret