Extended the algoa-bay-forecast workflow to include the post-processi…

…ng "load" step, though I'm not sure how this will work specifying an out-db raster... TBC (#12)
SAEON · Jul 6, 2022 · 1d1be82 · 1d1be82
1 parent c1b0cc8
commit 1d1be82
Show file tree

Hide file tree

Showing 2 changed files with 33 additions and 19 deletions.
diff --git a/.github/workflows/run_algoa-bay-forecast.yml b/.github/workflows/run_algoa-bay-forecast.yml
@@ -109,7 +109,7 @@ jobs:
           echo "::set-output name=PG_PASSWORD_ENCRYPTED::$(gpg --symmetric --batch --passphrase "${{ env.GPG_PASSPHRASE }}" --output - <(echo "${{ secrets[steps._PG_PASSWORD_.outputs.uppercase] }}") | base64 -w0)"
 
   # Compile CROCO model for the Algoa Bay forecast
-  compile-model:
+  compile-croco:
     needs: [envs]
     runs-on: ubuntu-latest
     env:
@@ -149,7 +149,7 @@ jobs:
   # In the context of a testing environment, where updates to these
   # scripts is part of the development workflow, the toolkit needs
   # to be compiled on every run
-  bundle-python-toolkit:
+  python:
     needs: [envs]
     runs-on: ubuntu-latest
     outputs:
@@ -226,8 +226,8 @@ jobs:
   # These downloads are used to create forcing files that are fed as input to the CROCO model
   # => marine.copernicus.eu: This is ocean data that forms the boundary of our model run
   # => ncei.noaa.gov: This is weather data used to create sea-surface conditions for our model run
-  download-boundary-data:
-    needs: [bundle-python-toolkit, workdir]
+  boundary-data:
+    needs: [python, workdir]
     runs-on: github-runner.saeon.int
     env:
       WORKDIR: ${{ needs.workdir.outputs.WORKDIR }}
@@ -239,13 +239,13 @@ jobs:
             -v $WORKDIR/:/tmp/somisana/current \
             -e COPERNICUS_USERNAME=${{ env.COPERNICUS_USERNAME }} \
             -e COPERNICUS_PASSWORD=${{ env.COPERNICUS_PASSWORD }} \
-            ${{ needs.bundle-python-toolkit.outputs.image }} \
+            ${{ needs.python.outputs.image }} \
                download-boundary-data
 
   # CROCOTOOLS is a collection of MatLab scripts for converting environmental data (i.e. the boundary data downloaded previously)
   # into NetCDF files that can be used as input to the CROCO model. https://www.croco-ocean.org/documentation/crocotools-documentation/
-  make-forcings:
-    needs: [download-boundary-data, workdir, envs]
+  crocotools:
+    needs: [boundary-data, workdir, envs]
     runs-on: github-runner.saeon.int
     env:
       WORKDIR: ${{ needs.workdir.outputs.WORKDIR }}
@@ -269,7 +269,7 @@ jobs:
             --rm \
             --mac-address 02:42:ff:ff:ff:ff \
             -v /opt/licenses/matlab-r2022a/license.lic:/licenses/license.lic \
-            -v $(pwd)/models/algoa-bay-forecast/toolkit/make-forcings:/crocotools/ \
+            -v $(pwd)/models/algoa-bay-forecast/toolkit/crocotools:/crocotools/ \
             -v $(pwd)/models/algoa-bay-forecast/lib/grd.nc:/crocotools/croco/forcing/grd.nc \
             -v $WORKDIR:/tmp/somisana/current \
             -e MLM_LICENSE_FILE=/licenses/license.lic \
@@ -279,8 +279,8 @@ jobs:
   # Execute the CROCO model using the forcing files created previously
   # The CROCO model executable is compiled a part of a Docker build, and is baked into a docker image.
   # As such the CROCO model run must be in the context of a container instantiated from that Docker image
-  run-model:
-    needs: [compile-model, make-forcings, envs, workdir]
+  croco:
+    needs: [compile-croco, crocotools, envs, workdir]
     runs-on: github-runner.saeon.int
     env:
       WORKDIR: ${{ needs.workdir.outputs.WORKDIR }}
@@ -297,7 +297,7 @@ jobs:
             -v $WORKDIR:/algoa-bay-forecast/current \
             -v $(pwd)/models/algoa-bay-forecast/lib/grd.nc:/algoa-bay-forecast/current/croco/forcing/grd.nc \
             --cpus 10 \
-            ${{ needs.compile-model.outputs.image }} \
+            ${{ needs.compile-croco.outputs.image }} \
               ./run_croco.bash \
                 /algoa-bay-forecast/current \
                 ${{ needs.envs.outputs.MODEL_RUN_DATE }} \
@@ -307,8 +307,8 @@ jobs:
   # area mapped to a grid. The post-processing step is for extracting data into
   # a PostGIS instance where it is useful in the context of web applications
   # (specifically as the backend of a WMS server)
-  process-model-output:
-    needs: [bundle-python-toolkit, run-model, envs, workdir]
+  post-processing:
+    needs: [python, croco, envs, workdir]
     runs-on: github-runner.saeon.int
     env:
       WORKDIR: ${{ needs.workdir.outputs.WORKDIR }}
@@ -326,7 +326,7 @@ jobs:
             --rm \
             -v $WORKDIR:/tmp/somisana/current \
             -v $(pwd)/models/algoa-bay-forecast/lib/grd.nc:/tmp/somisana/current/croco/forcing/grd.nc \
-            ${{ needs.bundle-python-toolkit.outputs.image }} \
+            ${{ needs.python.outputs.image }} \
               post-processing \
                 -t \
                 -g /tmp/somisana/current/croco/forcing/grd.nc \
@@ -335,8 +335,8 @@ jobs:
 
   # TODO - this will likely be to move the processed nc file
   # to another server and configure WMS layers from it
-  handle-processed-data:
-    needs: [process-model-output, workdir, _secrets_]
+  register-postgis-rasters:
+    needs: [post-processing, python, workdir, _secrets_]
     runs-on: github-runner.saeon.int
     env:
       WORKDIR: ${{ needs.workdir.outputs.WORKDIR }}
@@ -352,12 +352,24 @@ jobs:
       - name: Archive NetCDF files
         run: |
           echo "Where should we put the NetCDF (avg.nc) files?"
-      - name: Update PostGIS
+      - name: Load processed NetCDF output to PostGIS
+        continue-on-error: true
         env:
           PG_HOST: ${{ steps.decrypt-secrets.outputs.PG_HOST_DECRYPTED }}
           PG_PORT: ${{ steps.decrypt-secrets.outputs.PG_PORT_DECRYPTED }}
           PG_DB: ${{ steps.decrypt-secrets.outputs.PG_DB_DECRYPTED }}
           PG_USERNAME: ${{ steps.decrypt-secrets.outputs.PG_USERNAME_DECRYPTED }}
           PG_PASSWORD: ${{ steps.decrypt-secrets.outputs.PG_PASSWORD_DECRYPTED }}
-        run: |
-          echo "TODO"
+        run: >-
+          docker run \
+            --rm \
+            -v $WORKDIR:/tmp/somisana/current \
+            -e PG_HOST=${{ env.PG_HOST }} \
+            -e PG_PORT=${{ env.PG_PORT }} \
+            -e PG_USERNAME=${{ env.PG_USERNAME }} \
+            -e PG_PASSWORD=${{ env.PG_PASSWORD }} \
+            -e PG_DB=${{ env.PG_DB }} \
+            ${{ needs.python.outputs.image }} \
+              post-processing \
+                -l \
+                -i /tmp/somisana/current/croco/forecast/hourly-avg-${{ env.RUN_DATE }}-normalized.nc
diff --git a/models/algoa-bay-forecast/toolkit.Dockerfile b/models/algoa-bay-forecast/toolkit.Dockerfile
@@ -6,6 +6,7 @@ ARG PG_HOST
 ARG PG_PORT=5432
 ARG PG_USERNAME=admin
 ARG PG_PASSWORD=password
+ARG PG_DB=somisana_local
 ARG MONGO_HOST=localhost:27017
 ARG MONGO_USERNAME=admin
 ARG MONGO_PASSWORD=password
@@ -18,6 +19,7 @@ ENV PG_HOST=$PG_HOST
 ENV PG_PORT=$PG_PORT
 ENV PG_USERNAME=$PG_USERNAME
 ENV PG_PASSWORD=$PG_PASSWORD
+ENV PG_DB=$PG_DB
 ENV MONGO_HOST=$MONGO_HOST
 ENV MONGO_USERNAME=$MONGO_USERNAME
 ENV MONGO_PASSWORD=$MONGO_PASSWORD