Re-organized toolkit CLI so that load operation into postgres doesn't…

… require any flags (all info comes from the netcdf file being processed), and refactored structure (#30)
SAEON · Apr 24, 2023 · 607aa95 · 607aa95
1 parent 77de03c
commit 607aa95
Show file tree

Hide file tree

Showing 35 changed files with 526 additions and 1,059 deletions.
diff --git a/.github/workflows/run_algoa-bay-forecast.yml b/.github/workflows/run_algoa-bay-forecast.yml
@@ -28,6 +28,24 @@ jobs:
         id: BRANCH_REF
         run: echo "value=${GITHUB_REF##*/}" >> $GITHUB_OUTPUT
 
+  # Cleanup old runs temp files
+  cleanup-old-run-temp-files:
+    needs: [branch-ref]
+    runs-on: somisana
+    env:
+      BRANCH: ${{ needs.branch-ref.outputs.value }}
+    steps:
+      - name: Clean /home/runner/somisana/algoa-bay-forecast/${{ env.BRANCH }}
+        run: >-
+          find \
+            /home/runner/somisana/algoa-bay-forecast/${{ env.BRANCH }}/* \
+            -maxdepth 0 \
+            -type d \
+            -ctime +7 \
+            -exec \
+              rm \
+                -rf {} \;
+
   # Many of the env variables reference the current branch
   # Set the environment variables using the current branch reference
   # (which is set dynamically)
@@ -349,6 +367,7 @@ jobs:
             ${{ needs.toolkit.outputs.image }}:${{ env.SHA }} \
               croco \
                 post-process-v1 \
+                  --id algoa-bay-forecast \
                   --grid /tmp/somisana/current/croco/forcing/grd.nc \
                   --input /tmp/somisana/current/croco/forecast/hourly-avg-${{ env.RUN_DATE }}.nc \
                   --output /tmp/somisana/current/croco/forecast/algoa-bay-forecast-${{ env.RUN_DATE }}.nc
@@ -449,151 +468,7 @@ jobs:
             -e PG_PASSWORD=$PG_PASSWORD \
             -e PG_DB=$PG_DB \
             ${{ needs.toolkit.outputs.image }}:${{ env.SHA }} \
-              ops \
-                load \
-                  --upsert-rasters \
-                  --model-data /tmp/somisana/current/croco/forecast/algoa-bay-forecast-${{ env.RUN_DATE }}.nc \
-                  --model algoa-bay-forecast \
-                  --reload-data \
-                  --run-date ${{ env.RUN_DATE }}
-
-  # Refresh the values table with new data
-  refresh-values:
-    needs: [load-postgis, branch-ref, toolkit, envs, workdir]
-    runs-on: somisana
-    env:
-      WORKDIR: ${{ needs.workdir.outputs.WORKDIR }}
-      RUN_DATE: ${{ needs.envs.outputs.MODEL_RUN_DATE }}
-      SHA: sha-${{ github.sha }}
-    steps:
-      - name: Get PG_HOST secret name
-        id: _PG_HOST_
-        run: |
-          STRING=PG_HOST_${{ needs.branch-ref.outputs.value }}
-          STRING_UPPERCASE=$(echo $STRING | tr '[:lower:]' '[:upper:]')
-          echo "uppercase=$STRING_UPPERCASE" >> $GITHUB_OUTPUT
-      - name: Get PG_PORT secret name
-        id: _PG_PORT_
-        run: |
-          STRING=PG_PORT_${{ needs.branch-ref.outputs.value }}
-          STRING_UPPERCASE=$(echo $STRING | tr '[:lower:]' '[:upper:]')
-          echo "uppercase=$STRING_UPPERCASE" >> $GITHUB_OUTPUT
-      - name: Get PG_DB secret name
-        id: _PG_DB_
-        run: |
-          STRING=PG_DB_${{ needs.branch-ref.outputs.value }}
-          STRING_UPPERCASE=$(echo $STRING | tr '[:lower:]' '[:upper:]')
-          echo "uppercase=$STRING_UPPERCASE" >> $GITHUB_OUTPUT
-      - name: Get PG_USERNAME secret name
-        id: _PG_USERNAME_
-        run: |
-          STRING=PG_USERNAME_${{ needs.branch-ref.outputs.value }}
-          STRING_UPPERCASE=$(echo $STRING | tr '[:lower:]' '[:upper:]')
-          echo "uppercase=$STRING_UPPERCASE" >> $GITHUB_OUTPUT
-      - name: Get PG_PASSWORD secret name
-        id: _PG_PASSWORD_
-        run: |
-          STRING=PG_PASSWORD_${{ needs.branch-ref.outputs.value }}
-          STRING_UPPERCASE=$(echo $STRING | tr '[:lower:]' '[:upper:]')
-          echo "uppercase=$STRING_UPPERCASE" >> $GITHUB_OUTPUT
-      - name: Load normalized NetCDF output to PostGIS
-        env:
-          PG_HOST: ${{ secrets[steps._PG_HOST_.outputs.uppercase] }}
-          PG_PORT: ${{ secrets[steps._PG_PORT_.outputs.uppercase] }}
-          PG_DB: ${{ secrets[steps._PG_DB_.outputs.uppercase] }}
-          PG_USERNAME: ${{ secrets[steps._PG_USERNAME_.outputs.uppercase] }}
-          PG_PASSWORD: ${{ secrets[steps._PG_PASSWORD_.outputs.uppercase] }}
-        run: >-
-          docker run \
-            --rm \
-            -v $WORKDIR:/tmp/somisana/current \
-            -e PG_HOST=$PG_HOST \
-            -e PG_PORT=$PG_PORT \
-            -e PG_USERNAME=$PG_USERNAME \
-            -e PG_PASSWORD=$PG_PASSWORD \
-            -e PG_DB=$PG_DB \
-            ${{ needs.toolkit.outputs.image }}:${{ env.SHA }} \
-              ops \
-                load \
-                  --upsert-values \
-                  --depths 1,20 \
-                  --parallelization 32 \
-                  --model algoa-bay-forecast \
-                  --model-data /tmp/somisana/current/croco/forecast/algoa-bay-forecast-${{ env.RUN_DATE }}.nc \
-                  --run-date ${{ env.RUN_DATE }}
-
-  # Cleanup old runs temp files
-  cleanup-old-run-temp-files:
-    needs: [branch-ref]
-    runs-on: somisana
-    env:
-      BRANCH: ${{ needs.branch-ref.outputs.value }}
-    steps:
-      - name: Clean /home/runner/somisana/algoa-bay-forecast/${{ env.BRANCH }}
-        run: >-
-          find \
-            /home/runner/somisana/algoa-bay-forecast/${{ env.BRANCH }}/* \
-            -maxdepth 0 \
-            -type d \
-            -ctime +7 \
-            -exec \
-              rm \
-                -rf {} \;
-
-  # Cleanup rasters, and mark run as complete
-  finalize-run:
-    needs: [branch-ref, toolkit, refresh-values]
-    runs-on: somisana
-    env:
-      SHA: sha-${{ github.sha }}
-    steps:
-      - name: Get PG_HOST secret name
-        id: _PG_HOST_
-        run: |
-          STRING=PG_HOST_${{ needs.branch-ref.outputs.value }}
-          STRING_UPPERCASE=$(echo $STRING | tr '[:lower:]' '[:upper:]')
-          echo "uppercase=$STRING_UPPERCASE" >> $GITHUB_OUTPUT
-      - name: Get PG_PORT secret name
-        id: _PG_PORT_
-        run: |
-          STRING=PG_PORT_${{ needs.branch-ref.outputs.value }}
-          STRING_UPPERCASE=$(echo $STRING | tr '[:lower:]' '[:upper:]')
-          echo "uppercase=$STRING_UPPERCASE" >> $GITHUB_OUTPUT
-      - name: Get PG_DB secret name
-        id: _PG_DB_
-        run: |
-          STRING=PG_DB_${{ needs.branch-ref.outputs.value }}
-          STRING_UPPERCASE=$(echo $STRING | tr '[:lower:]' '[:upper:]')
-          echo "uppercase=$STRING_UPPERCASE" >> $GITHUB_OUTPUT
-      - name: Get PG_USERNAME secret name
-        id: _PG_USERNAME_
-        run: |
-          STRING=PG_USERNAME_${{ needs.branch-ref.outputs.value }}
-          STRING_UPPERCASE=$(echo $STRING | tr '[:lower:]' '[:upper:]')
-          echo "uppercase=$STRING_UPPERCASE" >> $GITHUB_OUTPUT
-      - name: Get PG_PASSWORD secret name
-        id: _PG_PASSWORD_
-        run: |
-          STRING=PG_PASSWORD_${{ needs.branch-ref.outputs.value }}
-          STRING_UPPERCASE=$(echo $STRING | tr '[:lower:]' '[:upper:]')
-          echo "uppercase=$STRING_UPPERCASE" >> $GITHUB_OUTPUT
-      - name: Cleanup rasters that have been loaded into values table and mark run as complete
-        env:
-          PG_HOST: ${{ secrets[steps._PG_HOST_.outputs.uppercase] }}
-          PG_PORT: ${{ secrets[steps._PG_PORT_.outputs.uppercase] }}
-          PG_DB: ${{ secrets[steps._PG_DB_.outputs.uppercase] }}
-          PG_USERNAME: ${{ secrets[steps._PG_USERNAME_.outputs.uppercase] }}
-          PG_PASSWORD: ${{ secrets[steps._PG_PASSWORD_.outputs.uppercase] }}
-        run: >-
-          docker run \
-            --rm \
-            -e PG_HOST=$PG_HOST \
-            -e PG_PORT=$PG_PORT \
-            -e PG_USERNAME=$PG_USERNAME \
-            -e PG_PASSWORD=$PG_PASSWORD \
-            -e PG_DB=$PG_DB \
-            ${{ needs.toolkit.outputs.image }}:${{ env.SHA }} \
-              ops \
-                load \
-                  --finalize-run \
-                  --model algoa-bay-forecast
+              croco \
+                load-pp-v1-output-to-pg \
+                  --input /tmp/somisana/current/croco/forecast/algoa-bay-forecast-${{ env.RUN_DATE }}.nc \
+                  --parallelization 32