refactor cli building to speed up operational workflow, add compilati…

…on of fortran tools to the cli image build

.github/workflows/build_cli_base_image.yml

@@ -0,0 +1,25 @@
+name: Build the base cli docker image
+
+# this workflow only gets executed when changes are pushed to Dockerfile.cli_base and environment.yml
+# i.e. the files which would impact the environment inside the cli base image
+# you can also optionally execute the image manually (in case you want to update to more recent packages)
+
+on:
+  push:
+    paths:
+      - 'Dockerfile.cli_base'
+      - 'environment.yml'
+  workflow_dispatch:  # Allows the workflow to be triggered manually
+
+jobs:
+  build_cli_base_image:
+    uses: ./.github/workflows/build_images.yml  # Path to your reusable workflow
+    with:
+      IMAGE_ID: cli_base
+
+  build_cli_image:
+    # update the cli since the base image has been updated too
+    needs: [build_cli_base_image]
+    uses: ./.github/workflows/build_images.yml  # Path to your reusable workflow
+    with:
+      IMAGE_ID: cli

.github/workflows/build_run_image.yml

@@ -0,0 +1,15 @@
+name: Build the run docker image
+
+# this workflow only gets executed when changes are pushed to Dockerfile.run
+
+on:
+  push:
+    paths:
+      - 'Dockerfile.run'
+
+jobs:
+  build_images:
+    uses: ./.github/workflows/build_images.yml  # Path to your reusable workflow
+    with:
+      IMAGE_ID: run
+

.github/workflows/run_ops.yml

@@ -12,28 +12,22 @@ on:
         required: false
         default: ''
         type: string
-      build_images:
-        description: 'Run the image builds?'
-        required: true
-        default: 'true'
-        type: boolean
+      #build_images:
+      #  description: 'Run the image builds?'
+      #  required: true
+      #  default: 'true'
+      #  type: boolean
 
 jobs:
   # note that the jobs aren't executed in the order they are written below
   # they are executed in the order depending on the 'needs' attribute of each job
 
-  # start by building our docker images needed in the workflow
-  build_images:
-    # conditional execution when run manually as a workflow_dispatch 
-    # saves time when testing something and we don't want to build the images every time
-    # but it'll get executed every time when run as a scheduled workflow
-    if: ${{ github.event.schedule || github.event.inputs.build_images == 'true' }}  
-    strategy:
-      matrix: # the 'matrix' strategy allows us to build all three docker images in parallel using the same reusable workflow
-        image_id: ['cli', 'run', 'matlab']
+  build_cli:
+    # do we still need this to be conditional?
+    # if: ${{ github.event.schedule || github.event.inputs.build_images == 'true' }}  
     uses: ./.github/workflows/build_images.yml  # Path to your reusable workflow
     with:
-      IMAGE_ID: ${{ matrix.image_id }}
+      IMAGE_ID: cli
 
   # set some environment variables
   envs:
@@ -77,7 +71,7 @@ jobs:
   #
 
   cleanup:
-    needs: [envs,build_images]
+    needs: [envs,build_cli]
     uses: ./.github/workflows/cleanup.yml  # Path to your reusable workflow
     with:
       RUNNER_NAME: mims1
@@ -86,8 +80,7 @@ jobs:
   # download all the data we'll need to force the models
   # no need to provide any model specific inputs as we hard code the extent to cover the whole EEZ
   get_forcing:
-    needs: [envs, build_images]
-    if: ${{ always() }}  # Always run even if build_images is not executed (but it'll wait for build_images if it is)
+    needs: [envs, build_cli]
     uses: ./.github/workflows/get_forcing.yml  # Path to your reusable workflow
     with:
       RUNNER_NAME: mims1
@@ -100,7 +93,6 @@ jobs:
   # prepare croco config dirs for each domain
   prep_domains:
     needs: [envs,get_forcing]
-    if: ${{ always() }}  # Always run even if build_images is not executed (but it'll wait for build_images if it is) (this job needs get_forcing, which needs build_images)
     strategy:
       matrix:
         # running as a matrix strategy allows us to prepare different domains in parallel inside a single job
@@ -120,7 +112,6 @@ jobs:
   # prepare croco boundary forcing for each domain
   make_forcing:
     needs: [envs,get_forcing,prep_domains]
-    if: ${{ always() }}  # Always run even if build_images is not executed (but it'll wait for build_images if it is) (this job needs get_forcing, which needs build_images)
     strategy:
       matrix:
         # running as a matrix strategy allows us to prepare different domains in parallel inside a single job

Dockerfile.cli

@@ -1,29 +1,23 @@
 # This Dockerfile provides instructions to build a docker image 
-# which can be used to run the comman line interface  - cli.py in the root of this repo
+# which is the command line interface  - cli.py in the root of this repo
 # the cli allows us to call whatever python functions in this repo that we
 # need for our SOMISANA operations
 #
-# The docker image is built by .github/workflows/build_images.yml
+# The docker image is built by .github/workflows/build_cli_image.yml (which calles .github/workflows/build_images.yml)
+# which gets triggered whenever the operational workflow is run (either manually or as part of the scheduled workflow)
 #
-# This Dockerfile is based on the opendrift Dockerfile - See https://opendrift.github.io
-# We bake the somisana_croco environment into the image and use the cli.py as the entry point
-#
-# TODO: add a user and run the image as a user rather than as root?
+# All we do here is to over-write the code files inside the cli_base image, without having to recreate the environment.
+# We intentionally separate the cli_base image and this one, as the base image takes quite a lot of time to build
+# which gets annoying when testing new features of the operational workflow
 
-# FROM condaforge/mambaforge
-FROM condaforge/mambaforge:22.9.0-1
+# build from the cli_base image
+FROM ghcr.io/saeon/somisana-croco_cli_base_main:latest
 
 ENV DEBIAN_FRONTEND noninteractive
 
-RUN mkdir /somisana-croco
+# over-write the somisana-croco code
 WORKDIR /somisana-croco
-
-# Install somisana-croco environment into base conda environment
-COPY environment.yml .
-RUN mamba env update -n base -f environment.yml
-
-# Install somisana-croco
 ADD . /somisana-croco
-RUN pip install -e .
 
+# Set the cli.py as the entry point
 ENTRYPOINT ["python", "cli.py"]

Dockerfile.cli_base

@@ -0,0 +1,44 @@
+# This Dockerfile provides instructions to build a docker image 
+# which forms the base image for running the command line interface  - cli.py in the root of this repo
+# the cli allows us to call whatever python functions in this repo that we
+# need for our SOMISANA operations
+#
+# The docker image is built as  part of our operational workflow by 
+# .github/workflows/build_cli_base_image.yml (which calles .github/workflows/build_images.yml)
+# and gets triggered whenever any changes are made toi the Dockerfile.cli_base or environment.yml files
+# i.e. the files needed to create the environment needed by the cli
+#
+# the .github/workflows/build_cli_image.yml is automatically run as part of the operational workflow
+# but all this does is to over-write the somisana-croco code in the image, to account for any code edits
+# which don't impact the environment.
+# We intentionally separate the two steps out, as the base image takes quite a lot of time to build
+# which gets annoying when testing new features of the operational workflow
+#
+# TODO: maybe add a user and run the image as a user rather than as root?
+
+# start with a mambaforge base image
+# We're fixing the mambaforge version due to dependency issues which get introduced if left open ended
+FROM condaforge/mambaforge:22.9.0-1
+
+ENV DEBIAN_FRONTEND noninteractive
+
+# Install dependencies for compiling fortran tools
+RUN apt-get update && apt-get install -y \
+    make \
+    build-essential
+
+RUN mkdir /somisana-croco
+WORKDIR /somisana-croco
+
+# Install somisana-croco environment into base conda environment
+COPY environment.yml .
+RUN mamba env update -n base -f environment.yml
+
+# add the somisana-croco code and install into the base environment
+ADD . /somisana-croco
+RUN pip install -e .
+
+# Now compile the fortran tools
+WORKDIR /somisana-croco/crocotools_py/croco_pytools/prepro/Modules/tools_fort_routines/
+RUN make clean
+RUN make

environment.yml

@@ -47,3 +47,5 @@ dependencies:
   - gcc>=8.3.0
   - gfortran>=8.3.0
   - netcdf-fortran=4.6.0
+  # this is to try and handle dependency issues with compiling with f2py 
+  - setuptools=72.1.0