diff --git a/.github/workflows/build_cli_base_image.yml b/.github/workflows/build_cli_base_image.yml new file mode 100644 index 0000000..d14d64e --- /dev/null +++ b/.github/workflows/build_cli_base_image.yml @@ -0,0 +1,25 @@ +name: Build the base cli docker image + +# this workflow only gets executed when changes are pushed to Dockerfile.cli_base and environment.yml +# i.e. the files which would impact the environment inside the cli base image +# you can also optionally execute the image manually (in case you want to update to more recent packages) + +on: + push: + paths: + - 'Dockerfile.cli_base' + - 'environment.yml' + workflow_dispatch: # Allows the workflow to be triggered manually + +jobs: + build_cli_base_image: + uses: ./.github/workflows/build_images.yml # Path to your reusable workflow + with: + IMAGE_ID: cli_base + + build_cli_image: + # update the cli since the base image has been updated too + needs: [build_cli_base_image] + uses: ./.github/workflows/build_images.yml # Path to your reusable workflow + with: + IMAGE_ID: cli diff --git a/.github/workflows/build_run_image.yml b/.github/workflows/build_run_image.yml new file mode 100644 index 0000000..c0ee25a --- /dev/null +++ b/.github/workflows/build_run_image.yml @@ -0,0 +1,15 @@ +name: Build the run docker image + +# this workflow only gets executed when changes are pushed to Dockerfile.run + +on: + push: + paths: + - 'Dockerfile.run' + +jobs: + build_images: + uses: ./.github/workflows/build_images.yml # Path to your reusable workflow + with: + IMAGE_ID: run + diff --git a/.github/workflows/run_ops.yml b/.github/workflows/run_ops.yml index 0588d56..f045ba3 100644 --- a/.github/workflows/run_ops.yml +++ b/.github/workflows/run_ops.yml @@ -12,28 +12,22 @@ on: required: false default: '' type: string - build_images: - description: 'Run the image builds?' - required: true - default: 'true' - type: boolean + #build_images: + # description: 'Run the image builds?' + # required: true + # default: 'true' + # type: boolean jobs: # note that the jobs aren't executed in the order they are written below # they are executed in the order depending on the 'needs' attribute of each job - # start by building our docker images needed in the workflow - build_images: - # conditional execution when run manually as a workflow_dispatch - # saves time when testing something and we don't want to build the images every time - # but it'll get executed every time when run as a scheduled workflow - if: ${{ github.event.schedule || github.event.inputs.build_images == 'true' }} - strategy: - matrix: # the 'matrix' strategy allows us to build all three docker images in parallel using the same reusable workflow - image_id: ['cli', 'run', 'matlab'] + build_cli: + # do we still need this to be conditional? + # if: ${{ github.event.schedule || github.event.inputs.build_images == 'true' }} uses: ./.github/workflows/build_images.yml # Path to your reusable workflow with: - IMAGE_ID: ${{ matrix.image_id }} + IMAGE_ID: cli # set some environment variables envs: @@ -77,7 +71,7 @@ jobs: # cleanup: - needs: [envs,build_images] + needs: [envs,build_cli] uses: ./.github/workflows/cleanup.yml # Path to your reusable workflow with: RUNNER_NAME: mims1 @@ -86,8 +80,7 @@ jobs: # download all the data we'll need to force the models # no need to provide any model specific inputs as we hard code the extent to cover the whole EEZ get_forcing: - needs: [envs, build_images] - if: ${{ always() }} # Always run even if build_images is not executed (but it'll wait for build_images if it is) + needs: [envs, build_cli] uses: ./.github/workflows/get_forcing.yml # Path to your reusable workflow with: RUNNER_NAME: mims1 @@ -100,7 +93,6 @@ jobs: # prepare croco config dirs for each domain prep_domains: needs: [envs,get_forcing] - if: ${{ always() }} # Always run even if build_images is not executed (but it'll wait for build_images if it is) (this job needs get_forcing, which needs build_images) strategy: matrix: # running as a matrix strategy allows us to prepare different domains in parallel inside a single job @@ -120,7 +112,6 @@ jobs: # prepare croco boundary forcing for each domain make_forcing: needs: [envs,get_forcing,prep_domains] - if: ${{ always() }} # Always run even if build_images is not executed (but it'll wait for build_images if it is) (this job needs get_forcing, which needs build_images) strategy: matrix: # running as a matrix strategy allows us to prepare different domains in parallel inside a single job diff --git a/Dockerfile.cli b/Dockerfile.cli index 53dab64..3c0663d 100644 --- a/Dockerfile.cli +++ b/Dockerfile.cli @@ -1,29 +1,23 @@ # This Dockerfile provides instructions to build a docker image -# which can be used to run the comman line interface - cli.py in the root of this repo +# which is the command line interface - cli.py in the root of this repo # the cli allows us to call whatever python functions in this repo that we # need for our SOMISANA operations # -# The docker image is built by .github/workflows/build_images.yml +# The docker image is built by .github/workflows/build_cli_image.yml (which calles .github/workflows/build_images.yml) +# which gets triggered whenever the operational workflow is run (either manually or as part of the scheduled workflow) # -# This Dockerfile is based on the opendrift Dockerfile - See https://opendrift.github.io -# We bake the somisana_croco environment into the image and use the cli.py as the entry point -# -# TODO: add a user and run the image as a user rather than as root? +# All we do here is to over-write the code files inside the cli_base image, without having to recreate the environment. +# We intentionally separate the cli_base image and this one, as the base image takes quite a lot of time to build +# which gets annoying when testing new features of the operational workflow -# FROM condaforge/mambaforge -FROM condaforge/mambaforge:22.9.0-1 +# build from the cli_base image +FROM ghcr.io/saeon/somisana-croco_cli_base_main:latest ENV DEBIAN_FRONTEND noninteractive -RUN mkdir /somisana-croco +# over-write the somisana-croco code WORKDIR /somisana-croco - -# Install somisana-croco environment into base conda environment -COPY environment.yml . -RUN mamba env update -n base -f environment.yml - -# Install somisana-croco ADD . /somisana-croco -RUN pip install -e . +# Set the cli.py as the entry point ENTRYPOINT ["python", "cli.py"] diff --git a/Dockerfile.cli_base b/Dockerfile.cli_base new file mode 100644 index 0000000..471cf25 --- /dev/null +++ b/Dockerfile.cli_base @@ -0,0 +1,44 @@ +# This Dockerfile provides instructions to build a docker image +# which forms the base image for running the command line interface - cli.py in the root of this repo +# the cli allows us to call whatever python functions in this repo that we +# need for our SOMISANA operations +# +# The docker image is built as part of our operational workflow by +# .github/workflows/build_cli_base_image.yml (which calles .github/workflows/build_images.yml) +# and gets triggered whenever any changes are made toi the Dockerfile.cli_base or environment.yml files +# i.e. the files needed to create the environment needed by the cli +# +# the .github/workflows/build_cli_image.yml is automatically run as part of the operational workflow +# but all this does is to over-write the somisana-croco code in the image, to account for any code edits +# which don't impact the environment. +# We intentionally separate the two steps out, as the base image takes quite a lot of time to build +# which gets annoying when testing new features of the operational workflow +# +# TODO: maybe add a user and run the image as a user rather than as root? + +# start with a mambaforge base image +# We're fixing the mambaforge version due to dependency issues which get introduced if left open ended +FROM condaforge/mambaforge:22.9.0-1 + +ENV DEBIAN_FRONTEND noninteractive + +# Install dependencies for compiling fortran tools +RUN apt-get update && apt-get install -y \ + make \ + build-essential + +RUN mkdir /somisana-croco +WORKDIR /somisana-croco + +# Install somisana-croco environment into base conda environment +COPY environment.yml . +RUN mamba env update -n base -f environment.yml + +# add the somisana-croco code and install into the base environment +ADD . /somisana-croco +RUN pip install -e . + +# Now compile the fortran tools +WORKDIR /somisana-croco/crocotools_py/croco_pytools/prepro/Modules/tools_fort_routines/ +RUN make clean +RUN make diff --git a/environment.yml b/environment.yml index 604a090..93c7f05 100644 --- a/environment.yml +++ b/environment.yml @@ -47,3 +47,5 @@ dependencies: - gcc>=8.3.0 - gfortran>=8.3.0 - netcdf-fortran=4.6.0 + # this is to try and handle dependency issues with compiling with f2py + - setuptools=72.1.0