-
Notifications
You must be signed in to change notification settings - Fork 0
173 lines (159 loc) · 6.6 KB
/
run_ops.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
name: Run SOMISANA forecast models on MIMS
on:
schedule:
# the workflow designed to every 6 hours. Optionally it can also be run every 12 or 24 hours
# initialise once a day at 3am
- cron: '0 3 * * *'
# initialise twice a day
# - cron: '0 0,12 * * *'
workflow_dispatch:
inputs:
run_date:
description: 'Date and time for T0 for the run in format YYYYMMDD_HH'
required: false
default: ''
type: string
jobs:
# note that the jobs aren't executed in the order they are written below
# they are executed in the order depending on the 'needs' attribute of each job
build_cli:
uses: ./.github/workflows/build_images.yml
with:
IMAGE_ID: cli
# set some environment variables
envs:
runs-on: ubuntu-latest
outputs:
BRANCH_REF: ${{ steps.BRANCH_REF.outputs.value }}
RUN_DATE: ${{ steps.calculate_date.outputs.value }}
steps:
- name: Calculate run_date
id: calculate_date
run: |
input_run_date=${{ github.event.inputs.run_date || 'unspecified' }}
if [[ ${{ github.event_name }} == 'workflow_dispatch' && ${input_run_date} != 'unspecified' ]]
then
run_date="${{ github.event.inputs.run_date }}" # Use provided run_date
else
# automatically set the run_date by finding an appropriate time stamp in the past (corresponding to our cron schedule)
# Get the current time in UTC
current_time=$(date -u +'%Y%m%d_%H')
# Extract the hour and calculate the nearest multiple of 12 in the past (as per our cron schedule above)
hour=$(echo ${current_time:9:2} | awk '{print int($1 - ($1%12))}')
# Correct hour formatting (ensure leading zero)
hour=$(printf "%02d" $hour)
# Assemble the run_date
run_date=$(echo ${current_time:0:8}_${hour})
fi
echo "value=$run_date" >> $GITHUB_OUTPUT
# Dynamically set the branch ref to the currently executing branch
- name: Set the BRANCH_REF
id: BRANCH_REF
run: |
echo "value=${GITHUB_REF##*/}" >> $GITHUB_OUTPUT
# everything below here runs using the `mims1` self-hosted runner
# This is a server with 120 cpu's and 256 G ram, dedictated to running SOMISANA's operational models
# It is possible that we may want to use an additional modelling server (which will be a separate node on MIMS)
# In that event, we could put all the code below here in a new reusable workflow called run_ops_mims1.yml
# And then set up another one called run_ops_mims2.yml set up in the same way but obviously running different models/domains
# (note you'll also have to include another `git pull` command at the end of build_images.yml to make sure the latest images are available on the new server)
#
cleanup:
needs: [envs,build_cli]
uses: ./.github/workflows/cleanup.yml
with:
RUNNER_NAME: mims1
BRANCH_REF: ${{ needs.envs.outputs.BRANCH_REF }}
# prepare croco config dirs for each domain
prep_domains:
needs: [envs]
strategy:
matrix:
# running as a matrix strategy allows us to prepare different domains in parallel inside a single job
domain: ['sa_southeast_01','sa_west_02']
uses: ./.github/workflows/prep_domain.yml
with:
RUN_DATE: ${{ needs.envs.outputs.RUN_DATE }}
BRANCH_REF: ${{ needs.envs.outputs.BRANCH_REF }}
RUNNER_NAME: mims1
MODEL: croco_v1.3.1
DOMAIN: ${{ matrix.domain }}
COMP: C06
# in future we may want to use different compile options for each domain
# one way of handling that may be to add the compile option to each string in the domain variable under the matrix strategy
# and then tease out the domain and compile option inside prep_domain.yml
# download the surface data we'll need to force the models
download_surface:
needs: [envs, build_cli]
uses: ./.github/workflows/download_surface.yml
with:
RUNNER_NAME: mims1
RUN_DATE: ${{ needs.envs.outputs.RUN_DATE }}
BRANCH_REF: ${{ needs.envs.outputs.BRANCH_REF }}
HDAYS: 5
FDAYS: 5
secrets: inherit
# download the OGCM data we'll need to force the models
download_mercator:
needs: [envs, build_cli]
uses: ./.github/workflows/download_mercator.yml
with:
RUNNER_NAME: mims1
RUN_DATE: ${{ needs.envs.outputs.RUN_DATE }}
BRANCH_REF: ${{ needs.envs.outputs.BRANCH_REF }}
HDAYS: 5
FDAYS: 5
secrets: inherit
download_hycom:
needs: [envs, build_cli]
uses: ./.github/workflows/download_hycom.yml
with:
RUNNER_NAME: mims1
RUN_DATE: ${{ needs.envs.outputs.RUN_DATE }}
BRANCH_REF: ${{ needs.envs.outputs.BRANCH_REF }}
HDAYS: 5
FDAYS: 5
secrets: inherit
# prepare tidal forcing for each domain
make_tide_frc:
needs: [envs, build_cli, prep_domains]
strategy:
matrix:
# running as a matrix strategy allows us to prepare different domains in parallel inside a single job
domain: ['sa_southeast_01','sa_west_02']
uses: ./.github/workflows/make_tides.yml # Path to your reusable workflow
with:
RUN_DATE: ${{ needs.envs.outputs.RUN_DATE }}
BRANCH_REF: ${{ needs.envs.outputs.BRANCH_REF }}
RUNNER_NAME: mims1
MODEL: croco_v1.3.1
DOMAIN: ${{ matrix.domain }}
TIDE_FRC: TPXO10
HDAYS: 5
# run the model, do the postprocessing, archive the output
#
# we must ensure that run_all_domains.yml must use less than 120 processors in parallel - this is our limit
#
# we've also found that you can't run 4 domains in parallel, even if they use less than 120 processors in totoal
# the runs slow down hugely, so it's better to use more processors per domain, and maybe 2 domains max in parallel
run_mercator_domains:
needs: [envs,prep_domains,download_surface,download_mercator]
uses: ./.github/workflows/run_all_domains.yml
with:
RUN_DATE: ${{ needs.envs.outputs.RUN_DATE }}
BRANCH_REF: ${{ needs.envs.outputs.BRANCH_REF }}
RUNNER_NAME: mims1
OGCM: MERCATOR
SAWS_OK: ${{ needs.download_surface.outputs.SAWS_OK }}
secrets: inherit
run_hycom_domains:
needs: [envs,prep_domains,download_surface,download_hycom,run_mercator_domains]
if: ${{ always() }} # execute even if there was an error with the mercator runs
uses: ./.github/workflows/run_all_domains.yml
with:
RUN_DATE: ${{ needs.envs.outputs.RUN_DATE }}
BRANCH_REF: ${{ needs.envs.outputs.BRANCH_REF }}
RUNNER_NAME: mims1
OGCM: HYCOM
SAWS_OK: ${{ needs.download_surface.outputs.SAWS_OK }}
secrets: inherit