Ocean/ice product generation for GFS and GEFS (NOAA-EMC#2286)

This PR does several things: 1. the model output for ocean and ice in the `COM/` directory are now named per EE2 convention for coupled model. E.g `gfs.ocean.t12z.6hr_avg.f120.nc` and `gfs.ocean.t12z.daily.f120.nc` 2. The products are generated using the `ocnicepost.fd` utility developed by @DeniseWorthen in https://github.com/NOAA-EMC/gfs-utils and converted to grib2 using example scripts provided by @GwenChen-NOAA using `wgrib2`. 3. NetCDF products on the native grid are also generated by subsetting variables from the raw model output. This is done with `xarray`. 4. updates the hash of https://github.com/NOAA-EMC/gfs-utils to include fixes in `ocnicepost.fd` 5. removes NCL related scripting that was previously used for ocean/ice interpolation and `reg2grb2` used for converting to grib2. 6. updates archive scripts to accommodate updated file names 7. removes intermediate ocean processed files such as 2D/3D/xsect data- sets 8. separate jobs are added for ocean and ice product generation. 9. removes intermediate restarts for the mediator and only saves the medi- ator restart at the end of the forecast in `COM`. 10. Increases memory for offline UPP when run at C768. The program segfaults with an OOM when memory is self allocated based on PEs by the scheduler on Hera. 11. Enables ocean/ice ensemble product generation for GEFS 12. Some minor clean-ups Fixes NOAA-EMC#935 Fixes NOAA-EMC#1317 Fixes NOAA-EMC#1864
kayeekayee · Feb 13, 2024 · 6404892 · 6404892
1 parent 3f99f70
commit 6404892
Show file tree

Hide file tree

Showing 47 changed files with 1,356 additions and 1,599 deletions.
diff --git a/.gitignore b/.gitignore
@@ -36,7 +36,6 @@ fix/gsi
 fix/lut
 fix/mom6
 fix/orog
-fix/reg2grb2
 fix/sfc_climo
 fix/ugwd
 fix/verif
@@ -99,6 +98,9 @@ parm/post/postxconfig-NT-GFS-WAFS.txt
 parm/post/postxconfig-NT-GFS.txt
 parm/post/postxconfig-NT-gefs-aerosol.txt
 parm/post/postxconfig-NT-gefs-chem.txt
+parm/post/ocean.csv
+parm/post/ice.csv
+parm/post/ocnicepost.nml.jinja2
 parm/ufs/noahmptable.tbl
 parm/ufs/model_configure.IN
 parm/ufs/MOM_input_*.IN
@@ -137,7 +139,6 @@ sorc/radmon_bcor.fd
 sorc/radmon_time.fd
 sorc/rdbfmsua.fd
 sorc/recentersigp.fd
-sorc/reg2grb2.fd
 sorc/supvit.fd
 sorc/syndat_getjtbul.fd
 sorc/syndat_maksynrc.fd
@@ -147,6 +148,7 @@ sorc/tocsbufr.fd
 sorc/upp.fd
 sorc/vint.fd
 sorc/webtitle.fd
+sorc/ocnicepost.fd
 
 # Ignore scripts from externals
 #------------------------------

diff --git a/env/HERA.env b/env/HERA.env
@@ -199,6 +199,13 @@ elif [[ "${step}" = "atmos_products" ]]; then
 
     export USE_CFP="YES"  # Use MPMD for downstream product generation on Hera
 
+elif [[ "${step}" = "oceanice_products" ]]; then
+
+    nth_max=$((npe_node_max / npe_node_oceanice_products))
+
+    export NTHREADS_OCNICEPOST=${nth_oceanice_products:-1}
+    export APRUN_OCNICEPOST="${launcher} -n 1 --cpus-per-task=${NTHREADS_OCNICEPOST}"
+
 elif [[ "${step}" = "ecen" ]]; then
 
     nth_max=$((npe_node_max / npe_node_ecen))

diff --git a/env/HERCULES.env b/env/HERCULES.env
@@ -207,10 +207,20 @@ case ${step} in
     [[ ${NTHREADS_UPP} -gt ${nth_max} ]] && export NTHREADS_UPP=${nth_max}
     export APRUN_UPP="${launcher} -n ${npe_upp} --cpus-per-task=${NTHREADS_UPP}"
  ;;
+
  "atmos_products")
 
     export USE_CFP="YES"  # Use MPMD for downstream product generation
  ;;
+
+"oceanice_products")
+
+    nth_max=$((npe_node_max / npe_node_oceanice_products))
+
+    export NTHREADS_OCNICEPOST=${nth_oceanice_products:-1}
+    export APRUN_OCNICEPOST="${launcher} -n 1 --cpus-per-task=${NTHREADS_OCNICEPOST}"
+;;
+
  "ecen")
 
     nth_max=$((npe_node_max / npe_node_ecen))

diff --git a/env/JET.env b/env/JET.env
@@ -190,6 +190,13 @@ elif [[ "${step}" = "atmos_products" ]]; then
 
     export USE_CFP="YES"  # Use MPMD for downstream product generation
 
+elif [[ "${step}" = "oceanice_products" ]]; then
+
+    nth_max=$((npe_node_max / npe_node_oceanice_products))
+
+    export NTHREADS_OCNICEPOST=${nth_oceanice_products:-1}
+    export APRUN_OCNICEPOST="${launcher} -n 1 --cpus-per-task=${NTHREADS_OCNICEPOST}"
+
 elif [[ "${step}" = "ecen" ]]; then
 
     nth_max=$((npe_node_max / npe_node_ecen))

diff --git a/env/ORION.env b/env/ORION.env
@@ -209,6 +209,13 @@ elif [[ "${step}" = "atmos_products" ]]; then
 
     export USE_CFP="YES"  # Use MPMD for downstream product generation
 
+elif [[ "${step}" = "oceanice_products" ]]; then
+
+    nth_max=$((npe_node_max / npe_node_oceanice_products))
+
+    export NTHREADS_OCNICEPOST=${nth_oceanice_products:-1}
+    export APRUN_OCNICEPOST="${launcher} -n 1 --cpus-per-task=${NTHREADS_OCNICEPOST}"
+
 elif [[ "${step}" = "ecen" ]]; then
 
     nth_max=$((npe_node_max / npe_node_ecen))

diff --git a/env/S4.env b/env/S4.env
@@ -177,6 +177,13 @@ elif [[ "${step}" = "atmos_products" ]]; then
 
     export USE_CFP="YES"  # Use MPMD for downstream product generation
 
+elif [[ "${step}" = "oceanice_products" ]]; then
+
+    nth_max=$((npe_node_max / npe_node_oceanice_products))
+
+    export NTHREADS_OCNICEPOST=${nth_oceanice_products:-1}
+    export APRUN_OCNICEPOST="${launcher} -n 1 --cpus-per-task=${NTHREADS_OCNICEPOST}"
+
 elif [[ "${step}" = "ecen" ]]; then
 
     nth_max=$((npe_node_max / npe_node_ecen))

diff --git a/env/WCOSS2.env b/env/WCOSS2.env
@@ -193,6 +193,13 @@ elif [[ "${step}" = "atmos_products" ]]; then
 
     export USE_CFP="YES"  # Use MPMD for downstream product generation
 
+elif [[ "${step}" = "oceanice_products" ]]; then
+
+    nth_max=$((npe_node_max / npe_node_oceanice_products))
+
+    export NTHREADS_OCNICEPOST=${nth_oceanice_products:-1}
+    export APRUN_OCNICEPOST="${launcher} -n 1 -ppn ${npe_node_oceanice_products} --cpu-bind depth --depth ${NTHREADS_OCNICEPOST}"
+
 elif [[ "${step}" = "ecen" ]]; then
 
     nth_max=$((npe_node_max / npe_node_ecen))

diff --git a/jobs/JGLOBAL_ARCHIVE b/jobs/JGLOBAL_ARCHIVE
@@ -14,16 +14,15 @@ YMD=${PDY} HH=${cyc} generate_com -rx COM_ATMOS_ANALYSIS COM_ATMOS_BUFR COM_ATMO
   COM_ATMOS_TRACK COM_ATMOS_WMO \
   COM_CHEM_HISTORY COM_CHEM_ANALYSIS\
   COM_MED_RESTART \
-  COM_ICE_HISTORY COM_ICE_INPUT COM_ICE_RESTART \
+  COM_ICE_HISTORY COM_ICE_INPUT COM_ICE_RESTART COM_ICE_GRIB \
   COM_OBS COM_TOP \
-  COM_OCEAN_HISTORY COM_OCEAN_INPUT COM_OCEAN_RESTART COM_OCEAN_XSECT COM_OCEAN_2D COM_OCEAN_3D \
+  COM_OCEAN_HISTORY COM_OCEAN_INPUT COM_OCEAN_RESTART COM_OCEAN_GRIB COM_OCEAN_NETCDF \
   COM_OCEAN_ANALYSIS \
   COM_WAVE_GRID COM_WAVE_HISTORY COM_WAVE_STATION \
   COM_ATMOS_OZNMON COM_ATMOS_RADMON COM_ATMOS_MINMON COM_CONF
 
 for grid in "0p25" "0p50" "1p00"; do
   YMD=${PDY} HH=${cyc} GRID=${grid} generate_com -rx "COM_ATMOS_GRIB_${grid}:COM_ATMOS_GRIB_GRID_TMPL"
-  YMD=${PDY} HH=${cyc} GRID=${grid} generate_com -rx "COM_OCEAN_GRIB_${grid}:COM_OCEAN_GRIB_GRID_TMPL"
 done
 
 ###############################################################

diff --git a/jobs/JGLOBAL_OCEANICE_PRODUCTS b/jobs/JGLOBAL_OCEANICE_PRODUCTS
@@ -0,0 +1,40 @@
+#! /usr/bin/env bash
+
+source "${HOMEgfs}/ush/preamble.sh"
+source "${HOMEgfs}/ush/jjob_header.sh" -e "oceanice_products" -c "base oceanice_products"
+
+
+##############################################
+# Begin JOB SPECIFIC work
+##############################################
+
+# Construct COM variables from templates
+YMD="${PDY}" HH="${cyc}" generate_com -rx "COM_${COMPONENT^^}_HISTORY"
+YMD="${PDY}" HH="${cyc}" generate_com -rx "COM_${COMPONENT^^}_GRIB"
+YMD="${PDY}" HH="${cyc}" generate_com -rx "COM_${COMPONENT^^}_NETCDF"
+
+###############################################################
+# Run exglobal script
+"${HOMEgfs}/scripts/exglobal_oceanice_products.py"
+status=$?
+(( status != 0 )) && exit "${status}"
+
+##############################################
+# End JOB SPECIFIC work
+##############################################
+
+##############################################
+# Final processing
+##############################################
+if [[ -e "${pgmout}" ]]; then
+  cat "${pgmout}"
+fi
+
+##########################################
+# Remove the Temporary working directory
+##########################################
+cd "${DATAROOT}" || exit 1
+[[ "${KEEPDATA:-NO}" == "NO" ]] && rm -rf "${DATA}"
+
+
+exit 0
diff --git a/jobs/rocoto/oceanice_products.sh b/jobs/rocoto/oceanice_products.sh
@@ -0,0 +1,37 @@
+#! /usr/bin/env bash
+
+source "${HOMEgfs}/ush/preamble.sh"
+
+###############################################################
+## ocean ice products driver script
+## FHRLST : forecast hour list to post-process (e.g. f000, f000_f001_f002, ...)
+###############################################################
+
+# Source FV3GFS workflow modules
+. "${HOMEgfs}/ush/load_fv3gfs_modules.sh"
+status=$?
+if (( status != 0 )); then exit "${status}"; fi
+
+###############################################################
+# setup python path for workflow utilities and tasks
+wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src"
+PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}"
+export PYTHONPATH
+
+export job="oceanice_products"
+export jobid="${job}.$$"
+
+###############################################################
+# shellcheck disable=SC2153,SC2001
+IFS='_' read -ra fhrs <<< "${FHRLST//f}" # strip off the 'f's and convert to array
+
+#---------------------------------------------------------------
+# Execute the JJOB
+for fhr in "${fhrs[@]}"; do
+    export FORECAST_HOUR=$(( 10#${fhr} ))
+    "${HOMEgfs}/jobs/JGLOBAL_OCEANICE_PRODUCTS"
+    status=$?
+    if (( status != 0 )); then exit "${status}"; fi
+done
+
+exit 0
diff --git a/jobs/rocoto/ocnpost.sh b/jobs/rocoto/ocnpost.sh
diff --git a/modulefiles/module_base.hera.lua b/modulefiles/module_base.hera.lua
@@ -33,6 +33,7 @@ load(pathJoin("wgrib2", (os.getenv("wgrib2_ver") or "None")))
 load(pathJoin("py-netcdf4", (os.getenv("py_netcdf4_ver") or "None")))
 load(pathJoin("py-pyyaml", (os.getenv("py_pyyaml_ver") or "None")))
 load(pathJoin("py-jinja2", (os.getenv("py_jinja2_ver") or "None")))
+load(pathJoin("py-xarray", (os.getenv("py_xarray_ver") or "None")))
 
 -- MET/METplus are not available for use with spack-stack, yet
 --load(pathJoin("met", (os.getenv("met_ver") or "None")))

diff --git a/modulefiles/module_base.hercules.lua b/modulefiles/module_base.hercules.lua
@@ -35,6 +35,7 @@ load(pathJoin("wgrib2", (os.getenv("wgrib2_ver") or "None")))
 load(pathJoin("py-netcdf4", (os.getenv("py_netcdf4_ver") or "None")))
 load(pathJoin("py-pyyaml", (os.getenv("py_pyyaml_ver") or "None")))
 load(pathJoin("py-jinja2", (os.getenv("py_jinja2_ver") or "None")))
+load(pathJoin("py-xarray", (os.getenv("py_xarray_ver") or "None")))
 
 setenv("WGRIB2","wgrib2")
 setenv("UTILROOT",(os.getenv("prod_util_ROOT") or "None"))

diff --git a/modulefiles/module_base.jet.lua b/modulefiles/module_base.jet.lua
@@ -33,6 +33,7 @@ load(pathJoin("wgrib2", (os.getenv("wgrib2_ver") or "None")))
 load(pathJoin("py-netcdf4", (os.getenv("py_netcdf4_ver") or "None")))
 load(pathJoin("py-pyyaml", (os.getenv("py_pyyaml_ver") or "None")))
 load(pathJoin("py-jinja2", (os.getenv("py_jinja2_ver") or "None")))
+load(pathJoin("py-xarray", (os.getenv("py_xarray_ver") or "None")))
 
 setenv("WGRIB2","wgrib2")
 setenv("UTILROOT",(os.getenv("prod_util_ROOT") or "None"))

diff --git a/modulefiles/module_base.orion.lua b/modulefiles/module_base.orion.lua
@@ -31,6 +31,7 @@ load(pathJoin("wgrib2", (os.getenv("wgrib2_ver") or "None")))
 load(pathJoin("py-netcdf4", (os.getenv("py_netcdf4_ver") or "None")))
 load(pathJoin("py-pyyaml", (os.getenv("py_pyyaml_ver") or "None")))
 load(pathJoin("py-jinja2", (os.getenv("py_jinja2_ver") or "None")))
+load(pathJoin("py-xarray", (os.getenv("py_xarray_ver") or "None")))
 
 -- MET/METplus are not yet supported with spack-stack
 --load(pathJoin("met", (os.getenv("met_ver") or "None")))

diff --git a/modulefiles/module_base.s4.lua b/modulefiles/module_base.s4.lua
@@ -30,6 +30,7 @@ load(pathJoin("wgrib2", (os.getenv("wgrib2_ver") or "None")))
 load(pathJoin("py-netcdf4", (os.getenv("py_netcdf4_ver") or "None")))
 load(pathJoin("py-pyyaml", (os.getenv("py_pyyaml_ver") or "None")))
 load(pathJoin("py-jinja2", (os.getenv("py_jinja2_ver") or "None")))
+load(pathJoin("py-xarray", (os.getenv("py_xarray_ver") or "None")))
 
 setenv("WGRIB2","wgrib2")
 setenv("UTILROOT",(os.getenv("prod_util_ROOT") or "None"))

diff --git a/parm/config/gefs/config.oceanice_products b/parm/config/gefs/config.oceanice_products
@@ -0,0 +1 @@
+../gfs/config.oceanice_products