Merge branch 'NOAA-EMC:develop' into develop

.gitignore

@@ -48,6 +48,8 @@ parm/gdas/io
 parm/gdas/ioda
 parm/gdas/snow
 parm/gdas/soca
+parm/gdas/jcb-gdas
+parm/gdas/jcb-algorithms
 parm/monitor
 parm/post/AEROSOL_LUTS.dat
 parm/post/nam_micro_lookup.dat
@@ -106,6 +108,7 @@ parm/post/ice.csv
 parm/post/ocnicepost.nml.jinja2
 parm/ufs/noahmptable.tbl
 parm/ufs/model_configure.IN
+parm/ufs/model_configure_nest.IN
 parm/ufs/MOM_input_*.IN
 parm/ufs/MOM6_data_table.IN
 parm/ufs/ice_in.IN
@@ -194,3 +197,8 @@ versions/run.ver
 ush/python/wxflow
 workflow/wxflow
 ci/scripts/wxflow
+
+# jcb checkout and symlinks
+ush/python/jcb
+workflow/jcb
+ci/scripts/jcb

.gitmodules

@@ -30,3 +30,7 @@
   path = sorc/upp.fd
   url = https://github.com/NOAA-EMC/UPP.git
   ignore = dirty
+[submodule "sorc/jcb"]
+  path = sorc/jcb
+  url = https://github.com/noaa-emc/jcb
+  fetchRecurseSubmodules = false

ci/Jenkinsfile

@@ -78,7 +78,7 @@ pipeline {
                     ws("${custom_workspace[machine]}/${env.CHANGE_ID}") {
                         properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in', 'Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
                         HOME = "${WORKSPACE}"
-                        sh(script: "mkdir -p ${HOME}/RUNTESTS;rm -Rf ${HOME}/RUNTESTS/error.logs")
+                        sh(script: "mkdir -p ${HOME}/RUNTESTS;rm -Rf ${HOME}/RUNTESTS/*")
                         sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-Building" --remove-label "CI-${Machine}-Ready" """)
                     }
                     echo "Building and running on ${Machine} in directory ${HOME}"
@@ -149,7 +149,7 @@ pipeline {
                                                 }
                                             }
                                             sh(script: './link_workflow.sh')
-                                            sh(script: "echo ${HOMEgfs} > BUILT_semaphor")
+                                            //sh(script: "echo ${HOMEgfs} > BUILT_semaphor")
                                         }
                                     }
                                     if (env.CHANGE_ID && system == 'gfs') {

ci/cases/pr/C96_atm3DVar.yaml

@@ -15,3 +15,6 @@ arguments:
   gfs_cyc: 1
   start: cold
   yaml: {{ HOMEgfs }}/ci/cases/yamls/gfs_defaults_ci.yaml
+
+skip_ci_on_hosts:
+  - wcoss2

ci/cases/pr/C96_atm3DVar_extended.yaml

@@ -0,0 +1,22 @@
+experiment:
+  system: gfs
+  mode: cycled
+
+arguments:
+  pslot: {{ 'pslot' | getenv }}
+  app: ATM
+  resdetatmos: 96
+  comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
+  expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
+  icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48
+  idate: 2021122018
+  edate: 2021122118
+  nens: 0
+  gfs_cyc: 4
+  start: cold
+  yaml: {{ HOMEgfs }}/ci/cases/yamls/gfs_extended_ci.yaml
+
+skip_ci_on_hosts:
+  - hera
+  - orion
+  - hercules

ci/cases/yamls/gfs_extended_ci.yaml

@@ -0,0 +1,12 @@
+defaults:
+  !INC {{ HOMEgfs }}/parm/config/gfs/yaml/defaults.yaml
+
+base:
+  ACCOUNT: {{ 'SLURM_ACCOUNT' | getenv }}
+  DO_GOES: "YES"
+  DO_BUFRSND: "YES"
+  DO_GEMPAK: "YES"
+  DO_AWIPS: "NO"
+  DO_NPOESS: "YES"
+  DO_GENESIS_FSU: "NO"
+  FHMAX_GFS: 384

ci/platforms/config.hera

@@ -3,6 +3,7 @@
 export GFS_CI_ROOT=/scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT
 export ICSDIR_ROOT=/scratch1/NCEPDEV/global/glopara/data/ICSDIR
 export STMP="/scratch1/NCEPDEV/stmp2/${USER}"
+export PTMP="/scratch1/NCEPDEV/stmp2/${USER}"
 export SLURM_ACCOUNT=nems
 export max_concurrent_cases=5
 export max_concurrent_pr=4

ci/platforms/config.hercules

@@ -2,7 +2,8 @@
 
 export GFS_CI_ROOT=/work2/noaa/stmp/GFS_CI_ROOT/HERCULES
 export ICSDIR_ROOT=/work/noaa/global/glopara/data/ICSDIR
-export STMP="/work2/noaa/stmp/${USER}"
+export STMP="/work2/noaa/stmp/${USER}/HERCULES"
+export PTMP="/work2/noaa/stmp/${USER}/HERCULES"
 export SLURM_ACCOUNT=nems
 export max_concurrent_cases=5
 export max_concurrent_pr=4

ci/platforms/config.orion

@@ -2,7 +2,8 @@
 
 export GFS_CI_ROOT=/work2/noaa/stmp/GFS_CI_ROOT/ORION
 export ICSDIR_ROOT=/work/noaa/global/glopara/data/ICSDIR
-export STMP="/work2/noaa/stmp/${USER}"
+export STMP="/work2/noaa/stmp/${USER}/ORION"
+export PTMP="/work2/noaa/stmp/${USER}/ORION"
 export SLURM_ACCOUNT=nems
 export max_concurrent_cases=5
 export max_concurrent_pr=4

ci/platforms/config.wcoss2

@@ -3,6 +3,7 @@
 export GFS_CI_ROOT=/lfs/h2/emc/global/noscrub/globalworkflow.ci/GFS_CI_ROOT
 export ICSDIR_ROOT=/lfs/h2/emc/global/noscrub/emc.global/data/ICSDIR
 export STMP="/lfs/h2/emc/stmp/${USER}"
+export PTMP="/lfs/h2/emc/ptmp/${USER}"
 export SLURM_ACCOUNT=GFS-DEV
 export max_concurrent_cases=5
 export max_concurrent_pr=4

ci/scripts/driver.sh

@@ -220,18 +220,20 @@ for pr in ${pr_list}; do
       set +e
       export LOGFILE_PATH="${HOMEgfs}/ci/scripts/create_experiment.log"
       rm -f "${LOGFILE_PATH}"
+      yaml_case_file="${HOMEgfs}/ci/cases/pr/${case}.yaml"
+      skip_hosts=$("${HOMEgfs}/ci/scripts/utils/parse_yaml.py" --yaml "${yaml_case_file}" --key skip_ci_on_hosts --string)
+      if [[ "${skip_hosts}" == *"${MACHINE_ID}"* ]]; then
+        {
+          echo "Case setup: Skipped for experiment ${pslot}" || true
+        } >> "${output_ci}"
+        continue
+      fi
       "${HOMEgfs}/workflow/create_experiment.py" --yaml "${HOMEgfs}/ci/cases/pr/${case}.yaml"  > "${LOGFILE_PATH}" 2>&1
       ci_status=$?
       set -e
       if [[ ${ci_status} -eq 0 ]]; then
-        last_line=$(tail -1 "${LOGFILE_PATH}")
-        if [[ "${last_line}" == *"Skipping creation"* ]]; then
-          action="Skipped"
-        else
-          action="Completed"
-        fi
         {
-          echo "Case setup: ${action} for experiment ${pslot}" || true
+          echo "Case setup: Completed for experiment ${pslot}" || true
         } >> "${output_ci}"
       else
         {

ci/scripts/utils/parse_yaml.py

@@ -0,0 +1,70 @@
+#!/usr/bin/env python3
+
+"""
+This script parses a yaml file and returns the value of a specified key.
+"""
+
+import os
+import sys
+from wxflow import AttrDict, parse_j2yaml
+from argparse import ArgumentParser
+from pathlib import Path
+
+_here = os.path.dirname(__file__)
+_top = os.path.abspath(os.path.join(os.path.abspath(_here), '../../..'))
+
+description = """parse yaml file and return value of key"""
+
+
+def parse_args():
+    """
+    Parse command-line arguments.
+
+    Returns:
+        argparse.Namespace: The parsed command-line arguments.
+    """
+
+    parser = ArgumentParser(description=description)
+    parser.add_argument('-y', '--yaml', help='full path to yaml file to parse', type=Path, required=True)
+    parser.add_argument('-k', '--key', help='key to return value of', type=str, required=True)
+    parser.add_argument('-s', '--string', help='output results as strings', action="store_true", required=False)
+    return parser.parse_args()
+
+
+def yq(yamlfile, key):
+    """
+    Parse a yaml file and return the value of a specified key.
+
+    Args:
+        yamlfile (Path): The path to the yaml file.
+        key (str): The key to return the value of.
+
+    Returns:
+        The value of the specified key in the yaml file.
+    """
+
+    data = AttrDict(HOMEgfs=_top)
+    data.update({'HOMEgfs': _top})
+    ydict = parse_j2yaml(path=yamlfile, data=data)
+    if key == 'all':
+        return ydict
+    list_keys = key.split('.')
+    for k in list_keys:
+        ydict = ydict.get(k, None)
+        if ydict is None:
+            break
+    return ydict
+
+
+if __name__ == '__main__':
+    """
+    Main function. Parses command-line arguments and prints the value of the specified key in the specified yaml file.
+    """
+
+    args = parse_args()
+    values = yq(args.yaml, args.key)
+    if args.string and isinstance(values, list):
+        for value in values:
+            print(value)
+    else:
+        print(values)

env/HERA.env

@@ -49,6 +49,10 @@ elif [[ "${step}" = "prepsnowobs" ]]; then
 
     export APRUN_CALCFIMS="${launcher} -n 1"
 
+elif [[ "${step}" = "prep_emissions" ]]; then
+
+    export APRUN="${launcher} -n 1"
+
 elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}" = "wavepostsbs" ]] || [[ "${step}" = "wavepostbndpnt" ]] || [[ "${step}" = "wavepostbndpntbll" ]] || [[ "${step}" = "wavepostpnt" ]]; then
 
     export CFP_MP="YES"
@@ -126,6 +130,14 @@ elif [[ "${step}" = "ocnanalecen" ]]; then
     [[ ${NTHREADS_OCNANALECEN} -gt ${nth_max} ]] && export NTHREADS_OCNANALECEN=${nth_max}
     export APRUN_OCNANALECEN="${launcher} -n ${npe_ocnanalecen} --cpus-per-task=${NTHREADS_OCNANALECEN}"
 
+elif [[ "${step}" = "ocnanalletkf" ]]; then
+
+    nth_max=$((npe_node_max / npe_node_ocnanalletkf))
+
+    export NTHREADS_OCNANALLETKF=${nth_ocnanalletkf:-${nth_max}}
+    [[ ${NTHREADS_OCNANALLETKF} -gt ${nth_max} ]] && export NTHREADS_OCNANALLETKF=${nth_max}
+    export APRUN_OCNANALLETKF="${launcher} -n ${npe_ocnanalletkf} --cpus-per-task=${NTHREADS_OCNANALLETKF}"
+
 elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then
 
     export MKL_NUM_THREADS=4
@@ -288,15 +300,13 @@ elif [[ "${step}" = "gempak" ]]; then
     export CFP_MP="YES"
 
     if [[ ${CDUMP} == "gfs" ]]; then
-        npe_gempak=${npe_gempak_gfs}
         npe_node_gempak=${npe_node_gempak_gfs}
     fi
 
     nth_max=$((npe_node_max / npe_node_gempak))
 
     export NTHREADS_GEMPAK=${nth_gempak:-1}
     [[ ${NTHREADS_GEMPAK} -gt ${nth_max} ]] && export NTHREADS_GEMPAK=${nth_max}
-    export APRUN="${launcher} -n ${npe_gempak} ${mpmd_opt}"
 
 
 elif [[ "${step}" = "fit2obs" ]]; then

env/HERCULES.env

@@ -45,6 +45,10 @@ case ${step} in
 
     export APRUN_CALCFIMS="${launcher} -n 1"
  ;;
+ "prep_emissions")
+
+    export APRUN="${launcher} -n 1"
+ ;;
  "waveinit" | "waveprep" | "wavepostsbs" | "wavepostbndpnt" | "wavepostpnt" | "wavepostbndpntbll")
 
     export CFP_MP="YES"

env/JET.env

@@ -37,6 +37,10 @@ elif [[ "${step}" = "prepsnowobs" ]]; then
 
     export APRUN_CALCFIMS="${launcher} -n 1"
 
+elif [[ "${step}" = "prep_emissions" ]]; then
+
+    export APRUN="${launcher} -n 1"
+
 elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}" = "wavepostsbs" ]] || [[ "${step}" = "wavepostbndpnt" ]] || [[ "${step}" = "wavepostbndpntbll" ]] || [[ "${step}" = "wavepostpnt" ]]; then
 
     export CFP_MP="YES"

env/ORION.env

@@ -44,6 +44,10 @@ elif [[ "${step}" = "prepsnowobs" ]]; then
 
     export APRUN_CALCFIMS="${launcher} -n 1"
 
+elif [[ "${step}" = "prep_emissions" ]]; then
+
+    export APRUN="${launcher} -n 1"
+
 elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}" = "wavepostsbs" ]] || \
     [[ "${step}" = "wavepostbndpnt" ]] || [[ "${step}" = "wavepostpnt" ]] || [[ "${step}" == "wavepostbndpntbll" ]]; then
 
@@ -134,6 +138,14 @@ elif [[ "${step}" = "ocnanalecen" ]]; then
     [[ ${NTHREADS_OCNANALECEN} -gt ${nth_max} ]] && export NTHREADS_OCNANALECEN=${nth_max}
     export APRUN_OCNANALECEN="${launcher} -n ${npe_ocnanalecen} --cpus-per-task=${NTHREADS_OCNANALECEN}"
 
+elif [[ "${step}" = "ocnanalletkf" ]]; then
+
+    nth_max=$((npe_node_max / npe_node_ocnanalletkf))
+
+    export NTHREADS_OCNANALLETKF=${nth_ocnanalletkf:-${nth_max}}
+    [[ ${NTHREADS_OCNANALLETKF} -gt ${nth_max} ]] && export NTHREADS_OCNANALLETKF=${nth_max}
+    export APRUN_OCNANALLETKF="${launcher} -n ${npe_ocnanalletkf} --cpus-per-task=${NTHREADS_OCNANALLETKF}"
+
 elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then
 
     export MKL_NUM_THREADS=4

env/S4.env

@@ -37,6 +37,10 @@ elif [[ "${step}" = "prepsnowobs" ]]; then
 
     export APRUN_CALCFIMS="${launcher} -n 1"
 
+elif [[ "${step}" = "prep_emissions" ]]; then
+
+    export APRUN="${launcher} -n 1"
+
 elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}" = "wavepostsbs" ]] || [[ "${step}" = "wavepostbndpnt" ]] || [[ "${step}" = "wavepostbndpntbll" ]] || [[ "${step}" = "wavepostpnt" ]]; then
 
     export CFP_MP="YES"

env/WCOSS2.env

@@ -31,6 +31,10 @@ elif [[ "${step}" = "prepsnowobs" ]]; then
 
     export APRUN_CALCFIMS="${launcher} -n 1"
 
+elif [[ "${step}" = "prep_emissions" ]]; then
+
+    export APRUN="${launcher} -n 1"
+
 elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}" = "wavepostsbs" ]] || [[ "${step}" = "wavepostbndpnt" ]] || [[ "${step}" = "wavepostbndpntbll"  ]] || [[ "${step}" = "wavepostpnt" ]]; then
 
     export USE_CFP="YES"

gempak/ush/gdas_ecmwf_meta_ver.sh

@@ -24,7 +24,7 @@ fi
 
 export COMIN="gdas.${PDY}${cyc}"
 if [[ ! -L ${COMIN} ]]; then
-    ln -sf "${COM_ATMOS_GEMPAK_1p00}" "${COMIN}"
+    ${NLN} "${COM_ATMOS_GEMPAK_1p00}" "${COMIN}"
 fi
 vergrid="F-GDAS | ${PDY:2}/0600"
 fcsthr="0600f006"
@@ -49,7 +49,7 @@ for area in ${areas}; do
         sdatenum=$(date --utc +%y%m%d -d "${PDY} ${cyc2} - ${fhr} hours")
 
         if [[ ! -L "ecmwf.20${sdatenum}" ]]; then
-            ln -sf "${COMINecmwf}/ecmwf.20${sdatenum}/gempak" "ecmwf.20${sdatenum}"
+            ${NLN} "${COMINecmwf}/ecmwf.20${sdatenum}/gempak" "ecmwf.20${sdatenum}"
         fi
         gdfile="ecmwf.20${sdatenum}/ecmwf_glob_20${sdatenum}12"
 

gempak/ush/gdas_meta_loop.sh

@@ -13,7 +13,7 @@ device="nc | gdasloop.meta"
 #
 export COMIN="${RUN}.${PDY}${cyc}"
 if [[ ! -L "${COMIN}" ]]; then
-    ln -sf "${COM_ATMOS_GEMPAK_1p00}" "${COMIN}"
+    ${NLN} "${COM_ATMOS_GEMPAK_1p00}" "${COMIN}"
 fi
 
 if [[ "${envir}" == "para" ]] ; then
@@ -41,7 +41,7 @@ for (( fhr=24; fhr<=144; fhr+=24 )); do
         YMD=${day} HH=${cyc} GRID=1p00 declare_from_tmpl "COM_ATMOS_GEMPAK_1p00_past:COM_ATMOS_GEMPAK_TMPL"
         export COMIN="${RUN}.${day}${cycle}"
         if [[ ! -L "${COMIN}" ]]; then
-            ln -sf "${COM_ATMOS_GEMPAK_1p00_past}" "${COMIN}"
+            ${NLN} "${COM_ATMOS_GEMPAK_1p00_past}" "${COMIN}"
         fi
         gdfile="${COMIN}/gdas_1p00_${day}${cycle}f000"
 

gempak/ush/gdas_meta_na.sh

@@ -13,7 +13,7 @@ device="nc | gdas.meta"
 #
 export COMIN="${RUN}.${PDY}${cyc}"
 if [[ ! -L "${COMIN}" ]]; then
-    ln -sf "${COM_ATMOS_GEMPAK_1p00}" "${COMIN}"
+    ${NLN} "${COM_ATMOS_GEMPAK_1p00}" "${COMIN}"
 fi
 
 if [[ "${envir}" == "para" ]] ; then