Feature/updates for gdas app (#1091)

Updates the GDASapp version to a newer version that builds correctly. The former version no longer builds because submodules point to develop branches that are no longer compatible. Moves module loads out of the j-jobs and into their appropriate place in the rocoto job. A new load module script is added to handle the different module set needed for UFSDA. Also temporarily turns off strict mode for the UFSDA jobs to avoid PS1 unbound error in conda. Fixes #1043 Fixes #1067
NOAA-EMC · Oct 25, 2022 · c8c6994 · c8c6994
1 parent 26c23be
commit c8c6994
Show file tree

Hide file tree

Showing 20 changed files with 110 additions and 83 deletions.
diff --git a/Externals.cfg b/Externals.cfg
@@ -29,7 +29,7 @@ protocol = git
 required = True
 
 [GSI-EnKF]
-hash = 67f5ab4
+hash = 48d8676
 local_path = sorc/gsi_enkf.fd
 repo_url = https://github.com/NOAA-EMC/GSI.git
 protocol = git
@@ -50,7 +50,7 @@ protocol = git
 required = False
 
 [GDASApp]
-hash = 5952c9d
+hash = 843d3a9
 local_path = sorc/gdas.cd
 repo_url = https://github.com/NOAA-EMC/GDASApp.git
 protocol = git

diff --git a/jobs/JGDAS_GLOBAL_ATMOS_ANALYSIS_POST b/jobs/JGDAS_GLOBAL_ATMOS_ANALYSIS_POST
@@ -79,20 +79,13 @@ mkdir -p $COMOUT
 export COMIN_GES="$ROTDIR/$GDUMP.$gPDY/$gcyc/$COMPONENT"
 export COMIN_GES_ENS="$ROTDIR/enkf$GDUMP.$gPDY/$gcyc/$COMPONENT"
 
-# NOTE BELOW IS A HACK FOR TESTING
-# PLEASE FIX THIS LATER
-# ASK @aerorahul
-# HOW TO HANDLE DIFFERENT COMPILERS/ETC. FOR MODEL VS DA
-# PROD_UTIL, ETC. DO NOT EXIST FOR JEDI MODULE VERSIONS
-module purge
-module use $HOMEgfs/sorc/gdas.cd/modulefiles
-module load GDAS/orion
+# Add UFSDA to PYTHONPATH
 export PYTHONPATH=$HOMEgfs/sorc/gdas.cd/ush/:$PYTHONPATH
 
 ###############################################################
 # Run relevant script
 
-EXSCRIPT=${GDASPOSTPY:-$HOMEgfs/sorc/gdas.cd/scripts/exgdas_global_atmos_analysis_post.py}
+EXSCRIPT=${GDASPOSTPY:-$HOMEgfs/scripts/exgdas_global_atmos_analysis_post.py}
 $EXSCRIPT
 status=$?
 [[ $status -ne 0 ]] && exit $status

diff --git a/jobs/JGDAS_GLOBAL_ATMOS_ANALYSIS_PREP b/jobs/JGDAS_GLOBAL_ATMOS_ANALYSIS_PREP
@@ -79,20 +79,12 @@ mkdir -p $COMOUT
 export COMIN_GES="$ROTDIR/$GDUMP.$gPDY/$gcyc/$COMPONENT"
 export COMIN_GES_ENS="$ROTDIR/enkf$GDUMP.$gPDY/$gcyc/$COMPONENT"
 
-# NOTE BELOW IS A HACK FOR TESTING
-# PLEASE FIX THIS LATER
-# ASK @aerorahul
-# HOW TO HANDLE DIFFERENT COMPILERS/ETC. FOR MODEL VS DA
-# PROD_UTIL, ETC. DO NOT EXIST FOR JEDI MODULE VERSIONS
-module purge
-module use $HOMEgfs/sorc/gdas.cd/modulefiles
-module load GDAS/orion
+# Add UFSDA to PYTHONPATH
 export PYTHONPATH=$HOMEgfs/sorc/gdas.cd/ush/:$PYTHONPATH
 
 ###############################################################
 # Run relevant script
-
-EXSCRIPT=${GDASPREPPY:-$HOMEgfs/sorc/gdas.cd/scripts/exgdas_global_atmos_analysis_prep.py}
+EXSCRIPT=${GDASPREPPY:-$HOMEgfs/scripts/exgdas_global_atmos_analysis_prep.py}
 $EXSCRIPT
 status=$?
 [[ $status -ne 0 ]] && exit $status

diff --git a/jobs/JGDAS_GLOBAL_ATMOS_ANALYSIS_RUN b/jobs/JGDAS_GLOBAL_ATMOS_ANALYSIS_RUN
@@ -79,20 +79,13 @@ mkdir -p $COMOUT
 export COMIN_GES="$ROTDIR/$GDUMP.$gPDY/$gcyc/$COMPONENT"
 export COMIN_GES_ENS="$ROTDIR/enkf$GDUMP.$gPDY/$gcyc/$COMPONENT"
 
-# NOTE BELOW IS A HACK FOR TESTING
-# PLEASE FIX THIS LATER
-# ASK @aerorahul
-# HOW TO HANDLE DIFFERENT COMPILERS/ETC. FOR MODEL VS DA
-# PROD_UTIL, ETC. DO NOT EXIST FOR JEDI MODULE VERSIONS
-module purge
-module use $HOMEgfs/sorc/gdas.cd/modulefiles
-module load GDAS/orion
+# Add UFSDA to PYTHONPATH
 export PYTHONPATH=$HOMEgfs/sorc/gdas.cd/ush/:$PYTHONPATH
 
 ###############################################################
 # Run relevant script
 
-EXSCRIPT=${GDASRUNSH:-$HOMEgfs/sorc/gdas.cd/scripts/exgdas_global_atmos_analysis_run.sh}
+EXSCRIPT=${GDASRUNSH:-$HOMEgfs/scripts/exgdas_global_atmos_analysis_run.sh}
 $EXSCRIPT
 status=$?
 [[ $status -ne 0 ]] && exit $status

diff --git a/jobs/JGDAS_GLOBAL_ATMOS_ENSANAL_POST b/jobs/JGDAS_GLOBAL_ATMOS_ENSANAL_POST
@@ -82,20 +82,13 @@ export COMIN="$ROTDIR/$CDUMP.$PDY/$cyc/$COMPONENT"
 export COMIN_GES="$ROTDIR/$GDUMP.$gPDY/$gcyc/$COMPONENT"
 export COMIN_GES_ENS="$ROTDIR/enkf$GDUMP.$gPDY/$gcyc/$COMPONENT"
 
-# NOTE BELOW IS A HACK FOR TESTING
-# PLEASE FIX THIS LATER
-# ASK @aerorahul
-# HOW TO HANDLE DIFFERENT COMPILERS/ETC. FOR MODEL VS DA
-# PROD_UTIL, ETC. DO NOT EXIST FOR JEDI MODULE VERSIONS
-module purge
-module use $HOMEgfs/sorc/gdas.cd/modulefiles
-module load GDAS/orion
+# Add UFSDA to PYTHONPATH
 export PYTHONPATH=$HOMEgfs/sorc/gdas.cd/ush/:$PYTHONPATH
 
 ###############################################################
 # Run relevant script
 
-EXSCRIPT=${GDASPOSTPY:-$HOMEgfs/sorc/gdas.cd/scripts/exgdas_global_atmos_ensanal_post.py}
+EXSCRIPT=${GDASPOSTPY:-$HOMEgfs/scripts/exgdas_global_atmos_ensanal_post.py}
 $EXSCRIPT
 status=$?
 [[ $status -ne 0 ]] && exit $status

diff --git a/jobs/JGDAS_GLOBAL_ATMOS_ENSANAL_PREP b/jobs/JGDAS_GLOBAL_ATMOS_ENSANAL_PREP
@@ -82,20 +82,13 @@ export COMIN="$ROTDIR/$CDUMP.$PDY/$cyc/$COMPONENT"
 export COMIN_GES="$ROTDIR/$GDUMP.$gPDY/$gcyc/$COMPONENT"
 export COMIN_GES_ENS="$ROTDIR/enkf$GDUMP.$gPDY/$gcyc/$COMPONENT"
 
-# NOTE BELOW IS A HACK FOR TESTING
-# PLEASE FIX THIS LATER
-# ASK @aerorahul
-# HOW TO HANDLE DIFFERENT COMPILERS/ETC. FOR MODEL VS DA
-# PROD_UTIL, ETC. DO NOT EXIST FOR JEDI MODULE VERSIONS
-module purge
-module use $HOMEgfs/sorc/gdas.cd/modulefiles
-module load GDAS/orion
+# Add UFSDA to PYTHONPATH
 export PYTHONPATH=$HOMEgfs/sorc/gdas.cd/ush/:$PYTHONPATH
 
 ###############################################################
 # Run relevant script
 
-EXSCRIPT=${GDASPREPPY:-$HOMEgfs/sorc/gdas.cd/scripts/exgdas_global_atmos_analysis_prep.py}
+EXSCRIPT=${GDASPREPPY:-$HOMEgfs/scripts/exgdas_global_atmos_analysis_prep.py}
 $EXSCRIPT
 status=$?
 [[ $status -ne 0 ]] && exit $status

diff --git a/jobs/JGDAS_GLOBAL_ATMOS_ENSANAL_RUN b/jobs/JGDAS_GLOBAL_ATMOS_ENSANAL_RUN
@@ -82,20 +82,13 @@ export COMIN="$ROTDIR/$CDUMP.$PDY/$cyc/$COMPONENT"
 export COMIN_GES="$ROTDIR/$GDUMP.$gPDY/$gcyc/$COMPONENT"
 export COMIN_GES_ENS="$ROTDIR/enkf$GDUMP.$gPDY/$gcyc/$COMPONENT"
 
-# NOTE BELOW IS A HACK FOR TESTING
-# PLEASE FIX THIS LATER
-# ASK @aerorahul
-# HOW TO HANDLE DIFFERENT COMPILERS/ETC. FOR MODEL VS DA
-# PROD_UTIL, ETC. DO NOT EXIST FOR JEDI MODULE VERSIONS
-module purge
-module use $HOMEgfs/sorc/gdas.cd/modulefiles
-module load GDAS/orion
+# Add UFSDA to PYTHONPATH
 export PYTHONPATH=$HOMEgfs/sorc/gdas.cd/ush/:$PYTHONPATH
 
 ###############################################################
 # Run relevant script
 
-EXSCRIPT=${GDASRUNSH:-$HOMEgfs/sorc/gdas.cd/scripts/exgdas_global_atmos_ensanal_run.sh}
+EXSCRIPT=${GDASRUNSH:-$HOMEgfs/scripts/exgdas_global_atmos_ensanal_run.sh}
 $EXSCRIPT
 status=$?
 [[ $status -ne 0 ]] && exit $status

diff --git a/jobs/rocoto/atmanalpost.sh b/jobs/rocoto/atmanalpost.sh
@@ -1,10 +1,12 @@
 #! /usr/bin/env bash
 
+export STRICT="NO"
 source "$HOMEgfs/ush/preamble.sh"
+export STRICT="YES"
 
 ###############################################################
-# Source FV3GFS workflow modules
-. $HOMEgfs/ush/load_fv3gfs_modules.sh
+# Source UFSDA workflow modules
+. $HOMEgfs/ush/load_ufsda_modules.sh
 status=$?
 [[ $status -ne 0 ]] && exit $status
 

diff --git a/jobs/rocoto/atmanalprep.sh b/jobs/rocoto/atmanalprep.sh
@@ -1,10 +1,12 @@
 #! /usr/bin/env bash
 
+export STRICT="NO"
 source "$HOMEgfs/ush/preamble.sh"
+export STRICT="YES"
 
 ###############################################################
-# Source FV3GFS workflow modules
-. $HOMEgfs/ush/load_fv3gfs_modules.sh
+# Source UFSDA workflow modules
+. $HOMEgfs/ush/load_ufsda_modules.sh
 status=$?
 [[ $status -ne 0 ]] && exit $status
 

diff --git a/jobs/rocoto/atmanalrun.sh b/jobs/rocoto/atmanalrun.sh
@@ -1,10 +1,12 @@
 #! /usr/bin/env bash
 
+export STRICT="NO"
 source "$HOMEgfs/ush/preamble.sh"
+export STRICT="YES"
 
 ###############################################################
-# Source FV3GFS workflow modules
-. $HOMEgfs/ush/load_fv3gfs_modules.sh
+# Source UFSDA workflow modules
+. $HOMEgfs/ush/load_ufsda_modules.sh
 status=$?
 [[ $status -ne 0 ]] && exit $status
 

diff --git a/jobs/rocoto/atmensanalpost.sh b/jobs/rocoto/atmensanalpost.sh
@@ -1,10 +1,12 @@
 #! /usr/bin/env bash
 
+export STRICT="NO"
 source "$HOMEgfs/ush/preamble.sh"
+export STRICT="YES"
 
 ###############################################################
-# Source FV3GFS workflow modules
-. $HOMEgfs/ush/load_fv3gfs_modules.sh
+# Source UFSDA workflow modules
+. $HOMEgfs/ush/load_ufsda_modules.sh
 status=$?
 [[ $status -ne 0 ]] && exit $status
 

diff --git a/jobs/rocoto/atmensanalprep.sh b/jobs/rocoto/atmensanalprep.sh
@@ -1,10 +1,12 @@
 #! /usr/bin/env bash
 
+export STRICT="NO"
 source "$HOMEgfs/ush/preamble.sh"
+export STRICT="YES"
 
 ###############################################################
-# Source FV3GFS workflow modules
-. $HOMEgfs/ush/load_fv3gfs_modules.sh
+# Source UFSDA workflow modules
+. $HOMEgfs/ush/load_ufsda_modules.sh
 status=$?
 [[ $status -ne 0 ]] && exit $status
 

diff --git a/jobs/rocoto/atmensanalrun.sh b/jobs/rocoto/atmensanalrun.sh
@@ -1,10 +1,12 @@
 #! /usr/bin/env bash
 
+export STRICT="NO"
 source "$HOMEgfs/ush/preamble.sh"
+export STRICT="YES"
 
 ###############################################################
-# Source FV3GFS workflow modules
-. $HOMEgfs/ush/load_fv3gfs_modules.sh
+# Source UFSDA workflow modules
+. $HOMEgfs/ush/load_ufsda_modules.sh
 status=$?
 [[ $status -ne 0 ]] && exit $status
 

diff --git a/parm/config/config.atmanal b/parm/config/config.atmanal
@@ -8,7 +8,7 @@ echo "BEGIN: config.atmanal"
 export OBS_YAML_DIR=$HOMEgfs/sorc/gdas.cd/parm/atm/obs/config/
 export OBS_LIST=$HOMEgfs/sorc/gdas.cd/parm/atm/obs/lists/gdas_prototype.yaml
 export ATMVARYAML=$HOMEgfs/sorc/gdas.cd/parm/atm/variational/3dvar_dripcg.yaml
-export BERROR_YAML=$HOMEgfs/sorc/gdas.cd/parm/atm/berror/hybvar_bump.yaml
+export BERROR_YAML=$HOMEgfs/sorc/gdas.cd/parm/atm/berror/staticb_identity.yaml
 export FV3JEDI_FIX=$HOMEgfs/fix/gdas
 export R2D2_OBS_DB='ufsda_test'
 export R2D2_OBS_DUMP='oper_gdas'

diff --git a/parm/config/config.atmensanal b/parm/config/config.atmensanal
@@ -14,8 +14,7 @@ export R2D2_OBS_DB='ufsda_test'
 export R2D2_OBS_DUMP='oper_gdas'
 export R2D2_OBS_SRC='ncdiag'
 export R2D2_BC_SRC='gsi'
-##export R2D2_BC_DUMP='oper_gdas'
-export R2D2_BC_DUMP='prgdasens'
+export R2D2_BC_DUMP='oper_gdas'
 export R2D2_ARCH_DB='local'
 export INTERP_METHOD='barycentric'
 

diff --git a/scripts/exgdas_global_atmos_analysis_run.sh b/scripts/exgdas_global_atmos_analysis_run.sh
@@ -131,17 +131,19 @@ $NLN $JEDIVAREXE $DATA/fv3jedi_var.x
 
 ################################################################################
 # run executable
+export OMP_NUM_THREADS=$NTHREADS_ATMANAL
 export pgm=$JEDIVAREXE
 . prep_step
 $APRUN_ATMANAL $DATA/fv3jedi_var.x $DATA/fv3jedi_var.yaml 1>&1 2>&2
 export err=$?; err_chk
 
 ################################################################################
 # translate FV3-JEDI increment to FV3 readable format
+atmges_fv3=$COMIN_GES/${GPREFIX}atmf006.ensres.nc
 atminc_jedi=$DATA/anl/atminc.${PDY}_${cyc}0000z.nc4
 atminc_fv3=$COMOUT/${CDUMP}.${cycle}.atminc.nc
 if [ -s $atminc_jedi ]; then
-    $INCPY $atminc_jedi $atminc_fv3
+    $INCPY $atmges_fv3 $atminc_jedi $atminc_fv3
     export err=$?
 else
     echo "***WARNING*** missing $atminc_jedi   ABORT"
@@ -164,11 +166,11 @@ cp -rf $DATA/bc $COMOUT/
 # Deterministic abias used in enkf cycle
 alist="abias abias_air abias_int abias_pc"
 for abias in $alist; do
-    cp $COMIN_GES/${GPREFIX}${abias} $COMOUT/${APREFIX}${abias}
+    cp "${COMIN_GES}/${GPREFIX}${abias}" "${COMOUT}/${APREFIX}${abias}"
 done
 
 ################################################################################
 
-exit $err
+exit ${err}
 
 ################################################################################
diff --git a/scripts/exgdas_global_atmos_ensanal_run.sh b/scripts/exgdas_global_atmos_ensanal_run.sh
@@ -126,7 +126,8 @@ $NLN $JEDIENSEXE $DATA/fv3jedi_ens.x
 
 ################################################################################
 # run executable
-export pgm=$JEDIVAREXE
+export OMP_NUM_THREADS=$NTHREADS_ATMENSANAL
+export pgm=$JEDIENSEXE
 . prep_step
 $APRUN_ATMENSANAL $DATA/fv3jedi_ens.x $DATA/fv3jedi_ens.yaml 1>&1 2>&2
 export err=$?; err_chk
@@ -135,11 +136,12 @@ export err=$?; err_chk
 # translate FV3-JEDI increment to FV3 readable format
 for imem in $(seq 1 $NMEM_ENKF); do
     memchar="mem"$(printf %03i $imem)
+    atmges_fv3=$COMIN_GES_ENS/$memchar/${GPREFIX}atmf006.nc
     atminc_jedi=$DATA/anl/$memchar/atminc.${PDY}_${cyc}0000z.nc4
     atminc_fv3=$COMOUT_ENS/$memchar/${CDUMP}.${cycle}.atminc.nc
     mkdir -p $COMOUT_ENS/$memchar
     if [ -s $atminc_jedi ]; then
-	$INCPY $atminc_jedi $atminc_fv3
+	$INCPY $atmges_fv3 $atminc_jedi $atminc_fv3
 	export err=$?
     else
 	echo "***WARNING*** missing $atminc_jedi   ABORT"
@@ -154,12 +156,12 @@ echo "$CDUMP $CDATE atminc done at $(date)" > $COMOUT_ENS/${CDUMP}.${cycle}.logi
 
 ################################################################################
 # Copy diags and YAML to $COMOUT
-cp -r $DATA/fv3jedi_ens.yaml $COMOUT_ENS/${CDUMP}.${cycle}.fv3jedi_ens.yaml
-cp -rf $DATA/diags $COMOUT_ENS/
+cp -r ${DATA}/fv3jedi_ens.yaml ${COMOUT_ENS}/${CDUMP}.${cycle}.fv3jedi_ens.yaml
+cp -rf "${DATA}/diags" "${COMOUT_ENS}/"
 
 
 ################################################################################
 
-exit $err
+exit ${err}
 
 ################################################################################
diff --git a/sorc/build_ufs.sh b/sorc/build_ufs.sh
@@ -5,7 +5,7 @@ cwd=$(pwd)
 
 # Default settings
 APP="S2SWA"
-CCPP_SUITES="FV3_GFS_v16,FV3_GFS_v16_ugwpv1,FV3_GFS_v17_p8,FV3_GFS_v16_coupled_nsstNoahmpUGWPv1,FV3_GFS_v17_coupled_p8"
+CCPP_SUITES="FV3_GFS_v16,FV3_GFS_v16_no_nsst,FV3_GFS_v16_ugwpv1,FV3_GFS_v17_p8,FV3_GFS_v16_coupled_nsstNoahmpUGWPv1,FV3_GFS_v17_coupled_p8"
 
 export RT_COMPILER="intel"
 source $cwd/ufs_model.fd/tests/detect_machine.sh

diff --git a/sorc/checkout.sh b/sorc/checkout.sh
@@ -157,11 +157,11 @@ checkout "ufs_utils.fd"    "https://github.com/ufs-community/UFS_UTILS.git"
 checkout "verif-global.fd" "https://github.com/NOAA-EMC/EMC_verif-global.git"   "c267780"                    ; errs=$((errs + $?))
 
 if [[ ${checkout_gsi} == "YES" ]]; then
-  checkout "gsi_enkf.fd"     "https://github.com/NOAA-EMC/GSI.git"         "67f5ab4"; errs=$((errs + $?))
+  checkout "gsi_enkf.fd"     "https://github.com/NOAA-EMC/GSI.git"         "48d8676"; errs=$((errs + $?))
 fi
 
 if [[ ${checkout_gdas} == "YES" ]]; then
-  checkout "gdas.cd" "https://github.com/NOAA-EMC/GDASApp.git" "5952c9d"; errs=$((errs + $?))
+  checkout "gdas.cd" "https://github.com/NOAA-EMC/GDASApp.git" "843d3a9"; errs=$((errs + $?))
 fi
 
 if [[ ${checkout_gsi} == "YES" || ${checkout_gdas} == "YES" ]]; then