diff --git a/model/bin/switches.json b/model/bin/switches.json index 5f042e31d..f70878ea3 100644 --- a/model/bin/switches.json +++ b/model/bin/switches.json @@ -29,7 +29,7 @@ "valid-options": [ { "name": "SCRIP", - "build_files": ["wmscrpmd"] + "build_files": ["wmscrpmd.F90"] } ] }, @@ -103,15 +103,15 @@ }, { "name": "PR1", - "build_files": ["w3profsmd", "w3pro1md"] + "build_files": ["w3profsmd.F90", "w3pro1md.F90"] }, { "name": "PR2", - "build_files": ["w3profsmd", "w3pro2md"] + "build_files": ["w3profsmd.F90", "w3pro2md.F90"] }, { "name": "PR3", - "build_files": ["w3profsmd", "w3pro3md"] + "build_files": ["w3profsmd.F90", "w3pro3md.F90"] } ] }, @@ -128,11 +128,11 @@ }, { "name": "UQ", - "build_files": ["w3uqckmd"] + "build_files": ["w3uqckmd.F90"] }, { "name": "UNO", - "build_files": ["w3uno2md"] + "build_files": ["w3uno2md.F90"] } ] }, @@ -143,7 +143,7 @@ "valid-options": [ { "name": "SMC", - "build_files": ["w3psmcmd", "w3smcomd", "w3psmcmd"] + "build_files": ["w3psmcmd.F90", "w3smcomd.F90", "w3psmcmd.F90"] } ] }, @@ -154,27 +154,27 @@ "valid-options": [ { "name": "FLX0", - "build_files": ["w3flx1md"] + "build_files": ["w3flx1md.F90"] }, { "name": "FLX1", - "build_files": ["w3flx1md"] + "build_files": ["w3flx1md.F90"] }, { "name": "FLX2", - "build_files": ["w3flx2md"] + "build_files": ["w3flx2md.F90"] }, { "name": "FLX3", - "build_files": ["w3flx3md"] + "build_files": ["w3flx3md.F90"] }, { "name": "FLX4", - "build_files": ["w3flx4md"] + "build_files": ["w3flx4md.F90"] }, { "name": "FLX5", - "build_files": ["w3flx5md"] + "build_files": ["w3flx5md.F90"] } ] }, @@ -188,11 +188,11 @@ }, { "name": "FLD1", - "build_files": ["w3fld1md"] + "build_files": ["w3fld1md.F90"] }, { "name": "FLD2", - "build_files": ["w3fld1md", "w3fld2md"] + "build_files": ["w3fld1md.F90", "w3fld2md.F90"] } ] }, @@ -209,7 +209,7 @@ }, { "name": "LN1", - "build_files": ["w3sln1md"] + "build_files": ["w3sln1md.F90"] } ] }, @@ -220,32 +220,32 @@ "valid-options": [ { "name": "ST0", - "build_files": ["w3src0md"] + "build_files": ["w3src0md.F90"] }, { "name": "ST1", - "build_files": ["w3src1md"], + "build_files": ["w3src1md.F90"], "conflicts": ["FLX0", "FLX5"] }, { "name": "ST2", - "build_files": ["w3src2md"], + "build_files": ["w3src2md.F90"], "conflicts": ["FLX0", "FLX1", "FLX4", "FLX5"] }, { "name": "ST3", - "build_files": ["w3src3md"], + "build_files": ["w3src3md.F90"], "conflicts": ["FLX1", "FLX2", "FLX3", "FLX4"] }, { "name": "ST4", - "build_files": ["w3src4md"], + "build_files": ["w3src4md.F90"], "conflicts": ["FLX1", "FLX2", "FLX3", "FLX4"] }, { "name": "ST6", - "build_files": ["w3src6md", "w3swldmd"], + "build_files": ["w3src6md.F90", "w3swldmd.F90"], "conflicts": ["FLX0"] } ] @@ -279,24 +279,24 @@ }, { "name": "NL1", - "build_files": ["w3snl1md"] + "build_files": ["w3snl1md.F90"] }, { "name": "NL2", - "build_files": ["w3snl2md", "mod_xnl4v5", "serv_xnl4v5"], + "build_files": ["w3snl2md.F90", "mod_xnl4v5.f90", "serv_xnl4v5.f90", "mod_fileio.f90", "mod_constants.f90"], "conflicts": ["OMPG", "OMPH"] }, { "name": "NL3", - "build_files": ["w3snl3md"] + "build_files": ["w3snl3md.F90"] }, { "name": "NL4", - "build_files": ["w3snl4md"] + "build_files": ["w3snl4md.F90"] }, { "name": "NL5", - "build_files": ["w3snl5md", "w3gkemd"] + "build_files": ["w3snl5md.F90", "w3gkemd.F90"] } ] }, @@ -307,7 +307,7 @@ "valid-options": [ { "name": "NLS", - "build_files": ["w3snlsmd"] + "build_files": ["w3snlsmd.F90"] } ] }, @@ -321,19 +321,19 @@ }, { "name": "BT1", - "build_files": ["w3sbt1md"] + "build_files": ["w3sbt1md.F90"] }, { "name": "BT4", - "build_files": ["w3sbt4md"] + "build_files": ["w3sbt4md.F90"] }, { "name": "BT8", - "build_files": ["w3sbt8md"] + "build_files": ["w3sbt8md.F90"] }, { "name": "BT9", - "build_files": ["w3sbt9md"] + "build_files": ["w3sbt9md.F90"] } ] }, @@ -347,7 +347,7 @@ }, { "name": "DB1", - "build_files": ["w3sdb1md"] + "build_files": ["w3sdb1md.F90"] } ] }, @@ -371,7 +371,7 @@ }, { "name": "TR1", - "build_files": ["w3str1md"] + "build_files": ["w3str1md.F90"] } ] }, @@ -385,7 +385,7 @@ }, { "name": "BS1", - "build_files": ["w3sbs1md"] + "build_files": ["w3sbs1md.F90"] } ] }, @@ -399,23 +399,23 @@ }, { "name": "IC1", - "build_files": ["w3sic1md"] + "build_files": ["w3sic1md.F90"] }, { "name": "IC2", - "build_files": ["w3sic2md"] + "build_files": ["w3sic2md.F90"] }, { "name": "IC3", - "build_files": ["w3sic3md"] + "build_files": ["w3sic3md.F90"] }, { "name": "IC4", - "build_files": ["w3sic4md"] + "build_files": ["w3sic4md.F90"] }, { "name": "IC5", - "build_files": ["w3sic5md"] + "build_files": ["w3sic5md.F90"] } ] }, @@ -429,11 +429,11 @@ }, { "name": "IS1", - "build_files": ["w3sis1md"] + "build_files": ["w3sis1md.F90"] }, { "name": "IS2", - "build_files": ["w3sis2md"] + "build_files": ["w3sis2md.F90"] } ] }, @@ -447,7 +447,7 @@ }, { "name": "REF1", - "build_files": ["w3ref1md"] + "build_files": ["w3ref1md.F90"] } ] }, @@ -645,7 +645,7 @@ "valid-options": [ { "name": "IG1", - "build_files": ["w3gig1md", "w3canomd"] + "build_files": ["w3gig1md.F90", "w3canomd.F90"] } ] }, @@ -676,7 +676,7 @@ "valid-options": [ { "name": "MPRF", - "build_files": ["w3getmem"] + "build_files": ["w3getmem.F90"] } ] }, @@ -687,7 +687,7 @@ "valid-options": [ { "name": "OASACM", - "build_files": ["w3agcmmd"] + "build_files": ["w3agcmmd.F90"] } ] }, @@ -698,7 +698,7 @@ "valid-options": [ { "name": "OASOCM", - "build_files": ["w3ogcmmd"] + "build_files": ["w3ogcmmd.F90"] } ] }, @@ -709,7 +709,7 @@ "valid-options": [ { "name": "OASICM", - "build_files": ["w3igcmmd"] + "build_files": ["w3igcmmd.F90"] } ] }, @@ -730,7 +730,7 @@ "valid-options": [ { "name": "OASIS", - "build_files": ["w3oacpmd"], + "build_files": ["w3oacpmd.F90"], "conflicts": ["FLX1", "FLX2", "FLX3", "FLX4"] } @@ -764,7 +764,7 @@ "valid-options": [ { "name": "MEMCHECK", - "build_files": ["w3meminfo"] + "build_files": ["w3meminfo.F90"] } ] }, @@ -775,7 +775,7 @@ "valid-options": [ { "name": "SETUP", - "build_files": ["w3wavset"] + "build_files": ["w3wavset.F90"] } ] }, @@ -786,7 +786,7 @@ "valid-options": [ { "name": "UOST", - "build_files": ["w3uostmd"] + "build_files": ["w3uostmd.F90"] } ] }, diff --git a/model/src/CMakeLists.txt b/model/src/CMakeLists.txt index dc168d058..b359dfc5e 100644 --- a/model/src/CMakeLists.txt +++ b/model/src/CMakeLists.txt @@ -74,7 +74,7 @@ set(programs ww3_strt ww3_grid ww3_bound ww3_outf ww3_outp ww3_trck ww3_grib ww3_gint gx_outf gx_outp ww3_uprstr ww3_shel ww3_prep ww3_gspl ww3_multi ww3_systrk) # Executables to build if NetCDF is found -set(netcdf_programs ww3_ounf ww3_ounp ww3_bounc ww3_trnc) +set(netcdf_programs ww3_ounf ww3_ounp ww3_bounc ww3_trnc ww3_prnc) find_package(NetCDF COMPONENTS Fortran) if(NetCDF_Fortran_FOUND) @@ -134,9 +134,9 @@ if("TIDE" IN_LIST switches) add_executable(ww3_prtide ww3_prtide.F90) target_link_libraries(ww3_prtide PRIVATE ww3_lib) - target_sources(ww3_lib PRIVATE w3nmlprncmd.F90) - add_executable(ww3_prnc ww3_prnc.F90) - target_link_libraries(ww3_prnc PRIVATE ww3_lib) + #target_sources(ww3_lib PRIVATE w3nmlprncmd.F90) + #add_executable(ww3_prnc ww3_prnc.F90) + #target_link_libraries(ww3_prnc PRIVATE ww3_lib) endif() foreach(program ${programs}) diff --git a/model/src/cmake/check_switches.cmake b/model/src/cmake/check_switches.cmake index bb28fae94..debe67d42 100644 --- a/model/src/cmake/check_switches.cmake +++ b/model/src/cmake/check_switches.cmake @@ -80,7 +80,7 @@ function(check_switches switches switch_files) math(EXPR n_files "${n_files} - 1") foreach(i_files RANGE ${n_files}) string(JSON file GET ${category} valid-options ${j_options} build_files ${i_files}) - list(APPEND files "${file}.F90") + list(APPEND files "${file}") endforeach() endif() endif() diff --git a/model/src/cmake/src_list.cmake b/model/src/cmake/src_list.cmake index df6c1d377..d3fef0336 100644 --- a/model/src/cmake/src_list.cmake +++ b/model/src/cmake/src_list.cmake @@ -3,9 +3,6 @@ set(c_src w3getmem.c) # Core files always built set(ftn_src constants.F90 - ctest.F90 - mod_constants.f90 - mod_fileio.f90 w3adatmd.F90 w3arrymd.F90 w3bullmd.F90 @@ -39,7 +36,6 @@ set(ftn_src w3parall.F90 w3partmd.F90 w3servmd.F90 - w3src0md.F90 w3srcemd.F90 w3strkmd.F90 w3timemd.F90 diff --git a/model/src/wmesmfmd.F90 b/model/src/wmesmfmd.F90 index 7c7c4dbf3..6a2fae3fc 100644 --- a/model/src/wmesmfmd.F90 +++ b/model/src/wmesmfmd.F90 @@ -2558,7 +2558,9 @@ subroutine GetImport ( gcomp, rc ) call w3setw ( imod, mdse, mdst ) call w3seti ( imod, mdse, mdst ) call wmsetm ( imod, mdse, mdst ) +#ifdef W3_MPI if ( mpi_comm_grd .eq. mpi_comm_null ) cycle +#endif INPUTS(IMOD)%TW0(:) = INPUTS(impGridID)%TW0(:) INPUTS(IMOD)%TFN(:,3) = INPUTS(impGridID)%TFN(:,3) wxn = WXNwrst !replace with values from restart diff --git a/model/src/wminitmd.F90 b/model/src/wminitmd.F90 index 5f8e41cfd..9a14ed907 100644 --- a/model/src/wminitmd.F90 +++ b/model/src/wminitmd.F90 @@ -2517,9 +2517,9 @@ SUBROUTINE WMINIT ( IDSI, IDSO, IDSS, IDST, IDSE, IFNAME, & TSYNC(:,I) = TIME(:) ! #ifdef W3_SMC - !!Li Check GTYPE values after initialization. JGLi08Apr2021 - IF( IMPROC .EQ. CROOT ) WRITE(MDSE,*) " GRID CROOT GTYPE", & - I, CROOT, GRIDS(I)%GTYPE +! Check GTYPE values after initialization + IF ( IMPROC .EQ. NMPERR ) WRITE(MDSE,*) "GRID IMPROC GTYPE", & + I, IMPROC, GRIDS(I)%GTYPE #endif ! #ifdef W3_T diff --git a/regtests/bin/matrix_cmake_ncep b/regtests/bin/matrix_cmake_ncep new file mode 100755 index 000000000..c1eca4dcc --- /dev/null +++ b/regtests/bin/matrix_cmake_ncep @@ -0,0 +1,260 @@ +#!/bin/bash +# --------------------------------------------------------------------------- # +# matrix.go: Run matrix of regression tests on target machine. # +# # +# Remarks: # +# - This version is set up for automatic w3_setenv script and for the # +# NOAA RDHPC systems. When using this for your own setup and # +# computer, please copy rather than modify. # +# # +# Hendrik L. Tolman # +# August 2013 # +# December 2013 # +# April 2018 # +# March 2020 # +# # +# Copyright 2013 National Weather Service (NWS), # +# National Oceanic and Atmospheric Administration. All rights # +# reserved. WAVEWATCH III is a trademark of the NWS. # +# No unauthorized use without permission. # +# # +# --------------------------------------------------------------------------- # + +usage () +{ +cat 2>&1 << EOF + +Usage: $myname model_dir +Required: + model_dir : path to model dir of WW3 source +EOF +} + + +# 0.e Get required arguments +if [ ! $# = 0 ] +then + main_dir="$1" ; shift +else + usage + exit 1 +fi + +#convert main_dir to absolute path +main_dir="`cd $main_dir 1>/dev/null 2>&1 && pwd`" + +# Compiler option. Choose appropriate compiler and set cmplOption to +# y if using for the first time or using a different compiler + + cmplr=intel + export cmplOption='y' + +# Set batchq queue, choose modules and other custom variables to fit system and +# to define headers etc (default to original version if empty) + ishera=`hostname | grep hfe` + isorion=`hostname | grep Orion` + if [ $ishera ] + then + # If no other h, assuming Hera + cmplr='hera.intel' + batchq='slurm' + hpcstackpath='/scratch2/NCEPDEV/nwprod/hpc-stack/libs/hpc-stack/modulefiles/stack' + hpcstackversion='hpc/1.1.0' + modcomp='hpc-intel/18.0.5.274' + modmpi='hpc-impi/2018.0.4' + modnetcdf='netcdf/4.7.4' + metispath='/scratch2/COASTAL/coastal/save/Ali.Abdolali/hpc-stack/parmetis-4.0.3' + modjasper='jasper/2.0.25' + modzlib='zlib/1.2.11' + modpng='png/1.6.35' + modhdf5='hdf5/1.10.6' + modbacio='bacio/2.4.1' + modg2='g2/3.4.1' + modw3nco='w3nco/2.4.1' + modesmf='esmf/8_1_1' + modcmake='cmake/3.20.1' + elif [ $isorion ] + then + cmplr='orion.intel' + batchq='slurm' + hpcstackpath='/apps/contrib/NCEP/libs/hpc-stack/modulefiles/stack' + hpcstackversion='hpc/1.1.0' + modcomp='hpc-intel/2018.4' + modmpi='hpc-impi/2018.4' + modnetcdf='netcdf/4.7.4' + metispath='/work/noaa/marine/ali.abdolali/Source/hpc-stack/parmetis-4.0.3' + modjasper='jasper/2.0.25' + modzlib='zlib/1.2.11' + modpng='png/1.6.35' + modhdf5='hdf5/1.10.6' + modbacio='bacio/2.4.1' + modg2='g2/3.4.1' + modw3nco='w3nco/2.4.1' + modesmf='esmf/8_1_1' + modcmake='cmake/3.18.1' + else + batchq= + fi + +# 1. Set up + + export np='24' #number of mpi tasks + export npl='140' #number of mpi tasks for ufs applications and large setups + export npl1='100' #number of mpi tasks for ufs/large setups (b4b check) + export nr='4' #number of mpi tasks for hybrid + export nth='6' #number of threads + export nth1='4' #number of threads (b4b check) + +# 1.a Computer/ user dependent set up + + echo '#!/bin/sh --login' > matrix.head + echo ' ' >> matrix.head + if [ $batchq = "slurm" ] && [ $isorion ] + then + echo "#SBATCH -n ${np}" >> matrix.head + echo "##SBATCH --cpus-per-task=${nth}" >> matrix.head + echo '#SBATCH -q batch' >> matrix.head + echo '#SBATCH -t 08:00:00' >> matrix.head + echo '#SBATCH -A marine-cpu' >> matrix.head + echo '#SBATCH -J ww3_regtest' >> matrix.head + echo '#SBATCH -o matrix.out' >> matrix.head + echo '#SBATCH -p orion' >> matrix.head + echo 'ulimit -s unlimited' >> matrix.head + elif [ $batchq = "slurm" ] + then + echo "#SBATCH -n ${np}" >> matrix.head + echo "##SBATCH --cpus-per-task=${nth}" >> matrix.head + echo '#SBATCH -q batch' >> matrix.head + echo '#SBATCH -t 08:00:00' >> matrix.head + echo '#SBATCH -A marine-cpu' >> matrix.head + echo '#SBATCH -J ww3_regtest' >> matrix.head + echo '#SBATCH -o matrix.out' >> matrix.head + else + echo '#PBS -l procs=24' >> matrix.head + echo '#PBS -q batch' >> matrix.head + echo '#PBS -l walltime=08:00:00' >> matrix.head + echo '#PBS -A marine-cpu' >> matrix.head + echo '#PBS -N ww3_regtest' >> matrix.head + echo '#PBS -j oe' >> matrix.head + echo '#PBS -o matrix.out' >> matrix.head + fi + echo ' ' >> matrix.head + echo " cd $(dirname $main_dir)/regtests" >> matrix.head + echo ' ' >> matrix.head + +# Netcdf and Parmetis modules & variables + echo " module purge" >> matrix.head + echo " module load $modcmake" >> matrix.head + echo " module use $hpcstackpath" >> matrix.head + echo " module load $hpcstackversion" >> matrix.head + echo " module load $modcomp" >> matrix.head + echo " module load $modmpi" >> matrix.head + echo " module load $modnetcdf" >> matrix.head + echo " module load $modjasper" >> matrix.head + echo " module load $modzlib" >> matrix.head + echo " module load $modpng" >> matrix.head + echo " module load $modhdf5" >> matrix.head + echo " module load $modbacio" >> matrix.head + echo " module load $modg2" >> matrix.head + echo " module load $modw3nco" >> matrix.head + echo " module load $modesmf" >> matrix.head + + echo " export METIS_PATH=${metispath}" >> matrix.head + echo ' ' + + if [ "$batchq" = 'slurm' ] + then + export mpi='srun' + else + export mpi='mpirun' + fi + +# Compile option + if [ "$cmplOption" = 'y' ] + then + opt="-c $cmplr -S -T" + else + opt="-S" + fi +# Batch queue option + if [ "$batchq" = 'slurm' ] + then + opt="-b $batchq $opt" + fi + +# Base run_test command line + export rtst="./bin/run_cmake_test $opt" + + export ww3='../model' + +# 1.b Flags to do course selection - - - - - - - - - - - - - - - - - - - - - - +# Addition selection by commenting out lines as below + + export shrd='y' # Do shared architecture tests + export dist='y' # Do distributed architecture (MPI) tests + export omp='y' # Threaded (OpenMP) tests + export hybd='y' # Hybrid options + + export prop1D='y' # 1-D propagation tests (ww3_tp1.X) + export prop2D='y' # 2-D propagation tests (ww3_tp2.X) + export time='y' # time linmited growth + export fetch='y' # fetch linmited growth + export hur1mg='y' # Hurricane with one moving grid + export shwtr='y' # shallow water tests + export unstr='y' # unstructured grid tests + export pdlib='y' # unstr with pdlib for domain decomposition and implicit solver + export smcgr='y' # SMC grid test + export rtd='y' # Rotated pole test + export mudice='y' # Mud/Ice and wave interaction tests + export infgrv='y' # Second harmonic generation tests + export uost='y' # ww3_ts4 Unresolved Obstacles Source Term (UOST) + export assim='y' # Restart spectra update + export oasis='y' # Atmosphere, ocean, and ice coupling using OASIS + export calendar='y' # Calendar type + export confignc='y' # Configurable netCDF meta data (ww3_ounf) + + export multi01='y' # mww3_test_01 (wetting and drying) + export multi02='y' # mww3_test_02 (basic two-way nesting test)) + export multi03='y' # mww3_test_03 (three high and three low res grids). + export multi04='y' # mww3_test_04 (swell on sea mount and/or current) + export multi05='y' # mww3_test_05 (three-grid moving hurricane) + export multi06='y' # mww3_test_06 (curvilinear grid tests) + export multi07='y' # mww3_test_07 (unstructured grid tests) + export multi08='y' # mww3_test_08 (wind and ice tests) + export multi09='y' # mww3_test_09 (SMC multi grid test) + + export ufs='y' # The Unified Forecast System + export ufscoarse='y' # Option for small PCs + export grib='y' # grib file field output + export rstrt_b4b='y' # Restart Reproducibility + export npl_b4b='y' # MPI task Reproducibility + export nth_b4b='y' # Thread Reproducibility + export esmf='n' # ESMF coupling +# export filter='PR3 ST2 UQ' + # The filter does a set of consecutive greps on the + # command lines generated by filter.base with the above + # selected options. + +# --------------------------------------------------------------------------- # +# 2. Execute matrix.base ... # +# --------------------------------------------------------------------------- # + + $main_dir/../regtests/bin/matrix.base + + $main_dir/../regtests/bin/matrix_divider_cmake.sh + + + echo "#submit all of the diveded matrix files" > msuball.sh + if [ $batchq = "slurm" ] + then + files=`ls matrix? matrix??` + for file in $files + do + echo "sbatch < $file" >> msuball.sh + done + fi + + +# --------------------------------------------------------------------------- # +# End to the matrix # +# --------------------------------------------------------------------------- # diff --git a/regtests/bin/matrix_divider_cmake.sh b/regtests/bin/matrix_divider_cmake.sh new file mode 100755 index 000000000..d05cdd561 --- /dev/null +++ b/regtests/bin/matrix_divider_cmake.sh @@ -0,0 +1,167 @@ +#!/bin/bash +# --------------------------------------------------------------------------- # +# matrix_divider simply divides the main matrix into subsets with a given # +# number of test (i.e. maxlist1 for mpi tests, maxlist3 for OMP/OMPH tests # +# and maxlist3 for serial tests) # +# # +# Remarks: # +# - Once the matrix is generated, this script can be execute. The user should # +# define the maxlist and this script divide the matrix into matrix? and # +# submit the job using sbatch. # +# # +# Ali Abdolali # +# August 2018 # +# Updated: May 2021 # +# # +# Copyright 2013 National Weather Service (NWS), # +# National Oceanic and Atmospheric Administration. All rights # +# reserved. WAVEWATCH III is a trademark of the NWS. # +# No unauthorized use without permission. # +# # +# --------------------------------------------------------------------------- # +# --------------------------------------------------------------------------- # +# 1. clean up and definitions # +# --------------------------------------------------------------------------- # +cp matrix matrix.tmp +HOME=${PWD%/*} + +maxlist1=48 +maxlist2=35 +maxlist3=104 +#Put the job requirement/spec in "before" +sed -e "/run_cmake_test/,\$d" matrix.tmp > before +#Put the list of tests in "list" +command egrep 'ww3_ufs1.2|ww3_ufs1.3' matrix.tmp | cat >> list_ufs +command egrep 'ww3_tp2.14|ww3_tp2.15|ww3_tp2.17|ww3_tp2.21|ww3_ufs1.1' matrix.tmp | cat >> list_heavy +awk '!/ww3_tp2.14/' matrix.tmp > tmpfile && mv tmpfile matrix.tmp +awk '!/ww3_tp2.15/' matrix.tmp > tmpfile && mv tmpfile matrix.tmp +awk '!/ww3_tp2.17/' matrix.tmp > tmpfile && mv tmpfile matrix.tmp +awk '!/ww3_tp2.21/' matrix.tmp > tmpfile && mv tmpfile matrix.tmp +awk '!/ww3_ufs/' matrix.tmp > tmpfile && mv tmpfile matrix.tmp +command egrep 'mpirun|mpiexec|MPI_LAUNCH|srun|\-O' matrix.tmp | cat >> list_mpi +awk '!/mpirun|mpiexec|MPI_LAUNCH|srun|\-O/' matrix.tmp > tmpfile && mv tmpfile matrix.tmp +#Separate the OMP/OMPH tests from MPI tests +command egrep '\-t|\-O' list_mpi | cat >> list_omp +awk '!/\-t|\-O/' list_mpi > tmpfile && mv tmpfile list_mpi + +split -dl $maxlist1 list_mpi list_mpi_ +rm list_mpi +matrixno1=$(ls list_mpi_* | wc -l) +echo "Total nummber of matrix with parallel tests = $(($matrixno1 + 1)); each includes $maxlist1 tests" + +split -dl $maxlist2 list_omp list_omp_ +rm list_omp +matrixno2=$(ls list_omp_* | wc -l) +echo "Total nummber of matrix with parallel tests with OMP/OMPH = $matrixno2; each includes $maxlist2 tests" +#serial jobs +command egrep 'run_cmake_test' matrix.tmp | cat >> list_serial +split -dl $maxlist3 list_serial list_serial_ +rm list_serial +matrixno3=$(ls list_serial_* | wc -l) +echo "Total nummber of matrix with serial test = $matrixno3; each includes $maxlist3 tests" +rm matrix.tmp + +# --------------------------------------------------------------------------- # +# 2. Divide and dump in subsets # +# --------------------------------------------------------------------------- # +# parallel jobs +count=0 + for i in `seq -f '%02g' 0 "$((matrixno1 - 1))"`; do +#echo $i +#Replace matrix.out > matrix?.out, model > model? + (( count = count + 1 )) + if [ -f "matrix${count}" ]; then rm -f matrix${count}; fi + cat before >> matrix${count} + cat list_mpi_$i >> matrix${count} + sed -i 's/'matrix.out'/'matrix${count}.out'/gI' matrix${count} + echo " echo ' '" >> matrix${count} + echo " echo ' **************************************************************'" >> matrix${count} + echo " echo ' * end of WAVEWATCH III matrix${count} of regression tests *'" >> matrix${count} + echo " echo ' **************************************************************'" >> matrix${count} + echo " echo ' '" >> matrix${count} + echo " matrix${count} prepared" + done + +# --------------------------------------------------------------------------- # + +# parallel jobs +OMP/OMPH + for i in `seq -f '%02g' 0 "$((matrixno2 - 1))"`; do +#echo $i +#Replace matrix.out > matrix?.out, model > model? + (( count = count + 1 )) + if [ -f "matrix${count}" ]; then rm -f matrix${count}; fi + cat before >> matrix${count} + cat list_omp_$i >> matrix${count} + sed -i 's/'matrix.out'/'matrix${count}.out'/gI' matrix${count} + echo " echo ' '" >> matrix${count} + echo " echo ' **************************************************************'" >> matrix${count} + echo " echo ' * end of WAVEWATCH III matrix${count} of regression tests *'" >> matrix${count} + echo " echo ' **************************************************************'" >> matrix${count} + echo " echo ' '" >> matrix${count} + echo " matrix${count} prepared" + done + +# --------------------------------------------------------------------------- # + +#serial jobs + for i in `seq -f '%02g' 0 "$((matrixno3 - 1))"`; do +#echo $i +#Replace matrix.out > matrix?.out, model > model? + (( count = count + 1 )) + if [ -f "matrix${count}" ]; then rm -f matrix${count}; fi + cat before >> matrix${count} + cat list_serial_$i >> matrix${count} + sed -i 's/'matrix.out'/'matrix${count}.out'/gI' matrix${count} + echo " echo ' '" >> matrix${count} + echo " echo ' **************************************************************'" >> matrix${count} + echo " echo ' * end of WAVEWATCH III matrix${count} of regression tests *'" >> matrix${count} + echo " echo ' **************************************************************'" >> matrix${count} + echo " echo ' '" >> matrix${count} + echo " matrix${count} prepared" + done + +# --------------------------------------------------------------------------- # + +#ww3_tp2.14 is separated, as it has dependency. +#ww3_tp2.17 and ww3_tp2.21 is separated, as it takes a long time to finish + (( count = count + 1 )) + if [ -f "matrix${count}" ]; then rm -f matrix${count}; fi + cat before >> matrix${count} + cat list_heavy >> matrix${count} + sed -i 's/'matrix.out'/'matrix${count}.out'/gI' matrix${count} + echo " echo ' '" >> matrix${count} + echo " echo ' **************************************************************'" >> matrix${count} + echo " echo ' * end of WAVEWATCH III matrix${count} of regression tests *'" >> matrix${count} + echo " echo ' **************************************************************'" >> matrix${count} + echo " echo ' '" >> matrix${count} + echo " matrix${count} prepared" + +# --------------------------------------------------------------------------- # + +#ncep operational tests including ww3_ufs and gfsv16 which require a large number of processor/esmf coupler and the ones for grib test are separated + (( count = count + 1 )) + if [ -f "matrix${count}" ]; then rm -f matrix${count}; fi + cat before >> matrix${count} + sed -i 's/'n\ 24'/'n\ 140'/gI' matrix${count} + cat list_ufs >> matrix${count} + sed -i 's/'matrix.out'/'matrix${count}.out'/gI' matrix${count} + sed -i 's/'##SBATCH'/'#SBATCH'/gI' matrix${count} + echo " echo ' '" >> matrix${count} + echo " echo ' **************************************************************'" >> matrix${count} + echo " echo ' * end of WAVEWATCH III matrix${count} of regression tests *'" >> matrix${count} + echo " echo ' **************************************************************'" >> matrix${count} + echo " echo ' '" >> matrix${count} + + echo " matrix${count} prepared" + +# --------------------------------------------------------------------------- # + +rm before +rm list* + + echo "file matrix is divided into ${count} subsets ...." + +# --------------------------------------------------------------------------- # +# End to matrix_divider # +# --------------------------------------------------------------------------- # + diff --git a/regtests/bin/run_cmake_test b/regtests/bin/run_cmake_test new file mode 100755 index 000000000..d25d1b6e2 --- /dev/null +++ b/regtests/bin/run_cmake_test @@ -0,0 +1,2384 @@ +#!/bin/bash +# --------------------------------------------------------------------------- # +# # +# Script for running WW-III tests. # +# # +# Last update : 21-May-2020 # +# --------------------------------------------------------------------------- # +# Modification history +# 27-Jan-2014 : Adapts ww3_ounf section for multigrid ( version 4.18 ) +# 04-May-2020 : F. Ardhuin added step 3.b2 for CDL input files ( version 7.12 ) +# 20-April-2021 : A. Abdolali added ww3_grib bulid and execution ( version 7.12 ) +# 21-May-2021 : C. Bunney add support for ALPS job placement ( version 7.12 ) +# +# Limitations: +# - For each ww3_grid_*.inp, run_test process *all* ww3_prep_*.inp files. +# For example, when processing ww3_grid_1.inp, it does not know that +# it needs to process ww3_prep_a.inp and *not* process ww3_prep_b.inp. +# This can be addressed in the future by adding instructions for prep +# to the grdset file. +# - When running through ww3_prep, run_test is not smart enough to process +# multiple input files of the same type. For example, for wind it wants +# a file ww3_prep_wind.inp and does not know what to do if you have two +# files, ww3_prep_wind_hwna_15m.inp and ww3_prep_wind_gfs_30m.inp. +# It needs to rename wind.ww3 as wind.wind_gfs_30m, for example, but +# looks for wind_gfs_30m.ww3 where it should look for wind.ww3. Another +# loop is needed to make this work. + +# --------------------------------------------------------------------------- # +# 1. Function definitions # +# --------------------------------------------------------------------------- # + +# 1.a Error message function +errmsg () +{ + echo "" 2>&1 + while [ $# != 0 ] + do + echo "ERROR: $1" 2>&1 + shift + done + echo "" 2>&1 +} + +# 1.b Usage function +myname="`basename $0`" #name of script +optstr="a:b:c:C:defg:Ghi:m:n:No:Op:q:r:s:t:STw:" #option string for getopt function +usage () +{ + +cat 2>&1 << EOF + +Usage: $myname [options] source_dir test_name +Required: + source_dir : path to top-level of WW3 source + test_name : name of test case (directory) +Options: + -a ww3_env : use WW3 environment setup file + : *default is /bin/wwatch3.env + : *file will be created if it does not already exist + -b batchq : optional setting to determine batch queue type (slurm or alps) + -c cmplr : setup comp & link files for specified cmplr + -C coupl : invoke test using coupled application + : OASIS : OASIS3-mct ww3_shel coupled application + : ESMF : ESMF ww3_multi coupled application + -d : invoke main program using gdb (non-parallel) + -e : prompt for changes to existing WW3 environment + -f : force pre- and post-processing programs to be compiled + : non-MPI (i.e., with SHRD switch); default is all programs + : compiled with unmodified switch settings + -g grid_string : use ww3_grid_.inp + -G : create GrADS data files using gx_outX.inp + -h : print usage and exit + -i inpdir : use inputs in test_name/ (default test_name/input) + -m grid_set : execute multi-model test + : *grid names are obtained from input/ + : *ww3_multi_ will execute instead of ww3_shel + : *to execute a single model test case with ww3_multi use + : grid_set = none + -n nproc : specify processors for parallel run + : *some programs do not require + : *ignored if -p or -O is not specified + -N : use namelist (.nml) input instead of .inp (if available) + -o outopt : limit output post-processing based on + : native : post-process only native output + : netcdf : post-process only NetCDF output + : both : post-process both native and NetCDF output + : grib : post-process only GRIB2 output + : all : post-process in native, NetCDF and GRIB2 formats + : * default is native + : * note that required input files must be present for + : selected output post-processing to occur + -O : parallel run using OpenMP paradigm and OMP_NUM_THREADS + environment variable and number of processors defined with + the -n np option + -p runcmd : run in parallel using to start program + : *MPICH or OpenMPI: mpirun or mpiexec (default = 1) + : *IBM with Loadleveler: poe (no required) + : *LSF: mpirun.lsf (no required) + -q program : exit script after program executes + -r program : only execute program + -s switch_string : use switch_ + -S : create stub file . with end data and time. + tests not executed if file is found. + -t nthrd : Threading option. (this is system dependant and can be used + : only for the hybrid option) + -T : Run w3_make under time command to create compile-time metric + -w work_dir : run test case in test_name/work_dir (default test_name/work) + +EOF +} + + +# --------------------------------------------------------------------------- # +# 2. Preparations # +# --------------------------------------------------------------------------- # + +echo ' ' +echo " Running now options: run_test $*" +echo ' ' + +# 2.a Setup array of command-line arguments +args=`getopt $optstr $*` +if [ $? != 0 ] +then + usage + exit 1 +fi +set -- $args + +ARGS=$args + +# 2.b Process command-line options +exit_p=none +exec_p=none +batchq=none +multi=0 +dist=0 +inpdir=input +outopt=native +grads=0 +coupl=none +batchq=none +while : +do + case "$1" in + -a) shift; ww3_env="$1" ;; + -b) shift; batchq="$1" ;; + -c) shift; cmplr="$1" ;; + -C) shift; coupl="$1" ;; + -d) use_gdb=1 ;; + -e) prompt=1 ;; + -f) force_shrd=1 ;; + -g) shift; grdstr="$1" ;; + -G) grads=1 ;; + -h) help=1 ;; + -i) shift; inpdir="$1" ;; + -m) shift; grdset="$1" ; + if [ $grdset = none ] + then + multi=1 + else + multi=2 + fi ;; + -n) shift; nproc="$1" ;; + -N) nml_input=1 ;; + -o) shift; outopt="$1" ;; + -O) pomp=1 ;; + -p) shift; runcmd="$1" ; pmpi=1 ;; + -q) shift; exit_p="$1" ;; + -r) shift; exec_p="$1" ;; + -s) shift; swtstr="$1" ;; + -S) stub=1 ;; + -t) shift; nthrd="$1" ;; + -T) time_count=1 ;; + -w) shift; wrkdir="$1" ;; + --) break ;; + esac + shift +done +shift #remove the trailing -- +if [ $help ] +then + usage + exit 1 +fi +#uncomment next line to add S & T switches to every test +#testST=1 +if [ ! $exec_p = "none" ] +then + exit_p=$exec_p +fi +case $outopt in + native|netcdf|both|grib|all) ;; + *) errmsg "outopt = $outopt not supported" ; usage ; exit 1 ;; +esac +case $coupl in + none) ;; + OASIS) + if [ $multi -ne 0 ] + then + errmsg "coupl = $coupl with ww3_multi is not supported" + usage + exit 1 + fi + ;; + ESMF) + if [ $multi -eq 0 ] + then + errmsg "coupl = $coupl with ww3_shel is not supported" + usage + exit 1 + fi + if [ -z "$ESMFMKFILE" ] + then + errmsg "ESMFMKFILE must be defined with coupl = $coupl" + usage + exit 1 + fi + ;; + *) errmsg "coupl = $coupl not supported" ; usage ; exit 1 ;; +esac + +# 2.c Get required arguments +if [ ! $# = 0 ] +then + path_s="$1" ; shift +else + usage + exit 1 +fi +if [ ! $# = 0 ] +then + testnm="$1" ; shift +else + usage + exit 1 +fi + +# 2.d Convert source path from "relative" to "absolute" +if [ ! -d $path_s ] +then + errmsg "$path_s not found" + usage + exit 1 +fi +path_s="`cd $path_s 1>/dev/null 2>&1 && pwd`" + +# 2.e Path to test directory +path_t="`pwd`/$testnm" +if [ ! -d $path_t ] +then + errmsg "$path_t not found" + usage + exit 1 +fi + +# 2.f Path to input files +path_i="$path_t/$inpdir" +if [ ! -d $path_i ] +then + errmsg "$path_i not found" + usage + exit 1 +fi + +# 2.g Path to working directory +if [ $wrkdir ] +then + path_w="$path_t/$wrkdir" +else + path_w="$path_t/work" +fi + +# 2.h Paths to source subdirectories +path_a="$path_s/tools" +path_b="$path_s/bin" +if [ ! -d $path_a ] +then + errmsg "$path_a not found" + exit 1 +fi +if [ ! -d $path_b ] +then + errmsg "$path_b not found" + exit 1 +fi + +# 2.i Check for switch file +if [ $swtstr ] +then + file_c="$path_i/switch_$swtstr" +else + file_c="$path_i/switch" +fi + +if [ ! -f $file_c ] +then + errmsg "switch file $file_c not found" + exit 1 +fi + +# 2.j If parallel execution, then check for proper switches +# Also, set default run command +if [ $pmpi ] +then + if [ "`grep MPI $file_c`" ] + then : + else + errmsg "Improper switch setup for MPI run" + exit 1 + fi +else + if [ $use_gdb ] + then + runcmd='gdb' + else + runcmd='' + fi +fi + +if [ $pomp ] +then + if [ "`grep OMP $file_c`" ] + then : + else + errmsg "Improper switch setup for OpenMP run" + exit 1 + fi +fi + +# 2.k Setup for multi-model (and defaults for non-multi-model) +gu="" +if [ $multi -eq 2 ] +then + if [ -f $path_i/$grdset ] + then + model_grids="`awk '/^MODEL:/' $path_i/$grdset | sed 's/MODEL\://'`" + input_grids="`awk '/^INPUT:/' $path_i/$grdset | sed 's/INPUT\://'`" + point_grids="`awk '/^POINT:/' $path_i/$grdset | sed 's/POINT\://'`" + intgl_grids="`awk '/^INTGL:/' $path_i/$grdset | sed 's/INTGL\://'`" + else + errmsg "grid_set file $path_i/$grdset not found" + exit 1 + fi +else + model_grids="none" + input_grids="none" + point_grids="none" + intgl_grids="none" +fi +all_grids=$model_grids +for g in $input_grids $point_grids $intgl_grids +do + if [ -z "`echo $all_grids | grep $g`" ] + then + all_grids="$all_grids $g" + fi +done + +# 2.m Initialize time counter if time_count option +if [ $time_count ] +then # Add time counter if -T + cumult_comp=0 + cumult_run=0 +fi + +# --------------------------------------------------------------------------- # +# Build all executables +# --------------------------------------------------------------------------- # + +cd $path_s +cd ../ +path_cmake="`pwd`" + +path_e=$path_w/exe +mkdir -p $path_e + +ofile=$path_w/build.log + +echo " Building WW3, exes will be in $path_e" + +echo "Exe directory is $path_e" > $ofile +if [ $force_shrd ] +then + + # build pre- & post-processing programs with SHRD only + echo "Forcing a SHRD build" >> $ofile + path_build=$path_w/build_SHRD + mkdir -p $path_build + cd $path_build + cat $file_c | sed 's/DIST/SHRD/' | sed 's/MPI //' | \ + sed 's/OMPG //' | \ + sed 's/OMPH //' | sed 's/PDLIB //' | \ + sed 's/B4B //' > $path_build/switch + echo "Switch file is $path_build/switch with switches:" >> $ofile + cat $path_build/switch >> $ofile + cmake $path_cmake -DSWITCH=$path_build/switch -DCMAKE_INSTALL_PREFIX=install > $ofile 2>&1 + rc=$? + if [[ $rc -ne 0 ]] ; then + echo "Fatal error in cmake." + echo "The build log is in $ofile" + exit + fi + make -j 8 > $ofile 2>&1 + rc=$? + if [[ $rc -ne 0 ]] ; then + echo "Fatal error in make." + echo "The build log is in $ofile" + exit + fi + make install > $ofile 2>&1 + if [[ $rc -ne 0 ]] ; then + echo "Fatal error in make install." + echo "The build log is in $ofile" + exit + fi + + cp $path_build/install/bin/* $path_e/ + + echo "non-SHRD build" >> $ofile + #build without SHRD + path_build=$path_w/build + mkdir -p $path_build + cd $path_build + \cp -f $file_c $path_build/switch + echo "Switch file is $path_build/switch with switches:" >> $ofile + cat $path_build/switch >> $ofile + cmake $path_cmake -DSWITCH=$path_build/switch -DCMAKE_INSTALL_PREFIX=install > $ofile 2>&1 + rc=$? + if [[ $rc -ne 0 ]] ; then + echo "Fatal error in cmake." + echo "The build log is in $ofile" + exit + fi + make -j 8 > $ofile 2>&1 + if [[ $rc -ne 0 ]] ; then + echo "Fatal error in make." + echo "The build log is in $ofile" + exit + fi + make install > $ofile 2>&1 + if [[ $rc -ne 0 ]] ; then + echo "Fatal error in make install." + echo "The build log is in $ofile" + exit + fi + path_e=$path_w/exe + cp $path_build/install/bin/ww3_shel $path_e/ + cp $path_build/install/bin/ww3_multi $path_e/ + cp $path_build/install/bin/ww3_systrk $path_e/ + +else + path_build=$path_w/build + mkdir -p $path_build + cd $path_build + \cp -f $file_c $path_build/switch + echo "Switch file is $path_build/switch with switches:" >> $ofile + cat $path_build/switch >> $ofile + cmake $path_cmake -DSWITCH=$path_build/switch -DCMAKE_INSTALL_PREFIX=install > $ofile 2>&1 + rc=$? + if [[ $rc -ne 0 ]] ; then + echo "Fatal error in cmake." + echo "The build log is in $ofile" + exit + fi + make -j 8 > $ofile 2>&1 + if [[ $rc -ne 0 ]] ; then + echo "Fatal error in make." + echo "The build log is in $ofile" + exit + fi + make install > $ofile 2>&1 + if [[ $rc -ne 0 ]] ; then + echo "Fatal error in make install." + echo "The build log is in $ofile" + exit + fi + + cp $path_build/install/bin/* $path_e/ + +fi +echo " Build log is in $ofile" + +# --------------------------------------------------------------------------- # +# 3. Execute Test # +# --------------------------------------------------------------------------- # +# 3.a Go to work directory and clean-up +mkdir -p $path_w +cd $path_w +if [ $exec_p = "none" ] +then + ncfiles=`\ls *.nc 2>/dev/null | grep -v rmp_` + if [ ! $stub ] + then + # restart, nest, etc. may have been placed there manually, so don't remove them + \rm -f *.inp *.out *.txt $ncfiles finished + \ls *.ww3 2>/dev/null | grep -v 'restart.ww3' | grep -v 'nest.ww3' | grep -v 'wind.ww3' | grep -v 'ice.ww3' | grep -v 'ice1.ww3' | xargs \rm 2>/dev/null + fi +fi + +if [ $time_count ] +then # Add time counter if -T + echo " REGTESTS Time counter: run_test $ARGS" >> time_count.txt +fi + +if [ $multi -eq 0 ] && [ $coupl = "OASIS" ] +then + if $path_i/prep_env.sh $path_i $path_w $cmplr $swtstr + then : + else + errmsg "Error occured during WW3 $prog build" + exit 1 + fi + export OASISDIR=$path_i/../work_oasis3-mct +fi + +# 3.b Header +echo ' ' ; echo ' ' +echo ' ================================== ' +echo ' ======> TEST RUN WAVEWATCH III <====== ' +echo ' ================================== ' +echo ' ' +if [ -f $path_t/info ] +then + cat $path_t/info +fi +echo ' ' +echo " Input directory: $path_i" +echo " Switch file: $file_c" +echo ' ' + +if [ $stub ] && [ -f finished ] +then + echo " Test already finished, skipping ..." + echo ' ' + exit 0 +fi + +# 3.b2 Preprocess cdl files into nc files ----------------------------------- # + +filescdl=$path_i/*.cdl +for f in $filescdl +do + echo "Converting $f to NetCDF" + # take action on each file. Note that converted file is in $path_w + ncgen -b $f +done + +# 3.c Grid pre-processor ---------------------------------------------------- # + +prog=ww3_grid +if [ $exec_p = $prog -o $exec_p = "none" ] +then + + echo ' ' + echo '+--------------------+' + echo '| Grid preprocessor |' + echo '+--------------------+' + echo ' ' + + for g in $all_grids + do + + if [ $multi -eq 2 ] + then + gu="_$g" + fi + + if [ $grdstr ] + then + fileconf="${prog}_${grdstr}${gu}" + else + fileconf="${prog}${gu}" + fi + + # select inp/nml files + if [ $nml_input ] && [ ! -z "`ls ${path_i}/${fileconf}.nml 2>/dev/null`" ] + then + ifile="`ls ${path_i}/${fileconf}.nml 2>/dev/null`" + else + ifile="`ls $path_i/${fileconf}.inp 2>/dev/null`" + fi + + + if [ ! -f $path_e/$prog ] + then + errmsg "$path_e/$prog not found" + exit 1 + fi + if [ ! -f $ifile ] + then + errmsg "$ifile not found" + exit 1 + fi + + # link conf file + if [ $nml_input ] && [ ! -z "`echo ${ifile} | grep -o nml`" ] + then + \rm -f $prog.nml + \ln -s $ifile $prog.nml + ofile="$path_w/`basename $ifile .nml`.out" + else + \rm -f $prog.inp + \ln -s $ifile $prog.inp + ofile="$path_w/`basename $ifile .inp`.out" + fi + + echo " Processing $ifile" + echo " Screen output routed to $ofile" + + if [ $time_count ] + then # Add time counter if -T + Tstart=`date +"%s.%2N"` + fi + + if $path_e/$prog > $ofile + then + \rm -f $prog.inp + \rm -f $prog.nml + if [ $multi -eq 2 ] + then + mv mod_def.ww3 mod_def.$g + if [ $nml_input ] && [ ! -z "`echo ${ifile} | grep -o nml`" ] + then + mv $prog.nml.log ${prog}_$g.nml.log + fi + fi + else + errmsg "Error occured during $path_e/$prog execution" + exit 1 + fi + + if [ $time_count ] + then # Add time counter if -T + Tend=`date +"%s.%2N"` + Maketime=`echo "$Tend - $Tstart" | bc` + cumult_run=`echo "$Maketime + $cumult_run" | bc` + printf "\n %8.2f sec run time \n" $Maketime >> time_count.txt + fi + + done + +fi + +if [ $exit_p = $prog ] +then + exit +fi + +# 3.d Initial conditions ---------------------------------------------------- # + +prog=ww3_strt +if [ $exec_p = $prog -o $exec_p = "none" ] +then + +# select inp/nml files + if [ $nml_input ] && [ ! -z "`ls ${path_i}/${prog}.nml 2>/dev/null`" ] + then + ifile="`ls ${path_i}/${prog}.nml 2>/dev/null`" + else + ifile="`ls $path_i/$prog.inp 2>/dev/null`" + fi + + if [ $? = 0 ] + then + + echo ' ' + echo '+--------------------+' + echo '| Initial conditions |' + echo '+--------------------+' + echo ' ' + + if [ ! -f $path_e/$prog ] + then + errmsg "$path_e/$prog not found" + exit 1 + fi + + for g in $model_grids + do + + if [ $multi -eq 2 ] + then + gu="_$g" + fi + + # link conf file + if [ $nml_input ] && [ ! -z "`echo ${ifile} | grep -o nml`" ] + then + \rm -f $prog.nml + \ln -s $ifile $prog.nml + ofile="$path_w/`basename $ifile .nml`${gu}.out" + else + \rm -f $prog.inp + \ln -s $ifile $prog.inp + ofile="$path_w/`basename $ifile .inp`${gu}.out" + fi + + echo " Processing $ifile" + echo " Screen output routed to $ofile" + + if [ $multi -eq 2 ] + then + \rm -f mod_def.ww3 + \ln -s mod_def.$g mod_def.ww3 + fi + + if [ $time_count ] + then # Add time counter if -T + Tstart=`date +"%s.%2N"` + fi + + if $path_e/$prog > $ofile + then + \rm -f $prog.inp + \rm -f $prog.nml + if [ $multi -eq 2 ] + then + mv restart.ww3 restart.$g + \rm -f mod_def.ww3 + if [ $nml_input ] && [ ! -z "`echo ${ifile} | grep -o nml`" ] + then + mv $prog.nml.log ${prog}_$g.nml.log + fi + fi + else + errmsg "Error occured during $path_e/$prog execution" + exit 1 + fi + + if [ $time_count ] + then # Add time counter if -T + Tend=`date +"%s.%2N"` + Maketime=`echo "$Tend - $Tstart" | bc` + cumult_run=`echo "$Maketime + $cumult_run" | bc` + printf "\n %8.2f sec run time \n" $Maketime >> time_count.txt + fi + + done + + fi + +fi + +if [ $exit_p = $prog ] +then + exit +fi + +# 3.d.1 boundary conditions -------------------------------------------------- # + +prog=ww3_bound +if [ $exec_p = $prog -o $exec_p = "none" ] +then + +# select inp/nml files + if [ $nml_input ] && [ ! -z "`ls ${path_i}/${prog}.nml 2>/dev/null`" ] + then + ifile="`ls ${path_i}/${prog}.nml 2>/dev/null`" + else + ifile="`ls $path_i/$prog.inp 2>/dev/null`" + fi + + if [ $? = 0 ] + then + + echo ' ' + echo '+---------------------+' + echo '| Boundary conditions |' + echo '+---------------------+' + echo ' ' + + if [ ! -f $path_e/$prog ] + then + errmsg "$path_e/$prog not found" + exit 1 + fi + + for g in $model_grids + do + if [ $multi -eq 2 ] + then + gu="_$g" + fi + + # link conf file + if [ $nml_input ] && [ ! -z "`echo ${ifile} | grep -o nml`" ] + then + \rm -f $prog.nml + \ln -s $ifile $prog.nml + ofile="$path_w/`basename $ifile .nml`${gu}.out" + else + \rm -f $prog.inp + \ln -s $ifile $prog.inp + ofile="$path_w/`basename $ifile .inp`${gu}.out" + fi + + echo " Processing $ifile" + echo " Screen output routed to $ofile" + + if [ $multi -eq 2 ] + then + \rm -f mod_def.ww3 + \ln -s mod_def.$g mod_def.ww3 + fi + + if [ $time_count ] + then # Add time counter if -T + Tstart=`date +"%s.%2N"` + fi + + if $path_e/$prog > $ofile + then + \rm -f $prog.inp + \rm -f $prog.nml + if [ $multi -eq 2 ] + then + mv nest.ww3 nest.$g + \rm -f mod_def.ww3 + if [ $nml_input ] && [ ! -z "`echo ${ifile} | grep -o nml`" ] + then + mv $prog.nml.log ${prog}_$g.nml.log + fi + fi + else + errmsg "Error occured during $path_e/$prog execution" + exit 1 + fi + + if [ $time_count ] + then # Add time counter if -T + Tend=`date +"%s.%2N"` + Maketime=`echo "$Tend - $Tstart" | bc` + cumult_run=`echo "$Maketime + $cumult_run" | bc` + printf "\n %8.2f sec run time \n" $Maketime >> time_count.txt + fi + + done + + fi + +fi + +if [ $exit_p = $prog ] +then + exit +fi + +# 3.d.2 boundary conditions -------------------------------------------------- # + +prog=ww3_bounc +if [ $exec_p = $prog -o $exec_p = "none" ] +then + +# select inp/nml files + if [ $nml_input ] && [ ! -z "`ls ${path_i}/${prog}.nml 2>/dev/null`" ] + then + ifile="`ls ${path_i}/${prog}.nml 2>/dev/null`" + else + ifile="`ls $path_i/$prog.inp 2>/dev/null`" + fi + + if [ $? = 0 ] + then + + echo ' ' + echo '+---------------------+' + echo '| Boundary conditions |' + echo '+---------------------+' + echo ' ' + + if [ ! -f $path_e/$prog ] + then + errmsg "$path_e/$prog not found" + exit 1 + fi + + for g in $model_grids + do + if [ $multi -eq 2 ] + then + gu="_$g" + fi + + # link conf file + if [ $nml_input ] && [ ! -z "`echo ${ifile} | grep -o nml`" ] + then + \rm -f $prog.nml + \ln -s $ifile $prog.nml + ofile="$path_w/`basename $ifile .nml`${gu}.out" + else + \rm -f $prog.inp + \ln -s $ifile $prog.inp + ofile="$path_w/`basename $ifile .inp`${gu}.out" + fi + + echo " Processing $ifile" + echo " Screen output routed to $ofile" + + if [ $multi -eq 2 ] + then + \rm -f mod_def.ww3 + \ln -s mod_def.$g mod_def.ww3 + fi + + if [ $time_count ] + then # Add time counter if -T + Tstart=`date +"%s.%2N"` + fi + + if $path_e/$prog > $ofile + then + \rm -f $prog.inp + \rm -f $prog.nml + if [ $multi -eq 2 ] + then + mv nest.ww3 nest.$g + \rm -f mod_def.ww3 + if [ $nml_input ] && [ ! -z "`echo ${ifile} | grep -o nml`" ] + then + mv $prog.nml.log ${prog}_$g.nml.log + fi + fi + else + errmsg "Error occured during $path_e/$prog execution" + exit 1 + fi + + if [ $time_count ] + then # Add time counter if -T + Tend=`date +"%s.%2N"` + Maketime=`echo "$Tend - $Tstart" | bc` + cumult_run=`echo "$Maketime + $cumult_run" | bc` + printf "\n %8.2f sec run time \n" $Maketime >> time_count.txt + fi + + done + + fi + +fi + +if [ $exit_p = $prog ] +then + exit +fi + +# 3.e.1 Prep forcing fields --------------------------------------------------- # + +prog=ww3_prep +if [ $exec_p = $prog -o $exec_p = "none" ] +then + +# select inp/nml files + if [ $nml_input ] && [ ! -z "`ls ${path_i}/${prog}*.nml 2>/dev/null`" ] + then + inputs="`ls ${path_i}/${prog}*.nml 2>/dev/null`" + else + inputs="`ls $path_i/$prog*.inp 2>/dev/null`" + fi + + if [ $? = 0 ] + then + echo ' ' + echo '+---------------------+' + echo '| Prep forcing fields |' + echo '+---------------------+' + echo ' ' + + if [ ! -f $path_e/$prog ] + then + errmsg "$path_e/$prog not found" + exit 1 + fi + + for g in $input_grids + do + + if [ $multi -eq 2 ] + then + gu="_$g" + fi + + for ifile in $inputs + do + +# link conf file + if [ $nml_input ] && [ ! -z "`echo ${ifile} | grep -o nml`" ] + then + \rm -f $prog.nml + \ln -s $ifile $prog.nml + otype="`basename $ifile .nml | sed s/^${prog}_//`" + ofile="$path_w/`basename $ifile .nml`.out" + else + \rm -f $prog.inp + \ln -s $ifile $prog.inp + otype="`basename $ifile .inp | sed s/^${prog}_//`" + ofile="$path_w/`basename $ifile .inp`.out" + fi + + echo " Processing $ifile for $otype" + echo " Screen output routed to $ofile" + + if [ $multi -eq 2 ] + then + \rm -f mod_def.ww3 + \ln -s mod_def.$g mod_def.ww3 + fi + + if [ $time_count ] + then # Add time counter if -T + Tstart=`date +"%s.%2N"` + fi + + if $path_e/$prog > $ofile + then + \rm -f $prog.inp + \rm -f $prog.nml + if [ $multi -eq 2 ] + then + \rm -f mod_def.ww3 + mv $otype.ww3 $otype.$g + if [ $nml_input ] && [ ! -z "`echo ${ifile} | grep -o nml`" ] + then + mv $prog.nml.log ${prog}_$g.nml.log + fi + fi + else + errmsg "Error occured during $path_e/$prog execution" + exit 1 + fi + + if [ $time_count ] + then # Add time counter if -T + Tend=`date +"%s.%2N"` + Maketime=`echo "$Tend - $Tstart" | bc` + cumult_run=`echo "$Maketime + $cumult_run" | bc` + printf "\n %8.2f sec run time \n" $Maketime >> time_count.txt + fi + + done + + done + + fi + +fi + +if [ $exit_p = $prog ] +then + exit +fi + +# 3.e.2 Prep forcing fields --------------------------------------------------- # + +prog=ww3_prnc +if [ $exec_p = $prog -o $exec_p = "none" ] +then + +# select inp/nml files + if [ $nml_input ] && [ ! -z "`ls ${path_i}/${prog}*.nml 2>/dev/null`" ] + then + inputs="`ls ${path_i}/${prog}*.nml 2>/dev/null`" + else + inputs="`ls $path_i/$prog*.inp 2>/dev/null`" + fi + + if [ $? = 0 ] + then + echo ' ' + echo '+-------------------------------+' + echo '| Prep of NetCDF forcing fields |' + echo '+-------------------------------+' + echo ' ' + + runprog=$runcmd + if [ $pmpi ] + then + if [ $nproc ] + then + if [ $batchq = "slurm" ] || [ $batchq = "alps" ] + then + runprog="$runprog -n $nproc" + + else + runprog="$runprog -np $nproc" + fi + fi + fi + if [ -n "$(grep SHRD $path_b/switch)" ] + then + runprog='' + fi + + if [ ! -f $path_e/$prog ] + then + errmsg "$path_e/$prog not found" + exit 1 + fi + + for g in $input_grids + do + echo "grid= $g" + if [ $multi -eq 2 ] + then + gu="_$g" +# check if each grid has its own input + if [ $nml_input ] && [ ! -z "`ls ${path_i}/${prog}*.nml 2>/dev/null`" ] + then + inputs="`ls ${path_i}/${prog}*.nml 2>/dev/null`" + else + inputs="`ls $path_i/$prog*.inp 2>/dev/null`" + fi + inputs_tmp=`( ls ${path_i}/${prog}${gu}* )` + if [ ! -z "$inputs_tmp" ];then + inputs=$inputs_tmp + fi + fi + + for ifile in $inputs + do + +# link conf file + if [ $nml_input ] && [ ! -z "`echo ${ifile} | grep -o nml`" ] + then + \rm -f $prog.nml + \ln -s $ifile $prog.nml + otype="`basename $ifile .nml | sed s/^${prog}_// | sed s/^${g}_//`" + ofile="$path_w/`basename $ifile .nml`.out" + else + \rm -f $prog.inp + \ln -s $ifile $prog.inp + otype="`basename $ifile .inp | sed s/^${prog}_// | sed s/^${g}_//`" + ofile="$path_w/`basename $ifile .inp`.out" + fi + + echo " Processing $ifile" + echo " Screen output routed to $ofile" + + if [ $multi -eq 2 ] + then + \rm -f mod_def.ww3 + \ln -s mod_def.$g mod_def.ww3 + fi + + if [ $time_count ] + then # Add time counter if -T + Tstart=`date +"%s.%2N"` + fi + + if $runprog $path_e/$prog > $ofile + then + \rm -f $prog.inp + \rm -f $prog.nml + if [ $multi -eq 2 ] + then + \rm -f mod_def.ww3 + mv $otype.ww3 $otype.$g + if [ $nml_input ] && [ ! -z "`echo ${ifile} | grep -o nml`" ] + then + mv $prog.nml.log ${prog}_$g.nml.log + fi + fi + else + errmsg "Error occured during $path_e/$prog execution" + exit 1 + fi + + + if [ $time_count ] + then # Add time counter if -T + Tend=`date +"%s.%2N"` + Maketime=`echo "$Tend - $Tstart" | bc` + cumult_run=`echo "$Maketime + $cumult_run" | bc` + printf "\n %8.2f sec run time \n" $Maketime >> time_count.txt + fi + + done + + done + + fi + +fi + +if [ $exit_p = $prog ] +then + exit +fi + +# 3.e.2 Prtide forcing fields ------------------------------------------------- # + +prog=ww3_prtide +if [ $exec_p = $prog -o $exec_p = "none" ] +then + +# select inp/nml files + if [ $nml_input ] && [ ! -z "`ls ${path_i}/${prog}*.nml 2>/dev/null`" ] + then + inputs="`ls ${path_i}/${prog}*.nml 2>/dev/null`" + else + inputs="`ls $path_i/$prog*.inp 2>/dev/null`" + fi + + if [ $? = 0 ] + then + echo ' ' + echo '+---------------------------------+' + echo '| Prtide of NetCDF forcing fields |' + echo '+---------------------------------+' + echo ' ' + + runprog=$runcmd + if [ $pmpi ] + then + if [ $nproc ] + then + if [ $batchq = "slurm" ] || [ $batchq = "alps" ] + then + runprog="$runprog -n $nproc" + else + runprog="$runprog -np $nproc" + fi + fi + fi + + if [ ! -f $path_e/$prog ] + then + errmsg "$path_e/$prog not found" + exit 1 + fi + + for g in $input_grids + do + + if [ $multi -eq 2 ] + then + gu="_$g" + fi + + for ifile in $inputs + do + +# link conf file + if [ $nml_input ] && [ ! -z "`echo ${ifile} | grep -o nml`" ] + then + \rm -f $prog.nml + \ln -s $ifile $prog.nml + otype="`basename $ifile .nml | sed s/^${prog}_//`" + ofile="$path_w/`basename $ifile .nml`.out" + else + \rm -f $prog.inp + \ln -s $ifile $prog.inp + otype="`basename $ifile .inp | sed s/^${prog}_//`" + ofile="$path_w/`basename $ifile .inp`.out" + fi + + echo " Processing $ifile" + echo " Screen output routed to $ofile" + + if [ $multi -eq 2 ] + then + \rm -f mod_def.ww3 + \ln -s mod_def.$g mod_def.ww3 + fi + + mv $otype.ww3 $otype.ww3_tide + + if [ $time_count ] + then # Add time counter if -T + Tstart=`date +"%s.%2N"` + fi + + if $runprog $path_e/$prog > $ofile + then + \rm -f $prog.inp + \rm -f $prog.nml + if [ $multi -eq 2 ] + then + \rm -f mod_def.ww3 + mv $otype.ww3 $otype.$g + if [ $nml_input ] && [ ! -z "`echo ${ifile} | grep -o nml`" ] + then + mv $prog.nml.log ${prog}_$g.nml.log + fi + fi + else + errmsg "Error occured during $path_e/$prog execution" + exit 1 + fi + + if [ $time_count ] + then # Add time counter if -T + Tend=`date +"%s.%2N"` + Maketime=`echo "$Tend - $Tstart" | bc` + cumult_run=`echo "$Maketime + $cumult_run" | bc` + printf "\n %8.2f sec run time \n" $Maketime >> time_count.txt + fi + + done + + done + + fi + +fi + +if [ $exit_p = $prog ] +then + exit +fi + +# 3.f Main program ---------------------------------------------------------- # + +if [ -e $path_i/bottomspectrum.inp ] +then + cp $path_i/bottomspectrum.inp . +fi + +# copy rmp_src* files from input to work directory + rmpfile=`\ls $path_i/rmp_src*.nc 2>/dev/null` + if [ ! -z "$rmpfile" ]; then + for rmpnc in $rmpfile + do + cp $rmpnc . + echo "copying $rmpnc to $path_w" + done + fi + +if [ $multi -ge 1 ] +then + prog=ww3_multi + if [ $coupl = "ESMF" ] + then + prgb=ww3_multi_esmf + else + prgb=ww3_multi + fi +else + prog=ww3_shel + prgb=ww3_shel +fi + +if [ $exec_p = $prog -o $exec_p = "none" ] +then + +# track file - multigrid option (ge 1) + if [ $multi -ge 1 ] + then + for g in $all_grids + do + ifile="`ls $path_i/track_i.$g 2>/dev/null`" + if [ $? = 0 ] + then + \rm -f track_i.$g + \ln -s $ifile + fi + done + else + ifile="`ls $path_i/track_i.ww3 2>/dev/null`" + if [ $? = 0 ] + then + \rm -f track_i.ww3 + \ln -s $ifile + fi + fi + +# config filename - gridset option (eq 2) + if [ $multi -eq 2 ] + then + fileconf="${prog}_${grdset}" + else + fileconf="${prog}" + fi + +# select inp/nml files + if [ $nml_input ] && [ ! -z "`ls ${path_i}/${fileconf}.nml 2>/dev/null`" ] + then + ifile="`ls ${path_i}/${fileconf}.nml 2>/dev/null`" + else + ifile="`ls $path_i/${fileconf}.inp 2>/dev/null`" + fi + + + if [ $? = 0 ] + then + + echo ' ' + echo '+--------------------+' + echo '| Main program |' + echo '+--------------------+' + echo ' ' + + if [ $multi -ge 1 ] && [ $coupl = "ESMF" ] + then + if [ $cmplr ] + then + export WW3_COMP=$cmplr + fi + if make -C $path_s/esmf $prgb + then : + else + errmsg "Error occured during WW3 ESMF build" + exit 1 + fi + fi + + if [ ! -f $path_e/$prgb ] + then + errmsg "$path_e/$prgb not found" + exit 1 + fi + + ofile="$path_w/$prog.out" + + if [ $nml_input ] && [ ! -z "`echo ${ifile} | grep -o nml`" ] + then + \rm -f $prog.nml + \ln -s $ifile $prog.nml + else + \rm -f $prog.inp + \ln -s $ifile $prog.inp + fi + + if [ $multi -ge 1 ] && [ $coupl = "ESMF" ] + then + \rm -f PET*.ESMF_LogFile + \rm -f ww3_esmf.rc + \cp -f ${path_i}/ww3_esmf.rc ww3_esmf.rc + if [ ! -z "`echo ${ifile} | grep -o nml`" ] + then + echo "WAV_input_file_name: $prog.nml" >> ww3_esmf.rc + fi + if [ $nproc ] + then + echo "pet_count: $nproc" >> ww3_esmf.rc + else + echo "pet_count: 1" >> ww3_esmf.rc + fi + fi + + echo " Processing $ifile" + echo " Screen output copied to $ofile" + + runprog=$runcmd + if [ $pmpi ] + then + if [ $nproc ] + then + if [ $batchq = "slurm" ] || [ $batchq = "alps" ] + then + runprog="$runprog -n $nproc" + else + runprog="$runprog -np $nproc" + fi + fi + if [ $nthrd ] + then + if ( which omplace ) ; then + runprog="$runprog omplace -nt $nthrd" + elif [ $batchq = "alps" ] + then + runprog="$runprog -d $nthrd /usr/bin/env OMP_NUM_THREADS=$nthrd" + else + runprog="$runprog /usr/bin/env OMP_NUM_THREADS=$nthrd" + fi + fi + fi + if [ $pomp ] + then + if [ $nproc ] + then + export OMP_NUM_THREADS=$nproc + fi + fi + + if [ $time_count ] + then # Add time counter if -T + Tstart=`date +"%s.%2N"` + fi + + if [ $multi -eq 0 ] && [ $coupl = "OASIS" ] + then + halfnproc=$(($nproc / 2)) + if [ $runcmd = "srun" ] + then + \rm -f $ofile + conf="srun.conf" + echo 0-$(( $halfnproc - 1 )) $path_e/$prgb > $conf + echo $halfnproc-$(( $nproc - 1 )) $path_w/toy_model >> $conf + if $runcmd -n $nproc --multi-prog $conf + then + \rm -f track_i.ww3 + \rm -f $prog.inp + \rm -f $conf + else + errmsg "Error occured during $path_e/$prog execution" + exit 1 + fi + elif $runcmd -np $halfnproc $path_e/$prgb : -np $halfnproc $path_w/toy_model | tee $ofile + then + \rm -f track_i.ww3 + \rm -f $prog.inp + else + errmsg "Error occured during $path_e/$prog execution" + exit 1 + fi + else + if $runprog $path_e/$prgb | tee $ofile + then + \rm -f track_i.ww3 + \rm -f $prog.inp + \rm -f $prog.nml + for file_p in ${files_p} + do + \rm -f ${file_p} + done + else + errmsg "Error occured during $path_e/$prog execution" + exit 1 + fi + fi + if [ $time_count ] + then # Add time counter if -T + Tend=`date +"%s.%2N"` + Maketime=`echo "$Tend - $Tstart" | bc` + cumult_run=`echo "$Maketime + $cumult_run" | bc` + printf "\n %8.2f sec run time \n" $Maketime >> time_count.txt + fi + fi +fi + +if [ $exit_p = $prog ] +then + exit +fi + +# 3.g Grid integration ------------------------------------------------------- # + +prog=ww3_gint +if [ $exec_p = $prog -o $exec_p = "none" ] +then + +# config filename - gridset option (eq 2) + if [ $multi -eq 2 ] + then + fileconf="${prog}_${grdset}" + else + fileconf="${prog}" + fi + +# select inp/nml files + if [ $nml_input ] && [ ! -z "`ls ${path_i}/${fileconf}.nml 2>/dev/null`" ] + then + ifile="`ls ${path_i}/${fileconf}.nml 2>/dev/null`" + else + ifile="`ls $path_i/${fileconf}.inp 2>/dev/null`" + fi + + + if [ $? = 0 ] + then + + echo ' ' + echo '+-------------------------+' + echo '| Integrated output |' + echo '+-------------------------+' + echo ' ' + + if [ $nml_input ] && [ ! -z "`echo ${ifile} | grep -o nml`" ] + then + \rm -f $prog.nml + \ln -s $ifile $prog.nml + ofile="$path_w/`basename $ifile .nml`.out" + else + \rm -f $prog.inp + \ln -s $ifile $prog.inp + ofile="$path_w/`basename $ifile .inp`.out" + fi + + echo " Processing $ifile" + echo " Screen output copied to $ofile" + + if [ $time_count ] + then # Add time counter if -T + Tstart=`date +"%s.%2N"` + fi + + if $path_e/$prog > $ofile + then + \rm -f $prog.inp + \rm -f $prog.nml + for g in $intgl_grids + do + if [ -f "out_grd.$g" ] + then + model_grids="$model_grids $g" + fi + done + else + errmsg "Error occured during $path_e/$prog execution" + exit 1 + fi + + if [ $time_count ] + then # Add time counter if -T + Tend=`date +"%s.%2N"` + Maketime=`echo "$Tend - $Tstart" | bc` + cumult_run=`echo "$Maketime + $cumult_run" | bc` + printf "\n %8.2f sec run time \n" $Maketime >> time_count.txt + fi + + fi + +fi + +if [ $exit_p = $prog ] +then + exit +fi + +# 3.g Gridded output -------------------------------------------------------- # + +case $outopt in + native) out_progs="ww3_outf" ;; + netcdf) out_progs="ww3_ounf" ;; + both) out_progs="ww3_outf ww3_ounf" ;; + grib) out_progs="ww3_grib" ;; + all) out_progs="ww3_outf ww3_ounf ww3_grib" ;; + *) out_progs="" ;; +esac + +if [ "$grads" = '1' ] +then + out_progs="$out_progs gx_outf" + if [ `ls $path_i | grep '.gs' | wc -l` -gt '0' ] + then + cp $path_i/*.gs . + cp $path_a/cbarn.gs . + cp $path_a/colorset.gs . + fi +fi + +for prog in $out_progs +do + + rline='| Gridded output |' + if [ $prog = ww3_ounf ] + then + rline='| NC Gridded output |' + fi + if [ $prog = ww3_grib ] + then + rline='| GRIB Gridded output |' + fi + if [ $prog = gx_outf ] + then + rline='|GrADS Gridded output|' + fi + + if [ $exec_p = $prog -o $exec_p = "none" ] + then + +# select inp/nml files + if [ $nml_input ] && [ ! -z "`ls ${path_i}/${prog}*.nml 2>/dev/null`" ] + then + inputs="`ls ${path_i}/${prog}*.nml 2>/dev/null`" + else + inputs="`ls $path_i/$prog*.inp 2>/dev/null`" + fi + + if [ $? = 0 ] + then + + echo ' ' + echo '+--------------------+' + echo "$rline" + echo '+--------------------+' + echo ' ' + + if [ ! -f $path_e/$prog ] + then + errmsg "$path_e/$prog not found" + exit 1 + fi + + for g in $model_grids + do + + if [ $multi -eq 2 ] + then + if [ ! -e out_grd.$g ] + then + continue + fi + gu="_$g" + fi +# for ww3_grib only + if [ $prog == 'ww3_grib' ] + then + ifile=`( ls ${path_i}/${prog}${gu}.??? )` + +# link conf file + if [ $nml_input ] && [ ! -z "`echo ${ifile} | grep -o nml`" ] + then + \rm -f $prog.nml + \ln -s $path_i/$prog${gu}.nml $prog.nml + otype="`basename $prog${gu}.nml | sed s/^${prog}_//`" + ofile="$path_w/`basename $prog.nml`${gu}.out" + else + \rm -f $prog.inp + \ln -s $path_i/$prog${gu}.inp $prog.inp + otype="`basename $prog${gu}.inp | sed s/^${prog}_//`" + ofile="$path_w/`basename $prog.inp`${gu}.out" + fi + + echo " Processing $path_i/$prog${gu}.inp" + echo " Screen output routed to $ofile" + + if [ $multi -eq 2 ] + then + \rm -f mod_def.ww3 + \rm -f out_grd.ww3 + \ln -s mod_def.$g mod_def.ww3 + \ln -s out_grd.$g out_grd.ww3 + \rm -fr ${otype}_$g + fi + if [ $time_count ] + then # Add time counter if -T + Tstart=`date +"%s.%2N"` + fi + + if $path_e/$prog > $ofile + then + \rm -f $prog.inp + \rm -f $prog.nml + if [ $multi -eq 2 ] + then + \rm -f mod_def.ww3 + \rm -f out_grd.ww3 + if [ $nml_input ] && [ ! -z "`echo ${ifile} | grep -o nml`" ] + then + mv $prog.nml.log ${prog}_$g.nml.log + fi + ofiles="`ls gribfile 2>/dev/null`" + if [ $? = 0 ] + then + mv -f $ofiles $g.grb2 + echo " GRIB output files moved to $g.grb2" + fi + fi + + else + errmsg "Error occured during $path_e/$prog execution" + fi +#for ww3_ounf, ww3_outp and ww3_gx_outf + else + for ifile in $inputs + do + +# link conf file + if [ $nml_input ] && [ ! -z "`echo ${ifile} | grep -o nml`" ] + then + \rm -f $prog.nml + \ln -s $ifile $prog.nml + otype="`basename $ifile .nml | sed s/^${prog}_//`" + ofile="$path_w/`basename $ifile .nml`${gu}.out" + else + \rm -f $prog.inp + \ln -s $ifile $prog.inp + otype="`basename $ifile .inp | sed s/^${prog}_//`" + ofile="$path_w/`basename $ifile .inp`${gu}.out" + fi + + echo " Processing $ifile" + echo " Screen output routed to $ofile" + + + if [ $multi -eq 2 ] + then + \rm -f mod_def.ww3 + \rm -f out_grd.ww3 + \ln -s mod_def.$g mod_def.ww3 + \ln -s out_grd.$g out_grd.ww3 + \rm -f ww3.????????.* + \rm -fr ${otype}_$g + fi + + if [ $time_count ] + then # Add time counter if -T + Tstart=`date +"%s.%2N"` + fi + + # Link in ounfmeta.inp for ww3_ounf, if exists: + if [ $prog == 'ww3_ounf' ] && [ ! -z ${path_i}/ounfmeta.inp ]; then + ln -sf ${path_i}/ounfmeta.inp . + fi + + if $path_e/$prog > $ofile + then + \rm -f $prog.inp + \rm -f $prog.nml + if [ $multi -eq 2 ] + then + \rm -f mod_def.ww3 + \rm -f out_grd.ww3 + if [ $nml_input ] && [ ! -z "`echo ${ifile} | grep -o nml`" ] + then + mv $prog.nml.log ${prog}_$g.nml.log + fi + ofiles="`ls ww3.????????.* 2>/dev/null`" + if [ $? = 0 ] + then + mkdir ${otype}_$g + mv -f $ofiles ${otype}_$g/. + echo " ASCII output files moved to ${otype}_$g" + fi + ofiles="`ls ww3.????*.nc 2>/dev/null`" + if [ $? = 0 ] + then + mkdir ${otype}_$g + mv -f $ofiles ${otype}_$g/. + echo " NetCDF output files moved to ${otype}_$g" + fi +# + if [ "$prog" = 'gx_outf' ] + then + case $g in + 'grd2' ) sed -e "s/ww3\.grads/ww3\.$g/g" \ + -e "s/37\.5/3\.75/g" \ + -e "s/1\.50/0\.15/g" \ + ww3.ctl > $g.ctl ;; + 'grd3' ) sed -e "s/ww3\.grads/ww3\.$g/g" \ + -e "s/12\.5/1\.25/g" \ + -e "s/0\.50/0\.05/g" \ + ww3.ctl > $g.ctl ;; + * ) sed -e "s/ww3\.grads/ww3\.$g/g" \ + -e "s/0\.25/2\.50/g" ww3.ctl > $g.ctl ;; + esac + rm -f ww3.ctl + echo " ww3.ctl moved to $g.ctl" + mv ww3.grads ww3.$g + echo " ww3.grads moved to ww3.$g" + fi + fi + else + errmsg "Error occured during $path_e/$prog execution" + exit 1 + fi + + if [ $time_count ] + then # Add time counter if -T + Tend=`date +"%s.%2N"` + Maketime=`echo "$Tend - $Tstart" | bc` + cumult_run=`echo "$Maketime + $cumult_run" | bc` + printf "\n %8.2f sec run time \n" $Maketime >> time_count.txt + fi + + + done + fi + + done + + + fi + + fi + + if [ $exit_p = $prog ] + then + exit + fi + +done # end of loop on progs + +# 3.h Point output ---------------------------------------------------------- # + +case $outopt in + native) out_progs="ww3_outp" ;; + netcdf) out_progs="ww3_ounp" ;; + both) out_progs="ww3_outp ww3_ounp" ;; + all) out_progs="ww3_outp ww3_ounp" ;; + *) out_progs="" ;; +esac + +if [ "$grads" = '1' ] +then + out_progs="$out_progs gx_outp" + if [ `ls $path_i | grep 'gx_outp' | wc -l` -gt '0' ] + then + cp $path_a/cbarn.gs . + cp $path_a/colorset.gs . + cp $path_a/source.gs . + cp $path_a/1source.gs . + cp $path_a/spec.gs . + fi +fi + +for prog in $out_progs +do + + rline='| Point output |' + if [ $prog = ww3_ounp ] + then + rline='| NC Point output |' + fi + + if [ $exec_p = $prog -o $exec_p = "none" ] + then + +# select inp/nml format for input file to program $prog + if [ $nml_input ] && [ ! -z "`ls ${path_i}/${prog}*.nml 2>/dev/null`" ] + then + inputs="`ls ${path_i}/${prog}*.nml 2>/dev/null`" + else + inputs="`ls $path_i/$prog*.inp 2>/dev/null`" + fi + + if [ $? = 0 ] + then + + echo ' ' + echo '+--------------------+' + echo "$rline" + echo '+--------------------+' + echo ' ' + + if [ ! -f $path_e/$prog ] + then + errmsg "$path_e/$prog not found" + exit 1 + fi + + for g in $point_grids + do + + if [ $multi -eq 2 ] + then + if [ ! -e out_pnt.$g ] + then + continue + fi + gu="_$g" + fi + + for ifile in $inputs + do + + # link conf file + if [ $nml_input ] && [ ! -z "`echo ${ifile} | grep -o nml`" ] + then + \rm -f $prog.nml + \ln -s $ifile $prog.nml + otype="`basename $ifile .nml | sed s/^${prog}_//`" + ofile="$path_w/`basename $ifile .nml`${gu}.out" + else + \rm -f $prog.inp + \ln -s $ifile $prog.inp + otype="`basename $ifile .inp | sed s/^${prog}_//`" + ofile="$path_w/`basename $ifile .inp`${gu}.out" + fi + + echo " Processing $ifile" + echo " Screen output routed to $ofile" + + if [ $multi -eq 2 ] + then + \rm -f mod_def.ww3 + \rm -f out_pnt.ww3 + \ln -s mod_def.$g mod_def.ww3 + \ln -s out_pnt.$g out_pnt.ww3 + fi + + if [ $time_count ] + then # Add time counter if -T + Tstart=`date +"%s.%2N"` + fi + + if $path_e/$prog > $ofile + then + \rm -f $prog.inp + \rm -f $prog.nml + if [ $multi -eq 2 ] + then + \rm -f mod_def.ww3 + \rm -f out_pnt.ww3 + if [ $nml_input ] && [ ! -z "`echo ${ifile} | grep -o nml`" ] + then + mv $prog.nml.log ${prog}_$g.nml.log + fi + fi + else + errmsg "Error occured during $path_e/$prog execution" + exit 1 + fi + + if [ $time_count ] + then # Add time counter if -T + Tend=`date +"%s.%2N"` + Maketime=`echo "$Tend - $Tstart" | bc` + cumult_run=`echo "$Maketime + $cumult_run" | bc` + printf "\n %8.2f sec run time \n" $Maketime >> time_count.txt + fi + + done + + done + + fi + + fi + + if [ $exit_p = $prog ] + then + exit + fi + +done # end of loop on progs + +# 3.i Track output ---------------------------------------------------------- # + +case $outopt in + native) out_progs="ww3_trck" ;; + netcdf) out_progs="ww3_trnc" ;; + both) out_progs="ww3_trck ww3_trnc" ;; + *) out_progs="" ;; +esac + +for prog in $out_progs +do + + if [ $exec_p = $prog -o $exec_p = "none" ] + then + +# select inp/nml files + if [ $nml_input ] && [ ! -z "`ls ${path_i}/${prog}*.nml 2>/dev/null`" ] + then + inputs="`ls ${path_i}/${prog}*.nml 2>/dev/null`" + else + inputs="`ls $path_i/$prog*.inp 2>/dev/null`" + fi + + + if [ $? = 0 ] + then + + echo ' ' + echo '+--------------------+' + echo '| Track output |' + echo '+--------------------+' + echo ' ' + + if [ ! -f $path_e/$prog ] + then + errmsg "$path_e/$prog not found" + exit 1 + fi + + for g in $point_grids + do + + if [ $multi -eq 2 ] + then + if [ ! -e track_o.$g ] + then + continue + fi + gu="_$g" + fileconf="$prog${gu}" + else + fileconf="$prog" + fi + +# select inp/nml files + if [ $nml_input ] && [ ! -z "`ls ${path_i}/${fileconf}.nml 2>/dev/null`" ] + then + ifile="`ls ${path_i}/${fileconf}.nml 2>/dev/null`" + else + ifile="`ls $path_i/${fileconf}.inp 2>/dev/null`" + fi + + if [ ! -f $ifile ] + then + errmsg "$ifile not found" + exit 1 + fi + +# link conf file + if [ $nml_input ] && [ ! -z "`echo ${ifile} | grep -o nml`" ] + then + \rm -f $prog.nml + \ln -s $ifile $prog.nml + otype="`basename $ifile .nml | sed s/^${prog}_//`" + ofile="$path_w/`basename $ifile .nml`.out" + else + \rm -f $prog.inp + \ln -s $ifile $prog.inp + otype="`basename $ifile .inp | sed s/^${prog}_//`" + ofile="$path_w/`basename $ifile .inp`.out" + fi + + echo " Processing $ifile" + echo " Screen output routed to $ofile" + + if [ $multi -eq 2 ] + then + \rm -f track_o.ww3 + \ln -s track_o.$g track_o.ww3 + fi + + if [ $time_count ] + then # Add time counter if -T + Tstart=`date +"%s.%2N"` + fi + + if $path_e/$prog > $ofile + then + \rm -f $prog.inp + \rm -f $prog.nml + if [ $multi -eq 2 ] + then + \rm -f track_o.ww3 + if [ $nml_input ] && [ ! -z "`echo ${ifile} | grep -o nml`" ] + then + mv $prog.nml.log ${prog}_$g.nml.log + fi + if [ -e track.ww3 ] + then + mv track.ww3 track.$g + elif [ -e track.nc ] + then + mv track.nc track_$g.nc + fi + fi + else + errmsg "Error occured during $path_e/$prog execution" + exit 1 + fi + + if [ $time_count ] + then # Add time counter if -T + Tend=`date +"%s.%2N"` + Maketime=`echo "$Tend - $Tstart" | bc` + cumult_run=`echo "$Maketime + $cumult_run" | bc` + printf "\n %8.2f sec run time \n" $Maketime >> time_count.txt + fi + done + + fi + + fi + + if [ $exit_p = $prog ] + then + exit + fi + +done # end of loop on progs + +# 3.j Wave system tracking -------------------------------------------------- # + +prog=ww3_systrk +if [ $exec_p = $prog -o $exec_p = "none" ] +then + +# select inp/nml files + if [ $nml_input ] && [ ! -z "`ls ${path_i}/${prog}.nml 2>/dev/null`" ] + then + ifile="`ls ${path_i}/${prog}.nml 2>/dev/null`" + else + ifile="`ls $path_i/$prog.inp 2>/dev/null`" + fi + + + if [ $? = 0 ] + then + + echo ' ' + echo '+-------------------------+' + echo '| Wave system tracking |' + echo '+-------------------------+' + echo ' ' + + if [ $nml_input ] && [ ! -z "`echo ${ifile} | grep -o nml`" ] + then + \rm -f $prog.nml + \ln -s $ifile $prog.nml + ofile="$path_w/`basename $ifile .nml`.out" + else + \rm -f $prog.inp + \ln -s $ifile $prog.inp + ofile="$path_w/`basename $ifile .inp`.out" + fi + + echo " Processing $ifile" + echo " Screen output copied to $ofile" + + runprog=$runcmd + if [ $pmpi ] + then + if [ $nproc ] + then + if [ $batchq = "slurm" ] || [ $batchq = "alps" ] + then + runprog="$runprog -n $nproc" + + else + runprog="$runprog -np $nproc" + fi + fi + fi + if [ $time_count ] + then # Add time counter if -T + Tstart=`date +"%s.%2N"` + fi + if $runprog $path_e/$prog | tee $ofile + then + \rm -f $prog.inp + \rm -f $prog.nml + else + errmsg "Error occured during $path_e/$prog execution" + exit 1 + fi + if [ $time_count ] + then # Add time counter if -T + Tend=`date +"%s.%2N"` + Maketime=`echo "$Tend - $Tstart" | bc` + cumult_run=`echo "$Maketime + $cumult_run" | bc` + printf "\n %8.2f sec run time \n" $Maketime >> time_count.txt + fi +# if [ $time_count ] +# then # Add time counter if -T +# Tstart=`date +"%s.%2N"` +# fi +# +# if $path_e/$prog > $ofile +# then +# \rm -f $prog.inp +# for g in $intgl_grids +# do +# if [ -f "out_grd.$g" ] +# then +# model_grids="$model_grids $g" +# fi +# done +# else +# errmsg "Error occured during $path_e/$prog execution" +# exit 1 +# fi +# +# if [ $time_count ] +# then # Add time counter if -T +# Tend=`date +"%s.%2N"` +# Maketime=`echo "$Tend - $Tstart" | bc` +# cumult_run=`echo "$Maketime + $cumult_run" | bc` +# printf "\n %8.2f sec run time \n" $Maketime >> time_count.txt +# fi + + fi + +fi + +if [ $exit_p = $prog ] +then + exit +fi + +# 3.k Update Restart File ------------------------------------------ # + +prog=ww3_uprstr +if [ $exec_p = $prog -o $exec_p = "none" ] +then + +# Check for input file + +# select inp/nml files + if [ $nml_input ] && [ ! -z "`ls ${path_i}/${prog}.nml 2>/dev/null`" ] + then + ifile="`ls ${path_i}/${prog}.nml 2>/dev/null`" + else + ifile="`ls $path_i/$prog.inp 2>/dev/null`" + fi + + if [ $? = 0 ] + then + + echo ' ' + echo '+-------------------------+' + echo '| Update Restart File |' + echo '+-------------------------+' + echo ' ' + +# link conf file + if [ $nml_input ] && [ ! -z "`echo ${ifile} | grep -o nml`" ] + then + \rm -f $prog.nml + \ln -s $ifile $prog.nml + ofile="$path_w/`basename $ifile .nml`.out" + else + \rm -f $prog.inp + \ln -s $ifile $prog.inp + ofile="$path_w/`basename $ifile .inp`.out" + fi + + echo " Processing $ifile" + echo " Screen output copied to $ofile" + +# Additional Files + \rm -f anl.grbtxt + \ln -s "$path_i/anl.grbtxt" anl.grbtxt + + mv -f restart001.ww3 restart.ww3 + + runprog=$runcmd + if [ $pmpi ] + then + if [ $nproc ] + then + if [ $batchq = "slurm" ] || [ $batchq = "alps" ] + then + runprog="$runprog -n $nproc" + else + runprog="$runprog -np $nproc" + fi + fi + fi + if [ $time_count ] + then # Add time counter if -T + Tstart=`date +"%s.%2N"` + fi + + if $runprog $path_e/$prog | tee $ofile + then + \rm -f $prog.inp + \rm -f $prog.nml + else + errmsg "Error occured during $path_e/$prog execution" + exit 1 + fi + if [ $time_count ] + then # Add time counter if -T + Tend=`date +"%s.%2N"` + Maketime=`echo "$Tend - $Tstart" | bc` + cumult_run=`echo "$Maketime + $cumult_run" | bc` + printf "\n %8.2f sec run time \n" $Maketime >> time_count.txt + fi + + fi + +fi + +if [ $exit_p = $prog ] +then + exit +fi + +# 3.k End ------------------------------------------------------------------- # + +if [ "$stub" ] +then + date > finished +fi + +if [ $time_count ] +then # Export cumultive time if time_count + printf "\n\n Total compile time: %8.2f sec" $cumult_comp >> time_count.txt + printf "\n Total run time : %8.2f sec" $cumult_run >> time_count.txt + printf "\n" >> time_count.txt +fi + +echo ' ' ; echo ' ' ; echo "Files in `pwd` :" ; echo ' ' +ls -l + +echo ' ' ; echo ' ' +echo ' ================================== ' +echo ' ======> END OF WAVEWATCH III <====== ' +echo ' ================================== ' +echo ' ' + +# --------------------------------------------------------------------------- # +# End of script # +# --------------------------------------------------------------------------- # +