diff --git a/model/bin/cmplr.env b/model/bin/cmplr.env index dab1a054c..c6d615fa3 100644 --- a/model/bin/cmplr.env +++ b/model/bin/cmplr.env @@ -4,19 +4,19 @@ # cmplr.env : Defines the compiler executable and its options for WW3 # # from the compiler keyword given in argument # # # -# - supported compiler are SGI-MPT, INTEL, GNU, PORTLAND # +# - supported compiler are SGI-MPT, INTEL, GNU, PORTLAND, CRAY # # # # # # use : cmplr.env cmplr # # # # cmplr : keyword based on a value and optional suffix, prefix and extras # # # -# value : mpt / intel / gnu / pgi # +# value : mpt / intel / gnu / pgi / cray # # suffix : _debug # -# prefix : datarmor_ # +# prefix : datarmor_ # # extras : so_ # # # -# - examples : intel / pgi_debug / datarmor_mpt_debug / so_intel # +# - examples : intel / pgi_debug / datarmor_mpt_debug / so_intel # # # # remarks : # # # @@ -176,7 +176,8 @@ fi if [ "$cmplr" == "gnu" ] || [ "$cmplr" == "gnu_debug" ] || \ [ "$cmplr" == "so_gnu" ] || [ "$cmplr" == "so_gnu_debug" ] || \ - [ "$cmplr" == "datarmor_gnu" ] || [ "$cmplr" == "datarmor_gnu_debug" ] ; then + [ "$cmplr" == "datarmor_gnu" ] || [ "$cmplr" == "datarmor_gnu_debug" ] || \ + [ "$cmplr" == "ukmo_cray_gnu" ] || [ "$cmplr" == "ukmo_cray_gnu_debug" ] ; then # COMPILER - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -185,7 +186,7 @@ if [ "$cmplr" == "gnu" ] || [ "$cmplr" == "gnu_debug" ] || \ comp_mpi='mpif90' # Cray compiler - if [ ! -z "$(echo $cmplr | grep wcoss_cray)" ] ; then + if [ ! -z "$(echo $cmplr | grep cray)" ] ; then comp_seq='ftn' comp_mpi='ftn' fi @@ -216,6 +217,9 @@ if [ "$cmplr" == "gnu" ] || [ "$cmplr" == "gnu_debug" ] || \ if [ ! -z "$(echo $cmplr | grep datarmor)" ] ; then optc="$optc -march=core-avx2" optl="$optl -march=core-avx2" + elif [ ! -z "$(echo $cmplr | grep ukmo_cray)" ]; then + # don't specify -march for Cray; processor specific tuning handled by ftn wrapper script + optc="$optc -frecord-marker=4" else optc="$optc -march=native" optl="$optl -march=native" @@ -281,7 +285,62 @@ if [ "$cmplr" == "pgi" ] || [ "$cmplr" == "pgi_debug" ] || \ fi +############################### +# Crayftn (CCE) +############################### + +if [ "$cmplr" == "ukmo_cray" ] || [ "$cmplr" == "ukmo_cray_debug" ] || \ + [ "$cmplr" == "ukmo_cray_regtest" ]; then + + # COMPILER - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + # common compiler + comp_seq='ftn' + comp_mpi='ftn' + + # OPTIONS - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + # common options + optc='-c -J$path_m -sdefault32 -eg' + optl='-o $prog' + + # list options + if [ "$list" == 'yes' ] ; then optc="$optc -hlist=a"; fi + + # omp options + if [ "$omp_mod" = 'yes' ] ; then + optc="$optc -h omp" + optl="$optl -h omp" + else + # by default, OMP is enabled on Cray ftn: + optc="$optc -h noomp" + # ... but don't disable at link stage as is currently needed by SCRIP + #optl="$optl -h noomp" + fi + + # optimized options + if [ -z "$(echo $cmplr | grep debug)" ] ; then + optc="$optc -O3" + optl="$optl -O3" + fi + + # debugging options + if [ ! -z "$(echo $cmplr | grep debug)" ] ; then + optc="$optc -O0 -g -Rbcps" + optl="$optl -O0 -g" + fi + + # regtest options: + if [ ! -z "$(echo $cmplr | grep regtest)" ] ; then + # -O1 best balance between compile time and runtime for regtests + optc="$optc -O1" + optl="$optl -O1" + fi + + # system-dependant options + # N/A +fi diff --git a/model/bin/w3_setup b/model/bin/w3_setup index b9cd4d402..081e79b39 100755 --- a/model/bin/w3_setup +++ b/model/bin/w3_setup @@ -419,7 +419,9 @@ then [ "$cmplr" == "wcoss_phase2" ] || [ "$cmplr" == "wcoss_dell_p3" ] || \ [ "$cmplr" == "datarmor_gnu" ] || [ "$cmplr" == "datarmor_gnu_debug" ] || \ [ "$cmplr" == "pgi" ] || [ "$cmplr" == "pgi_debug" ] || \ - [ "$cmplr" == "datarmor_pgi" ] || [ "$cmplr" == "datarmor_pgi_debug" ] ; then + [ "$cmplr" == "datarmor_pgi" ] || [ "$cmplr" == "datarmor_pgi_debug" ] || \ + [ "$cmplr" == "ukmo_cray" ] || [ "$cmplr" == "ukmo_cray_debug" ] || \ + [ "$cmplr" == "ukmo_cray_gnu" ] || [ "$cmplr" == "ukmo_cray_gnu_debug" ]; then source $path_b/cmplr.env sed -e "s//$optc/" -e "s//$comp_seq/" -e "s//$comp_mpi/" $path_b/comp.tmpl > $path_b/comp echo " sed $path_b/comp.tmpl => $path_b/comp" @@ -443,7 +445,9 @@ then [ "$cmplr" == "wcoss_phase2" ] || [ "$cmplr" == "wcoss_dell_p3" ] || \ [ "$cmplr" == "datarmor_gnu" ] || [ "$cmplr" == "datarmor_gnu_debug" ] || \ [ "$cmplr" == "pgi" ] || [ "$cmplr" == "pgi_debug" ] || \ - [ "$cmplr" == "datarmor_pgi" ] || [ "$cmplr" == "datarmor_pgi_debug" ] ; then + [ "$cmplr" == "datarmor_pgi" ] || [ "$cmplr" == "datarmor_pgi_debug" ] || \ + [ "$cmplr" == "ukmo_cray" ] || [ "$cmplr" == "ukmo_cray_debug" ] || \ + [ "$cmplr" == "ukmo_cray_gnu" ] || [ "$cmplr" == "ukmo_cray_gnu_debug" ]; then source $path_b/cmplr.env sed -e "s//$optl/" -e "s//$comp_seq/" -e "s//$comp_mpi/" $path_b/link.tmpl > $path_b/link echo " sed $path_b/link.tmpl => $path_b/link" diff --git a/model/ftn/w3iogrmd.ftn b/model/ftn/w3iogrmd.ftn index afe6cd145..dc365c92e 100644 --- a/model/ftn/w3iogrmd.ftn +++ b/model/ftn/w3iogrmd.ftn @@ -1260,8 +1260,8 @@ !/DB1 IF ( FLTEST ) WRITE (NDST,9053) SDBC1, SDBC2, FDONLY !/UOST IF ( WRITE ) THEN -!/UOST WRITE (NDSM), UOSTFILELOCAL, UOSTFILESHADOW, & -!/UOST UOSTFACTORLOCAL, UOSTFACTORSHADOW +!/UOST WRITE (NDSM) UOSTFILELOCAL, UOSTFILESHADOW, & +!/UOST UOSTFACTORLOCAL, UOSTFACTORSHADOW !/UOST ELSE !/UOST READ (NDSM,END=801,ERR=802,IOSTAT=IERR) & !/UOST UOSTFILELOCAL, UOSTFILESHADOW, & diff --git a/model/ftn/w3psmcmd.ftn b/model/ftn/w3psmcmd.ftn index dcd59a77e..6269ffcab 100644 --- a/model/ftn/w3psmcmd.ftn +++ b/model/ftn/w3psmcmd.ftn @@ -359,31 +359,31 @@ VLCFLY = 0. !Li Pass spectral element VQ to CQ and define size-1 cell CFL -!$OMP Parallel DO Private(ISEA) +!/OMPG!$OMP Parallel DO Private(ISEA) DO ISEA=1, NSEA !Li Transported variable is divided by CG as in WW3 (???) CQ(ISEA) = VQ(ISEA)/CG(IK,ISEA) !Li Resetting NaNQ VQ to zero if any. JGLi18Mar2013 IF( .NOT. (CQ(ISEA) .EQ. CQ(ISEA)) ) CQ(ISEA) = 0.0 END DO -!$OMP END Parallel DO +!/OMPG!$OMP END Parallel DO !Li Add current components if any to wave velocity. IF ( FLCUR ) THEN -!$OMP Parallel DO Private(ISEA) +!/OMPG!$OMP Parallel DO Private(ISEA) DO ISEA=1, NSEA CXTOT(ISEA) = (CGCOS * CG(IK,ISEA) + CX(ISEA)) CYTOT(ISEA) = (CGSIN * CG(IK,ISEA) + CY(ISEA)) ENDDO -!$OMP END Parallel DO +!/OMPG!$OMP END Parallel DO ELSE !Li No current case use group speed only. -!$OMP Parallel DO Private(ISEA) +!/OMPG!$OMP Parallel DO Private(ISEA) DO ISEA=1, NSEA CXTOT(ISEA) = CGCOS * CG(IK,ISEA) CYTOT(ISEA) = CGSIN * CG(IK,ISEA) END DO -!$OMP END Parallel DO +!/OMPG!$OMP END Parallel DO !Li End of IF( FLCUR ) block. ENDIF @@ -404,12 +404,12 @@ !/ARC !Li Convert velocity components into CFL factors. -!$OMP Parallel DO Private(ISEA) +!/OMPG!$OMP Parallel DO Private(ISEA) DO ISEA=1, NSEA UCFL(ISEA) = DTLDX*CXTOT(ISEA)/CLATS(ISEA) VCFL(ISEA) = DTLDY*CYTOT(ISEA) ENDDO -!$OMP END Parallel DO +!/OMPG!$OMP END Parallel DO !Li Initialise boundary cell CQ and Velocity values. CQ(-9:0)=0.0 @@ -437,7 +437,7 @@ ENDIF ! Store conservative flux in FCNt advective one in AFCN -!$OMP Parallel DO Private(i, M, N, FUTRN) +!/OMPG!$OMP Parallel DO Private(i, M, N, FUTRN) DO i=1, NUFc M=IJKUFc(5,i) N=IJKUFc(6,i) @@ -445,7 +445,7 @@ !! Add sub-grid transparency for input flux update. JGLi16May2011 !! Transparency is also applied on diffusion flux. JGLi12Mar2012 -!$OMP CRITICAL +!/OMPG!$OMP CRITICAL IF( (CTRNX(M)+CTRNX(N)) .GE. 1.96 ) THEN FCNt(M) = FCNt(M) - FUTRN FCNt(N) = FCNt(N) + FUTRN @@ -459,19 +459,19 @@ ! Also divided by another cell length as UCFL is in basic unit. AFCN(M) = AFCN(M) - FUMD(i)*UCFL(M) + FUDIFX(i) AFCN(N) = AFCN(N) + FUMD(i)*UCFL(N) - FUDIFX(i) -!$OMP END CRITICAL +!/OMPG!$OMP END CRITICAL ENDDO -!$OMP END Parallel DO +!/OMPG!$OMP END Parallel DO ! Store conservative update in CQA and advective update in CQ ! The side length in MF value has to be cancelled with cell length ! Note ULCFLX has been divided by the cell size inside SMCxUNO2. -!$OMP Parallel DO Private(n) +!/OMPG!$OMP Parallel DO Private(n) DO n=1, NSEA CQA(n)=CQ(n) + FCNt(n)/FLOAT(IJKCel(3,n)) CQ (n)=CQ(n) + AFCN(n)/FLOAT(IJKCel(3,n)) ENDDO -!$OMP END Parallel DO +!/OMPG!$OMP END Parallel DO ! Call advection subs. IF( FUNO3 ) THEN @@ -482,7 +482,7 @@ CALL SMCyUNO2r(1, NVFc, CQ, VCFL, VLCFLY, DSSD, FVMD, FVDIFY) ENDIF -!$OMP Parallel DO Private(j, M, N, FVTRN) +!/OMPG!$OMP Parallel DO Private(j, M, N, FVTRN) DO j=1, NVFc M=IJKVFc(5,j) N=IJKVFc(6,j) @@ -490,7 +490,7 @@ !! Add sub-grid transparency for input flux update. JGLi16May2011 !! Transparency is also applied on diffusion flux. JGLi12Mar2012 -!$OMP CRITICAL +!/OMPG!$OMP CRITICAL IF( (CTRNY(M)+CTRNY(N)) .GE. 1.96 ) THEN BCNt(M) = BCNt(M) - FVTRN BCNt(N) = BCNt(N) + FVTRN @@ -501,18 +501,18 @@ BCNt(M) = BCNt(M) - FVTRN*CTRNY(N)*CTRNY(M) BCNt(N) = BCNt(N) + FVTRN*CTRNY(N) ENDIF -!$OMP END CRITICAL +!/OMPG!$OMP END CRITICAL ENDDO -!$OMP END Parallel DO +!/OMPG!$OMP END Parallel DO ! Store conservative update of CQA in CQ ! The v side length in MF value has to be cancelled with cell length !! One cosine factor is also needed to be divided for SMC grid -!$OMP Parallel DO Private(n) +!/OMPG!$OMP Parallel DO Private(n) DO n=1, NSEA CQ(n)=CQA(n) + BCNt(n)/( CLATS(n)*FLOAT(IJKCel(3,n)) ) ENDDO -!$OMP END Parallel DO +!/OMPG!$OMP END Parallel DO !/ARC !Li Polar cell needs a special area factor, one-level case. !/ARC CQ(NSEA) = CQA(NSEA) + BCNt(NSEA)*PCArea @@ -552,12 +552,12 @@ ENDIF ! Store fineset level conservative flux in FCNt advective one in AFCN -!$OMP Parallel DO Private(i, L, M, FUTRN) +!/OMPG!$OMP Parallel DO Private(i, L, M, FUTRN) DO i=iuf, juf L=IJKUFc(5,i) M=IJKUFc(6,i) FUTRN = FUMD(i)*ULCFLX(i) - FUDIFX(i) -!$OMP CRITICAL +!/OMPG!$OMP CRITICAL !! Add sub-grid blocking for refined cells. JGLi18Apr2018 IF( (CTRNX(M)+CTRNX(L)) .GE. 1.96 ) THEN FCNt(L) = FCNt(L) - FUTRN @@ -571,21 +571,21 @@ ENDIF AFCN(L) = AFCN(L) - FUMD(i)*UCFL(L)*FMR + FUDIFX(i) AFCN(M) = AFCN(M) + FUMD(i)*UCFL(M)*FMR - FUDIFX(i) -!$OMP END CRITICAL +!/OMPG!$OMP END CRITICAL ENDDO -!$OMP END Parallel DO +!/OMPG!$OMP END Parallel DO ! Store conservative update in CQA and advective update in CQ ! The side length in MF value has to be cancelled with cell y-length. ! Also divided by another cell x-size as UCFL is in size-1 unit. -!$OMP Parallel DO Private(n) +!/OMPG!$OMP Parallel DO Private(n) DO n=icl, jcl CQA(n)=CQ(n) + FCNt(n)/FLOAT( IJKCel(3, n)*IJKCel(4, n) ) CQ (n)=CQ(n) + AFCN(n)/FLOAT( IJKCel(3, n)*IJKCel(4, n) ) FCNt(n)=0.0 AFCN(n)=0.0 ENDDO -!$OMP END Parallel DO +!/OMPG!$OMP END Parallel DO ! ! Use 3rd order UNO3 scheme. JGLi03Sep2015 IF( FUNO3 ) THEN @@ -596,12 +596,12 @@ ENDIF ! ! Store conservative flux in BCNt -!$OMP Parallel DO Private(j, L, M, FVTRN) +!/OMPG!$OMP Parallel DO Private(j, L, M, FVTRN) DO j=ivf, jvf L=IJKVFc(5,j) M=IJKVFc(6,j) FVTRN = FVMD(j)*VLCFLY(j) - FVDIFY(j) -!$OMP CRITICAL +!/OMPG!$OMP CRITICAL !! Add sub-grid blocking for refined cells. JGLi18Apr2018 IF( (CTRNY(M)+CTRNY(L)) .GE. 1.96 ) THEN BCNt(L) = BCNt(L) - FVTRN @@ -613,21 +613,21 @@ BCNt(L) = BCNt(L) - FVTRN*CTRNY(L)*CTRNY(M) BCNt(M) = BCNt(M) + FVTRN*CTRNY(M) ENDIF -!$OMP END CRITICAL +!/OMPG!$OMP END CRITICAL ENDDO -!$OMP END Parallel DO +!/OMPG!$OMP END Parallel DO ! Store conservative update of CQA in CQ ! The v side length in MF value has to be cancelled with x-size. ! Also divided by cell y-size as VCFL is in size-1 unit. !! One cosine factor is also needed to be divided for SMC grid. -!$OMP Parallel DO Private(n) +!/OMPG!$OMP Parallel DO Private(n) DO n=icl, jcl CQ(n)=CQA(n) + BCNt(n)/( CLATS(n)* & & FLOAT( IJKCel(3, n)*IJKCel(4, n) ) ) BCNt(n)=0.0 ENDDO -!$OMP END Parallel DO +!/OMPG!$OMP END Parallel DO !/ARC !Li Polar cell needs a special area factor, multi-level case. !/ARC IF( jcl .EQ. NSEA ) THEN !/ARC CQ(NSEA) = CQA(NSEA) + BCNt(NSEA)*PCArea @@ -673,11 +673,11 @@ ! ! 4. Store results in VQ in proper format --------------------------- * ! -!$OMP Parallel DO Private(ISEA) +!/OMPG!$OMP Parallel DO Private(ISEA) DO ISEA=1, NSEA VQ(ISEA) = MAX ( 0. , CQ(ISEA)*CG(IK,ISEA) ) END DO -!$OMP END Parallel DO +!/OMPG!$OMP END Parallel DO ! RETURN ! @@ -1030,14 +1030,14 @@ ! Uniform diffusion coefficient for all sizes. JGLi24Feb2012 ! CNST0=AKDif*MRFct*FTS -!$OMP Parallel Default(Shared), Private(i, ij, K, L, M, N), & -!$OMP& Private(CNST,CNST1,CNST2,CNST3,CNST4,CNST5,CNST6,CNST8,CNST9) +!/OMPG!$OMP Parallel Default(Shared), Private(i, ij, K, L, M, N), & +!/OMPG!$OMP& Private(CNST,CNST1,CNST2,CNST3,CNST4,CNST5,CNST6,CNST8,CNST9) ! Notice an extra side length L is multiplied to mid-flux to give correct ! proportion of flux into the cells. This length will be removed by the ! cell length when the tracer concentration is updated. -!$OMP DO +!/OMPG!$OMP DO DO i=NUA, NUB @@ -1105,9 +1105,9 @@ END DO -!$OMP END DO +!/OMPG!$OMP END DO -!$OMP END Parallel +!/OMPG!$OMP END Parallel ! 999 PRINT*, ' Sub SMCxUNO2 ended.' @@ -1142,10 +1142,10 @@ ! Uniform diffusion coefficient for all sizes. JGLi24Feb2012 ! CNST0=AKDif*MRFct*FTS -!$OMP Parallel Default(Shared), Private(j, K, L, M, N), & -!$OMP& Private(CNST,CNST1,CNST2,CNST3,CNST4,CNST5,CNST6,CNST8) +!/OMPG!$OMP Parallel Default(Shared), Private(j, K, L, M, N), & +!/OMPG!$OMP& Private(CNST,CNST1,CNST2,CNST3,CNST4,CNST5,CNST6,CNST8) -!$OMP DO +!/OMPG!$OMP DO DO j=NVA, NVB @@ -1220,9 +1220,9 @@ END DO -!$OMP END DO +!/OMPG!$OMP END DO -!$OMP END Parallel +!/OMPG!$OMP END Parallel ! 999 PRINT*, ' Sub SMCyUNO2 ended.' @@ -1253,10 +1253,10 @@ ! proportion of flux into the cells. This length will be removed by the ! cell length when the tracer concentration is updated. -!$OMP Parallel Default(Shared), Private(i, ij, K, L, M, N), & -!$OMP& Private(CNST,CNST0,CNST1,CNST2,CNST3,CNST4,CNST5,CNST6) +!/OMPG!$OMP Parallel Default(Shared), Private(i, ij, K, L, M, N), & +!/OMPG!$OMP& Private(CNST,CNST0,CNST1,CNST2,CNST3,CNST4,CNST5,CNST6) -!$OMP DO +!/OMPG!$OMP DO DO i=NUA, NUB @@ -1317,9 +1317,9 @@ END DO -!$OMP END DO +!/OMPG!$OMP END DO -!$OMP END Parallel +!/OMPG!$OMP END Parallel ! 999 PRINT*, ' Sub SMCxUNO2r ended.' @@ -1346,10 +1346,10 @@ ! proportion of flux into the cells. This length will be removed by the ! cell length when the tracer concentration is updated. -!$OMP Parallel Default(Shared), Private(j, K, L, M, N), & -!$OMP& Private(CNST,CNST4,CNST5,CNST6,CNST8) +!/OMPG!$OMP Parallel Default(Shared), Private(j, K, L, M, N), & +!/OMPG!$OMP& Private(CNST,CNST4,CNST5,CNST6,CNST8) -!$OMP DO +!/OMPG!$OMP DO DO j=NVA, NVB @@ -1407,9 +1407,9 @@ END DO -!$OMP END DO +!/OMPG!$OMP END DO -!$OMP END Parallel +!/OMPG!$OMP END Parallel ! 999 PRINT*, ' Sub SMCyUNO2r ended.' @@ -1447,10 +1447,10 @@ ! proportion of flux into the cells. This length will be removed by the ! cell length when the tracer concentration is updated. -!$OMP Parallel Default(Shared), Private(i, ij, K, L, M, N), & -!$OMP& Private(CNST,CNST1,CNST2,CNST3,CNST4,CNST5,CNST6,CNST7,CNST8,CNST9) +!/OMPG!$OMP Parallel Default(Shared), Private(i, ij, K, L, M, N), & +!/OMPG!$OMP& Private(CNST,CNST1,CNST2,CNST3,CNST4,CNST5,CNST6,CNST7,CNST8,CNST9) -!$OMP DO +!/OMPG!$OMP DO DO i=NUA, NUB @@ -1547,9 +1547,9 @@ END DO -!$OMP END DO +!/OMPG!$OMP END DO -!$OMP END Parallel +!/OMPG!$OMP END Parallel ! 999 PRINT*, ' Sub SMCxUNO3 ended.' @@ -1585,10 +1585,10 @@ ! Uniform diffusion coefficient for all sizes. JGLi24Feb2012 ! CNST0=AKDif*MRFct*FTS -!$OMP Parallel Default(Shared), Private(j, K, L, M, N), & -!$OMP& Private(CNST,CNST1,CNST2,CNST3,CNST4,CNST5,CNST6,CNST7,CNST8,CNST9) +!/OMPG!$OMP Parallel Default(Shared), Private(j, K, L, M, N), & +!/OMPG!$OMP& Private(CNST,CNST1,CNST2,CNST3,CNST4,CNST5,CNST6,CNST7,CNST8,CNST9) -!$OMP DO +!/OMPG!$OMP DO DO j=NVA, NVB @@ -1695,9 +1695,9 @@ END DO -!$OMP END DO +!/OMPG!$OMP END DO -!$OMP END Parallel +!/OMPG!$OMP END Parallel ! 999 PRINT*, ' Sub SMCyUNO3 ended.' @@ -1728,10 +1728,10 @@ ! proportion of flux into the cells. This length will be removed by the ! cell length when the tracer concentration is updated. -!$OMP Parallel Default(Shared), Private(i, ij, K, L, M, N), & -!$OMP& Private(CNST,CNST0,CNST1,CNST2,CNST3,CNST4,CNST5,CNST6,CNST7,CNST8,CNST9) +!/OMPG!$OMP Parallel Default(Shared), Private(i, ij, K, L, M, N), & +!/OMPG!$OMP& Private(CNST,CNST0,CNST1,CNST2,CNST3,CNST4,CNST5,CNST6,CNST7,CNST8,CNST9) -!$OMP DO +!/OMPG!$OMP DO DO i=NUA, NUB @@ -1811,9 +1811,9 @@ END DO -!$OMP END DO +!/OMPG!$OMP END DO -!$OMP END Parallel +!/OMPG!$OMP END Parallel ! 999 PRINT*, ' Sub SMCxUNO3r ended.' @@ -1841,10 +1841,10 @@ ! proportion of flux into the cells. This length will be removed by the ! cell length when the tracer concentration is updated. -!$OMP Parallel Default(Shared), Private(j, K, L, M, N), & -!$OMP& Private(CNST,CNST4,CNST5,CNST6,CNST7,CNST8,CNST9) +!/OMPG!$OMP Parallel Default(Shared), Private(j, K, L, M, N), & +!/OMPG!$OMP& Private(CNST,CNST4,CNST5,CNST6,CNST7,CNST8,CNST9) -!$OMP DO +!/OMPG!$OMP DO DO j=NVA, NVB @@ -1925,9 +1925,9 @@ END DO -!$OMP END DO +!/OMPG!$OMP END DO -!$OMP END Parallel +!/OMPG!$OMP END Parallel ! 999 PRINT*, ' Sub SMCyUNO3r ended.' @@ -1980,10 +1980,10 @@ DX0I = MRFct/ ( SX * DERA * RADIUS ) DY0I = MRFct/ ( SY * DERA * RADIUS ) -!$OMP Parallel Default(Shared), Private(i, j, K, L, M, N), & -!$OMP& Private(CNST,CNST0,CNST1,CNST2,CNST3,CNST4,CNST5,CNST6) +!/OMPG!$OMP Parallel Default(Shared), Private(i, j, K, L, M, N), & +!/OMPG!$OMP& Private(CNST,CNST0,CNST1,CNST2,CNST3,CNST4,CNST5,CNST6) -!$OMP DO +!/OMPG!$OMP DO !! Calculate x-gradient by averaging U-face gradients. DO i=1, NUFc @@ -2006,22 +2006,22 @@ ! Face size factor is also included for average. CNST5=CNST1*(CVF(M)-CVF(L))/(CNST2+CNST3) -!$OMP CRITICAL +!/OMPG!$OMP CRITICAL ! Store side gradient in two neighbouring cells AUN(L) = AUN(L) + CNST5 AUN(M) = AUN(M) + CNST5 -!$OMP END CRITICAL +!/OMPG!$OMP END CRITICAL ENDIF END DO -!$OMP END DO +!/OMPG!$OMP END DO ! Assign averaged side-gradient to GrdX, plus latitude factor ! Note averaging over 2 times of cell y-width factor but AUN ! has already been divied by two cell lengths. -!$OMP DO +!/OMPG!$OMP DO DO n=1, NSEA ! Cell y-size IJKCel(4,i) is used to cancel the face size-factor in AUN. @@ -2031,9 +2031,9 @@ ENDDO -!$OMP END DO +!/OMPG!$OMP END DO -!$OMP DO +!/OMPG!$OMP DO !! Calculate y-gradient by averaging V-face gradients. DO j=1, NVFc @@ -2056,18 +2056,18 @@ ! Face size factor is also included for average. CNST6=CNST1*(CVF(M)-CVF(L))/(CNST2+CNST3) -!$OMP CRITICAL +!/OMPG!$OMP CRITICAL ! Store side gradient in two neighbouring cells AVN(L) = AVN(L) + CNST6 AVN(M) = AVN(M) + CNST6 -!$OMP END CRITICAL +!/OMPG!$OMP END CRITICAL ENDIF END DO -!$OMP END DO +!/OMPG!$OMP END DO -!$OMP DO +!/OMPG!$OMP DO ! Assign averaged side-gradient to GrdY. DO n=1, NSEA @@ -2077,9 +2077,9 @@ END DO -!$OMP END DO +!/OMPG!$OMP END DO -!$OMP END Parallel +!/OMPG!$OMP END Parallel !/ARC !!Li Polar cell (if any) y-gradient is set to zero. !/ARC IF( NSEA .GT. NGLO ) GrdY(NSEA) = 0.0 @@ -2121,10 +2121,10 @@ !/ARC !!Li Save polar cell value !/ARC CNST0 = CVQ(NSEA) -!$OMP Parallel Default(Shared), Private(i, j, L, M, n), & -!$OMP& Private(CNST3,CNST4,CNST5,CNST6) +!/OMPG!$OMP Parallel Default(Shared), Private(i, j, L, M, n), & +!/OMPG!$OMP& Private(CNST3,CNST4,CNST5,CNST6) -!$OMP DO +!/OMPG!$OMP DO !! Calculate x-gradient by averaging U-face gradients. DO i=1, NUFc @@ -2136,17 +2136,17 @@ ! Multi-resolution SMC grid requires flux multiplied by face factor. CNST5=Real( IJKUFc(3,i) )*(CVF(M)+CVF(L)) -!$OMP CRITICAL +!/OMPG!$OMP CRITICAL ! Store side gradient in two neighbouring cells AUN(L) = AUN(L) + CNST5 AUN(M) = AUN(M) + CNST5 -!$OMP END CRITICAL +!/OMPG!$OMP END CRITICAL END DO -!$OMP END DO +!/OMPG!$OMP END DO -!$OMP DO +!/OMPG!$OMP DO !! Calculate y-gradient by averaging V-face gradients. DO j=1, NVFc @@ -2158,17 +2158,17 @@ ! Face size is required for multi-resolution grid. CNST6=Real( IJKVfc(3,j) )*(CVF(M)+CVF(L)) -!$OMP CRITICAL +!/OMPG!$OMP CRITICAL ! Store side gradient in two neighbouring cells AVN(L) = AVN(L) + CNST6 AVN(M) = AVN(M) + CNST6 -!$OMP END CRITICAL +!/OMPG!$OMP END CRITICAL END DO -!$OMP END DO +!/OMPG!$OMP END DO -!$OMP DO +!/OMPG!$OMP DO ! Assign averaged value back to CVQ. DO n=1, NSEA @@ -2181,9 +2181,9 @@ END DO -!$OMP END DO +!/OMPG!$OMP END DO -!$OMP END Parallel +!/OMPG!$OMP END Parallel !/ARC !!Li Polar cell (if any) keep original value. !/ARC IF( NSEA .GT. NGLO ) CVQ(NSEA) = CNST0 @@ -2391,11 +2391,11 @@ HCel(1:NSEA)= DW(1:NSEA) !! Reset shallow water depth with minimum depth -!$OMP Parallel DO Private(k) +!/OMPG!$OMP Parallel DO Private(k) DO k=1, NSEA IF(DW(k) .LT. DMIN) HCel(k)=DMIN ENDDO -!$OMP END Parallel DO +!/OMPG!$OMP END Parallel DO !! Initialize full grid gradient arrays DDDX = 0. @@ -2412,7 +2412,7 @@ DHDY(1:NSEA) = GrHy !! Apply limiter to depth-gradient and copy to full grid. -!$OMP Parallel DO Private(i,j,k,m,n, CNST0, CNST1, CNST2) +!/OMPG!$OMP Parallel DO Private(i,j,k,m,n, CNST0, CNST1, CNST2) DO n=1,NSEA ! A limiter of gradient <= 0.1 is applied. @@ -2438,12 +2438,12 @@ !/ARC ENDIF END DO -!$OMP END Parallel DO +!/OMPG!$OMP END Parallel DO !! Calculate the depth gradient limiter for refraction. L = 0 -!$OMP Parallel DO Private(i, n, CNST4, CNST6) +!/OMPG!$OMP Parallel DO Private(i, n, CNST4, CNST6) DO n=1,NSEA !Li Work out magnitude of depth gradient @@ -2452,9 +2452,9 @@ !Li Directional depedent depth gradient limiter. JGLi16Jun2011 IF ( CNST4 .GT. 1.0E-5 ) THEN -!$OMP ATOMIC Update +!/OMPG!$OMP ATOMIC Update L = L + 1 -!$OMP END ATOMIC +!/OMPG!$OMP END ATOMIC DO i=1, NTH !Li Refraction is done only when depth gradient is non-zero. @@ -2472,7 +2472,7 @@ ENDIF ENDDO -!$OMP END Parallel DO +!/OMPG!$OMP END Parallel DO !/T WRITE(NDST,*) ' No. Refraction points =', L @@ -2516,7 +2516,7 @@ CALL SMCGradn(CXCY, GrHx, GrHy, L) !! Apply limiter to CX-gradient and copy to full grid. -!$OMP Parallel DO Private(i, j, k, m, n, CNST0, CNST1, CNST2) +!/OMPG!$OMP Parallel DO Private(i, j, k, m, n, CNST0, CNST1, CNST2) DO n=1,NSEA ! A limiter of gradient <= 0.01 is applied. @@ -2542,7 +2542,7 @@ DCXDY(j:j+m-1,i:i+k-1) = GrHy(n) END DO -!$OMP END Parallel DO +!/OMPG!$OMP END Parallel DO !/DEBUGDCXDX WRITE(740+IAPROC,*) 'After non-trivial assination to DCXDX array' @@ -2560,7 +2560,7 @@ CALL SMCGradn(CXCY, GrHx, GrHy, L) !! Apply limiter to CX-gradient and copy to full grid. -!$OMP Parallel DO Private(i, j, k, m, n, CNST0, CNST1, CNST2) +!/OMPG!$OMP Parallel DO Private(i, j, k, m, n, CNST0, CNST1, CNST2) DO n=1,NSEA ! A limiter of gradient <= 0.1 is applied. @@ -2586,7 +2586,7 @@ DCYDY(j:j+m-1,i:i+k-1) = GrHy(n) END DO -!$OMP END Parallel DO +!/OMPG!$OMP END Parallel DO !/T 999 PRINT*, ' Sub SMCDCXY ended.' diff --git a/model/ftn/w3smcomd.ftn b/model/ftn/w3smcomd.ftn index ea227124c..526bd5e89 100644 --- a/model/ftn/w3smcomd.ftn +++ b/model/ftn/w3smcomd.ftn @@ -526,7 +526,7 @@ ALLOCATE(NNIDX(NXO,NYO), XDIST(NXO,NYO), YDIST(NXO,NYO)) ! Indices and weights: - READ(NDSMC) (((NNIDX(I,J), xdist(i,j), ydist(i,j)),i=1,NXO),j=1,NYO) + READ(NDSMC)((NNIDX(I,J), XDIST(I,J), YDIST(I,J),I=1,NXO),J=1,NYO) CLOSE(NDSMC) diff --git a/model/ftn/w3uostmd.ftn b/model/ftn/w3uostmd.ftn index 9228a1e8a..0bf5cf608 100755 --- a/model/ftn/w3uostmd.ftn +++ b/model/ftn/w3uostmd.ftn @@ -9,6 +9,8 @@ !/ +-----------------------------------+ !/ !/ Aug-2018 : Origination. ( version 6.07 ) +!/ 18-Sep-2019 : Added UNIT_AB and changed unit ( version 7.06 ) +!/ number to 110 (C. Bunney, UKMO) !/ !/ Copyright 2010 National Weather Service (NWS), !/ National Oceanic and Atmospheric Administration. All rights @@ -72,6 +74,8 @@ ! srctrm: global singleton source term CLASS(UOST_SOURCETERM), ALLOCATABLE :: SRCTRM + INTEGER, PARAMETER :: UNIT_AB = 110 ! Unit number for LOAD_ALPHABETA + CONTAINS @@ -156,7 +160,7 @@ IF ( (IGRID .GT. 0) .AND. ( ALLOCATED(SRCTRM)) ) THEN ! loading local/shadow alpha/beta - CALL LOAD_ALPHABETA(GRD, SGD, 100) + CALL LOAD_ALPHABETA(GRD, SGD, UNIT_AB) ! warning the user that for cells too small UOST may be inaccurate CGMAX = 20 ! simply taking a high value for the max group velocity to give an indication of this threshold @@ -429,7 +433,7 @@ FILESTART = .FALSE. ELSEIF (HEADER) THEN ! reading the position of an obstructed cell - READ(LINE, *), IX, IY + READ(LINE, *) IX, IY ISOBSTRUCTED(IX, IY) = .TRUE. IF ((IX .GT. NX) .OR. (IY .GT. NY)) THEN WRITE(NDSE,*) '*** WAVEWATCH III ERROR IN W3UOST: '// & diff --git a/model/ftn/wminitmd.ftn b/model/ftn/wminitmd.ftn index 8b93efdb2..e8b122ea4 100644 --- a/model/ftn/wminitmd.ftn +++ b/model/ftn/wminitmd.ftn @@ -3265,7 +3265,7 @@ !/ USE W3SERVMD, ONLY: ITRACE, EXTCDE, NEXTLN, WWDATE, WWTIME !/S USE W3SERVMD, ONLY: STRACE -!/MPRF USE W3SERVMD, ONLY: PRINIT, PRTIME +!/MPRF USE W3TIMEMD, ONLY: PRINIT, PRTIME USE W3TIMEMD, ONLY: STME21, DSEC21, TICK21, TDIFF USE WMUNITMD, ONLY: WMUINI, WMUDMP, WMUSET, WMUGET, WMUINQ !/ diff --git a/model/ftn/ww3_grid.ftn b/model/ftn/ww3_grid.ftn index 3624ce92e..101b1443d 100644 --- a/model/ftn/ww3_grid.ftn +++ b/model/ftn/ww3_grid.ftn @@ -4551,7 +4551,7 @@ !/O2b WRITE (NDSO,1104) !/O2b END IF ! -!/O2c OPEN (NDSM,FILE=TRIM(FNMPRE)//'mapsta.ww3') +!/O2c OPEN (NDSM,FILE=TRIM(FNMPRE)//'mapsta.ww3', RECL=2*NX*NY*50+1) !/O2c DO IY=NY,1, -1 !/O2c DO IX=1,NX !/O2c DO I=1,50 diff --git a/regtests/bin/matrix_ukmo_cray b/regtests/bin/matrix_ukmo_cray new file mode 100755 index 000000000..f40eec117 --- /dev/null +++ b/regtests/bin/matrix_ukmo_cray @@ -0,0 +1,148 @@ +#!/bin/bash +# --------------------------------------------------------------------------- # +# matrix_ukmo_cray: Run matrix of regression tests on Cray XC architecture # +# as currently used at the UK Met Office. # +# # +# Remarks: # +# Currently, programs using the PDLIB switch (METIS library) crash when # +# compiled with the Cray CCE compiler. For those regtests that use PDLIB # +# it is required to use the GNU compiler. # +# # +# Chris Bunney # +# April 2019 # +# # +# Hendrik L. Tolman # +# August 2013 # +# December 2013 # +# April 2018 # +# # +# Copyright 2013 National Weather Service (NWS), # +# National Oceanic and Atmospheric Administration. All rights # +# reserved. WAVEWATCH III is a trademark of the NWS. # +# No unauthorized use without permission. # +# # +# --------------------------------------------------------------------------- # +# 0. Environment file + + source $(dirname $0)/../../model/bin/w3_setenv + main_dir=$WWATCH3_DIR + temp_dir=$WWATCH3_TMP + source=$WWATCH3_SOURCE + list=$WWATCH3_LIST + + echo "Main directory : $main_dir" + echo "Scratch directory : $temp_dir" + echo "Save source codes : $source" + echo "Save listings : $list" + + # Set Cray compiler variant; CCE (Cray Compiler Environment) or GNU. + # Use GNU for programs compiled with PDLIB, CCE for all others. + comp="CCE" + #comp = "GNU" + + +# 1. Set up for compilation environemnt on Cray XC (broadwell processors) + + echo '#!/bin/bash' > matrix.head + echo ' ' >> matrix.head + +# Run everything on single node in shared queue (compilation on compute +# nodes via parallel queue is inefficient and wasteful of resources): + echo '#PBS -l ncpus=16' >> matrix.head + echo '#PBS -l mem=16GB' >> matrix.head + echo '#PBS -q shared' >> matrix.head + echo '#PBS -l walltime=04:00:00' >> matrix.head + echo '#PBS -N ww3_regtest' >> matrix.head + echo '#PBS -j oe' >> matrix.head + echo '#PBS -o matrix.out' >> matrix.head + echo ' ' >> matrix.head + + echo " cd $(dirname $main_dir)/regtests" >> matrix.head + echo ' ' >> matrix.head + +if [[ $comp == "CCE" ]]; then + # Load targetted versions of Cray Development Tools (bug in Fortran StreamIO + # for older versions) and netCDF/HDF5 modules: + echo " module load cdt/18.12" >> matrix.head + echo " module load cray-netcdf/4.6.1.3" >> matrix.head + echo " module load cray-hdf5/1.10.2.0" >> matrix.head + echo " export METIS_PATH=/home/d02/frey/WW3/ParMETIS" >> matrix.head +elif [[ $comp == "GNU" ]]; then + # ParMETIS library not currently working with Cray compiler. + # Use GNU compiler for programs that use PDLIB. + echo " module switch PrgEnv-cray PrgEnv-gnu/5.2.82" >> matrix.head + echo " module load cray-netcdf" >> matrix.head + echo " export METIS_PATH=/home/d02/frey/WW3/ParMETIS_GNU" >> matrix.head +fi + +# SNP Launcher 7.7.4 allows -np switch: + echo " module load cray-snplauncher/7.7.4" >> matrix.head + + echo " export WWATCH3_NETCDF=NC4" >> matrix.head + echo " export NETCDF_CONFIG=\$(which nc-config)" >> matrix.head + +# Compiler option. Choose appropriate compiler and set cmplOption to +# y if using for the first time or using a different compiler + + cmplr=cray_xc.${comp} + export cmplOption='n' + + export mpi='mpiexec' + export np='16' + export nr='4' + export nth='4' + + if [ "$cmplOption" = 'y' ] + then + export rtst="./bin/run_test -c $cmplr -S" + else + export rtst="./bin/run_test -S" + fi + + export ww3='../model' + +# 1.b Flags to do course selection - - - - - - - - - - - - - - - - - - - - - - +# Addition selection by commenting out lines as below + + export shrd='y' # Do shared architecture tests + export dist='y' # Do distributed architecture (MPI) tests + export omp='y' # Threaded (OpenMP) tests + export hybd='y' # Hybrid options + + export prop1D='y' # 1-D propagation tests (ww3_tp1.X) + export prop2D='y' # 2-D propagation tests (ww3_tp2.X) + export time='y' # time linmited growth + export fetch='y' # fetch linmited growth + export hur1mg='y' # Hurricane with one moving grid + export shwtr='y' # shallow water tests + export unstr='y' # unstructured grid tests + export pdlib='y' # unstr with pdlib for domain decomposition and implicit solver + export smcgr='y' # SMC/Rotated grid test + export mudice='y' # Mud/Ice and wave interaction tests + export infgrv='y' # Second harmonic generation tests + export uost='y' # ww3_ts4 Unresolved Obstacles Source Term (UOST) + export assim='y' # Restart spectra update + + export multi01='y' # mww3_test_01 (wetting and drying) + export multi02='y' # mww3_test_02 (basic two-way nesting test)) + export multi03='y' # mww3_test_03 (three high and three low res grids). + export multi04='y' # mww3_test_04 (swell on sea mount and/or current) + export multi05='y' # mww3_test_05 (three-grid moving hurricane) + export multi06='y' # mww3_test_06 (curvilinear grid tests) + export multi07='y' # mww3_test_07 (unstructured grid tests) + export multi08='y' # mww3_test_08 (wind and ice tests) + +# export filter='PR3 ST2 UQ' + # The filter does a set of consecutinve greps on the + # command lines generated by filter.base with the above + # selected options. + +# --------------------------------------------------------------------------- # +# 2. Execute matrix.base ... # +# --------------------------------------------------------------------------- # + + $main_dir/../regtests/bin/matrix.base + +# --------------------------------------------------------------------------- # +# End to the matrix # +# --------------------------------------------------------------------------- # diff --git a/regtests/ww3_tp2.10/input/ww3_grid.inp b/regtests/ww3_tp2.10/input/ww3_grid.inp index 45136fc9f..3baf0d038 100644 --- a/regtests/ww3_tp2.10/input/ww3_grid.inp +++ b/regtests/ww3_tp2.10/input/ww3_grid.inp @@ -2,7 +2,7 @@ $ -------------------------------------------------------------------- $ $ WAVEWATCH III Grid preprocessor input file $ $ -------------------------------------------------------------------- $ $ Adapted for SMC grid test. JGLi18Dec2012 -$ Last modified: JGLi23Nov2015 +$ Last modified: C. Bunney, 10 Sep 2019 $ 'SMC0512 Grid' $ @@ -13,7 +13,9 @@ $ $ SMC625 model time steps. JGLi18Dec2012 $ 1800. 600. 450. 300. $ SMC0512 GtLakes model time steps. JGLi28Oct2015 - 450. 90. 90. 90. +$ 450. 90. 90. 90. +$ CFL timestep set to 60s to avoid instability. C.Bunney 20190910 + 450. 60. 60. 60. $ $ SMC grid swell age to 11. hr with refined cell-scaling. JGLi18Dec2012 $ ST4 option with new parameters in SIN4 and SDS4. JGLi13Feb2013