From fd769499e1bb4d0f27594f269bab2b87d10adede Mon Sep 17 00:00:00 2001 From: Slavko Brdar Date: Wed, 15 Mar 2023 11:20:40 +0100 Subject: [PATCH 01/45] add Atlas data structures to be used in a separate exe: cloudsc-fortran-atlas --- CMakeLists.txt | 1 + src/CMakeLists.txt | 1 + src/cloudsc_fortran_atlas/CMakeLists.txt | 77 + src/cloudsc_fortran_atlas/cloudsc.F90 | 2867 +++++++++++++++++ .../cloudsc_driver_mod.F90 | 210 ++ .../cloudsc_global_atlas_state_mod.F90 | 313 ++ .../dwarf_cloudsc_atlas.F90 | 105 + .../expand_atlas_mod.F90 | 151 + .../validate_atlas_mod.F90 | 182 ++ 9 files changed, 3907 insertions(+) create mode 100644 src/cloudsc_fortran_atlas/CMakeLists.txt create mode 100644 src/cloudsc_fortran_atlas/cloudsc.F90 create mode 100644 src/cloudsc_fortran_atlas/cloudsc_driver_mod.F90 create mode 100644 src/cloudsc_fortran_atlas/cloudsc_global_atlas_state_mod.F90 create mode 100644 src/cloudsc_fortran_atlas/dwarf_cloudsc_atlas.F90 create mode 100644 src/cloudsc_fortran_atlas/expand_atlas_mod.F90 create mode 100644 src/cloudsc_fortran_atlas/validate_atlas_mod.F90 diff --git a/CMakeLists.txt b/CMakeLists.txt index 9eafd77e..04939da8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -92,6 +92,7 @@ if( HAVE_SERIALBOX ) endif() ecbuild_find_package( NAME loki ) +ecbuild_find_package( NAME atlas ) # Add option for single-precision builds ecbuild_add_option( FEATURE SINGLE_PRECISION diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index b0a1c105..eac0dbb8 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -9,6 +9,7 @@ add_subdirectory(prototype1) add_subdirectory(common) add_subdirectory(cloudsc_fortran) +add_subdirectory(cloudsc_fortran_atlas) add_subdirectory(cloudsc_c) add_subdirectory(cloudsc_cuda) add_subdirectory(cloudsc_gpu) diff --git a/src/cloudsc_fortran_atlas/CMakeLists.txt b/src/cloudsc_fortran_atlas/CMakeLists.txt new file mode 100644 index 00000000..dc21c6ae --- /dev/null +++ b/src/cloudsc_fortran_atlas/CMakeLists.txt @@ -0,0 +1,77 @@ +# (C) Copyright 1988- ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation +# nor does it submit to any jurisdiction. + +# Define this dwarf variant as an ECBuild feature +ecbuild_add_option( FEATURE CLOUDSC_FORTRAN_ATLAS + DESCRIPTION "Build the Fortran version CLOUDSC using Atlas and Serialbox" DEFAULT ON + CONDITION atlas_FOUND AND (Serialbox_FOUND OR HDF5_FOUND) +) + +if( HAVE_CLOUDSC_FORTRAN_ATLAS ) + ecbuild_add_executable( + TARGET dwarf-cloudsc-fortran-atlas + SOURCES + cloudsc_global_atlas_state_mod.F90 + expand_atlas_mod.F90 + validate_atlas_mod.F90 + cloudsc_driver_mod.F90 + cloudsc.F90 + dwarf_cloudsc_atlas.F90 + LIBS + cloudsc-common-lib + atlas_f + DEFINITIONS ${CLOUDSC_DEFINITIONS} + ) +endif() + + # Create symlink for the input data + if( HAVE_SERIALBOX ) + execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink + ${CMAKE_CURRENT_SOURCE_DIR}/../../data ${CMAKE_CURRENT_BINARY_DIR}/../../../data ) + endif() + + if( HAVE_HDF5 ) + execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink + ${CMAKE_CURRENT_SOURCE_DIR}/../../config-files/input.h5 ${CMAKE_CURRENT_BINARY_DIR}/../../../input.h5 ) + execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink + ${CMAKE_CURRENT_SOURCE_DIR}/../../config-files/reference.h5 ${CMAKE_CURRENT_BINARY_DIR}/../../../reference.h5 ) + endif() + + ecbuild_add_test( + TARGET dwarf-cloudsc-fortran-atlas-serial + COMMAND bin/dwarf-cloudsc-atlas-fortran + ARGS 1 100 16 + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/../../.. + OMP 1 + ) + ecbuild_add_test( + TARGET dwarf-cloudsc-fortran-atlas-omp + COMMAND bin/dwarf-cloudsc-fortran-atlas + ARGS 4 100 16 + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/../../.. + OMP 4 + CONDITION HAVE_OMP + ) + ecbuild_add_test( + TARGET dwarf-cloudsc-fortran-atlas-mpi + COMMAND bin/dwarf-cloudsc-fortran-atlas + ARGS 1 100 16 + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/../../.. + MPI 2 + OMP 1 + CONDITION HAVE_MPI + ) + ecbuild_add_test( + TARGET dwarf-cloudsc-fortran-mpi-atlas-omp + COMMAND bin/dwarf-cloudsc-fortran-atlas + ARGS 4 100 16 + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/../../.. + MPI 2 + OMP 4 + CONDITION HAVE_OMP AND HAVE_MPI + ) diff --git a/src/cloudsc_fortran_atlas/cloudsc.F90 b/src/cloudsc_fortran_atlas/cloudsc.F90 new file mode 100644 index 00000000..0c712de4 --- /dev/null +++ b/src/cloudsc_fortran_atlas/cloudsc.F90 @@ -0,0 +1,2867 @@ +! (C) Copyright 1988- ECMWF. +! +! This software is licensed under the terms of the Apache Licence Version 2.0 +! which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +! +! In applying this licence, ECMWF does not waive the privileges and immunities +! granted to it by virtue of its status as an intergovernmental organisation +! nor does it submit to any jurisdiction. + +SUBROUTINE CLOUDSC & + !---input + & (KIDIA, KFDIA, KLON, KLEV,& + & PTSPHY,& + & PT, PQ, tendency_cml,tendency_tmp,tendency_loc, & + & PVFA, PVFL, PVFI, PDYNA, PDYNL, PDYNI, & + & PHRSW, PHRLW,& + & PVERVEL, PAP, PAPH,& + & PLSM, LDCUM, KTYPE, & + & PLU, PLUDE, PSNDE, PMFU, PMFD,& + !---prognostic fields + & PA,& + & PCLV, & + & PSUPSAT,& +!-- arrays for aerosol-cloud interactions +!!! & PQAER, KAER, & + & PLCRIT_AER,PICRIT_AER,& + & PRE_ICE,& + & PCCN, PNICE,& + !---diagnostic output + & PCOVPTOT, PRAINFRAC_TOPRFZ,& + !---resulting fluxes + & PFSQLF, PFSQIF , PFCQNNG, PFCQLNG,& + & PFSQRF, PFSQSF , PFCQRNG, PFCQSNG,& + & PFSQLTUR, PFSQITUR , & + & PFPLSL, PFPLSN, PFHPSL, PFHPSN,& + & KFLDX) + +!=============================================================================== +!**** *CLOUDSC* - ROUTINE FOR PARAMATERIZATION OF CLOUD PROCESSES +! FOR PROGNOSTIC CLOUD SCHEME +!! +! M.Tiedtke, C.Jakob, A.Tompkins, R.Forbes (E.C.M.W.F.) +!! +! PURPOSE +! ------- +! THIS ROUTINE UPDATES THE CONV/STRAT CLOUD FIELDS. +! THE FOLLOWING PROCESSES ARE CONSIDERED: +! - Detrainment of cloud water from convective updrafts +! - Evaporation/condensation of cloud water in connection +! with heating/cooling such as by subsidence/ascent +! - Erosion of clouds by turbulent mixing of cloud air +! with unsaturated environmental air +! - Deposition onto ice when liquid water present (Bergeron-Findeison) +! - Conversion of cloud water into rain (collision-coalescence) +! - Conversion of cloud ice to snow (aggregation) +! - Sedimentation of rain, snow and ice +! - Evaporation of rain and snow +! - Melting of snow and ice +! - Freezing of liquid and rain +! Note: Turbulent transports of s,q,u,v at cloud tops due to +! buoyancy fluxes and lw radiative cooling are treated in +! the VDF scheme +!! +! INTERFACE. +! ---------- +! *CLOUDSC* IS CALLED FROM *CALLPAR* +! THE ROUTINE TAKES ITS INPUT FROM THE LONG-TERM STORAGE: +! T,Q,L,PHI AND DETRAINMENT OF CLOUD WATER FROM THE +! CONVECTIVE CLOUDS (MASSFLUX CONVECTION SCHEME), BOUNDARY +! LAYER TURBULENT FLUXES OF HEAT AND MOISTURE, RADIATIVE FLUXES, +! OMEGA. +! IT RETURNS ITS OUTPUT TO: +! 1.MODIFIED TENDENCIES OF MODEL VARIABLES T AND Q +! AS WELL AS CLOUD VARIABLES L AND C +! 2.GENERATES PRECIPITATION FLUXES FROM STRATIFORM CLOUDS +!! +! EXTERNALS. +! ---------- +! NONE +!! +! MODIFICATIONS. +! ------------- +! M. TIEDTKE E.C.M.W.F. 8/1988, 2/1990 +! CH. JAKOB E.C.M.W.F. 2/1994 IMPLEMENTATION INTO IFS +! A.TOMPKINS E.C.M.W.F. 2002 NEW NUMERICS +! 01-05-22 : D.Salmond Safety modifications +! 02-05-29 : D.Salmond Optimisation +! 03-01-13 : J.Hague MASS Vector Functions J.Hague +! 03-10-01 : M.Hamrud Cleaning +! 04-12-14 : A.Tompkins New implicit solver and physics changes +! 04-12-03 : A.Tompkins & M.Ko"hler moist PBL +! G.Mozdzynski 09-Jan-2006 EXP security fix +! 19-01-09 : P.Bechtold Changed increased RCLDIFF value for KTYPE=2 +! 07-07-10 : A.Tompkins/R.Forbes 4-Phase flexible microphysics +! 01-03-11 : R.Forbes Mixed phase changes and tidy up +! 01-10-11 : R.Forbes Melt ice to rain, allow rain to freeze +! 01-10-11 : R.Forbes Limit supersat to avoid excessive values +! 31-10-11 : M.Ahlgrimm Add rain, snow and PEXTRA to DDH output +! 17-02-12 : F.Vana Simplified/optimized LU factorization +! 18-05-12 : F.Vana Cleaning + better support of sequential physics +! N.Semane+P.Bechtold 04-10-2012 Add RVRFACTOR factor for small planet +! 01-02-13 : R.Forbes New params of autoconv/acc,rain evap,snow riming +! 15-03-13 : F. Vana New dataflow + more tendencies from the first call +! K. Yessad (July 2014): Move some variables. +! F. Vana 05-Mar-2015 Support for single precision +! 15-01-15 : R.Forbes Added new options for snow evap & ice deposition +! 10-01-15 : R.Forbes New physics for rain freezing +! 23-10-14 : P. Bechtold remove zeroing of convection arrays +! +! SWITCHES. +! -------- +!! +! MODEL PARAMETERS +! ---------------- +! RCLDIFF: PARAMETER FOR EROSION OF CLOUDS +! RCLCRIT_SEA: THRESHOLD VALUE FOR RAIN AUTOCONVERSION OVER SEA +! RCLCRIT_LAND: THRESHOLD VALUE FOR RAIN AUTOCONVERSION OVER LAND +! RLCRITSNOW: THRESHOLD VALUE FOR SNOW AUTOCONVERSION +! RKCONV: PARAMETER FOR AUTOCONVERSION OF CLOUDS (KESSLER) +! RCLDMAX: MAXIMUM POSSIBLE CLW CONTENT (MASON,1971) +!! +! REFERENCES. +! ---------- +! TIEDTKE MWR 1993 +! JAKOB PhD 2000 +! GREGORY ET AL. QJRMS 2000 +! TOMPKINS ET AL. QJRMS 2007 +!! +!=============================================================================== + +USE PARKIND1 , ONLY : JPIM, JPRB +!USE YOMHOOK , ONLY : LHOOK, DR_HOOK +USE YOMCST , ONLY : RG, RD, RCPD, RETV, RLVTT, RLSTT, RLMLT, RTT, RV +USE YOETHF , ONLY : R2ES, R3LES, R3IES, R4LES, R4IES, R5LES, R5IES, & + & R5ALVCP, R5ALSCP, RALVDCP, RALSDCP, RALFDCP, RTWAT, RTICE, RTICECU, & + & RTWAT_RTICE_R, RTWAT_RTICECU_R, RKOOP1, RKOOP2 +USE YOECLDP , ONLY : YRECLDP, NCLDQV, NCLDQL, NCLDQR, NCLDQI, NCLDQS, NCLV +USE YOMPHYDER ,ONLY : STATE_TYPE + +IMPLICIT NONE + +!------------------------------------------------------------------------------- +! Declare input/output arguments +!------------------------------------------------------------------------------- + +! PLCRIT_AER : critical liquid mmr for rain autoconversion process +! PICRIT_AER : critical liquid mmr for snow autoconversion process +! PRE_LIQ : liq Re +! PRE_ICE : ice Re +! PCCN : liquid cloud condensation nuclei +! PNICE : ice number concentration (cf. CCN) + +REAL(KIND=JPRB) ,INTENT(IN) :: PLCRIT_AER(KLON,KLEV) +REAL(KIND=JPRB) ,INTENT(IN) :: PICRIT_AER(KLON,KLEV) +REAL(KIND=JPRB) ,INTENT(IN) :: PRE_ICE(KLON,KLEV) +REAL(KIND=JPRB) ,INTENT(IN) :: PCCN(KLON,KLEV) ! liquid cloud condensation nuclei +REAL(KIND=JPRB) ,INTENT(IN) :: PNICE(KLON,KLEV) ! ice number concentration (cf. CCN) + +INTEGER(KIND=JPIM),INTENT(IN) :: KLON ! Number of grid points +INTEGER(KIND=JPIM),INTENT(IN) :: KLEV ! Number of levels +INTEGER(KIND=JPIM),INTENT(IN) :: KIDIA +INTEGER(KIND=JPIM),INTENT(IN) :: KFDIA +REAL(KIND=JPRB) ,INTENT(IN) :: PTSPHY ! Physics timestep +REAL(KIND=JPRB) ,INTENT(IN) :: PT(KLON,KLEV) ! T at start of callpar +REAL(KIND=JPRB) ,INTENT(IN) :: PQ(KLON,KLEV) ! Q at start of callpar +TYPE (STATE_TYPE) , INTENT (IN) :: tendency_cml ! cumulative tendency used for final output +TYPE (STATE_TYPE) , INTENT (IN) :: tendency_tmp ! cumulative tendency used as input +TYPE (STATE_TYPE) , INTENT (OUT) :: tendency_loc ! local tendency from cloud scheme +REAL(KIND=JPRB) ,INTENT(IN) :: PVFA(KLON,KLEV) ! CC from VDF scheme +REAL(KIND=JPRB) ,INTENT(IN) :: PVFL(KLON,KLEV) ! Liq from VDF scheme +REAL(KIND=JPRB) ,INTENT(IN) :: PVFI(KLON,KLEV) ! Ice from VDF scheme +REAL(KIND=JPRB) ,INTENT(IN) :: PDYNA(KLON,KLEV) ! CC from Dynamics +REAL(KIND=JPRB) ,INTENT(IN) :: PDYNL(KLON,KLEV) ! Liq from Dynamics +REAL(KIND=JPRB) ,INTENT(IN) :: PDYNI(KLON,KLEV) ! Liq from Dynamics +REAL(KIND=JPRB) ,INTENT(IN) :: PHRSW(KLON,KLEV) ! Short-wave heating rate +REAL(KIND=JPRB) ,INTENT(IN) :: PHRLW(KLON,KLEV) ! Long-wave heating rate +REAL(KIND=JPRB) ,INTENT(IN) :: PVERVEL(KLON,KLEV) !Vertical velocity +REAL(KIND=JPRB) ,INTENT(IN) :: PAP(KLON,KLEV) ! Pressure on full levels +REAL(KIND=JPRB) ,INTENT(IN) :: PAPH(KLON,KLEV+1)! Pressure on half levels +REAL(KIND=JPRB) ,INTENT(IN) :: PLSM(KLON) ! Land fraction (0-1) +LOGICAL ,INTENT(IN) :: LDCUM(KLON) ! Convection active +INTEGER(KIND=JPIM),INTENT(IN) :: KTYPE(KLON) ! Convection type 0,1,2 +REAL(KIND=JPRB) ,INTENT(IN) :: PLU(KLON,KLEV) ! Conv. condensate +REAL(KIND=JPRB) ,INTENT(INOUT) :: PLUDE(KLON,KLEV) ! Conv. detrained water +REAL(KIND=JPRB) ,INTENT(IN) :: PSNDE(KLON,KLEV) ! Conv. detrained snow +REAL(KIND=JPRB) ,INTENT(IN) :: PMFU(KLON,KLEV) ! Conv. mass flux up +REAL(KIND=JPRB) ,INTENT(IN) :: PMFD(KLON,KLEV) ! Conv. mass flux down +REAL(KIND=JPRB) ,INTENT(IN) :: PA(KLON,KLEV) ! Original Cloud fraction (t) + +INTEGER(KIND=JPIM),INTENT(IN) :: KFLDX + +REAL(KIND=JPRB) ,INTENT(IN) :: PCLV(KLON,KLEV,NCLV) + + ! Supersat clipped at previous time level in SLTEND +REAL(KIND=JPRB) ,INTENT(IN) :: PSUPSAT(KLON,KLEV) +REAL(KIND=JPRB) ,INTENT(OUT) :: PCOVPTOT(KLON,KLEV) ! Precip fraction +REAL(KIND=JPRB) ,INTENT(OUT) :: PRAINFRAC_TOPRFZ(KLON) +! Flux diagnostics for DDH budget +REAL(KIND=JPRB) ,INTENT(OUT) :: PFSQLF(KLON,KLEV+1) ! Flux of liquid +REAL(KIND=JPRB) ,INTENT(OUT) :: PFSQIF(KLON,KLEV+1) ! Flux of ice +REAL(KIND=JPRB) ,INTENT(OUT) :: PFCQLNG(KLON,KLEV+1) ! -ve corr for liq +REAL(KIND=JPRB) ,INTENT(OUT) :: PFCQNNG(KLON,KLEV+1) ! -ve corr for ice +REAL(KIND=JPRB) ,INTENT(OUT) :: PFSQRF(KLON,KLEV+1) ! Flux diagnostics +REAL(KIND=JPRB) ,INTENT(OUT) :: PFSQSF(KLON,KLEV+1) ! for DDH, generic +REAL(KIND=JPRB) ,INTENT(OUT) :: PFCQRNG(KLON,KLEV+1) ! rain +REAL(KIND=JPRB) ,INTENT(OUT) :: PFCQSNG(KLON,KLEV+1) ! snow +REAL(KIND=JPRB) ,INTENT(OUT) :: PFSQLTUR(KLON,KLEV+1) ! liquid flux due to VDF +REAL(KIND=JPRB) ,INTENT(OUT) :: PFSQITUR(KLON,KLEV+1) ! ice flux due to VDF +REAL(KIND=JPRB) ,INTENT(OUT) :: PFPLSL(KLON,KLEV+1) ! liq+rain sedim flux +REAL(KIND=JPRB) ,INTENT(OUT) :: PFPLSN(KLON,KLEV+1) ! ice+snow sedim flux +REAL(KIND=JPRB) ,INTENT(OUT) :: PFHPSL(KLON,KLEV+1) ! Enthalpy flux for liq +REAL(KIND=JPRB) ,INTENT(OUT) :: PFHPSN(KLON,KLEV+1) ! Enthalp flux for ice + +!------------------------------------------------------------------------------- +! Declare local variables +!------------------------------------------------------------------------------- + +REAL(KIND=JPRB) :: & +! condensation and evaporation terms + & ZLCOND1(KLON), ZLCOND2(KLON),& + & ZLEVAP, ZLEROS,& + & ZLEVAPL(KLON), ZLEVAPI(KLON),& +! autoconversion terms + & ZRAINAUT(KLON), ZSNOWAUT(KLON), & + & ZLIQCLD(KLON), ZICECLD(KLON) +REAL(KIND=JPRB) :: ZFOKOOP(KLON), ZFOEALFA(KLON,KLEV+1) +REAL(KIND=JPRB) :: ZICENUCLEI(KLON) ! number concentration of ice nuclei + +REAL(KIND=JPRB) :: ZLICLD(KLON) +REAL(KIND=JPRB) :: ZACOND +REAL(KIND=JPRB) :: ZAEROS +REAL(KIND=JPRB) :: ZLFINALSUM(KLON) +REAL(KIND=JPRB) :: ZDQS(KLON) +REAL(KIND=JPRB) :: ZTOLD(KLON) +REAL(KIND=JPRB) :: ZQOLD(KLON) +REAL(KIND=JPRB) :: ZDTGDP(KLON) +REAL(KIND=JPRB) :: ZRDTGDP(KLON) +REAL(KIND=JPRB) :: ZTRPAUS(KLON) +REAL(KIND=JPRB) :: ZCOVPCLR(KLON) +REAL(KIND=JPRB) :: ZPRECLR +REAL(KIND=JPRB) :: ZCOVPTOT(KLON) +REAL(KIND=JPRB) :: ZCOVPMAX(KLON) +REAL(KIND=JPRB) :: ZQPRETOT(KLON) +REAL(KIND=JPRB) :: ZDPEVAP +REAL(KIND=JPRB) :: ZDTFORC +REAL(KIND=JPRB) :: ZDTDIAB +REAL(KIND=JPRB) :: ZTP1(KLON,KLEV) +REAL(KIND=JPRB) :: ZLDEFR(KLON) +REAL(KIND=JPRB) :: ZLDIFDT(KLON) +REAL(KIND=JPRB) :: ZDTGDPF(KLON) +REAL(KIND=JPRB) :: ZLCUST(KLON,NCLV) +REAL(KIND=JPRB) :: ZACUST(KLON) +REAL(KIND=JPRB) :: ZMF(KLON) + +REAL(KIND=JPRB) :: ZRHO(KLON) +REAL(KIND=JPRB) :: ZTMP1(KLON),ZTMP2(KLON),ZTMP3(KLON) +REAL(KIND=JPRB) :: ZTMP4(KLON),ZTMP5(KLON),ZTMP6(KLON),ZTMP7(KLON) +REAL(KIND=JPRB) :: ZALFAWM(KLON) + +! Accumulators of A,B,and C factors for cloud equations +REAL(KIND=JPRB) :: ZSOLAB(KLON) ! -ve implicit CC +REAL(KIND=JPRB) :: ZSOLAC(KLON) ! linear CC +REAL(KIND=JPRB) :: ZANEW +REAL(KIND=JPRB) :: ZANEWM1(KLON) + +REAL(KIND=JPRB) :: ZGDP(KLON) + +!---for flux calculation +REAL(KIND=JPRB) :: ZDA(KLON) +REAL(KIND=JPRB) :: ZLI(KLON,KLEV), ZA(KLON,KLEV) +REAL(KIND=JPRB) :: ZAORIG(KLON,KLEV) ! start of scheme value for CC + +LOGICAL :: LLFLAG(KLON) +LOGICAL :: LLO1 + +INTEGER(KIND=JPIM) :: ICALL, IK, JK, JL, JM, JN, JO, JLEN, IS + +REAL(KIND=JPRB) :: ZDP(KLON), ZPAPHD(KLON) + +REAL(KIND=JPRB) :: ZALFA +! & ZALFACU, ZALFALS +REAL(KIND=JPRB) :: ZALFAW +REAL(KIND=JPRB) :: ZBETA,ZBETA1 +!REAL(KIND=JPRB) :: ZBOTT +REAL(KIND=JPRB) :: ZCFPR +REAL(KIND=JPRB) :: ZCOR +REAL(KIND=JPRB) :: ZCDMAX +REAL(KIND=JPRB) :: ZMIN(KLON) +REAL(KIND=JPRB) :: ZLCONDLIM +REAL(KIND=JPRB) :: ZDENOM +REAL(KIND=JPRB) :: ZDPMXDT +REAL(KIND=JPRB) :: ZDPR +REAL(KIND=JPRB) :: ZDTDP +REAL(KIND=JPRB) :: ZE +REAL(KIND=JPRB) :: ZEPSEC +REAL(KIND=JPRB) :: ZFAC, ZFACI, ZFACW +REAL(KIND=JPRB) :: ZGDCP +REAL(KIND=JPRB) :: ZINEW +REAL(KIND=JPRB) :: ZLCRIT +REAL(KIND=JPRB) :: ZMFDN +REAL(KIND=JPRB) :: ZPRECIP +REAL(KIND=JPRB) :: ZQE +REAL(KIND=JPRB) :: ZQSAT, ZQTMST, ZRDCP +REAL(KIND=JPRB) :: ZRHC, ZSIG, ZSIGK +REAL(KIND=JPRB) :: ZWTOT +REAL(KIND=JPRB) :: ZZCO, ZZDL, ZZRH, ZZZDT, ZQADJ +REAL(KIND=JPRB) :: ZQNEW, ZTNEW +REAL(KIND=JPRB) :: ZRG_R,ZGDPH_R,ZCONS1,ZCOND,ZCONS1A +REAL(KIND=JPRB) :: ZLFINAL +REAL(KIND=JPRB) :: ZMELT +REAL(KIND=JPRB) :: ZEVAP +REAL(KIND=JPRB) :: ZFRZ +REAL(KIND=JPRB) :: ZVPLIQ, ZVPICE +REAL(KIND=JPRB) :: ZADD, ZBDD, ZCVDS, ZICE0, ZDEPOS +REAL(KIND=JPRB) :: ZSUPSAT(KLON) +REAL(KIND=JPRB) :: ZFALL +REAL(KIND=JPRB) :: ZRE_ICE +REAL(KIND=JPRB) :: ZRLDCP +REAL(KIND=JPRB) :: ZQP1ENV + +!---------------------------- +! Arrays for new microphysics +!---------------------------- +INTEGER(KIND=JPIM) :: IPHASE(NCLV) ! marker for water phase of each species + ! 0=vapour, 1=liquid, 2=ice + +INTEGER(KIND=JPIM) :: IMELT(NCLV) ! marks melting linkage for ice categories + ! ice->liquid, snow->rain + +LOGICAL :: LLFALL(NCLV) ! marks falling species + ! LLFALL=0, cloud cover must > 0 for zqx > 0 + ! LLFALL=1, no cloud needed, zqx can evaporate + +LOGICAL :: LLINDEX1(KLON,NCLV) ! index variable +LOGICAL :: LLINDEX3(KLON,NCLV,NCLV) ! index variable +REAL(KIND=JPRB) :: ZMAX +REAL(KIND=JPRB) :: ZRAT +INTEGER(KIND=JPIM) :: IORDER(KLON,NCLV) ! array for sorting explicit terms + +REAL(KIND=JPRB) :: ZLIQFRAC(KLON,KLEV) ! cloud liquid water fraction: ql/(ql+qi) +REAL(KIND=JPRB) :: ZICEFRAC(KLON,KLEV) ! cloud ice water fraction: qi/(ql+qi) +REAL(KIND=JPRB) :: ZQX(KLON,KLEV,NCLV) ! water variables +REAL(KIND=JPRB) :: ZQX0(KLON,KLEV,NCLV) ! water variables at start of scheme +REAL(KIND=JPRB) :: ZQXN(KLON,NCLV) ! new values for zqx at time+1 +REAL(KIND=JPRB) :: ZQXFG(KLON,NCLV) ! first guess values including precip +REAL(KIND=JPRB) :: ZQXNM1(KLON,NCLV) ! new values for zqx at time+1 at level above +REAL(KIND=JPRB) :: ZFLUXQ(KLON,NCLV) ! fluxes convergence of species (needed?) +! Keep the following for possible future total water variance scheme? +!REAL(KIND=JPRB) :: ZTL(KLON,KLEV) ! liquid water temperature +!REAL(KIND=JPRB) :: ZABETA(KLON,KLEV) ! cloud fraction +!REAL(KIND=JPRB) :: ZVAR(KLON,KLEV) ! temporary variance +!REAL(KIND=JPRB) :: ZQTMIN(KLON,KLEV) +!REAL(KIND=JPRB) :: ZQTMAX(KLON,KLEV) + +REAL(KIND=JPRB) :: ZPFPLSX(KLON,KLEV+1,NCLV) ! generalized precipitation flux +REAL(KIND=JPRB) :: ZLNEG(KLON,KLEV,NCLV) ! for negative correction diagnostics +REAL(KIND=JPRB) :: ZMELTMAX(KLON) +REAL(KIND=JPRB) :: ZFRZMAX(KLON) +REAL(KIND=JPRB) :: ZICETOT(KLON) + +REAL(KIND=JPRB) :: ZQXN2D(KLON,KLEV,NCLV) ! water variables store + +REAL(KIND=JPRB) :: ZQSMIX(KLON,KLEV) ! diagnostic mixed phase saturation +!REAL(KIND=JPRB) :: ZQSBIN(KLON,KLEV) ! binary switched ice/liq saturation +REAL(KIND=JPRB) :: ZQSLIQ(KLON,KLEV) ! liquid water saturation +REAL(KIND=JPRB) :: ZQSICE(KLON,KLEV) ! ice water saturation + +!REAL(KIND=JPRB) :: ZRHM(KLON,KLEV) ! diagnostic mixed phase RH +!REAL(KIND=JPRB) :: ZRHL(KLON,KLEV) ! RH wrt liq +!REAL(KIND=JPRB) :: ZRHI(KLON,KLEV) ! RH wrt ice + +REAL(KIND=JPRB) :: ZFOEEWMT(KLON,KLEV) +REAL(KIND=JPRB) :: ZFOEEW(KLON,KLEV) +REAL(KIND=JPRB) :: ZFOEELIQT(KLON,KLEV) +!REAL(KIND=JPRB) :: ZFOEEICET(KLON,KLEV) + +REAL(KIND=JPRB) :: ZDQSLIQDT(KLON), ZDQSICEDT(KLON), ZDQSMIXDT(KLON) +REAL(KIND=JPRB) :: ZCORQSLIQ(KLON) +REAL(KIND=JPRB) :: ZCORQSICE(KLON) +!REAL(KIND=JPRB) :: ZCORQSBIN(KLON) +REAL(KIND=JPRB) :: ZCORQSMIX(KLON) +REAL(KIND=JPRB) :: ZEVAPLIMLIQ(KLON), ZEVAPLIMICE(KLON), ZEVAPLIMMIX(KLON) + +!------------------------------------------------------- +! SOURCE/SINK array for implicit and explicit terms +!------------------------------------------------------- +! a POSITIVE value entered into the arrays is a... +! Source of this variable +! | +! | Sink of this variable +! | | +! V V +! ZSOLQA(JL,IQa,IQb) = explicit terms +! ZSOLQB(JL,IQa,IQb) = implicit terms +! Thus if ZSOLAB(JL,NCLDQL,IQV)=K where K>0 then this is +! a source of NCLDQL and a sink of IQV +! put 'magic' source terms such as PLUDE from +! detrainment into explicit source/sink array diagnognal +! ZSOLQA(NCLDQL,NCLDQL)= -PLUDE +! i.e. A positive value is a sink!????? weird... +!------------------------------------------------------- + +REAL(KIND=JPRB) :: ZSOLQA(KLON,NCLV,NCLV) ! explicit sources and sinks +REAL(KIND=JPRB) :: ZSOLQB(KLON,NCLV,NCLV) ! implicit sources and sinks + ! e.g. microphysical pathways between ice variables. +REAL(KIND=JPRB) :: ZQLHS(KLON,NCLV,NCLV) ! n x n matrix storing the LHS of implicit solver +REAL(KIND=JPRB) :: ZVQX(NCLV) ! fall speeds of three categories +REAL(KIND=JPRB) :: ZEXPLICIT, ZRATIO(KLON,NCLV), ZSINKSUM(KLON,NCLV) + +! for sedimentation source/sink terms +REAL(KIND=JPRB) :: ZFALLSINK(KLON,NCLV) +REAL(KIND=JPRB) :: ZFALLSRCE(KLON,NCLV) + +! for convection detrainment source and subsidence source/sink terms +REAL(KIND=JPRB) :: ZCONVSRCE(KLON,NCLV) +REAL(KIND=JPRB) :: ZCONVSINK(KLON,NCLV) + +! for supersaturation source term from previous timestep +REAL(KIND=JPRB) :: ZPSUPSATSRCE(KLON,NCLV) + +! Numerical fit to wet bulb temperature +REAL(KIND=JPRB),PARAMETER :: ZTW1 = 1329.31_JPRB +REAL(KIND=JPRB),PARAMETER :: ZTW2 = 0.0074615_JPRB +REAL(KIND=JPRB),PARAMETER :: ZTW3 = 0.85E5_JPRB +REAL(KIND=JPRB),PARAMETER :: ZTW4 = 40.637_JPRB +REAL(KIND=JPRB),PARAMETER :: ZTW5 = 275.0_JPRB + +REAL(KIND=JPRB) :: ZSUBSAT ! Subsaturation for snow melting term +REAL(KIND=JPRB) :: ZTDMTW0 ! Diff between dry-bulb temperature and + ! temperature when wet-bulb = 0degC + +! Variables for deposition term +REAL(KIND=JPRB) :: ZTCG ! Temperature dependent function for ice PSD +REAL(KIND=JPRB) :: ZFACX1I, ZFACX1S! PSD correction factor +REAL(KIND=JPRB) :: ZAPLUSB,ZCORRFAC,ZCORRFAC2,ZPR02,ZTERM1,ZTERM2 ! for ice dep +REAL(KIND=JPRB) :: ZCLDTOPDIST(KLON) ! Distance from cloud top +REAL(KIND=JPRB) :: ZINFACTOR ! No. of ice nuclei factor for deposition + +! Autoconversion/accretion/riming/evaporation +INTEGER(KIND=JPIM) :: IWARMRAIN +INTEGER(KIND=JPIM) :: IEVAPRAIN +INTEGER(KIND=JPIM) :: IEVAPSNOW +INTEGER(KIND=JPIM) :: IDEPICE +REAL(KIND=JPRB) :: ZRAINACC(KLON) +REAL(KIND=JPRB) :: ZRAINCLD(KLON) +REAL(KIND=JPRB) :: ZSNOWRIME(KLON) +REAL(KIND=JPRB) :: ZSNOWCLD(KLON) +REAL(KIND=JPRB) :: ZESATLIQ +REAL(KIND=JPRB) :: ZFALLCORR +REAL(KIND=JPRB) :: ZLAMBDA +REAL(KIND=JPRB) :: ZEVAP_DENOM +REAL(KIND=JPRB) :: ZCORR2 +REAL(KIND=JPRB) :: ZKA +REAL(KIND=JPRB) :: ZCONST +REAL(KIND=JPRB) :: ZTEMP + +! Rain freezing +LOGICAL :: LLRAINLIQ(KLON) ! True if majority of raindrops are liquid (no ice core) + +!---------------------------- +! End: new microphysics +!---------------------------- + +!---------------------- +! SCM budget statistics +!---------------------- +REAL(KIND=JPRB) :: ZRAIN + +REAL(KIND=JPRB) :: Z_TMP1(KFDIA-KIDIA+1) +REAL(KIND=JPRB) :: Z_TMP2(KFDIA-KIDIA+1) +REAL(KIND=JPRB) :: Z_TMP3(KFDIA-KIDIA+1) +REAL(KIND=JPRB) :: Z_TMP4(KFDIA-KIDIA+1) +!REAL(KIND=JPRB) :: Z_TMP5(KFDIA-KIDIA+1) +REAL(KIND=JPRB) :: Z_TMP6(KFDIA-KIDIA+1) +REAL(KIND=JPRB) :: Z_TMP7(KFDIA-KIDIA+1) +REAL(KIND=JPRB) :: Z_TMPK(KFDIA-KIDIA+1,KLEV) +!REAL(KIND=JPRB) :: ZCON1,ZCON2 +REAL(KIND=JPRB) :: ZHOOK_HANDLE +REAL(KIND=JPRB) :: ZTMPL,ZTMPI,ZTMPA + +REAL(KIND=JPRB) :: ZMM,ZRR +REAL(KIND=JPRB) :: ZRG(KLON) + +REAL(KIND=JPRB) :: ZBUDCC(KLON,KFLDX) ! extra fields +REAL(KIND=JPRB) :: ZBUDL(KLON,KFLDX) ! extra fields +REAL(KIND=JPRB) :: ZBUDI(KLON,KFLDX) ! extra fields + +REAL(KIND=JPRB) :: ZZSUM, ZZRATIO +REAL(KIND=JPRB) :: ZEPSILON + +REAL(KIND=JPRB) :: ZCOND1, ZQP + + +#include "abor1.intfb.h" + +!DIR$ VFUNCTION EXPHF +#include "fcttre.func.h" +#include "fccld.func.h" + +!=============================================================================== +!IF (LHOOK) CALL DR_HOOK('CLOUDSC',0,ZHOOK_HANDLE) +ASSOCIATE(LAERICEAUTO=>YRECLDP%LAERICEAUTO, LAERICESED=>YRECLDP%LAERICESED, & + & LAERLIQAUTOLSP=>YRECLDP%LAERLIQAUTOLSP, LAERLIQCOLL=>YRECLDP%LAERLIQCOLL, & + & LCLDBUDGET=>YRECLDP%LCLDBUDGET, NCLDTOP=>YRECLDP%NCLDTOP, & + & NSSOPT=>YRECLDP%NSSOPT, RAMID=>YRECLDP%RAMID, RAMIN=>YRECLDP%RAMIN, & + & RCCN=>YRECLDP%RCCN, RCLCRIT_LAND=>YRECLDP%RCLCRIT_LAND, & + & RCLCRIT_SEA=>YRECLDP%RCLCRIT_SEA, RCLDIFF=>YRECLDP%RCLDIFF, & + & RCLDIFF_CONVI=>YRECLDP%RCLDIFF_CONVI, RCLDTOPCF=>YRECLDP%RCLDTOPCF, & + & RCL_APB1=>YRECLDP%RCL_APB1, RCL_APB2=>YRECLDP%RCL_APB2, & + & RCL_APB3=>YRECLDP%RCL_APB3, RCL_CDENOM1=>YRECLDP%RCL_CDENOM1, & + & RCL_CDENOM2=>YRECLDP%RCL_CDENOM2, RCL_CDENOM3=>YRECLDP%RCL_CDENOM3, & + & RCL_CONST1I=>YRECLDP%RCL_CONST1I, RCL_CONST1R=>YRECLDP%RCL_CONST1R, & + & RCL_CONST1S=>YRECLDP%RCL_CONST1S, RCL_CONST2I=>YRECLDP%RCL_CONST2I, & + & RCL_CONST2R=>YRECLDP%RCL_CONST2R, RCL_CONST2S=>YRECLDP%RCL_CONST2S, & + & RCL_CONST3I=>YRECLDP%RCL_CONST3I, RCL_CONST3R=>YRECLDP%RCL_CONST3R, & + & RCL_CONST3S=>YRECLDP%RCL_CONST3S, RCL_CONST4I=>YRECLDP%RCL_CONST4I, & + & RCL_CONST4R=>YRECLDP%RCL_CONST4R, RCL_CONST4S=>YRECLDP%RCL_CONST4S, & + & RCL_CONST5I=>YRECLDP%RCL_CONST5I, RCL_CONST5R=>YRECLDP%RCL_CONST5R, & + & RCL_CONST5S=>YRECLDP%RCL_CONST5S, RCL_CONST6I=>YRECLDP%RCL_CONST6I, & + & RCL_CONST6R=>YRECLDP%RCL_CONST6R, RCL_CONST6S=>YRECLDP%RCL_CONST6S, & + & RCL_CONST7S=>YRECLDP%RCL_CONST7S, RCL_CONST8S=>YRECLDP%RCL_CONST8S, & + & RCL_FAC1=>YRECLDP%RCL_FAC1, RCL_FAC2=>YRECLDP%RCL_FAC2, & + & RCL_FZRAB=>YRECLDP%RCL_FZRAB, RCL_KA273=>YRECLDP%RCL_KA273, & + & RCL_KKAAC=>YRECLDP%RCL_KKAAC, RCL_KKAAU=>YRECLDP%RCL_KKAAU, & + & RCL_KKBAC=>YRECLDP%RCL_KKBAC, RCL_KKBAUN=>YRECLDP%RCL_KKBAUN, & + & RCL_KKBAUQ=>YRECLDP%RCL_KKBAUQ, & + & RCL_KK_CLOUD_NUM_LAND=>YRECLDP%RCL_KK_CLOUD_NUM_LAND, & + & RCL_KK_CLOUD_NUM_SEA=>YRECLDP%RCL_KK_CLOUD_NUM_SEA, RCL_X3I=>YRECLDP%RCL_X3I, & + & RCOVPMIN=>YRECLDP%RCOVPMIN, RDENSREF=>YRECLDP%RDENSREF, & + & RDEPLIQREFDEPTH=>YRECLDP%RDEPLIQREFDEPTH, & + & RDEPLIQREFRATE=>YRECLDP%RDEPLIQREFRATE, RICEHI1=>YRECLDP%RICEHI1, & + & RICEHI2=>YRECLDP%RICEHI2, RICEINIT=>YRECLDP%RICEINIT, RKCONV=>YRECLDP%RKCONV, & + & RKOOPTAU=>YRECLDP%RKOOPTAU, RLCRITSNOW=>YRECLDP%RLCRITSNOW, & + & RLMIN=>YRECLDP%RLMIN, RNICE=>YRECLDP%RNICE, RPECONS=>YRECLDP%RPECONS, & + & RPRC1=>YRECLDP%RPRC1, RPRECRHMAX=>YRECLDP%RPRECRHMAX, & + & RSNOWLIN1=>YRECLDP%RSNOWLIN1, RSNOWLIN2=>YRECLDP%RSNOWLIN2, & + & RTAUMEL=>YRECLDP%RTAUMEL, RTHOMO=>YRECLDP%RTHOMO, RVICE=>YRECLDP%RVICE, & + & RVRAIN=>YRECLDP%RVRAIN, RVRFACTOR=>YRECLDP%RVRFACTOR, & + & RVSNOW=>YRECLDP%RVSNOW) + +!=============================================================================== +! 0.0 Beginning of timestep book-keeping +!---------------------------------------------------------------------- + + +!###################################################################### +! 0. *** SET UP CONSTANTS *** +!###################################################################### + +ZEPSILON=100._JPRB*EPSILON(ZEPSILON) + +! --------------------------------------------------------------------- +! Set version of warm-rain autoconversion/accretion +! IWARMRAIN = 1 ! Sundquist +! IWARMRAIN = 2 ! Khairoutdinov and Kogan (2000) +! --------------------------------------------------------------------- +IWARMRAIN = 2 +! --------------------------------------------------------------------- +! Set version of rain evaporation +! IEVAPRAIN = 1 ! Sundquist +! IEVAPRAIN = 2 ! Abel and Boutle (2013) +! --------------------------------------------------------------------- +IEVAPRAIN = 2 +! --------------------------------------------------------------------- +! Set version of snow evaporation +! IEVAPSNOW = 1 ! Sundquist +! IEVAPSNOW = 2 ! New +! --------------------------------------------------------------------- +IEVAPSNOW = 1 +! --------------------------------------------------------------------- +! Set version of ice deposition +! IDEPICE = 1 ! Rotstayn (2001) +! IDEPICE = 2 ! New +! --------------------------------------------------------------------- +IDEPICE = 1 + +! --------------------- +! Some simple constants +! --------------------- +ZQTMST = 1.0_JPRB/PTSPHY +ZGDCP = RG/RCPD +ZRDCP = RD/RCPD +ZCONS1A = RCPD/(RLMLT*RG*RTAUMEL) +ZEPSEC = 1.E-14_JPRB +ZRG_R = 1.0_JPRB/RG +ZRLDCP = 1.0_JPRB/(RALSDCP-RALVDCP) + +! Note: Defined in module/yoecldp.F90 +! NCLDQL=1 ! liquid cloud water +! NCLDQI=2 ! ice cloud water +! NCLDQR=3 ! rain water +! NCLDQS=4 ! snow +! NCLDQV=5 ! vapour + +! ----------------------------------------------- +! Define species phase, 0=vapour, 1=liquid, 2=ice +! ----------------------------------------------- +IPHASE(NCLDQV)=0 +IPHASE(NCLDQL)=1 +IPHASE(NCLDQR)=1 +IPHASE(NCLDQI)=2 +IPHASE(NCLDQS)=2 + +! --------------------------------------------------- +! Set up melting/freezing index, +! if an ice category melts/freezes, where does it go? +! --------------------------------------------------- +IMELT(NCLDQV)=-99 +IMELT(NCLDQL)=NCLDQI +IMELT(NCLDQR)=NCLDQS +IMELT(NCLDQI)=NCLDQR +IMELT(NCLDQS)=NCLDQR + +! ----------------------------------------------- +! INITIALIZATION OF OUTPUT TENDENCIES +! ----------------------------------------------- +DO JK=1,KLEV + DO JL=KIDIA,KFDIA + tendency_loc%T(JL,JK)=0.0_JPRB + tendency_loc%q(JL,JK)=0.0_JPRB + tendency_loc%a(JL,JK)=0.0_JPRB + ENDDO +ENDDO +DO JM=1,NCLV-1 + DO JK=1,KLEV + DO JL=KIDIA,KFDIA + tendency_loc%cld(JL,JK,JM)=0.0_JPRB + ENDDO + ENDDO +ENDDO + +! ------------------------- +! set up fall speeds in m/s +! ------------------------- +ZVQX(NCLDQV)=0.0_JPRB +ZVQX(NCLDQL)=0.0_JPRB +ZVQX(NCLDQI)=RVICE +ZVQX(NCLDQR)=RVRAIN +ZVQX(NCLDQS)=RVSNOW +LLFALL(:)=.FALSE. +DO JM=1,NCLV + IF (ZVQX(JM)>0.0_JPRB) LLFALL(JM)=.TRUE. ! falling species +ENDDO +! Set LLFALL to false for ice (but ice still sediments!) +! Need to rationalise this at some point +LLFALL(NCLDQI)=.FALSE. + + +!###################################################################### +! 1. *** INITIAL VALUES FOR VARIABLES *** +!###################################################################### + + +! ---------------------- +! non CLV initialization +! ---------------------- +DO JK=1,KLEV + DO JL=KIDIA,KFDIA + ZTP1(JL,JK) = PT(JL,JK)+PTSPHY*tendency_tmp%T(JL,JK) + ZQX(JL,JK,NCLDQV) = PQ(JL,JK)+PTSPHY*tendency_tmp%q(JL,JK) + ZQX0(JL,JK,NCLDQV) = PQ(JL,JK)+PTSPHY*tendency_tmp%q(JL,JK) + ZA(JL,JK) = PA(JL,JK)+PTSPHY*tendency_tmp%a(JL,JK) + ZAORIG(JL,JK) = PA(JL,JK)+PTSPHY*tendency_tmp%a(JL,JK) + ENDDO +ENDDO + +! ------------------------------------- +! initialization for CLV family +! ------------------------------------- +DO JM=1,NCLV-1 + DO JK=1,KLEV + DO JL=KIDIA,KFDIA + ZQX(JL,JK,JM) = PCLV(JL,JK,JM)+PTSPHY*tendency_tmp%cld(JL,JK,JM) + ZQX0(JL,JK,JM) = PCLV(JL,JK,JM)+PTSPHY*tendency_tmp%cld(JL,JK,JM) + ENDDO + ENDDO +ENDDO + +!------------- +! zero arrays +!------------- +ZPFPLSX(:,:,:) = 0.0_JPRB ! precip fluxes +ZQXN2D(:,:,:) = 0.0_JPRB ! end of timestep values in 2D +ZLNEG(:,:,:) = 0.0_JPRB ! negative input check +PRAINFRAC_TOPRFZ(:) =0.0_JPRB ! rain fraction at top of refreezing layer +LLRAINLIQ(:) = .TRUE. ! Assume all raindrops are liquid initially + +! ---------------------------------------------------- +! Tidy up very small cloud cover or total cloud water +! ---------------------------------------------------- +DO JK=1,KLEV + DO JL=KIDIA,KFDIA + IF (ZQX(JL,JK,NCLDQL)+ZQX(JL,JK,NCLDQI)273K + !--------------------------------------------- + ZALFA=FOEDELTA(ZTP1(JL,JK)) + ZFOEEW(JL,JK)=MIN((ZALFA*FOEELIQ(ZTP1(JL,JK))+ & + & (1.0_JPRB-ZALFA)*FOEEICE(ZTP1(JL,JK)))/PAP(JL,JK),0.5_JPRB) + ZFOEEW(JL,JK)=MIN(0.5_JPRB,ZFOEEW(JL,JK)) + ZQSICE(JL,JK)=ZFOEEW(JL,JK)/(1.0_JPRB-RETV*ZFOEEW(JL,JK)) + + !---------------------------------- + ! liquid water saturation + !---------------------------------- + ZFOEELIQT(JL,JK)=MIN(FOEELIQ(ZTP1(JL,JK))/PAP(JL,JK),0.5_JPRB) + ZQSLIQ(JL,JK)=ZFOEELIQT(JL,JK) + ZQSLIQ(JL,JK)=ZQSLIQ(JL,JK)/(1.0_JPRB-RETV*ZQSLIQ(JL,JK)) + +! !---------------------------------- +! ! ice water saturation +! !---------------------------------- +! ZFOEEICET(JL,JK)=MIN(FOEEICE(ZTP1(JL,JK))/PAP(JL,JK),0.5_JPRB) +! ZQSICE(JL,JK)=ZFOEEICET(JL,JK) +! ZQSICE(JL,JK)=ZQSICE(JL,JK)/(1.0_JPRB-RETV*ZQSICE(JL,JK)) + ENDDO + +ENDDO + +DO JK=1,KLEV + DO JL=KIDIA,KFDIA + + + !------------------------------------------ + ! Ensure cloud fraction is between 0 and 1 + !------------------------------------------ + ZA(JL,JK)=MAX(0.0_JPRB,MIN(1.0_JPRB,ZA(JL,JK))) + + !------------------------------------------------------------------- + ! Calculate liq/ice fractions (no longer a diagnostic relationship) + !------------------------------------------------------------------- + ZLI(JL,JK)=ZQX(JL,JK,NCLDQL)+ZQX(JL,JK,NCLDQI) + IF (ZLI(JL,JK)>RLMIN) THEN + ZLIQFRAC(JL,JK)=ZQX(JL,JK,NCLDQL)/ZLI(JL,JK) + ZICEFRAC(JL,JK)=1.0_JPRB-ZLIQFRAC(JL,JK) + ELSE + ZLIQFRAC(JL,JK)=0.0_JPRB + ZICEFRAC(JL,JK)=0.0_JPRB + ENDIF + + ENDDO +ENDDO + +!###################################################################### +! 2. *** CONSTANTS AND PARAMETERS *** +!###################################################################### +! Calculate L in updrafts of bl-clouds +! Specify QS, P/PS for tropopause (for c2) +! And initialize variables +!------------------------------------------ + +!--------------------------------- +! Find tropopause level (ZTRPAUS) +!--------------------------------- +DO JL=KIDIA,KFDIA + ZTRPAUS(JL)=0.1_JPRB + ZPAPHD(JL)=1.0_JPRB/PAPH(JL,KLEV+1) +ENDDO +DO JK=1,KLEV-1 + DO JL=KIDIA,KFDIA + ZSIG=PAP(JL,JK)*ZPAPHD(JL) + IF (ZSIG>0.1_JPRB.AND.ZSIG<0.4_JPRB.AND.ZTP1(JL,JK)>ZTP1(JL,JK+1)) THEN + ZTRPAUS(JL)=ZSIG + ENDIF + ENDDO +ENDDO + +!----------------------------- +! Reset single level variables +!----------------------------- + +ZANEWM1(:) = 0.0_JPRB +ZDA(:) = 0.0_JPRB +ZCOVPCLR(:) = 0.0_JPRB +ZCOVPMAX(:) = 0.0_JPRB +ZCOVPTOT(:) = 0.0_JPRB +ZCLDTOPDIST(:) = 0.0_JPRB + +!###################################################################### +! 3. *** PHYSICS *** +!###################################################################### + + +!---------------------------------------------------------------------- +! START OF VERTICAL LOOP +!---------------------------------------------------------------------- + +DO JK=NCLDTOP,KLEV + +!---------------------------------------------------------------------- +! 3.0 INITIALIZE VARIABLES +!---------------------------------------------------------------------- + + !--------------------------------- + ! First guess microphysics + !--------------------------------- + DO JM=1,NCLV + DO JL=KIDIA,KFDIA + ZQXFG(JL,JM)=ZQX(JL,JK,JM) + ENDDO + ENDDO + + !--------------------------------- + ! Set KLON arrays to zero + !--------------------------------- + + ZLICLD(:) = 0.0_JPRB + ZRAINAUT(:) = 0.0_JPRB ! currently needed for diags + ZRAINACC(:) = 0.0_JPRB ! currently needed for diags + ZSNOWAUT(:) = 0.0_JPRB ! needed + ZLDEFR(:) = 0.0_JPRB + ZACUST(:) = 0.0_JPRB ! set later when needed + ZQPRETOT(:) = 0.0_JPRB + ZLFINALSUM(:)= 0.0_JPRB + + ! Required for first guess call + ZLCOND1(:) = 0.0_JPRB + ZLCOND2(:) = 0.0_JPRB + ZSUPSAT(:) = 0.0_JPRB + ZLEVAPL(:) = 0.0_JPRB + ZLEVAPI(:) = 0.0_JPRB + + !------------------------------------- + ! solvers for cloud fraction + !------------------------------------- + ZSOLAB(:) = 0.0_JPRB + ZSOLAC(:) = 0.0_JPRB + + !------------------------------------------ + ! reset matrix so missing pathways are set + !------------------------------------------ + ZSOLQB(:,:,:) = 0.0_JPRB + ZSOLQA(:,:,:) = 0.0_JPRB + + !---------------------------------- + ! reset new microphysics variables + !---------------------------------- + ZFALLSRCE(:,:) = 0.0_JPRB + ZFALLSINK(:,:) = 0.0_JPRB + ZCONVSRCE(:,:) = 0.0_JPRB + ZCONVSINK(:,:) = 0.0_JPRB + ZPSUPSATSRCE(:,:) = 0.0_JPRB + ZRATIO(:,:) = 0.0_JPRB + ZICETOT(:) = 0.0_JPRB + + DO JL=KIDIA,KFDIA + + !------------------------- + ! derived variables needed + !------------------------- + + ZDP(JL) = PAPH(JL,JK+1)-PAPH(JL,JK) ! dp + ZGDP(JL) = RG/ZDP(JL) ! g/dp + ZRHO(JL) = PAP(JL,JK)/(RD*ZTP1(JL,JK)) ! p/RT air density + + ZDTGDP(JL) = PTSPHY*ZGDP(JL) ! dt g/dp + ZRDTGDP(JL) = ZDP(JL)*(1.0_JPRB/(PTSPHY*RG)) ! 1/(dt g/dp) + + IF (JK>1) ZDTGDPF(JL) = PTSPHY*RG/(PAP(JL,JK)-PAP(JL,JK-1)) + + !------------------------------------ + ! Calculate dqs/dT correction factor + !------------------------------------ + ! Reminder: RETV=RV/RD-1 + + ! liquid + ZFACW = R5LES/((ZTP1(JL,JK)-R4LES)**2) + ZCOR = 1.0_JPRB/(1.0_JPRB-RETV*ZFOEELIQT(JL,JK)) + ZDQSLIQDT(JL) = ZFACW*ZCOR*ZQSLIQ(JL,JK) + ZCORQSLIQ(JL) = 1.0_JPRB+RALVDCP*ZDQSLIQDT(JL) + + ! ice + ZFACI = R5IES/((ZTP1(JL,JK)-R4IES)**2) + ZCOR = 1.0_JPRB/(1.0_JPRB-RETV*ZFOEEW(JL,JK)) + ZDQSICEDT(JL) = ZFACI*ZCOR*ZQSICE(JL,JK) + ZCORQSICE(JL) = 1.0_JPRB+RALSDCP*ZDQSICEDT(JL) + + ! diagnostic mixed + ZALFAW = ZFOEALFA(JL,JK) + ZALFAWM(JL) = ZALFAW + ZFAC = ZALFAW*ZFACW+(1.0_JPRB-ZALFAW)*ZFACI + ZCOR = 1.0_JPRB/(1.0_JPRB-RETV*ZFOEEWMT(JL,JK)) + ZDQSMIXDT(JL) = ZFAC*ZCOR*ZQSMIX(JL,JK) + ZCORQSMIX(JL) = 1.0_JPRB+FOELDCPM(ZTP1(JL,JK))*ZDQSMIXDT(JL) + + ! evaporation/sublimation limits + ZEVAPLIMMIX(JL) = MAX((ZQSMIX(JL,JK)-ZQX(JL,JK,NCLDQV))/ZCORQSMIX(JL),0.0_JPRB) + ZEVAPLIMLIQ(JL) = MAX((ZQSLIQ(JL,JK)-ZQX(JL,JK,NCLDQV))/ZCORQSLIQ(JL),0.0_JPRB) + ZEVAPLIMICE(JL) = MAX((ZQSICE(JL,JK)-ZQX(JL,JK,NCLDQV))/ZCORQSICE(JL),0.0_JPRB) + + !-------------------------------- + ! in-cloud consensate amount + !-------------------------------- + ZTMPA = 1.0_JPRB/MAX(ZA(JL,JK),ZEPSEC) + ZLIQCLD(JL) = ZQX(JL,JK,NCLDQL)*ZTMPA + ZICECLD(JL) = ZQX(JL,JK,NCLDQI)*ZTMPA + ZLICLD(JL) = ZLIQCLD(JL)+ZICECLD(JL) + + ENDDO + + !------------------------------------------------ + ! Evaporate very small amounts of liquid and ice + !------------------------------------------------ + DO JL=KIDIA,KFDIA + + IF (ZQX(JL,JK,NCLDQL) < RLMIN) THEN + ZSOLQA(JL,NCLDQV,NCLDQL) = ZQX(JL,JK,NCLDQL) + ZSOLQA(JL,NCLDQL,NCLDQV) = -ZQX(JL,JK,NCLDQL) + ENDIF + + IF (ZQX(JL,JK,NCLDQI) < RLMIN) THEN + ZSOLQA(JL,NCLDQV,NCLDQI) = ZQX(JL,JK,NCLDQI) + ZSOLQA(JL,NCLDQI,NCLDQV) = -ZQX(JL,JK,NCLDQI) + ENDIF + + ENDDO + + !--------------------------------------------------------------------- + ! 3.1 ICE SUPERSATURATION ADJUSTMENT + !--------------------------------------------------------------------- + ! Note that the supersaturation adjustment is made with respect to + ! liquid saturation: when T>0C + ! ice saturation: when T<0C + ! with an adjustment made to allow for ice + ! supersaturation in the clear sky + ! Note also that the KOOP factor automatically clips the supersaturation + ! to a maximum set by the liquid water saturation mixing ratio + ! important for temperatures near to but below 0C + !----------------------------------------------------------------------- + +!DIR$ NOFUSION + DO JL=KIDIA,KFDIA + + !----------------------------------- + ! 3.1.1 Supersaturation limit (from Koop) + !----------------------------------- + ! Needs to be set for all temperatures + ZFOKOOP(JL)=FOKOOP(ZTP1(JL,JK)) + ENDDO + DO JL=KIDIA,KFDIA + + IF (ZTP1(JL,JK)>=RTT .OR. NSSOPT==0) THEN + ZFAC = 1.0_JPRB + ZFACI = 1.0_JPRB + ELSE + ZFAC = ZA(JL,JK)+ZFOKOOP(JL)*(1.0_JPRB-ZA(JL,JK)) + ZFACI = PTSPHY/RKOOPTAU + ENDIF + + !------------------------------------------------------------------- + ! 3.1.2 Calculate supersaturation wrt Koop including dqs/dT + ! correction factor + ! [#Note: QSICE or QSLIQ] + !------------------------------------------------------------------- + + ! Calculate supersaturation to add to cloud + IF (ZA(JL,JK) > 1.0_JPRB-RAMIN) THEN + ZSUPSAT(JL) = MAX((ZQX(JL,JK,NCLDQV)-ZFAC*ZQSICE(JL,JK))/ZCORQSICE(JL)& + & ,0.0_JPRB) + ELSE + ! Calculate environmental humidity supersaturation + ZQP1ENV = (ZQX(JL,JK,NCLDQV) - ZA(JL,JK)*ZQSICE(JL,JK))/ & + & MAX(1.0_JPRB-ZA(JL,JK),ZEPSILON) + !& SIGN(MAX(ABS(1.0_JPRB-ZA(JL,JK)),ZEPSILON),1.0_JPRB-ZA(JL,JK)) + ZSUPSAT(JL) = MAX((1.0_JPRB-ZA(JL,JK))*(ZQP1ENV-ZFAC*ZQSICE(JL,JK))& + & /ZCORQSICE(JL),0.0_JPRB) + ENDIF + + !------------------------------------------------------------------- + ! Here the supersaturation is turned into liquid water + ! However, if the temperature is below the threshold for homogeneous + ! freezing then the supersaturation is turned instantly to ice. + !-------------------------------------------------------------------- + + IF (ZSUPSAT(JL) > ZEPSEC) THEN + + IF (ZTP1(JL,JK) > RTHOMO) THEN + ! Turn supersaturation into liquid water + ZSOLQA(JL,NCLDQL,NCLDQV) = ZSOLQA(JL,NCLDQL,NCLDQV)+ZSUPSAT(JL) + ZSOLQA(JL,NCLDQV,NCLDQL) = ZSOLQA(JL,NCLDQV,NCLDQL)-ZSUPSAT(JL) + ! Include liquid in first guess + ZQXFG(JL,NCLDQL)=ZQXFG(JL,NCLDQL)+ZSUPSAT(JL) + ELSE + ! Turn supersaturation into ice water + ZSOLQA(JL,NCLDQI,NCLDQV) = ZSOLQA(JL,NCLDQI,NCLDQV)+ZSUPSAT(JL) + ZSOLQA(JL,NCLDQV,NCLDQI) = ZSOLQA(JL,NCLDQV,NCLDQI)-ZSUPSAT(JL) + ! Add ice to first guess for deposition term + ZQXFG(JL,NCLDQI)=ZQXFG(JL,NCLDQI)+ZSUPSAT(JL) + ENDIF + + ! Increase cloud amount using RKOOPTAU timescale + ZSOLAC(JL) = (1.0_JPRB-ZA(JL,JK))*ZFACI + + ENDIF + + !------------------------------------------------------- + ! 3.1.3 Include supersaturation from previous timestep + ! (Calculated in sltENDIF semi-lagrangian LDSLPHY=T) + !------------------------------------------------------- + IF (PSUPSAT(JL,JK)>ZEPSEC) THEN + IF (ZTP1(JL,JK) > RTHOMO) THEN + ! Turn supersaturation into liquid water + ZSOLQA(JL,NCLDQL,NCLDQL) = ZSOLQA(JL,NCLDQL,NCLDQL)+PSUPSAT(JL,JK) + ZPSUPSATSRCE(JL,NCLDQL) = PSUPSAT(JL,JK) + ! Add liquid to first guess for deposition term + ZQXFG(JL,NCLDQL)=ZQXFG(JL,NCLDQL)+PSUPSAT(JL,JK) + ! Store cloud budget diagnostics if required + ELSE + ! Turn supersaturation into ice water + ZSOLQA(JL,NCLDQI,NCLDQI) = ZSOLQA(JL,NCLDQI,NCLDQI)+PSUPSAT(JL,JK) + ZPSUPSATSRCE(JL,NCLDQI) = PSUPSAT(JL,JK) + ! Add ice to first guess for deposition term + ZQXFG(JL,NCLDQI)=ZQXFG(JL,NCLDQI)+PSUPSAT(JL,JK) + ! Store cloud budget diagnostics if required + ENDIF + + ! Increase cloud amount using RKOOPTAU timescale + ZSOLAC(JL)=(1.0_JPRB-ZA(JL,JK))*ZFACI + ! Store cloud budget diagnostics if required + ENDIF + + ENDDO ! on JL + + !--------------------------------------------------------------------- + ! 3.2 DETRAINMENT FROM CONVECTION + !--------------------------------------------------------------------- + ! * Diagnostic T-ice/liq split retained for convection + ! Note: This link is now flexible and a future convection + ! scheme can detrain explicit seperate budgets of: + ! cloud water, ice, rain and snow + ! * There is no (1-ZA) multiplier term on the cloud detrainment + ! term, since is now written in mass-flux terms + ! [#Note: Should use ZFOEALFACU used in convection rather than ZFOEALFA] + !--------------------------------------------------------------------- + IF (JK < KLEV .AND. JK>=NCLDTOP) THEN + + DO JL=KIDIA,KFDIA + + PLUDE(JL,JK)=PLUDE(JL,JK)*ZDTGDP(JL) + + IF(LDCUM(JL).AND.PLUDE(JL,JK) > RLMIN.AND.PLU(JL,JK+1)> ZEPSEC) THEN + + ZSOLAC(JL)=ZSOLAC(JL)+PLUDE(JL,JK)/PLU(JL,JK+1) + ! *diagnostic temperature split* + ZALFAW = ZFOEALFA(JL,JK) + ZCONVSRCE(JL,NCLDQL) = ZALFAW*PLUDE(JL,JK) + ZCONVSRCE(JL,NCLDQI) = (1.0_JPRB-ZALFAW)*PLUDE(JL,JK) + ZSOLQA(JL,NCLDQL,NCLDQL) = ZSOLQA(JL,NCLDQL,NCLDQL)+ZCONVSRCE(JL,NCLDQL) + ZSOLQA(JL,NCLDQI,NCLDQI) = ZSOLQA(JL,NCLDQI,NCLDQI)+ZCONVSRCE(JL,NCLDQI) + + ELSE + + PLUDE(JL,JK)=0.0_JPRB + + ENDIF + ! *convective snow detrainment source + IF (LDCUM(JL)) ZSOLQA(JL,NCLDQS,NCLDQS) = ZSOLQA(JL,NCLDQS,NCLDQS) + PSNDE(JL,JK)*ZDTGDP(JL) + + ENDDO + + ENDIF ! JK NCLDTOP) THEN + + DO JL=KIDIA,KFDIA + ZMF(JL)=MAX(0.0_JPRB,(PMFU(JL,JK)+PMFD(JL,JK))*ZDTGDP(JL)) + ZACUST(JL)=ZMF(JL)*ZANEWM1(JL) + ENDDO + + DO JM=1,NCLV + IF (.NOT.LLFALL(JM).AND.IPHASE(JM)>0) THEN + DO JL=KIDIA,KFDIA + ZLCUST(JL,JM)=ZMF(JL)*ZQXNM1(JL,JM) + ! record total flux for enthalpy budget: + ZCONVSRCE(JL,JM)=ZCONVSRCE(JL,JM)+ZLCUST(JL,JM) + ENDDO + ENDIF + ENDDO + + ! Now have to work out how much liquid evaporates at arrival point + ! since there is no prognostic memory for in-cloud humidity, i.e. + ! we always assume cloud is saturated. + + DO JL=KIDIA,KFDIA + ZDTDP=ZRDCP*0.5_JPRB*(ZTP1(JL,JK-1)+ZTP1(JL,JK))/PAPH(JL,JK) + ZDTFORC = ZDTDP*(PAP(JL,JK)-PAP(JL,JK-1)) + ![#Note: Diagnostic mixed phase should be replaced below] + ZDQS(JL)=ZANEWM1(JL)*ZDTFORC*ZDQSMIXDT(JL) + ENDDO + + DO JM=1,NCLV + IF (.NOT.LLFALL(JM).AND.IPHASE(JM)>0) THEN + DO JL=KIDIA,KFDIA + ZLFINAL=MAX(0.0_JPRB,ZLCUST(JL,JM)-ZDQS(JL)) !lim to zero + ! no supersaturation allowed incloud ---V + ZEVAP=MIN((ZLCUST(JL,JM)-ZLFINAL),ZEVAPLIMMIX(JL)) +! ZEVAP=0.0_JPRB + ZLFINAL=ZLCUST(JL,JM)-ZEVAP + ZLFINALSUM(JL)=ZLFINALSUM(JL)+ZLFINAL ! sum + + ZSOLQA(JL,JM,JM) = ZSOLQA(JL,JM,JM)+ZLCUST(JL,JM) ! whole sum + ZSOLQA(JL,NCLDQV,JM) = ZSOLQA(JL,NCLDQV,JM)+ZEVAP + ZSOLQA(JL,JM,NCLDQV) = ZSOLQA(JL,JM,NCLDQV)-ZEVAP + ENDDO + ENDIF + ENDDO + + ! Reset the cloud contribution if no cloud water survives to this level: + DO JL=KIDIA,KFDIA + IF (ZLFINALSUM(JL)NCLDTOP + + !--------------------------------------------------------------------- + ! Subsidence sink of cloud to the layer below + ! (Implicit - re. CFL limit on convective mass flux) + !--------------------------------------------------------------------- + + DO JL=KIDIA,KFDIA + + IF(JK 0 .AND. PLUDE(JL,JK) > ZEPSEC)& + & ZLDIFDT(JL)=RCLDIFF_CONVI*ZLDIFDT(JL) + ENDDO + + ! At the moment, works on mixed RH profile and partitioned ice/liq fraction + ! so that it is similar to previous scheme + ! Should apply RHw for liquid cloud and RHi for ice cloud separately + DO JL=KIDIA,KFDIA + IF(ZLI(JL,JK) > ZEPSEC) THEN + ! Calculate environmental humidity +! ZQE=(ZQX(JL,JK,NCLDQV)-ZA(JL,JK)*ZQSMIX(JL,JK))/& +! & MAX(ZEPSEC,1.0_JPRB-ZA(JL,JK)) +! ZE=ZLDIFDT(JL)*MAX(ZQSMIX(JL,JK)-ZQE,0.0_JPRB) + ZE=ZLDIFDT(JL)*MAX(ZQSMIX(JL,JK)-ZQX(JL,JK,NCLDQV),0.0_JPRB) + ZLEROS=ZA(JL,JK)*ZE + ZLEROS=MIN(ZLEROS,ZEVAPLIMMIX(JL)) + ZLEROS=MIN(ZLEROS,ZLI(JL,JK)) + ZAEROS=ZLEROS/ZLICLD(JL) !if linear term + + ! Erosion is -ve LINEAR in L,A + ZSOLAC(JL)=ZSOLAC(JL)-ZAEROS !linear + + ZSOLQA(JL,NCLDQV,NCLDQL) = ZSOLQA(JL,NCLDQV,NCLDQL)+ZLIQFRAC(JL,JK)*ZLEROS + ZSOLQA(JL,NCLDQL,NCLDQV) = ZSOLQA(JL,NCLDQL,NCLDQV)-ZLIQFRAC(JL,JK)*ZLEROS + ZSOLQA(JL,NCLDQV,NCLDQI) = ZSOLQA(JL,NCLDQV,NCLDQI)+ZICEFRAC(JL,JK)*ZLEROS + ZSOLQA(JL,NCLDQI,NCLDQV) = ZSOLQA(JL,NCLDQI,NCLDQV)-ZICEFRAC(JL,JK)*ZLEROS + + ENDIF + ENDDO + + !---------------------------------------------------------------------- + ! 3.4 CONDENSATION/EVAPORATION DUE TO DQSAT/DT + !---------------------------------------------------------------------- + ! calculate dqs/dt + ! Note: For the separate prognostic Qi and Ql, one would ideally use + ! Qsat/DT wrt liquid/Koop here, since the physics is that new clouds + ! forms by liquid droplets [liq] or when aqueous aerosols [Koop] form. + ! These would then instantaneous freeze if T<-38C or lead to ice growth + ! by deposition in warmer mixed phase clouds. However, since we do + ! not have a separate prognostic equation for in-cloud humidity or a + ! statistical scheme approach in place, the depositional growth of ice + ! in the mixed phase can not be modelled and we resort to supersaturation + ! wrt ice instanteously converting to ice over one timestep + ! (see Tompkins et al. QJRMS 2007 for details) + ! Thus for the initial implementation the diagnostic mixed phase is + ! retained for the moment, and the level of approximation noted. + !---------------------------------------------------------------------- + + DO JL=KIDIA,KFDIA + ZDTDP = ZRDCP*ZTP1(JL,JK)/PAP(JL,JK) + ZDPMXDT = ZDP(JL)*ZQTMST + ZMFDN = 0.0_JPRB + IF(JK < KLEV) ZMFDN=PMFU(JL,JK+1)+PMFD(JL,JK+1) + ZWTOT = PVERVEL(JL,JK)+0.5_JPRB*RG*(PMFU(JL,JK)+PMFD(JL,JK)+ZMFDN) + ZWTOT = MIN(ZDPMXDT,MAX(-ZDPMXDT,ZWTOT)) + ZZZDT = PHRSW(JL,JK)+PHRLW(JL,JK) + ZDTDIAB = MIN(ZDPMXDT*ZDTDP,MAX(-ZDPMXDT*ZDTDP,ZZZDT))& + & *PTSPHY+RALFDCP*ZLDEFR(JL) +! Note: ZLDEFR should be set to the difference between the mixed phase functions +! in the convection and cloud scheme, but this is not calculated, so is zero and +! the functions must be the same + ZDTFORC = ZDTDP*ZWTOT*PTSPHY+ZDTDIAB + ZQOLD(JL) = ZQSMIX(JL,JK) + ZTOLD(JL) = ZTP1(JL,JK) + ZTP1(JL,JK) = ZTP1(JL,JK)+ZDTFORC + ZTP1(JL,JK) = MAX(ZTP1(JL,JK),160.0_JPRB) + LLFLAG(JL) = .TRUE. + ENDDO + + ! Formerly a call to CUADJTQ(..., ICALL=5) + DO JL=KIDIA,KFDIA + ZQP = 1.0_JPRB/PAP(JL,JK) + ZQSAT = FOEEWM(ZTP1(JL,JK))*ZQP + ZQSAT = MIN(0.5_JPRB,ZQSAT) + ZCOR = 1.0_JPRB/(1.0_JPRB-RETV *ZQSAT) + ZQSAT = ZQSAT*ZCOR + ZCOND = (ZQSMIX(JL,JK)-ZQSAT)/(1.0_JPRB+ZQSAT*ZCOR*FOEDEM(ZTP1(JL,JK))) + ZTP1(JL,JK) = ZTP1(JL,JK)+FOELDCPM(ZTP1(JL,JK))*ZCOND + ZQSMIX(JL,JK) = ZQSMIX(JL,JK)-ZCOND + ZQSAT = FOEEWM(ZTP1(JL,JK))*ZQP + ZQSAT = MIN(0.5_JPRB,ZQSAT) + ZCOR = 1.0_JPRB/(1.0_JPRB-RETV *ZQSAT) + ZQSAT = ZQSAT*ZCOR + ZCOND1= (ZQSMIX(JL,JK)-ZQSAT)/(1.0_JPRB+ZQSAT*ZCOR*FOEDEM(ZTP1(JL,JK))) + ZTP1(JL,JK) = ZTP1(JL,JK)+FOELDCPM(ZTP1(JL,JK))*ZCOND1 + ZQSMIX(JL,JK) = ZQSMIX(JL,JK)-ZCOND1 + ENDDO + + DO JL=KIDIA,KFDIA + ZDQS(JL) = ZQSMIX(JL,JK)-ZQOLD(JL) + ZQSMIX(JL,JK) = ZQOLD(JL) + ZTP1(JL,JK) = ZTOLD(JL) + ENDDO + + !---------------------------------------------------------------------- + ! 3.4a ZDQS(JL) > 0: EVAPORATION OF CLOUDS + ! ---------------------------------------------------------------------- + ! Erosion term is LINEAR in L + ! Changed to be uniform distribution in cloud region + + DO JL=KIDIA,KFDIA + + ! Previous function based on DELTA DISTRIBUTION in cloud: + IF (ZDQS(JL) > 0.0_JPRB) THEN +! If subsidence evaporation term is turned off, then need to use updated +! liquid and cloud here? +! ZLEVAP = MAX(ZA(JL,JK)+ZACUST(JL),1.0_JPRB)*MIN(ZDQS(JL),ZLICLD(JL)+ZLFINALSUM(JL)) + ZLEVAP = ZA(JL,JK)*MIN(ZDQS(JL),ZLICLD(JL)) + ZLEVAP = MIN(ZLEVAP,ZEVAPLIMMIX(JL)) + ZLEVAP = MIN(ZLEVAP,MAX(ZQSMIX(JL,JK)-ZQX(JL,JK,NCLDQV),0.0_JPRB)) + + ! For first guess call + ZLEVAPL(JL) = ZLIQFRAC(JL,JK)*ZLEVAP + ZLEVAPI(JL) = ZICEFRAC(JL,JK)*ZLEVAP + + ZSOLQA(JL,NCLDQV,NCLDQL) = ZSOLQA(JL,NCLDQV,NCLDQL)+ZLIQFRAC(JL,JK)*ZLEVAP + ZSOLQA(JL,NCLDQL,NCLDQV) = ZSOLQA(JL,NCLDQL,NCLDQV)-ZLIQFRAC(JL,JK)*ZLEVAP + + ZSOLQA(JL,NCLDQV,NCLDQI) = ZSOLQA(JL,NCLDQV,NCLDQI)+ZICEFRAC(JL,JK)*ZLEVAP + ZSOLQA(JL,NCLDQI,NCLDQV) = ZSOLQA(JL,NCLDQI,NCLDQV)-ZICEFRAC(JL,JK)*ZLEVAP + + ENDIF + + ENDDO + + !---------------------------------------------------------------------- + ! 3.4b ZDQS(JL) < 0: FORMATION OF CLOUDS + !---------------------------------------------------------------------- + ! (1) Increase of cloud water in existing clouds + DO JL=KIDIA,KFDIA + IF(ZA(JL,JK) > ZEPSEC.AND.ZDQS(JL) <= -RLMIN) THEN + + ZLCOND1(JL)=MAX(-ZDQS(JL),0.0_JPRB) !new limiter + +!old limiter (significantly improves upper tropospheric humidity rms) + IF(ZA(JL,JK) > 0.99_JPRB) THEN + ZCOR=1.0_JPRB/(1.0_JPRB-RETV*ZQSMIX(JL,JK)) + ZCDMAX=(ZQX(JL,JK,NCLDQV)-ZQSMIX(JL,JK))/& + & (1.0_JPRB+ZCOR*ZQSMIX(JL,JK)*FOEDEM(ZTP1(JL,JK))) + ELSE + ZCDMAX=(ZQX(JL,JK,NCLDQV)-ZA(JL,JK)*ZQSMIX(JL,JK))/ZA(JL,JK) + ENDIF + ZLCOND1(JL)=MAX(MIN(ZLCOND1(JL),ZCDMAX),0.0_JPRB) +! end old limiter + + ZLCOND1(JL)=ZA(JL,JK)*ZLCOND1(JL) + IF(ZLCOND1(JL) < RLMIN) ZLCOND1(JL)=0.0_JPRB + + !------------------------------------------------------------------------- + ! All increase goes into liquid unless so cold cloud homogeneously freezes + ! Include new liquid formation in first guess value, otherwise liquid + ! remains at cold temperatures until next timestep. + !------------------------------------------------------------------------- + IF (ZTP1(JL,JK)>RTHOMO) THEN + ZSOLQA(JL,NCLDQL,NCLDQV)=ZSOLQA(JL,NCLDQL,NCLDQV)+ZLCOND1(JL) + ZSOLQA(JL,NCLDQV,NCLDQL)=ZSOLQA(JL,NCLDQV,NCLDQL)-ZLCOND1(JL) + ZQXFG(JL,NCLDQL)=ZQXFG(JL,NCLDQL)+ZLCOND1(JL) + ELSE + ZSOLQA(JL,NCLDQI,NCLDQV)=ZSOLQA(JL,NCLDQI,NCLDQV)+ZLCOND1(JL) + ZSOLQA(JL,NCLDQV,NCLDQI)=ZSOLQA(JL,NCLDQV,NCLDQI)-ZLCOND1(JL) + ZQXFG(JL,NCLDQI)=ZQXFG(JL,NCLDQI)+ZLCOND1(JL) + ENDIF + ENDIF + ENDDO + + ! (2) Generation of new clouds (da/dt>0) + + DO JL=KIDIA,KFDIA + + IF(ZDQS(JL) <= -RLMIN .AND. ZA(JL,JK)<1.0_JPRB-ZEPSEC) THEN + + !--------------------------- + ! Critical relative humidity + !--------------------------- + ZRHC=RAMID + ZSIGK=PAP(JL,JK)/PAPH(JL,KLEV+1) + ! Increase RHcrit to 1.0 towards the surface (eta>0.8) + IF(ZSIGK > 0.8_JPRB) THEN + ZRHC=RAMID+(1.0_JPRB-RAMID)*((ZSIGK-0.8_JPRB)/0.2_JPRB)**2 + ENDIF + +! Commented out for CY37R1 to reduce humidity in high trop and strat +! ! Increase RHcrit to 1.0 towards the tropopause (trop-0.2) and above +! ZBOTT=ZTRPAUS(JL)+0.2_JPRB +! IF(ZSIGK < ZBOTT) THEN +! ZRHC=RAMID+(1.0_JPRB-RAMID)*MIN(((ZBOTT-ZSIGK)/0.2_JPRB)**2,1.0_JPRB) +! ENDIF + + !--------------------------- + ! Supersaturation options + !--------------------------- + IF (NSSOPT==0) THEN + ! No scheme + ZQE=(ZQX(JL,JK,NCLDQV)-ZA(JL,JK)*ZQSICE(JL,JK))/& + & MAX(ZEPSEC,1.0_JPRB-ZA(JL,JK)) + ZQE=MAX(0.0_JPRB,ZQE) + ELSEIF (NSSOPT==1) THEN + ! Tompkins + ZQE=(ZQX(JL,JK,NCLDQV)-ZA(JL,JK)*ZQSICE(JL,JK))/& + & MAX(ZEPSEC,1.0_JPRB-ZA(JL,JK)) + ZQE=MAX(0.0_JPRB,ZQE) + ELSEIF (NSSOPT==2) THEN + ! Lohmann and Karcher + ZQE=ZQX(JL,JK,NCLDQV) + ELSEIF (NSSOPT==3) THEN + ! Gierens + ZQE=ZQX(JL,JK,NCLDQV)+ZLI(JL,JK) + ENDIF + + IF (ZTP1(JL,JK)>=RTT .OR. NSSOPT==0) THEN + ! No ice supersaturation allowed + ZFAC=1.0_JPRB + ELSE + ! Ice supersaturation + ZFAC=ZFOKOOP(JL) + ENDIF + + IF(ZQE >= ZRHC*ZQSICE(JL,JK)*ZFAC.AND.ZQERTHOMO) THEN + ZSOLQA(JL,NCLDQL,NCLDQV)=ZSOLQA(JL,NCLDQL,NCLDQV)+ZLCOND2(JL) + ZSOLQA(JL,NCLDQV,NCLDQL)=ZSOLQA(JL,NCLDQV,NCLDQL)-ZLCOND2(JL) + ZQXFG(JL,NCLDQL)=ZQXFG(JL,NCLDQL)+ZLCOND2(JL) + ELSE ! homogeneous freezing + ZSOLQA(JL,NCLDQI,NCLDQV)=ZSOLQA(JL,NCLDQI,NCLDQV)+ZLCOND2(JL) + ZSOLQA(JL,NCLDQV,NCLDQI)=ZSOLQA(JL,NCLDQV,NCLDQI)-ZLCOND2(JL) + ZQXFG(JL,NCLDQI)=ZQXFG(JL,NCLDQI)+ZLCOND2(JL) + ENDIF + + ENDIF + ENDIF + ENDDO + + !---------------------------------------------------------------------- + ! 3.7 Growth of ice by vapour deposition + !---------------------------------------------------------------------- + ! Following Rotstayn et al. 2001: + ! does not use the ice nuclei number from cloudaer.F90 + ! but rather a simple Meyers et al. 1992 form based on the + ! supersaturation and assuming clouds are saturated with + ! respect to liquid water (well mixed), (or Koop adjustment) + ! Growth considered as sink of liquid water if present so + ! Bergeron-Findeisen adjustment in autoconversion term no longer needed + !---------------------------------------------------------------------- + + !-------------------------------------------------------- + !- + !- Ice deposition following Rotstayn et al. (2001) + !- (monodisperse ice particle size distribution) + !- + !-------------------------------------------------------- + IF (IDEPICE == 1) THEN + + DO JL=KIDIA,KFDIA + + !-------------------------------------------------------------- + ! Calculate distance from cloud top + ! defined by cloudy layer below a layer with cloud frac <0.01 + ! ZDZ = ZDP(JL)/(ZRHO(JL)*RG) + !-------------------------------------------------------------- + + IF (ZA(JL,JK-1) < RCLDTOPCF .AND. ZA(JL,JK) >= RCLDTOPCF) THEN + ZCLDTOPDIST(JL) = 0.0_JPRB + ELSE + ZCLDTOPDIST(JL) = ZCLDTOPDIST(JL) + ZDP(JL)/(ZRHO(JL)*RG) + ENDIF + + !-------------------------------------------------------------- + ! only treat depositional growth if liquid present. due to fact + ! that can not model ice growth from vapour without additional + ! in-cloud water vapour variable + !-------------------------------------------------------------- + IF (ZTP1(JL,JK)RLMIN) THEN ! T<273K + + ZVPICE=FOEEICE(ZTP1(JL,JK))*RV/RD + ZVPLIQ=ZVPICE*ZFOKOOP(JL) + ZICENUCLEI(JL)=1000.0_JPRB*EXP(12.96_JPRB*(ZVPLIQ-ZVPICE)/ZVPLIQ-0.639_JPRB) + + !------------------------------------------------ + ! 2.4e-2 is conductivity of air + ! 8.8 = 700**1/3 = density of ice to the third + !------------------------------------------------ + ZADD=RLSTT*(RLSTT/(RV*ZTP1(JL,JK))-1.0_JPRB)/(2.4E-2_JPRB*ZTP1(JL,JK)) + ZBDD=RV*ZTP1(JL,JK)*PAP(JL,JK)/(2.21_JPRB*ZVPICE) + ZCVDS=7.8_JPRB*(ZICENUCLEI(JL)/ZRHO(JL))**0.666_JPRB*(ZVPLIQ-ZVPICE) / & + & (8.87_JPRB*(ZADD+ZBDD)*ZVPICE) + + !----------------------------------------------------- + ! RICEINIT=1.E-12_JPRB is initial mass of ice particle + !----------------------------------------------------- + ZICE0=MAX(ZICECLD(JL), ZICENUCLEI(JL)*RICEINIT/ZRHO(JL)) + + !------------------ + ! new value of ice: + !------------------ + ZINEW=(0.666_JPRB*ZCVDS*PTSPHY+ZICE0**0.666_JPRB)**1.5_JPRB + + !--------------------------- + ! grid-mean deposition rate: + !--------------------------- + ZDEPOS=MAX(ZA(JL,JK)*(ZINEW-ZICE0),0.0_JPRB) + + !-------------------------------------------------------------------- + ! Limit deposition to liquid water amount + ! If liquid is all frozen, ice would use up reservoir of water + ! vapour in excess of ice saturation mixing ratio - However this + ! can not be represented without a in-cloud humidity variable. Using + ! the grid-mean humidity would imply a large artificial horizontal + ! flux from the clear sky to the cloudy area. We thus rely on the + ! supersaturation check to clean up any remaining supersaturation + !-------------------------------------------------------------------- + ZDEPOS=MIN(ZDEPOS,ZQXFG(JL,NCLDQL)) ! limit to liquid water amount + + !-------------------------------------------------------------------- + ! At top of cloud, reduce deposition rate near cloud top to account for + ! small scale turbulent processes, limited ice nucleation and ice fallout + !-------------------------------------------------------------------- +! ZDEPOS = ZDEPOS*MIN(RDEPLIQREFRATE+ZCLDTOPDIST(JL)/RDEPLIQREFDEPTH,1.0_JPRB) + ! Change to include dependence on ice nuclei concentration + ! to increase deposition rate with decreasing temperatures + ZINFACTOR = MIN(ZICENUCLEI(JL)/15000._JPRB, 1.0_JPRB) + ZDEPOS = ZDEPOS*MIN(ZINFACTOR + (1.0_JPRB-ZINFACTOR)* & + & (RDEPLIQREFRATE+ZCLDTOPDIST(JL)/RDEPLIQREFDEPTH),1.0_JPRB) + + !-------------- + ! add to matrix + !-------------- + ZSOLQA(JL,NCLDQI,NCLDQL)=ZSOLQA(JL,NCLDQI,NCLDQL)+ZDEPOS + ZSOLQA(JL,NCLDQL,NCLDQI)=ZSOLQA(JL,NCLDQL,NCLDQI)-ZDEPOS + ZQXFG(JL,NCLDQI)=ZQXFG(JL,NCLDQI)+ZDEPOS + ZQXFG(JL,NCLDQL)=ZQXFG(JL,NCLDQL)-ZDEPOS + + ENDIF + ENDDO + + !-------------------------------------------------------- + !- + !- Ice deposition assuming ice PSD + !- + !-------------------------------------------------------- + ELSEIF (IDEPICE == 2) THEN + + DO JL=KIDIA,KFDIA + + !-------------------------------------------------------------- + ! Calculate distance from cloud top + ! defined by cloudy layer below a layer with cloud frac <0.01 + ! ZDZ = ZDP(JL)/(ZRHO(JL)*RG) + !-------------------------------------------------------------- + + IF (ZA(JL,JK-1) < RCLDTOPCF .AND. ZA(JL,JK) >= RCLDTOPCF) THEN + ZCLDTOPDIST(JL) = 0.0_JPRB + ELSE + ZCLDTOPDIST(JL) = ZCLDTOPDIST(JL) + ZDP(JL)/(ZRHO(JL)*RG) + ENDIF + + !-------------------------------------------------------------- + ! only treat depositional growth if liquid present. due to fact + ! that can not model ice growth from vapour without additional + ! in-cloud water vapour variable + !-------------------------------------------------------------- + IF (ZTP1(JL,JK)RLMIN) THEN ! T<273K + + ZVPICE = FOEEICE(ZTP1(JL,JK))*RV/RD + ZVPLIQ = ZVPICE*ZFOKOOP(JL) + ZICENUCLEI(JL)=1000.0_JPRB*EXP(12.96_JPRB*(ZVPLIQ-ZVPICE)/ZVPLIQ-0.639_JPRB) + + !----------------------------------------------------- + ! RICEINIT=1.E-12_JPRB is initial mass of ice particle + !----------------------------------------------------- + ZICE0=MAX(ZICECLD(JL), ZICENUCLEI(JL)*RICEINIT/ZRHO(JL)) + + ! Particle size distribution + ZTCG = 1.0_JPRB + ZFACX1I = 1.0_JPRB + + ZAPLUSB = RCL_APB1*ZVPICE-RCL_APB2*ZVPICE*ZTP1(JL,JK)+ & + & PAP(JL,JK)*RCL_APB3*ZTP1(JL,JK)**3._JPRB + ZCORRFAC = (1.0_JPRB/ZRHO(JL))**0.5_JPRB + ZCORRFAC2 = ((ZTP1(JL,JK)/273.0_JPRB)**1.5_JPRB) & + & *(393.0_JPRB/(ZTP1(JL,JK)+120.0_JPRB)) + + ZPR02 = ZRHO(JL)*ZICE0*RCL_CONST1I/(ZTCG*ZFACX1I) + + ZTERM1 = (ZVPLIQ-ZVPICE)*ZTP1(JL,JK)**2.0_JPRB*ZVPICE*ZCORRFAC2*ZTCG* & + & RCL_CONST2I*ZFACX1I/(ZRHO(JL)*ZAPLUSB*ZVPICE) + ZTERM2 = 0.65_JPRB*RCL_CONST6I*ZPR02**RCL_CONST4I+RCL_CONST3I & + & *ZCORRFAC**0.5_JPRB*ZRHO(JL)**0.5_JPRB & + & *ZPR02**RCL_CONST5I/ZCORRFAC2**0.5_JPRB + + ZDEPOS = MAX(ZA(JL,JK)*ZTERM1*ZTERM2*PTSPHY,0.0_JPRB) + + !-------------------------------------------------------------------- + ! Limit deposition to liquid water amount + ! If liquid is all frozen, ice would use up reservoir of water + ! vapour in excess of ice saturation mixing ratio - However this + ! can not be represented without a in-cloud humidity variable. Using + ! the grid-mean humidity would imply a large artificial horizontal + ! flux from the clear sky to the cloudy area. We thus rely on the + ! supersaturation check to clean up any remaining supersaturation + !-------------------------------------------------------------------- + ZDEPOS=MIN(ZDEPOS,ZQXFG(JL,NCLDQL)) ! limit to liquid water amount + + !-------------------------------------------------------------------- + ! At top of cloud, reduce deposition rate near cloud top to account for + ! small scale turbulent processes, limited ice nucleation and ice fallout + !-------------------------------------------------------------------- + ! Change to include dependence on ice nuclei concentration + ! to increase deposition rate with decreasing temperatures + ZINFACTOR = MIN(ZICENUCLEI(JL)/15000._JPRB, 1.0_JPRB) + ZDEPOS = ZDEPOS*MIN(ZINFACTOR + (1.0_JPRB-ZINFACTOR)* & + & (RDEPLIQREFRATE+ZCLDTOPDIST(JL)/RDEPLIQREFDEPTH),1.0_JPRB) + + !-------------- + ! add to matrix + !-------------- + ZSOLQA(JL,NCLDQI,NCLDQL) = ZSOLQA(JL,NCLDQI,NCLDQL)+ZDEPOS + ZSOLQA(JL,NCLDQL,NCLDQI) = ZSOLQA(JL,NCLDQL,NCLDQI)-ZDEPOS + ZQXFG(JL,NCLDQI) = ZQXFG(JL,NCLDQI)+ZDEPOS + ZQXFG(JL,NCLDQL) = ZQXFG(JL,NCLDQL)-ZDEPOS + ENDIF + ENDDO + + ENDIF ! on IDEPICE + + !###################################################################### + ! 4 *** PRECIPITATION PROCESSES *** + !###################################################################### + + !---------------------------------- + ! revise in-cloud consensate amount + !---------------------------------- + DO JL=KIDIA,KFDIA + ZTMPA = 1.0_JPRB/MAX(ZA(JL,JK),ZEPSEC) + ZLIQCLD(JL) = ZQXFG(JL,NCLDQL)*ZTMPA + ZICECLD(JL) = ZQXFG(JL,NCLDQI)*ZTMPA + ZLICLD(JL) = ZLIQCLD(JL)+ZICECLD(JL) + ENDDO + + !---------------------------------------------------------------------- + ! 4.2 SEDIMENTATION/FALLING OF *ALL* MICROPHYSICAL SPECIES + ! now that rain, snow, graupel species are prognostic + ! the precipitation flux can be defined directly level by level + ! There is no vertical memory required from the flux variable + !---------------------------------------------------------------------- + + DO JM = 1,NCLV + IF (LLFALL(JM) .OR. JM == NCLDQI) THEN + DO JL=KIDIA,KFDIA + !------------------------ + ! source from layer above + !------------------------ + IF (JK > NCLDTOP) THEN + ZFALLSRCE(JL,JM) = ZPFPLSX(JL,JK,JM)*ZDTGDP(JL) + ZSOLQA(JL,JM,JM) = ZSOLQA(JL,JM,JM)+ZFALLSRCE(JL,JM) + ZQXFG(JL,JM) = ZQXFG(JL,JM)+ZFALLSRCE(JL,JM) + ! use first guess precip----------V + ZQPRETOT(JL) = ZQPRETOT(JL)+ZQXFG(JL,JM) + ENDIF + !------------------------------------------------- + ! sink to next layer, constant fall speed + !------------------------------------------------- + ! if aerosol effect then override + ! note that for T>233K this is the same as above. + IF (LAERICESED .AND. JM == NCLDQI) THEN + ZRE_ICE=PRE_ICE(JL,JK) + ! The exponent value is from + ! Morrison et al. JAS 2005 Appendix + ZVQX(NCLDQI) = 0.002_JPRB*ZRE_ICE**1.0_JPRB + ENDIF + ZFALL=ZVQX(JM)*ZRHO(JL) + !------------------------------------------------- + ! modified by Heymsfield and Iaquinta JAS 2000 + !------------------------------------------------- + ! ZFALL = ZFALL*((PAP(JL,JK)*RICEHI1)**(-0.178_JPRB)) & + ! &*((ZTP1(JL,JK)*RICEHI2)**(-0.394_JPRB)) + + ZFALLSINK(JL,JM)=ZDTGDP(JL)*ZFALL + ! Cloud budget diagnostic stored at end as implicit + ENDDO ! jl + ENDIF ! LLFALL + ENDDO ! jm + + !--------------------------------------------------------------- + ! Precip cover overlap using MAX-RAN Overlap + ! Since precipitation is now prognostic we must + ! 1) apply an arbitrary minimum coverage (0.3) if precip>0 + ! 2) abandon the 2-flux clr/cld treatment + ! 3) Thus, since we have no memory of the clear sky precip + ! fraction, we mimic the previous method by reducing + ! ZCOVPTOT(JL), which has the memory, proportionally with + ! the precip evaporation rate, taking cloud fraction + ! into account + ! #3 above leads to much smoother vertical profiles of + ! precipitation fraction than the Klein-Jakob scheme which + ! monotonically increases precip fraction and then resets + ! it to zero in a step function once clear-sky precip reaches + ! zero. + !--------------------------------------------------------------- + DO JL=KIDIA,KFDIA + IF (ZQPRETOT(JL)>ZEPSEC) THEN + ZCOVPTOT(JL) = 1.0_JPRB - ((1.0_JPRB-ZCOVPTOT(JL))*& + & (1.0_JPRB - MAX(ZA(JL,JK),ZA(JL,JK-1)))/& + & (1.0_JPRB - MIN(ZA(JL,JK-1),1.0_JPRB-1.E-06_JPRB)) ) + ZCOVPTOT(JL) = MAX(ZCOVPTOT(JL),RCOVPMIN) + ZCOVPCLR(JL) = MAX(0.0_JPRB,ZCOVPTOT(JL)-ZA(JL,JK)) ! clear sky proportion + ZRAINCLD(JL) = ZQXFG(JL,NCLDQR)/ZCOVPTOT(JL) + ZSNOWCLD(JL) = ZQXFG(JL,NCLDQS)/ZCOVPTOT(JL) + ZCOVPMAX(JL) = MAX(ZCOVPTOT(JL),ZCOVPMAX(JL)) + ELSE + ZRAINCLD(JL) = 0.0_JPRB + ZSNOWCLD(JL) = 0.0_JPRB + ZCOVPTOT(JL) = 0.0_JPRB ! no flux - reset cover + ZCOVPCLR(JL) = 0.0_JPRB ! reset clear sky proportion + ZCOVPMAX(JL) = 0.0_JPRB ! reset max cover for ZZRH calc + ENDIF + ENDDO + + !---------------------------------------------------------------------- + ! 4.3a AUTOCONVERSION TO SNOW + !---------------------------------------------------------------------- + DO JL=KIDIA,KFDIA + + IF(ZTP1(JL,JK) <= RTT) THEN + !----------------------------------------------------- + ! Snow Autoconversion rate follow Lin et al. 1983 + !----------------------------------------------------- + IF (ZICECLD(JL)>ZEPSEC) THEN + + ZZCO=PTSPHY*RSNOWLIN1*EXP(RSNOWLIN2*(ZTP1(JL,JK)-RTT)) + + IF (LAERICEAUTO) THEN + ZLCRIT=PICRIT_AER(JL,JK) + ! 0.3 = N**0.333 with N=0.027 + ZZCO=ZZCO*(RNICE/PNICE(JL,JK))**0.333_JPRB + ELSE + ZLCRIT=RLCRITSNOW + ENDIF + + ZSNOWAUT(JL)=ZZCO*(1.0_JPRB-EXP(-(ZICECLD(JL)/ZLCRIT)**2)) + ZSOLQB(JL,NCLDQS,NCLDQI)=ZSOLQB(JL,NCLDQS,NCLDQI)+ZSNOWAUT(JL) + + ENDIF + ENDIF + + !---------------------------------------------------------------------- + ! 4.3b AUTOCONVERSION WARM CLOUDS + ! Collection and accretion will require separate treatment + ! but for now we keep this simple treatment + !---------------------------------------------------------------------- + + IF (ZLIQCLD(JL)>ZEPSEC) THEN + + !-------------------------------------------------------- + !- + !- Warm-rain process follow Sundqvist (1989) + !- + !-------------------------------------------------------- + IF (IWARMRAIN == 1) THEN + + ZZCO=RKCONV*PTSPHY + + IF (LAERLIQAUTOLSP) THEN + ZLCRIT=PLCRIT_AER(JL,JK) + ! 0.3 = N**0.333 with N=125 cm-3 + ZZCO=ZZCO*(RCCN/PCCN(JL,JK))**0.333_JPRB + ELSE + ! Modify autoconversion threshold dependent on: + ! land (polluted, high CCN, smaller droplets, higher threshold) + ! sea (clean, low CCN, larger droplets, lower threshold) + IF (PLSM(JL) > 0.5_JPRB) THEN + ZLCRIT = RCLCRIT_LAND ! land + ELSE + ZLCRIT = RCLCRIT_SEA ! ocean + ENDIF + ENDIF + + !------------------------------------------------------------------ + ! Parameters for cloud collection by rain and snow. + ! Note that with new prognostic variable it is now possible + ! to REPLACE this with an explicit collection parametrization + !------------------------------------------------------------------ + ZPRECIP=(ZPFPLSX(JL,JK,NCLDQS)+ZPFPLSX(JL,JK,NCLDQR))/MAX(ZEPSEC,ZCOVPTOT(JL)) + ZCFPR=1.0_JPRB + RPRC1*SQRT(MAX(ZPRECIP,0.0_JPRB)) +! ZCFPR=1.0_JPRB + RPRC1*SQRT(MAX(ZPRECIP,0.0_JPRB))*& +! &ZCOVPTOT(JL)/(MAX(ZA(JL,JK),ZEPSEC)) + + IF (LAERLIQCOLL) THEN + ! 5.0 = N**0.333 with N=125 cm-3 + ZCFPR=ZCFPR*(RCCN/PCCN(JL,JK))**0.333_JPRB + ENDIF + + ZZCO=ZZCO*ZCFPR + ZLCRIT=ZLCRIT/MAX(ZCFPR,ZEPSEC) + + IF(ZLIQCLD(JL)/ZLCRIT < 20.0_JPRB )THEN ! Security for exp for some compilers + ZRAINAUT(JL)=ZZCO*(1.0_JPRB-EXP(-(ZLIQCLD(JL)/ZLCRIT)**2)) + ELSE + ZRAINAUT(JL)=ZZCO + ENDIF + + ! rain freezes instantly + IF(ZTP1(JL,JK) <= RTT) THEN + ZSOLQB(JL,NCLDQS,NCLDQL)=ZSOLQB(JL,NCLDQS,NCLDQL)+ZRAINAUT(JL) + ELSE + ZSOLQB(JL,NCLDQR,NCLDQL)=ZSOLQB(JL,NCLDQR,NCLDQL)+ZRAINAUT(JL) + ENDIF + + !-------------------------------------------------------- + !- + !- Warm-rain process follow Khairoutdinov and Kogan (2000) + !- + !-------------------------------------------------------- + ELSEIF (IWARMRAIN == 2) THEN + + IF (PLSM(JL) > 0.5_JPRB) THEN ! land + ZCONST = RCL_KK_CLOUD_NUM_LAND + ZLCRIT = RCLCRIT_LAND + ELSE ! ocean + ZCONST = RCL_KK_CLOUD_NUM_SEA + ZLCRIT = RCLCRIT_SEA + ENDIF + + IF (ZLIQCLD(JL) > ZLCRIT) THEN + + ZRAINAUT(JL) = 1.5_JPRB*ZA(JL,JK)*PTSPHY* & + & RCL_KKAau * ZLIQCLD(JL)**RCL_KKBauq * ZCONST**RCL_KKBaun + + ZRAINAUT(JL) = MIN(ZRAINAUT(JL),ZQXFG(JL,NCLDQL)) + IF (ZRAINAUT(JL) < ZEPSEC) ZRAINAUT(JL) = 0.0_JPRB + + ZRAINACC(JL) = 2.0_JPRB*ZA(JL,JK)*PTSPHY* & + & RCL_KKAac * (ZLIQCLD(JL)*ZRAINCLD(JL))**RCL_KKBac + + ZRAINACC(JL) = MIN(ZRAINACC(JL),ZQXFG(JL,NCLDQL)) + IF (ZRAINACC(JL) < ZEPSEC) ZRAINACC(JL) = 0.0_JPRB + + ELSE + ZRAINAUT(JL) = 0.0_JPRB + ZRAINACC(JL) = 0.0_JPRB + ENDIF + + ! If temperature < 0, then autoconversion produces snow rather than rain + ! Explicit + IF(ZTP1(JL,JK) <= RTT) THEN + ZSOLQA(JL,NCLDQS,NCLDQL)=ZSOLQA(JL,NCLDQS,NCLDQL)+ZRAINAUT(JL) + ZSOLQA(JL,NCLDQS,NCLDQL)=ZSOLQA(JL,NCLDQS,NCLDQL)+ZRAINACC(JL) + ZSOLQA(JL,NCLDQL,NCLDQS)=ZSOLQA(JL,NCLDQL,NCLDQS)-ZRAINAUT(JL) + ZSOLQA(JL,NCLDQL,NCLDQS)=ZSOLQA(JL,NCLDQL,NCLDQS)-ZRAINACC(JL) + ELSE + ZSOLQA(JL,NCLDQR,NCLDQL)=ZSOLQA(JL,NCLDQR,NCLDQL)+ZRAINAUT(JL) + ZSOLQA(JL,NCLDQR,NCLDQL)=ZSOLQA(JL,NCLDQR,NCLDQL)+ZRAINACC(JL) + ZSOLQA(JL,NCLDQL,NCLDQR)=ZSOLQA(JL,NCLDQL,NCLDQR)-ZRAINAUT(JL) + ZSOLQA(JL,NCLDQL,NCLDQR)=ZSOLQA(JL,NCLDQL,NCLDQR)-ZRAINACC(JL) + ENDIF + + ENDIF ! on IWARMRAIN + + ENDIF ! on ZLIQCLD > ZEPSEC + ENDDO + + + !---------------------------------------------------------------------- + ! RIMING - COLLECTION OF CLOUD LIQUID DROPS BY SNOW AND ICE + ! only active if T<0degC and supercooled liquid water is present + ! AND if not Sundquist autoconversion (as this includes riming) + !---------------------------------------------------------------------- + IF (IWARMRAIN > 1) THEN + + DO JL=KIDIA,KFDIA + IF(ZTP1(JL,JK) <= RTT .AND. ZLIQCLD(JL)>ZEPSEC) THEN + + ! Fallspeed air density correction + ZFALLCORR = (RDENSREF/ZRHO(JL))**0.4_JPRB + + !------------------------------------------------------------------ + ! Riming of snow by cloud water - implicit in lwc + !------------------------------------------------------------------ + IF (ZSNOWCLD(JL)>ZEPSEC .AND. ZCOVPTOT(JL)>0.01_JPRB) THEN + + ! Calculate riming term + ! Factor of liq water taken out because implicit + ZSNOWRIME(JL) = 0.3_JPRB*ZCOVPTOT(JL)*PTSPHY*RCL_CONST7S*ZFALLCORR & + & *(ZRHO(JL)*ZSNOWCLD(JL)*RCL_CONST1S)**RCL_CONST8S + + ! Limit snow riming term + ZSNOWRIME(JL)=MIN(ZSNOWRIME(JL),1.0_JPRB) + + ZSOLQB(JL,NCLDQS,NCLDQL) = ZSOLQB(JL,NCLDQS,NCLDQL) + ZSNOWRIME(JL) + + ENDIF + + !------------------------------------------------------------------ + ! Riming of ice by cloud water - implicit in lwc + ! NOT YET ACTIVE + !------------------------------------------------------------------ +! IF (ZICECLD(JL)>ZEPSEC .AND. ZA(JL,JK)>0.01_JPRB) THEN +! +! ! Calculate riming term +! ! Factor of liq water taken out because implicit +! ZSNOWRIME(JL) = ZA(JL,JK)*PTSPHY*RCL_CONST7S*ZFALLCORR & +! & *(ZRHO(JL)*ZICECLD(JL)*RCL_CONST1S)**RCL_CONST8S +! +! ! Limit ice riming term +! ZSNOWRIME(JL)=MIN(ZSNOWRIME(JL),1.0_JPRB) +! +! ZSOLQB(JL,NCLDQI,NCLDQL) = ZSOLQB(JL,NCLDQI,NCLDQL) + ZSNOWRIME(JL) +! +! ENDIF + ENDIF + ENDDO + + ENDIF ! on IWARMRAIN > 1 + + + !---------------------------------------------------------------------- + ! 4.4a MELTING OF SNOW and ICE + ! with new implicit solver this also has to treat snow or ice + ! precipitating from the level above... i.e. local ice AND flux. + ! in situ ice and snow: could arise from LS advection or warming + ! falling ice and snow: arrives by precipitation process + !---------------------------------------------------------------------- + DO JL=KIDIA,KFDIA + + ZICETOT(JL)=ZQXFG(JL,NCLDQI)+ZQXFG(JL,NCLDQS) + ZMELTMAX(JL) = 0.0_JPRB + + ! If there are frozen hydrometeors present and dry-bulb temperature > 0degC + IF(ZICETOT(JL) > ZEPSEC .AND. ZTP1(JL,JK) > RTT) THEN + + ! Calculate subsaturation + ZSUBSAT = MAX(ZQSICE(JL,JK)-ZQX(JL,JK,NCLDQV),0.0_JPRB) + + ! Calculate difference between dry-bulb (ZTP1) and the temperature + ! at which the wet-bulb=0degC (RTT-ZSUBSAT*....) using an approx. + ! Melting only occurs if the wet-bulb temperature >0 + ! i.e. warming of ice particle due to melting > cooling + ! due to evaporation. + ZTDMTW0 = ZTP1(JL,JK)-RTT-ZSUBSAT* & + & (ZTW1+ZTW2*(PAP(JL,JK)-ZTW3)-ZTW4*(ZTP1(JL,JK)-ZTW5)) + ! Not implicit yet... + ! Ensure ZCONS1 is positive so that ZMELTMAX=0 if ZTDMTW0<0 + ZCONS1 = ABS(PTSPHY*(1.0_JPRB+0.5_JPRB*ZTDMTW0)/RTAUMEL) + ZMELTMAX(JL) = MAX(ZTDMTW0*ZCONS1*ZRLDCP,0.0_JPRB) + ENDIF + ENDDO + + ! Loop over frozen hydrometeors (ice, snow) + DO JM=1,NCLV + IF (IPHASE(JM) == 2) THEN + JN = IMELT(JM) + DO JL=KIDIA,KFDIA + IF(ZMELTMAX(JL)>ZEPSEC .AND. ZICETOT(JL)>ZEPSEC) THEN + ! Apply melting in same proportion as frozen hydrometeor fractions + ZALFA = ZQXFG(JL,JM)/ZICETOT(JL) + ZMELT = MIN(ZQXFG(JL,JM),ZALFA*ZMELTMAX(JL)) + ! needed in first guess + ! This implies that zqpretot has to be recalculated below + ! since is not conserved here if ice falls and liquid doesn't + ZQXFG(JL,JM) = ZQXFG(JL,JM)-ZMELT + ZQXFG(JL,JN) = ZQXFG(JL,JN)+ZMELT + ZSOLQA(JL,JN,JM) = ZSOLQA(JL,JN,JM)+ZMELT + ZSOLQA(JL,JM,JN) = ZSOLQA(JL,JM,JN)-ZMELT + ENDIF + ENDDO + ENDIF + ENDDO + + !---------------------------------------------------------------------- + ! 4.4b FREEZING of RAIN + !---------------------------------------------------------------------- + DO JL=KIDIA,KFDIA + + ! If rain present + IF (ZQX(JL,JK,NCLDQR) > ZEPSEC) THEN + + IF (ZTP1(JL,JK) <= RTT .AND. ZTP1(JL,JK-1) > RTT) THEN + ! Base of melting layer/top of refreezing layer so + ! store rain/snow fraction for precip type diagnosis + ! If mostly rain, then supercooled rain slow to freeze + ! otherwise faster to freeze (snow or ice pellets) + ZQPRETOT(JL) = MAX(ZQX(JL,JK,NCLDQS)+ZQX(JL,JK,NCLDQR),ZEPSEC) + PRAINFRAC_TOPRFZ(JL) = ZQX(JL,JK,NCLDQR)/ZQPRETOT(JL) + IF (PRAINFRAC_TOPRFZ(JL) > 0.8) THEN + LLRAINLIQ(JL) = .True. + ELSE + LLRAINLIQ(JL) = .False. + ENDIF + ENDIF + + ! If temperature less than zero + IF (ZTP1(JL,JK) < RTT) THEN + + IF (LLRAINLIQ(JL)) THEN + + ! Majority of raindrops completely melted + ! Refreezing is by slow heterogeneous freezing + + ! Slope of rain particle size distribution + ZLAMBDA = (RCL_FAC1/(ZRHO(JL)*ZQX(JL,JK,NCLDQR)))**RCL_FAC2 + + ! Calculate freezing rate based on Bigg(1953) and Wisner(1972) + ZTEMP = RCL_FZRAB * (ZTP1(JL,JK)-RTT) + ZFRZ = PTSPHY * (RCL_CONST5R/ZRHO(JL)) * (EXP(ZTEMP)-1._JPRB) & + & * ZLAMBDA**RCL_CONST6R + ZFRZMAX(JL) = MAX(ZFRZ,0.0_JPRB) + + ELSE + + ! Majority of raindrops only partially melted + ! Refreeze with a shorter timescale (reverse of melting...for now) + + ZCONS1 = ABS(PTSPHY*(1.0_JPRB+0.5_JPRB*(RTT-ZTP1(JL,JK)))/RTAUMEL) + ZFRZMAX(JL) = MAX((RTT-ZTP1(JL,JK))*ZCONS1*ZRLDCP,0.0_JPRB) + + ENDIF + + IF(ZFRZMAX(JL)>ZEPSEC) THEN + ZFRZ = MIN(ZQX(JL,JK,NCLDQR),ZFRZMAX(JL)) + ZSOLQA(JL,NCLDQS,NCLDQR) = ZSOLQA(JL,NCLDQS,NCLDQR)+ZFRZ + ZSOLQA(JL,NCLDQR,NCLDQS) = ZSOLQA(JL,NCLDQR,NCLDQS)-ZFRZ + ENDIF + ENDIF + + ENDIF + + ENDDO + + !---------------------------------------------------------------------- + ! 4.4c FREEZING of LIQUID + !---------------------------------------------------------------------- + DO JL=KIDIA,KFDIA + ! not implicit yet... + ZFRZMAX(JL)=MAX((RTHOMO-ZTP1(JL,JK))*ZRLDCP,0.0_JPRB) + ENDDO + + JM = NCLDQL + JN = IMELT(JM) + DO JL=KIDIA,KFDIA + IF(ZFRZMAX(JL)>ZEPSEC .AND. ZQXFG(JL,JM)>ZEPSEC) THEN + ZFRZ = MIN(ZQXFG(JL,JM),ZFRZMAX(JL)) + ZSOLQA(JL,JN,JM) = ZSOLQA(JL,JN,JM)+ZFRZ + ZSOLQA(JL,JM,JN) = ZSOLQA(JL,JM,JN)-ZFRZ + ENDIF + ENDDO + + !---------------------------------------------------------------------- + ! 4.5 EVAPORATION OF RAIN/SNOW + !---------------------------------------------------------------------- + + !---------------------------------------- + ! Rain evaporation scheme from Sundquist + !---------------------------------------- + IF (IEVAPRAIN == 1) THEN + + ! Rain + + DO JL=KIDIA,KFDIA + + ZZRH=RPRECRHMAX+(1.0_JPRB-RPRECRHMAX)*ZCOVPMAX(JL)/MAX(ZEPSEC,1.0_JPRB-ZA(JL,JK)) + ZZRH=MIN(MAX(ZZRH,RPRECRHMAX),1.0_JPRB) + + ZQE=(ZQX(JL,JK,NCLDQV)-ZA(JL,JK)*ZQSLIQ(JL,JK))/& + & MAX(ZEPSEC,1.0_JPRB-ZA(JL,JK)) + !--------------------------------------------- + ! humidity in moistest ZCOVPCLR part of domain + !--------------------------------------------- + ZQE=MAX(0.0_JPRB,MIN(ZQE,ZQSLIQ(JL,JK))) + LLO1=ZCOVPCLR(JL)>ZEPSEC .AND. & + & ZQXFG(JL,NCLDQR)>ZEPSEC .AND. & + & ZQE0.8) + !IF(ZSIGK > 0.8_JPRB) THEN + ! ZRHC=RAMID+(1.0_JPRB-RAMID)*((ZSIGK-0.8_JPRB)/0.2_JPRB)**2 + !ENDIF + !ZZRH = MIN(ZRHC,ZZRH) + + ! Further limit RH for rain evaporation to 80% (RHcrit in free troposphere) + ZZRH = MIN(0.8_JPRB,ZZRH) + + ZQE=MAX(0.0_JPRB,MIN(ZQX(JL,JK,NCLDQV),ZQSLIQ(JL,JK))) + + LLO1=ZCOVPCLR(JL)>ZEPSEC .AND. & + & ZQXFG(JL,NCLDQR)>ZEPSEC .AND. & + & ZQEZEPSEC .AND. & + & ZQXFG(JL,NCLDQS)>ZEPSEC .AND. & + & ZQEZEPSEC .AND. & + & ZQX(JL,JK,NCLDQS)>ZEPSEC .AND. & + & ZQE SELF%IN_VARS_3D_REAL64(1)%PTR + SELF%PICRIT_AER => SELF%IN_VARS_3D_REAL64(2)%PTR + SELF%PRE_ICE => SELF%IN_VARS_3D_REAL64(3)%PTR + SELF%PCCN => SELF%IN_VARS_3D_REAL64(4)%PTR + SELF%PNICE => SELF%IN_VARS_3D_REAL64(5)%PTR + SELF%PT => SELF%IN_VARS_3D_REAL64(6)%PTR + SELF%PQ => SELF%IN_VARS_3D_REAL64(7)%PTR + SELF%PVFA => SELF%IN_VARS_3D_REAL64(8)%PTR + SELF%PVFL => SELF%IN_VARS_3D_REAL64(9)%PTR + SELF%PVFI => SELF%IN_VARS_3D_REAL64(10)%PTR + SELF%PDYNA => SELF%IN_VARS_3D_REAL64(11)%PTR + SELF%PDYNL => SELF%IN_VARS_3D_REAL64(12)%PTR + SELF%PDYNI => SELF%IN_VARS_3D_REAL64(13)%PTR + SELF%PHRSW => SELF%IN_VARS_3D_REAL64(14)%PTR + SELF%PHRLW => SELF%IN_VARS_3D_REAL64(15)%PTR + SELF%PVERVEL => SELF%IN_VARS_3D_REAL64(16)%PTR + SELF%PAP => SELF%IN_VARS_3D_REAL64(17)%PTR + SELF%PLU => SELF%IN_VARS_3D_REAL64(18)%PTR + SELF%PLUDE => SELF%IN_VARS_3D_REAL64(19)%PTR + SELF%PSNDE => SELF%IN_VARS_3D_REAL64(20)%PTR + SELF%PMFU => SELF%IN_VARS_3D_REAL64(21)%PTR + SELF%PMFD => SELF%IN_VARS_3D_REAL64(22)%PTR + SELF%PA => SELF%IN_VARS_3D_REAL64(23)%PTR + SELF%PSUPSAT => SELF%IN_VARS_3D_REAL64(24)%PTR + + FIELD = FSET%FIELD("PLSM") + CALL FIELD%DATA(SELF%PLSM) + FIELD = FSET%FIELD("LDCUM") + CALL FIELD%DATA(SELF%LDCUM) + FIELD = FSET%FIELD("KTYPE") + CALL FIELD%DATA(SELF%KTYPE) + FIELD = FSET%FIELD("PAPH") + CALL FIELD%DATA(SELF%PAPH) + FIELD = FSET%FIELD("PEXTRA") + CALL FIELD%DATA(SELF%PEXTRA) + FIELD = FSET%FIELD("PCLV") + CALL FIELD%DATA(SELF%PCLV) + + DO IVAR = 1, SIZE(IN_VAR_NAMES) + CALL LOADVAR_ATLAS(FSET, TRIM(IN_VAR_NAMES(IVAR)), KLON, NGPTOTG) + ENDDO + + FIELD = FSPACE%CREATE_FIELD(NAME='TENDENCY_CML', KIND=ATLAS_REAL(JPRB), VARIABLES=3+NCLV) + CALL FIELD%DATA(SELF%B_CML) + CALL FSET%ADD(FIELD) + FIELD = FSPACE%CREATE_FIELD(NAME='TENDENCY_TMP', KIND=ATLAS_REAL(JPRB), VARIABLES=3+NCLV) + CALL FIELD%DATA(SELF%B_TMP) + CALL FSET%ADD(FIELD) + FIELD = FSPACE%CREATE_FIELD(NAME='TENDENCY_LOC', KIND=ATLAS_REAL(JPRB), VARIABLES=3+NCLV) + CALL FIELD%DATA(SELF%B_LOC) + CALL FSET%ADD(FIELD) + + ! The STATE_TYPE arrays are tricky, as the AOSOA layout needs to be expictly + ! unrolled at every step, and we rely on dirty hackery to do this. + CALL LOADSTATE_ATLAS(FSET, 'TENDENCY_CML', SELF%TENDENCY_CML, KLON, NGPTOTG) + CALL LOADSTATE_ATLAS(FSET, 'TENDENCY_TMP', SELF%TENDENCY_TMP, KLON, NGPTOTG) + ALLOCATE(SELF%TENDENCY_LOC(SELF%NBLOCKS)) + !$OMP PARALLEL DO DEFAULT(SHARED), PRIVATE(B) schedule(runtime) + DO B=1, SELF%NBLOCKS + SELF%TENDENCY_LOC(B)%T => SELF%B_LOC(:,:,1,B) + SELF%TENDENCY_LOC(B)%A => SELF%B_LOC(:,:,2,B) + SELF%TENDENCY_LOC(B)%Q => SELF%B_LOC(:,:,3,B) + SELF%TENDENCY_LOC(B)%CLD => SELF%B_LOC(:,:,4:,B) + END DO + !$OMP END PARALLEL DO + + ! Output fields are simply allocated and zero'd + DO IVAR = 1, SIZE(OUT_VAR_NAMES) - 2 + CALL FSET%ADD(FSPACE%CREATE_FIELD(NAME=TRIM(OUT_VAR_NAMES(IVAR)), KIND=ATLAS_REAL(JPRB), LEVELS=SELF%KLEV+1)) + ENDDO + CALL FSET%ADD(FSPACE%CREATE_FIELD(NAME="PCOVPTOT", KIND=ATLAS_REAL(JPRB))) + CALL FSET%ADD(FSPACE%CREATE_FIELD(NAME="PRAINFRAC_TOPRFZ", KIND=ATLAS_REAL(JPRB), LEVELS=0)) + + DO IVAR = 1, SIZE(OUT_VAR_NAMES) - 1 + FIELD = FSET%FIELD(TRIM(OUT_VAR_NAMES(IVAR))) + CALL FIELD%DATA(SELF%OUT_VARS_3D_REAL64(IVAR)%PTR) + !$OMP PARALLEL DO DEFAULT(SHARED), PRIVATE(B) schedule(runtime) + DO B=1, SELF%NBLOCKS + SELF%OUT_VARS_3D_REAL64(IVAR)%PTR(:,:,B) = 0.0_JPRB + END DO + !$omp end parallel do + ENDDO + + SELF%PFSQLF => SELF%OUT_VARS_3D_REAL64(1)%PTR + SELF%PFSQIF => SELF%OUT_VARS_3D_REAL64(2)%PTR + SELF%PFCQLNG => SELF%OUT_VARS_3D_REAL64(3)%PTR + SELF%PFCQNNG => SELF%OUT_VARS_3D_REAL64(4)%PTR + SELF%PFSQRF => SELF%OUT_VARS_3D_REAL64(5)%PTR + SELF%PFSQSF => SELF%OUT_VARS_3D_REAL64(6)%PTR + SELF%PFCQRNG => SELF%OUT_VARS_3D_REAL64(7)%PTR + SELF%PFCQSNG => SELF%OUT_VARS_3D_REAL64(8)%PTR + SELF%PFSQLTUR => SELF%OUT_VARS_3D_REAL64(9)%PTR + SELF%PFSQITUR => SELF%OUT_VARS_3D_REAL64(10)%PTR + SELF%PFPLSL => SELF%OUT_VARS_3D_REAL64(11)%PTR + SELF%PFPLSN => SELF%OUT_VARS_3D_REAL64(12)%PTR + SELF%PFHPSL => SELF%OUT_VARS_3D_REAL64(13)%PTR + SELF%PFHPSN => SELF%OUT_VARS_3D_REAL64(14)%PTR + SELF%PCOVPTOT => SELF%OUT_VARS_3D_REAL64(15)%PTR + + FIELD = FSET%FIELD("PRAINFRAC_TOPRFZ") + CALL FIELD%DATA(SELF%PRAINFRAC_TOPRFZ) + !$OMP PARALLEL DO DEFAULT(SHARED), PRIVATE(B) schedule(runtime) + DO B=1, SELF%NBLOCKS + SELF%PRAINFRAC_TOPRFZ(:,B) = 0.0_JPRB + END DO + !$OMP END PARALLEL DO + + ! Initialize global parameters from the input file + CALL LOAD_SCALAR('PTSPHY', SELF%PTSPHY) + CALL LOAD_SCALAR('LDSLPHY', SELF%LDSLPHY) + CALL LOAD_SCALAR('LDMAINCALL', SELF%LDMAINCALL) + CALL YOMCST_LOAD_PARAMETERS() + CALL YOETHF_LOAD_PARAMETERS() + CALL YRECLDP_LOAD_PARAMETERS() + CALL YREPHLI_LOAD_PARAMETERS() + + CALL INPUT_FINALIZE() + + END SUBROUTINE CLOUDSC_GLOBAL_ATLAS_STATE_LOAD + + SUBROUTINE CLOUDSC_GLOBAL_ATLAS_STATE_VALIDATE(SELF, NGPTOT, NGPTOTG) + ! Validate the correctness of output against reference data + CLASS(CLOUDSC_GLOBAL_ATLAS_STATE) :: SELF + INTEGER(KIND=JPIM), INTENT(IN) :: NGPTOT + INTEGER(KIND=JPIM), INTENT(IN), OPTIONAL :: NGPTOTG + + INTEGER(KIND=JPIM) :: KLON, IVAR + + CALL INPUT_INITIALIZE(NAME='reference') + CALL LOAD_SCALAR('KLON', KLON) + CALL INPUT_FINALIZE() + + ! Write variable validation header + IF (IRANK == 0) THEN + print '(1X,A20,1X,A3,5(1X,A20))', & + & 'Variable','Dim', 'MinValue','MaxValue','AbsMaxErr','AvgAbsErr/GP','MaxRelErr-%' + END IF + + + ! Actual variable validation + CALL VALIDATEVAR_ATLAS(FSET, 'PLUDE', KLON, NGPTOTG) + DO IVAR = 1, SIZE(OUT_VAR_NAMES) + CALL VALIDATEVAR_ATLAS(FSET, OUT_VAR_NAMES(IVAR), KLON, NGPTOTG) + ENDDO + CALL VALIDATESTATE_ATLAS(FSET, 'TENDENCY_LOC', KLON, NGPTOTG) + + END SUBROUTINE CLOUDSC_GLOBAL_ATLAS_STATE_VALIDATE + +END MODULE CLOUDSC_GLOBAL_ATLAS_STATE_MOD diff --git a/src/cloudsc_fortran_atlas/dwarf_cloudsc_atlas.F90 b/src/cloudsc_fortran_atlas/dwarf_cloudsc_atlas.F90 new file mode 100644 index 00000000..2504766c --- /dev/null +++ b/src/cloudsc_fortran_atlas/dwarf_cloudsc_atlas.F90 @@ -0,0 +1,105 @@ +! (C) Copyright 1988- ECMWF. +! +! This software is licensed under the terms of the Apache Licence Version 2.0 +! which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +! +! In applying this licence, ECMWF does not waive the privileges and immunities +! granted to it by virtue of its status as an intergovernmental organisation +! nor does it submit to any jurisdiction. + +PROGRAM DWARF_CLOUDSC + +USE PARKIND1, ONLY: JPIM, JPIB +USE CLOUDSC_MPI_MOD, ONLY: CLOUDSC_MPI_INIT, CLOUDSC_MPI_END, NUMPROC, IRANK +USE CLOUDSC_GLOBAL_ATLAS_STATE_MOD, ONLY: CLOUDSC_GLOBAL_ATLAS_STATE +USE CLOUDSC_DRIVER_MOD, ONLY: CLOUDSC_DRIVER +USE EC_PMON_MOD, ONLY: EC_PMON + +USE ATLAS_MODULE +USE, INTRINSIC :: ISO_C_BINDING + +IMPLICIT NONE + +CHARACTER(LEN=20) :: CLARG +INTEGER(KIND=JPIM) :: IARGS, LENARG, JARG, I + +INTEGER(KIND=JPIM) :: NUMOMP = 1 ! Number of OpenMP threads for this run +INTEGER(KIND=JPIM) :: NGPTOTG = 16384 ! Number of grid points (as read from command line) +INTEGER(KIND=JPIM) :: NPROMA = 32 ! NPROMA blocking factor (currently active) +INTEGER(KIND=JPIM) :: NGPTOT ! Local number of grid points + +TYPE(CLOUDSC_GLOBAL_ATLAS_STATE) :: GLOBAL_ATLAS_STATE + +INTEGER(KIND=JPIB) :: ENERGY, POWER +CHARACTER(LEN=1) :: CLEC_PMON + +CALL GET_ENVIRONMENT_VARIABLE('EC_PMON', CLEC_PMON) +IF (CLEC_PMON == '1') THEN + CALL EC_PMON(ENERGY, POWER) + print *, "EC_PMON:: Initial (idle) power: ", POWER +END IF + +IARGS = COMMAND_ARGUMENT_COUNT() + +! Get the number of OpenMP threads to use for the benchmark +if (IARGS >= 1) then + CALL GET_COMMAND_ARGUMENT(1, CLARG, LENARG) + READ(CLARG(1:LENARG),*) NUMOMP +end if + +! Initialize MPI environment +CALL ATLAS_LIBRARY%INITIALISE() +CALL CLOUDSC_MPI_INIT(NUMOMP) + +! Get total number of grid points (NGPTOT) with which to run the benchmark +IF (IARGS >= 2) THEN + CALL GET_COMMAND_ARGUMENT(2, CLARG, LENARG) + READ(CLARG(1:LENARG),*) NGPTOTG +END IF + +! Determine local number of grid points +NGPTOT = (NGPTOTG - 1) / NUMPROC + 1 +if (IRANK == NUMPROC - 1) then + NGPTOT = NGPTOTG - (NUMPROC - 1) * NGPTOT +end if + +! Get the block size (NPROMA) for which to run the benchmark +IF (IARGS >= 3) THEN + CALL GET_COMMAND_ARGUMENT(3, CLARG, LENARG) + READ(CLARG(1:LENARG),*) NPROMA +ENDIF + +! TODO: Create a global global memory state from serialized input data +CALL GLOBAL_ATLAS_STATE%LOAD(NPROMA, NGPTOT, NGPTOTG) + +! Call the driver to perform the parallel loop over our kernel +CALL CLOUDSC_DRIVER(NUMOMP, NPROMA, GLOBAL_ATLAS_STATE%KLEV, NGPTOT, NGPTOTG, & + & GLOBAL_ATLAS_STATE%KFLDX, GLOBAL_ATLAS_STATE%PTSPHY, & + & GLOBAL_ATLAS_STATE%PT, GLOBAL_ATLAS_STATE%PQ, & + & GLOBAL_ATLAS_STATE%TENDENCY_CML, GLOBAL_ATLAS_STATE%TENDENCY_TMP, GLOBAL_ATLAS_STATE%TENDENCY_LOC, & + & GLOBAL_ATLAS_STATE%PVFA, GLOBAL_ATLAS_STATE%PVFL, GLOBAL_ATLAS_STATE%PVFI, & + & GLOBAL_ATLAS_STATE%PDYNA, GLOBAL_ATLAS_STATE%PDYNL, GLOBAL_ATLAS_STATE%PDYNI, & + & GLOBAL_ATLAS_STATE%PHRSW, GLOBAL_ATLAS_STATE%PHRLW, & + & GLOBAL_ATLAS_STATE%PVERVEL, GLOBAL_ATLAS_STATE%PAP, GLOBAL_ATLAS_STATE%PAPH, & + & GLOBAL_ATLAS_STATE%PLSM, GLOBAL_ATLAS_STATE%LDCUM, GLOBAL_ATLAS_STATE%KTYPE, & + & GLOBAL_ATLAS_STATE%PLU, GLOBAL_ATLAS_STATE%PLUDE, GLOBAL_ATLAS_STATE%PSNDE, & + & GLOBAL_ATLAS_STATE%PMFU, GLOBAL_ATLAS_STATE%PMFD, & + & GLOBAL_ATLAS_STATE%PA, GLOBAL_ATLAS_STATE%PCLV, GLOBAL_ATLAS_STATE%PSUPSAT,& + & GLOBAL_ATLAS_STATE%PLCRIT_AER, GLOBAL_ATLAS_STATE%PICRIT_AER, GLOBAL_ATLAS_STATE%PRE_ICE, & + & GLOBAL_ATLAS_STATE%PCCN, GLOBAL_ATLAS_STATE%PNICE,& + & GLOBAL_ATLAS_STATE%PCOVPTOT, GLOBAL_ATLAS_STATE%PRAINFRAC_TOPRFZ, & + & GLOBAL_ATLAS_STATE%PFSQLF, GLOBAL_ATLAS_STATE%PFSQIF , GLOBAL_ATLAS_STATE%PFCQNNG, GLOBAL_ATLAS_STATE%PFCQLNG, & + & GLOBAL_ATLAS_STATE%PFSQRF, GLOBAL_ATLAS_STATE%PFSQSF , GLOBAL_ATLAS_STATE%PFCQRNG, GLOBAL_ATLAS_STATE%PFCQSNG, & + & GLOBAL_ATLAS_STATE%PFSQLTUR, GLOBAL_ATLAS_STATE%PFSQITUR, & + & GLOBAL_ATLAS_STATE%PFPLSL, GLOBAL_ATLAS_STATE%PFPLSN, GLOBAL_ATLAS_STATE%PFHPSL, GLOBAL_ATLAS_STATE%PFHPSN & + & ) + +! Validate the output against serialized reference data +!CALL GLOBAL_ATLAS_STATE%VALIDATE(NGPTOT, NGPTOTG) + +CALL ATLAS_LIBRARY%FINALISE() + +! Tear down MPI environment +CALL CLOUDSC_MPI_END() + +END PROGRAM DWARF_CLOUDSC diff --git a/src/cloudsc_fortran_atlas/expand_atlas_mod.F90 b/src/cloudsc_fortran_atlas/expand_atlas_mod.F90 new file mode 100644 index 00000000..1037f9c1 --- /dev/null +++ b/src/cloudsc_fortran_atlas/expand_atlas_mod.F90 @@ -0,0 +1,151 @@ +! (C) Copyright 1988- ECMWF. +! +! This software is licensed under the terms of the Apache Licence Version 2.0 +! which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +! +! In applying this licence, ECMWF does not waive the privileges and immunities +! granted to it by virtue of its status as an intergovernmental organisation +! nor does it submit to any jurisdiction. + +module expand_atlas_mod + use atlas_module + use atlas_fieldset_module + use atlas_functionspace_blockstructuredcolumns_module + + use parkind1 , only : jpim, jprb + use yomphyder, only : state_type + + use cloudsc_mpi_mod, only : irank, numproc + use file_io_mod, only: input_initialize, load_scalar, load_array + use expand_mod, only: get_offsets, expand + + use, intrinsic :: iso_c_binding, only : c_int, c_double + + implicit none + +contains + + subroutine loadvar_atlas(fset, name, nlon, ngptotg) + ! Load into the local memory buffer and expand to global field + type(atlas_fieldset), intent(inout) :: fset + character(len=*), intent(in) :: name + integer(kind=jpim), intent(in) :: nlon + integer(kind=jpim), intent(in), optional :: ngptotg + + integer(kind=jpim) :: start, end, size, nlev, nproma, ngptot, nblocks, ndim, frank + type(atlas_field) :: field + real(kind=jprb), allocatable :: buffer_r1(:), buffer_r2(:,:), buffer_r3(:,:,:) + integer(kind=jpim), allocatable :: buffer_i1(:) + logical, allocatable :: buffer_l1(:) + real(c_double), pointer :: field_r1(:,:), field_r2(:,:,:), field_r3(:,:,:,:) + integer(c_int), pointer :: field_i1(:,:) + logical, pointer :: field_l1(:,:) + type(atlas_functionspace_blockstructuredcolumns) :: fspace + logical :: lfield, rfield, ifield + + field = fset%field(name) + frank = field%rank() + lfield = (name == "LDCUM") + ifield = (name == "KTYPE") + rfield = ((.not. lfield) .and. (.not. ifield)) + + fspace = field%functionspace() + nlev = field%levels() + nproma = fspace%block_size(1) + ngptot = fspace%size() + nblocks = fspace%nblks() + + if (frank == 2) then + call get_offsets(start, end, size, nlon, 1, 1, ngptot, ngptotg) + if (rfield) then + allocate(buffer_r1(size)) + call field%data(field_r1) + call load_array(name, start, end, size, nlon, buffer_r1) + call expand(buffer_r1, field_r1, size, nproma, ngptot, nblocks) + deallocate(buffer_r1) + else if (lfield) then + allocate(buffer_l1(size)) + call field%data(field_l1) + call load_array(name, start, end, size, nlon, buffer_l1) + call expand(buffer_l1, field_l1, size, nproma, ngptot, nblocks) + deallocate(buffer_l1) + else + allocate(buffer_i1(size)) + call field%data(field_i1) + call load_array(name, start, end, size, nlon, buffer_i1) + call expand(buffer_i1, field_i1, size, nproma, ngptot, nblocks) + deallocate(buffer_i1) + endif + else if (frank == 3) then + call get_offsets(start, end, size, nlon, 1, nlev, ngptot, ngptotg) + if (rfield) then + call field%data(field_r2) + allocate(buffer_r2(size, nlev)) + call load_array(name, start, end, size, nlon, nlev, buffer_r2) + call expand(buffer_r2, field_r2, size, nproma, nlev, ngptot, nblocks) + deallocate(buffer_r2) + endif + else if (frank == 4) then + ndim = field%shape(3) + call get_offsets(start, end, size, nlon, ndim, nlev, ngptot, ngptotg) + if (rfield) then + call field%data(field_r3) + allocate(buffer_r3(size, nlev, ndim)) + call load_array(name, start, end, size, nlon, nlev, ndim, buffer_r3) + call expand(buffer_r3, field_r3, size, nproma, nlev, ndim, ngptot, nblocks) + deallocate(buffer_r3) + endif + endif + end subroutine loadvar_atlas + + subroutine loadstate_atlas(fset, name, state, nlon, ngptotg) + ! Load into the local memory buffer and expand to global field + type(atlas_fieldset), intent(inout) :: fset + character(len=*) :: name + type(state_type), allocatable, intent(inout) :: state(:) + integer(kind=jpim), intent(in) :: nlon + integer(kind=jpim), intent(in), optional :: ngptotg + + integer :: b + integer(kind=jpim) :: start, end, size, nlev, nproma, ngptot, nblocks, ndim + type(atlas_field) :: field + type(atlas_functionspace_blockstructuredcolumns) :: fspace + + real(kind=jprb), allocatable :: buffer(:,:,:) + real(c_double), pointer :: field_r3(:,:,:,:) + + field = fset%field(name) + fspace = field%functionspace() + nlev = field%levels() + nproma = fspace%block_size(1) + ngptot = fspace%size() + nblocks = fspace%nblks() + ndim = field%shape(3) - 3 + + call get_offsets(start, end, size, nlon, ndim, nlev, ngptot, ngptotg) + allocate(buffer(size, nlev, 3+ndim)) + if (.not. allocated(state)) allocate(state(nblocks)) + call field%data(field_r3) + + call load_array(name//'_T', start, end, size, nlon, nlev, buffer(:,:,1)) + call load_array(name//'_A', start, end, size, nlon, nlev, buffer(:,:,2)) + call load_array(name//'_Q', start, end, size, nlon, nlev, buffer(:,:,3)) + call load_array(name//'_CLD', start, end, size, nlon, nlev, ndim, buffer(:,:,4:)) + + call expand(buffer(:,:,1), field_r3(:,:,1,:), size, nproma, nlev, ngptot, nblocks) + call expand(buffer(:,:,2), field_r3(:,:,2,:), size, nproma, nlev, ngptot, nblocks) + call expand(buffer(:,:,3), field_r3(:,:,3,:), size, nproma, nlev, ngptot, nblocks) + call expand(buffer(:,:,4:), field_r3(:,:,4:,:), size, nproma, nlev, ndim, ngptot, nblocks) + deallocate(buffer) + +!$OMP PARALLEL DO DEFAULT(SHARED), PRIVATE(B) schedule(runtime) + do b=1, nblocks + state(b)%t => field_r3(:,:,1,b) + state(b)%a => field_r3(:,:,2,b) + state(b)%q => field_r3(:,:,3,b) + state(b)%cld => field_r3(:,:,4:3+ndim,b) + end do +!$OMP end parallel do + end subroutine loadstate_atlas + +end module expand_atlas_mod diff --git a/src/cloudsc_fortran_atlas/validate_atlas_mod.F90 b/src/cloudsc_fortran_atlas/validate_atlas_mod.F90 new file mode 100644 index 00000000..32365859 --- /dev/null +++ b/src/cloudsc_fortran_atlas/validate_atlas_mod.F90 @@ -0,0 +1,182 @@ +! (C) Copyright 1988- ECMWF. +! +! This software is licensed under the terms of the Apache Licence Version 2.0 +! which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +! +! In applying this licence, ECMWF does not waive the privileges and immunities +! granted to it by virtue of its status as an intergovernmental organisation +! nor does it submit to any jurisdiction. + +MODULE VALIDATE_ATLAS_MOD + USE PARKIND1, ONLY: JPIM, JPRB + USE CLOUDSC_MPI_MOD + USE VALIDATE_MOD, ONLY: VALIDATE, ERROR_PRINT + + USE ATLAS_MODULE + USE ATLAS_FIELDSET_MODULE + USE ATLAS_FUNCTIONSPACE_BLOCKSTRUCTUREDCOLUMNS_MODULE + USE, INTRINSIC :: ISO_C_BINDING + USE EXPAND_MOD, ONLY: LOAD_AND_EXPAND + USE FILE_IO_MOD, ONLY: INPUT_INITIALIZE, INPUT_FINALIZE + + IMPLICIT NONE + +CONTAINS + + SUBROUTINE VALIDATESTATE_ATLAS(FSET, NAME, NLON, NGPTOTG) + TYPE(ATLAS_FIELDSET), INTENT(INOUT) :: FSET + CHARACTER(*), INTENT(IN) :: NAME + INTEGER(KIND=JPIM), INTENT(IN) :: NLON + INTEGER(KIND=JPIM), INTENT(IN), OPTIONAL :: NGPTOTG + + CALL VALIDATEVAR_ATLAS(FSET, NAME, NLON, NGPTOTG, "A") + CALL VALIDATEVAR_ATLAS(FSET, NAME, NLON, NGPTOTG, "Q") + CALL VALIDATEVAR_ATLAS(FSET, NAME, NLON, NGPTOTG, "T") + CALL VALIDATEVAR_ATLAS(FSET, NAME, NLON, NGPTOTG, "CLD") + END SUBROUTINE VALIDATESTATE_ATLAS + + SUBROUTINE VALIDATEVAR_ATLAS(FSET, NAME, NLON, NGPTOTG, STATE_VAR) + ! Computes and prints errors "in the L1 norm sense" + TYPE(ATLAS_FIELDSET), INTENT(INOUT) :: FSET + CHARACTER(*), INTENT(IN) :: NAME + INTEGER(KIND=JPIM), INTENT(IN) :: NLON + INTEGER(KIND=JPIM), INTENT(IN), OPTIONAL :: NGPTOTG + CHARACTER(*), INTENT(IN), OPTIONAL :: STATE_VAR + + REAL(KIND=JPRB), ALLOCATABLE :: REF_R2(:,:), REF_R3(:,:,:), REF_R4(:,:,:,:) + REAL(C_DOUBLE), POINTER :: FIELD_R1(:,:), FIELD_R2(:,:,:), FIELD_R3(:,:,:,:) + TYPE(ATLAS_FUNCTIONSPACE_BLOCKSTRUCTUREDCOLUMNS) :: FSPACE + TYPE(ATLAS_FIELD) :: FIELD + INTEGER :: B, BSIZE, JL, JK, JM + REAL(KIND=JPRB) :: ZMINVAL(1), ZMAX_VAL_ERR(2), ZDIFF, ZSUM_ERR_ABS(2), ZRELERR, ZAVGPGP + INTEGER :: FRANK, NBLOCKS, NLEV, NGPTOT, NPROMA, VAR_ID, NDIM + CHARACTER(LEN=256) :: FULLNAME + + IF (PRESENT(STATE_VAR)) THEN + FULLNAME = NAME//'_'//STATE_VAR + ELSE + FULLNAME = NAME + ENDIF + + FIELD = FSET%FIELD(NAME) + FRANK = FIELD%RANK() + FSPACE = FIELD%FUNCTIONSPACE() + NLEV = FIELD%LEVELS() + NGPTOT = FSPACE%SIZE() + NBLOCKS = FSPACE%NBLKS() + NPROMA = FSPACE%BLOCK_SIZE(1) + + ZMINVAL(1) = +HUGE(ZMINVAL(1)) + ZMAX_VAL_ERR(1) = -HUGE(ZMAX_VAL_ERR(1)) + ZMAX_VAL_ERR(2) = 0.0_JPRB + ZSUM_ERR_ABS(:) = 0.0_JPRB + + CALL INPUT_INITIALIZE(NAME='reference') + IF (FRANK == 2) THEN + CALL LOAD_AND_EXPAND(NAME, REF_R2, NLON, NPROMA, NGPTOT, NBLOCKS, NGPTOTG) + CALL FIELD%DATA(FIELD_R1) + !OMP PARALLEL DO DEFAULT(SHARED) PRIVATE(B, BSIZE) & + !& REDUCTION(MIN:ZMINVAL, MAX:ZMAX_VAL_ERR, +:ZSUM_ERR_ABS) + DO B=1, NBLOCKS + BSIZE = FSPACE%BLOCK_SIZE(B) + ZMINVAL(1) = MIN(ZMINVAL(1),MINVAL(FIELD_R1(:,B))) + ZMAX_VAL_ERR(1) = MAX(ZMAX_VAL_ERR(1),MAXVAL(FIELD_R1(:,B))) + DO JK=1, BSIZE + ! Difference against reference result in one-norm sense + ZDIFF = ABS(FIELD_R1(JK,B) - REF_R2(JK,B)) + ZMAX_VAL_ERR(2) = MAX(ZMAX_VAL_ERR(2),ZDIFF) + ZSUM_ERR_ABS(1) = ZSUM_ERR_ABS(1) + ZDIFF + ZSUM_ERR_ABS(2) = ZSUM_ERR_ABS(2) + ABS(REF_R2(JK,B)) + ENDDO + END DO + ELSE IF (FRANK == 3) THEN + CALL LOAD_AND_EXPAND(NAME, REF_R3, NLON, FIELD%LEVELS(), NPROMA, NGPTOT, NBLOCKS, NGPTOTG) + CALL FIELD%DATA(FIELD_R2) + !OMP PARALLEL DO DEFAULT(SHARED) PRIVATE(B, BSIZE) & + !& REDUCTION(MIN:ZMINVAL, MAX:ZMAX_VAL_ERR, +:ZSUM_ERR_ABS) + DO B=1, NBLOCKS + BSIZE = FSPACE%BLOCK_SIZE(B) + ZMINVAL(1) = MIN(ZMINVAL(1),MINVAL(FIELD_R2(:,:,B))) + ZMAX_VAL_ERR(1) = MAX(ZMAX_VAL_ERR(1),MAXVAL(FIELD_R2(:,:,B))) + DO JL=1, NLEV + DO JK=1, BSIZE + ! Difference against reference result in one-norm sense + ZDIFF = ABS(FIELD_R2(JK,JL,B) - REF_R3(JK,JL,B)) + ZMAX_VAL_ERR(2) = MAX(ZMAX_VAL_ERR(2),ZDIFF) + ZSUM_ERR_ABS(1) = ZSUM_ERR_ABS(1) + ZDIFF + ZSUM_ERR_ABS(2) = ZSUM_ERR_ABS(2) + ABS(REF_R3(JK,JL,B)) + ENDDO + END DO + END DO + ELSE IF (FRANK == 4 .AND. PRESENT(STATE_VAR)) THEN + CALL FIELD%DATA(FIELD_R3) + NDIM = FIELD%SHAPE(3) - 3 + IF (STATE_VAR /= 'CLD') THEN + VAR_ID = 1 + IF (STATE_VAR == 'A') THEN + VAR_ID = 2 + ENDIF + IF (STATE_VAR == 'Q') THEN + VAR_ID = 3 + ENDIF + CALL LOAD_AND_EXPAND(NAME//'_'//STATE_VAR, REF_R3, NLON, NLEV, NPROMA, NGPTOT, NBLOCKS, NGPTOTG) + !OMP PARALLEL DO DEFAULT(SHARED) PRIVATE(B, BSIZE) & + !& REDUCTION(MIN:ZMINVAL, MAX:ZMAX_VAL_ERR, +:ZSUM_ERR_ABS) + DO B=1, NBLOCKS + BSIZE = FSPACE%BLOCK_SIZE(B) + ZMINVAL(1) = MIN(ZMINVAL(1),MINVAL(FIELD_R3(:,:,VAR_ID,B))) + ZMAX_VAL_ERR(1) = MAX(ZMAX_VAL_ERR(1),MAXVAL(FIELD_R3(:,:,VAR_ID,B))) + DO JL=1, NLEV + DO JK=1, BSIZE + ! Difference against reference result in one-norm sense + ZDIFF = ABS(FIELD_R3(JK,JL,VAR_ID,B) - REF_R3(JK,JL,B)) + ZMAX_VAL_ERR(2) = MAX(ZMAX_VAL_ERR(2),ZDIFF) + ZSUM_ERR_ABS(1) = ZSUM_ERR_ABS(1) + ZDIFF + ZSUM_ERR_ABS(2) = ZSUM_ERR_ABS(2) + ABS(REF_R3(JK,JL,B)) + ENDDO + END DO + END DO + ELSE IF (STATE_VAR == 'CLD') THEN + CALL LOAD_AND_EXPAND(NAME//'_CLD', REF_R4, NLON, NLEV, NDIM, NPROMA, NGPTOT, NBLOCKS, NGPTOTG) + !OMP PARALLEL DO DEFAULT(SHARED) PRIVATE(B, BSIZE) & + !& REDUCTION(MIN:ZMINVAL, MAX:ZMAX_VAL_ERR, +:ZSUM_ERR_ABS) + DO B=1, NBLOCKS + BSIZE = MIN(NLON, NGPTOT - (B-1)*NLON) ! Field block size + ZMINVAL(1) = MIN(ZMINVAL(1),MINVAL(FIELD_R3(:,:,4:,B))) + ZMAX_VAL_ERR(1) = MAX(ZMAX_VAL_ERR(1),MAXVAL(FIELD_R3(:,:,4:,B))) + DO JM=1, NDIM + DO JL=1, NLEV + DO JK=1, BSIZE + ! Difference against reference result in one-norm sense + ZDIFF = ABS(FIELD_R3(JK,JL,3+JM,B) - REF_R4(JK,JL,JM,B)) + ZMAX_VAL_ERR(2) = MAX(ZMAX_VAL_ERR(2),ZDIFF) + ZSUM_ERR_ABS(1) = ZSUM_ERR_ABS(1) + ZDIFF + ZSUM_ERR_ABS(2) = ZSUM_ERR_ABS(2) + ABS(REF_R4(JK,JL,JM,B)) + ENDDO + ENDDO + END DO + END DO + ENDIF + ELSE + PRINT *, "FIELD RANK NOT SUPPORTED" + CALL EXIT(1) + ENDIF + CALL INPUT_FINALIZE() + + CALL CLOUDSC_MPI_REDUCE_MIN(ZMINVAL, 1, 0) + CALL CLOUDSC_MPI_REDUCE_MAX(ZMAX_VAL_ERR, 2, 0) + CALL CLOUDSC_MPI_REDUCE_SUM(ZSUM_ERR_ABS, 2, 0) + + IF (PRESENT(NGPTOTG)) THEN + ZAVGPGP = ZSUM_ERR_ABS(1) / REAL(NGPTOTG,JPRB) + ELSE + ZAVGPGP = ZSUM_ERR_ABS(1) / REAL(NGPTOT,JPRB) + END IF + + IF (IRANK == 0) THEN + CALL ERROR_PRINT(FULLNAME, ZMINVAL(1), ZMAX_VAL_ERR(1), ZMAX_VAL_ERR(2), & + & ZSUM_ERR_ABS(1), ZSUM_ERR_ABS(2), ZAVGPGP, NDIM=FRANK-1) + END IF + END SUBROUTINE VALIDATEVAR_ATLAS + +END MODULE VALIDATE_ATLAS_MOD From d943e7f125614403497d642cc8f4c978725ea985 Mon Sep 17 00:00:00 2001 From: Slavko Brdar Date: Tue, 9 May 2023 00:17:22 +0200 Subject: [PATCH 02/45] using Atlas block view on FieldSet --- .../cloudsc_driver_mod.F90 | 204 +++++------ .../cloudsc_global_atlas_state_mod.F90 | 331 ++++++++++-------- .../dwarf_cloudsc_atlas.F90 | 37 +- .../expand_atlas_mod.F90 | 14 +- .../validate_atlas_mod.F90 | 2 +- 5 files changed, 304 insertions(+), 284 deletions(-) diff --git a/src/cloudsc_fortran_atlas/cloudsc_driver_mod.F90 b/src/cloudsc_fortran_atlas/cloudsc_driver_mod.F90 index 25b22254..8a84c043 100644 --- a/src/cloudsc_fortran_atlas/cloudsc_driver_mod.F90 +++ b/src/cloudsc_fortran_atlas/cloudsc_driver_mod.F90 @@ -14,84 +14,79 @@ MODULE CLOUDSC_DRIVER_MOD USE CLOUDSC_MPI_MOD, ONLY: NUMPROC, IRANK USE TIMER_MOD, ONLY : PERFORMANCE_TIMER, GET_THREAD_NUM USE EC_PMON_MOD, ONLY: EC_PMON + USE CLOUDSC_GLOBAL_ATLAS_STATE_MOD, ONLY: CLOUDSC_GLOBAL_ATLAS_STATE_BLOCK_VIEW + + USE ATLAS_MODULE + USE, INTRINSIC :: ISO_C_BINDING + USE ATLAS_FUNCTIONSPACE_BLOCKSTRUCTUREDCOLUMNS_MODULE IMPLICIT NONE CONTAINS - SUBROUTINE CLOUDSC_DRIVER( & - & NUMOMP, NPROMA, NLEV, NGPTOT, NGPTOTG, KFLDX, PTSPHY, & - & PT, PQ, TENDENCY_CML, TENDENCY_TMP, TENDENCY_LOC, & - & PVFA, PVFL, PVFI, PDYNA, PDYNL, PDYNI, & - & PHRSW, PHRLW, & - & PVERVEL, PAP, PAPH, & - & PLSM, LDCUM, KTYPE, & - & PLU, PLUDE, PSNDE, PMFU, PMFD, & - & PA, PCLV, PSUPSAT,& - & PLCRIT_AER,PICRIT_AER, PRE_ICE, & - & PCCN, PNICE,& - & PCOVPTOT, PRAINFRAC_TOPRFZ, & - & PFSQLF, PFSQIF , PFCQNNG, PFCQLNG, & - & PFSQRF, PFSQSF , PFCQRNG, PFCQSNG, & - & PFSQLTUR, PFSQITUR, & - & PFPLSL, PFPLSN, PFHPSL, PFHPSN & - & ) + SUBROUTINE CLOUDSC_DRIVER(FSET, NUMOMP, NGPTOT, NGPTOTG, KFLDX, PTSPHY) + ! Driver routine that performans the parallel NPROMA-blocking and ! invokes the CLOUDSC kernel - INTEGER(KIND=JPIM), INTENT(IN) :: NUMOMP, NPROMA, NLEV, NGPTOT, NGPTOTG - INTEGER(KIND=JPIM), INTENT(IN) :: KFLDX - REAL(KIND=JPRB), INTENT(IN) :: PTSPHY ! Physics timestep - REAL(KIND=JPRB), INTENT(IN) :: PT(:,:,:) ! T at start of callpar - REAL(KIND=JPRB), INTENT(IN) :: PQ(:,:,:) ! Q at start of callpar - TYPE(STATE_TYPE), INTENT(IN) :: TENDENCY_CML(:) ! cumulative tendency used for final output - TYPE(STATE_TYPE), INTENT(IN) :: TENDENCY_TMP(:) ! cumulative tendency used as input - TYPE(STATE_TYPE), INTENT(OUT) :: TENDENCY_LOC(:) ! local tendency from cloud scheme - REAL(KIND=JPRB), INTENT(IN) :: PVFA(:,:,:) ! CC from VDF scheme - REAL(KIND=JPRB), INTENT(IN) :: PVFL(:,:,:) ! Liq from VDF scheme - REAL(KIND=JPRB), INTENT(IN) :: PVFI(:,:,:) ! Ice from VDF scheme - REAL(KIND=JPRB), INTENT(IN) :: PDYNA(:,:,:) ! CC from Dynamics - REAL(KIND=JPRB), INTENT(IN) :: PDYNL(:,:,:) ! Liq from Dynamics - REAL(KIND=JPRB), INTENT(IN) :: PDYNI(:,:,:) ! Liq from Dynamics - REAL(KIND=JPRB), INTENT(IN) :: PHRSW(:,:,:) ! Short-wave heating rate - REAL(KIND=JPRB), INTENT(IN) :: PHRLW(:,:,:) ! Long-wave heating rate - REAL(KIND=JPRB), INTENT(IN) :: PVERVEL(:,:,:) !Vertical velocity - REAL(KIND=JPRB), INTENT(IN) :: PAP(:,:,:) ! Pressure on full levels - REAL(KIND=JPRB), INTENT(IN) :: PAPH(:,:,:) ! Pressure on half levels - REAL(KIND=JPRB), INTENT(IN) :: PLSM(:,:) ! Land fraction (0-1) - LOGICAL , INTENT(IN) :: LDCUM(:,:) ! Convection active - INTEGER(KIND=JPIM), INTENT(IN) :: KTYPE(:,:) ! Convection type 0,1,2 - REAL(KIND=JPRB), INTENT(IN) :: PLU(:,:,:) ! Conv. condensate - REAL(KIND=JPRB), INTENT(INOUT) :: PLUDE(:,:,:) ! Conv. detrained water - REAL(KIND=JPRB), INTENT(IN) :: PSNDE(:,:,:) ! Conv. detrained snow - REAL(KIND=JPRB), INTENT(IN) :: PMFU(:,:,:) ! Conv. mass flux up - REAL(KIND=JPRB), INTENT(IN) :: PMFD(:,:,:) ! Conv. mass flux down - REAL(KIND=JPRB), INTENT(IN) :: PA(:,:,:) ! Original Cloud fraction (t) - REAL(KIND=JPRB), INTENT(IN) :: PCLV(:,:,:,:) - REAL(KIND=JPRB), INTENT(IN) :: PSUPSAT(:,:,:) - REAL(KIND=JPRB), INTENT(IN) :: PLCRIT_AER(:,:,:) - REAL(KIND=JPRB), INTENT(IN) :: PICRIT_AER(:,:,:) - REAL(KIND=JPRB), INTENT(IN) :: PRE_ICE(:,:,:) - REAL(KIND=JPRB), INTENT(IN) :: PCCN(:,:,:) ! liquid cloud condensation nuclei - REAL(KIND=JPRB), INTENT(IN) :: PNICE(:,:,:) ! ice number concentration (cf. CCN) - - REAL(KIND=JPRB), INTENT(INOUT) :: PCOVPTOT(:,:,:) ! Precip fraction - REAL(KIND=JPRB), INTENT(OUT) :: PRAINFRAC_TOPRFZ(:,:) + TYPE(ATLAS_FIELDSET), INTENT(INOUT) :: FSET + INTEGER(KIND=JPIM), INTENT(IN) :: NUMOMP, NGPTOT, NGPTOTG, KFLDX + REAL(KIND=JPRB), INTENT(IN) :: PTSPHY ! Physics timestep + + TYPE(CLOUDSC_GLOBAL_ATLAS_STATE_BLOCK_VIEW) :: FBLOCK + TYPE(ATLAS_FUNCTIONSPACE_BLOCKSTRUCTUREDCOLUMNS) :: FSPACE + TYPE(ATLAS_FIELD) :: FIELD + INTEGER(KIND=JPIM) :: NPROMA, NLEV + REAL(KIND=JPRB), POINTER :: PT(:,:) ! T at start of callpar + REAL(KIND=JPRB), POINTER :: PQ(:,:) ! Q at start of callpar + TYPE(STATE_TYPE), POINTER :: TENDENCY_CML(:) ! cumulative tendency used for final output + TYPE(STATE_TYPE), POINTER :: TENDENCY_TMP(:) ! cumulative tendency used as input + TYPE(STATE_TYPE), POINTER :: TENDENCY_LOC(:) ! local tendency from cloud scheme + REAL(KIND=JPRB), POINTER:: PVFA(:,:) ! CC from VDF scheme + REAL(KIND=JPRB), POINTER:: PVFL(:,:) ! Liq from VDF scheme + REAL(KIND=JPRB), POINTER:: PVFI(:,:) ! Ice from VDF scheme + REAL(KIND=JPRB), POINTER:: PDYNA(:,:) ! CC from Dynamics + REAL(KIND=JPRB), POINTER:: PDYNL(:,:) ! Liq from Dynamics + REAL(KIND=JPRB), POINTER:: PDYNI(:,:) ! Liq from Dynamics + REAL(KIND=JPRB), POINTER:: PHRSW(:,:) ! Short-wave heating rate + REAL(KIND=JPRB), POINTER:: PHRLW(:,:) ! Long-wave heating rate + REAL(KIND=JPRB), POINTER:: PVERVEL(:,:) !Vertical velocity + REAL(KIND=JPRB), POINTER:: PAP(:,:) ! Pressure on full levels + REAL(KIND=JPRB), POINTER:: PAPH(:,:) ! Pressure on half levels + REAL(KIND=JPRB), POINTER:: PLSM(:) ! Land fraction (0-1) + LOGICAL, POINTER :: LDCUM(:) ! Convection active + INTEGER(KIND=JPIM), POINTER :: KTYPE(:) ! Convection type 0,1,2 + REAL(KIND=JPRB), POINTER:: PLU(:,:) ! Conv. condensate + REAL(KIND=JPRB), POINTER:: PLUDE(:,:) ! Conv. detrained water + REAL(KIND=JPRB), POINTER:: PSNDE(:,:) ! Conv. detrained snow + REAL(KIND=JPRB), POINTER:: PMFU(:,:) ! Conv. mass flux up + REAL(KIND=JPRB), POINTER:: PMFD(:,:) ! Conv. mass flux down + REAL(KIND=JPRB), POINTER:: PA(:,:) ! Original Cloud fraction (t) + REAL(KIND=JPRB), POINTER:: PCLV(:,:,:) + REAL(KIND=JPRB), POINTER:: PSUPSAT(:,:) + REAL(KIND=JPRB), POINTER:: PLCRIT_AER(:,:) + REAL(KIND=JPRB), POINTER:: PICRIT_AER(:,:) + REAL(KIND=JPRB), POINTER:: PRE_ICE(:,:) + REAL(KIND=JPRB), POINTER:: PCCN(:,:) ! liquid cloud condensation nuclei + REAL(KIND=JPRB), POINTER:: PNICE(:,:) ! ice number concentration (cf. CCN) + + REAL(KIND=JPRB), POINTER:: PCOVPTOT(:,:) ! Precip fraction + REAL(KIND=JPRB), POINTER:: PRAINFRAC_TOPRFZ(:) ! Flux diagnostics for DDH budget - REAL(KIND=JPRB), INTENT(OUT) :: PFSQLF(:,:,:) ! Flux of liquid - REAL(KIND=JPRB), INTENT(OUT) :: PFSQIF(:,:,:) ! Flux of ice - REAL(KIND=JPRB), INTENT(OUT) :: PFCQLNG(:,:,:) ! -ve corr for liq - REAL(KIND=JPRB), INTENT(OUT) :: PFCQNNG(:,:,:) ! -ve corr for ice - REAL(KIND=JPRB), INTENT(OUT) :: PFSQRF(:,:,:) ! Flux diagnostics - REAL(KIND=JPRB), INTENT(OUT) :: PFSQSF(:,:,:) ! for DDH, generic - REAL(KIND=JPRB), INTENT(OUT) :: PFCQRNG(:,:,:) ! rain - REAL(KIND=JPRB), INTENT(OUT) :: PFCQSNG(:,:,:) ! snow - REAL(KIND=JPRB), INTENT(OUT) :: PFSQLTUR(:,:,:) ! liquid flux due to VDF - REAL(KIND=JPRB), INTENT(OUT) :: PFSQITUR(:,:,:) ! ice flux due to VDF - REAL(KIND=JPRB), INTENT(OUT) :: PFPLSL(:,:,:) ! liq+rain sedim flux - REAL(KIND=JPRB), INTENT(OUT) :: PFPLSN(:,:,:) ! ice+snow sedim flux - REAL(KIND=JPRB), INTENT(OUT) :: PFHPSL(:,:,:) ! Enthalpy flux for liq - REAL(KIND=JPRB), INTENT(OUT) :: PFHPSN(:,:,:) ! Enthalp flux for ice + REAL(KIND=JPRB), POINTER :: PFSQLF(:,:) ! Flux of liquid + REAL(KIND=JPRB), POINTER :: PFSQIF(:,:) ! Flux of ice + REAL(KIND=JPRB), POINTER :: PFCQLNG(:,:) ! -ve corr for liq + REAL(KIND=JPRB), POINTER :: PFCQNNG(:,:) ! -ve corr for ice + REAL(KIND=JPRB), POINTER :: PFSQRF(:,:) ! Flux diagnostics + REAL(KIND=JPRB), POINTER :: PFSQSF(:,:) ! for DDH, generic + REAL(KIND=JPRB), POINTER :: PFCQRNG(:,:) ! rain + REAL(KIND=JPRB), POINTER :: PFCQSNG(:,:) ! snow + REAL(KIND=JPRB), POINTER :: PFSQLTUR(:,:) ! liquid flux due to VDF + REAL(KIND=JPRB), POINTER :: PFSQITUR(:,:) ! ice flux due to VDF + REAL(KIND=JPRB), POINTER :: PFPLSL(:,:) ! liq+rain sedim flux + REAL(KIND=JPRB), POINTER :: PFPLSN(:,:) ! ice+snow sedim flux + REAL(KIND=JPRB), POINTER :: PFHPSL(:,:) ! Enthalpy flux for liq + REAL(KIND=JPRB), POINTER :: PFHPSN(:,:) ! Enthalp flux for ice INTEGER(KIND=JPIM) :: JKGLO,IBL,ICEND,NGPBLKS @@ -108,6 +103,12 @@ SUBROUTINE CLOUDSC_DRIVER( & POWER_TOTAL = 0_JPIB POWER_COUNT = 0_JPIB + FIELD = FSET%FIELD("PEXTRA") + FSPACE = FIELD%FUNCTIONSPACE() + NPROMA = FSPACE%BLOCK_SIZE(1) + NLEV = FSPACE%LEVELS() + print *, "NPROMA, NUMOMP ", NPROMA, NUMOMP + NGPBLKS = (NGPTOT / NPROMA) + MIN(MOD(NGPTOT,NPROMA), 1) 1003 format(5x,'NUMPROC=',i0,', NUMOMP=',i0,', NGPTOTG=',i0,', NPROMA=',i0,', NGPBLKS=',i0) if (irank == 0) then @@ -129,46 +130,49 @@ SUBROUTINE CLOUDSC_DRIVER( & IBL=(JKGLO-1)/NPROMA+1 ICEND=MIN(NPROMA,NGPTOT-JKGLO+1) + ! get block views + call FBLOCK%GET_BLOCK(FSET, IBL) + !-- These were uninitialized : meaningful only when we compare error differences - PCOVPTOT(:,:,IBL) = 0.0_JPRB - TENDENCY_LOC(IBL)%cld(:,:,NCLV) = 0.0_JPRB - - !--- a future plan to replace the call to CLOUDSC ------ - ! - ! type( block_state_t ) - ! real(c_double), pointer :: PT(:,:) - ! type(state_type) :: tendency_LOC - ! type(state_type) :: tendency_TMP - ! type(state_type) :: tendency_CML - ! end type - - ! call extract_block( FSET, IBL, config, block_state ) - ! call FSET%FIELD("PT")%BLOCK_DATA(IBL,PT,CONFIG) - ! call FSET%FIELD("PQ")%BLOCK_DATA(IBL,PQ,CONFIG) - ! call cloudsc_atlas ( FSET, IBL, config ) + FBLOCK%PCOVPTOT(:,:) = 0.0_JPRB + FBLOCK%TENDENCY_LOC%cld(:,:,NCLV) = 0.0_JPRB + + !--- a future plan to replace the call to CLOUDSC ------ + ! + ! type( block_state_t ) + ! real(c_double), pointer :: PT(:,:) + ! type(state_type) :: tendency_LOC + ! type(state_type) :: tendency_TMP + ! type(state_type) :: tendency_CML + ! end type + ! call extract_block( FSET, IBL, config, block_state ) + ! call FSET%FIELD("PT")%BLOCK_DATA(IBL,PT,CONFIG) + ! call FSET%FIELD("PQ")%BLOCK_DATA(IBL,PQ,CONFIG) + ! call cloudsc_atlas ( FSET, IBL, config ) CALL CLOUDSC & & ( 1, ICEND, NPROMA, NLEV,& & PTSPHY,& - & PT(:,:,IBL), PQ(:,:,IBL), TENDENCY_CML(IBL), TENDENCY_TMP(IBL), TENDENCY_LOC(IBL), & - & PVFA(:,:,IBL), PVFL(:,:,IBL), PVFI(:,:,IBL), PDYNA(:,:,IBL), PDYNL(:,:,IBL), PDYNI(:,:,IBL), & - & PHRSW(:,:,IBL), PHRLW(:,:,IBL),& - & PVERVEL(:,:,IBL), PAP(:,:,IBL), PAPH(:,:,IBL),& - & PLSM(:,IBL), LDCUM(:,IBL), KTYPE(:,IBL), & - & PLU(:,:,IBL), PLUDE(:,:,IBL), PSNDE(:,:,IBL), PMFU(:,:,IBL), PMFD(:,:,IBL),& + & FBLOCK%PT, FBLOCK%PQ, FBLOCK%TENDENCY_CML, FBLOCK%TENDENCY_TMP, FBLOCK%TENDENCY_LOC, & + & FBLOCK%PVFA, FBLOCK%PVFL, FBLOCK%PVFI, FBLOCK%PDYNA, FBLOCK%PDYNL, FBLOCK%PDYNI, & + & FBLOCK%PHRSW, FBLOCK%PHRLW,& + & FBLOCK%PVERVEL, FBLOCK%PAP, FBLOCK%PAPH,& + & FBLOCK%PLSM, FBLOCK%LDCUM, FBLOCK%KTYPE, & + & FBLOCK%PLU, FBLOCK%PLUDE, & + & FBLOCK%PSNDE, FBLOCK%PMFU, FBLOCK%PMFD,& !---prognostic fields - & PA(:,:,IBL), PCLV(:,:,:,IBL), PSUPSAT(:,:,IBL),& + & FBLOCK%PA, FBLOCK%PCLV, FBLOCK%PSUPSAT,& !-- arrays for aerosol-cloud interactions - & PLCRIT_AER(:,:,IBL),PICRIT_AER(:,:,IBL),& - & PRE_ICE(:,:,IBL),& - & PCCN(:,:,IBL), PNICE(:,:,IBL),& + & FBLOCK%PLCRIT_AER, FBLOCK%PICRIT_AER,& + & FBLOCK%PRE_ICE,& + & FBLOCK%PCCN, FBLOCK%PNICE,& !---diagnostic output - & PCOVPTOT(:,:,IBL), PRAINFRAC_TOPRFZ(:,IBL),& + & FBLOCK%PCOVPTOT, FBLOCK%PRAINFRAC_TOPRFZ,& !---resulting fluxes - & PFSQLF(:,:,IBL), PFSQIF (:,:,IBL), PFCQNNG(:,:,IBL), PFCQLNG(:,:,IBL),& - & PFSQRF(:,:,IBL), PFSQSF (:,:,IBL), PFCQRNG(:,:,IBL), PFCQSNG(:,:,IBL),& - & PFSQLTUR(:,:,IBL), PFSQITUR (:,:,IBL), & - & PFPLSL(:,:,IBL), PFPLSN(:,:,IBL), PFHPSL(:,:,IBL), PFHPSN(:,:,IBL),& + & FBLOCK%PFSQLF, FBLOCK%PFSQIF, FBLOCK%PFCQNNG, FBLOCK%PFCQLNG,& + & FBLOCK%PFSQRF, FBLOCK%PFSQSF, FBLOCK%PFCQRNG, FBLOCK%PFCQSNG,& + & FBLOCK%PFSQLTUR, FBLOCK%PFSQITUR, & + & FBLOCK%PFPLSL, FBLOCK%PFPLSN, FBLOCK%PFHPSL, FBLOCK%PFHPSN,& & KFLDX) !--- end of a future plan to replace the call to CLOUDSC ------ diff --git a/src/cloudsc_fortran_atlas/cloudsc_global_atlas_state_mod.F90 b/src/cloudsc_fortran_atlas/cloudsc_global_atlas_state_mod.F90 index e210b165..713e833d 100644 --- a/src/cloudsc_fortran_atlas/cloudsc_global_atlas_state_mod.F90 +++ b/src/cloudsc_fortran_atlas/cloudsc_global_atlas_state_mod.F90 @@ -27,13 +27,12 @@ MODULE CLOUDSC_GLOBAL_ATLAS_STATE_MOD IMPLICIT NONE - TYPE(ATLAS_STRUCTUREDGRID) :: GRID - TYPE(ATLAS_FUNCTIONSPACE_BLOCKSTRUCTUREDCOLUMNS) :: FSPACE - TYPE(ATLAS_FIELDSET) :: FSET - TYPE VAR3D_PTR REAL(C_DOUBLE), POINTER :: PTR(:,:,:) END TYPE + TYPE VAR2D_PTR + REAL(C_DOUBLE), POINTER :: PTR(:,:) + END TYPE CHARACTER(LEN=10), PARAMETER, DIMENSION(30) :: IN_VAR_NAMES = (/ & "PLCRIT_AER", "PICRIT_AER", "PRE_ICE ", "PCCN ", "PNICE ", "PT ", "PQ ", & @@ -47,6 +46,68 @@ MODULE CLOUDSC_GLOBAL_ATLAS_STATE_MOD "PFPLSL ", "PFPLSN ", "PFHPSL ", "PFHPSN ", "PCOVPTOT ", & "PRAINFRAC_TOPRFZ" /) + TYPE CLOUDSC_GLOBAL_ATLAS_STATE_BLOCK_VIEW + TYPE(VAR2D_PTR), DIMENSION(24) :: IN_VARS_2D_REAL64 + TYPE(VAR2D_PTR), DIMENSION(15) :: OUT_VARS_2D_REAL64 + + ! Input field variables and tendencies + REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PLCRIT_AER(:,:) + REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PICRIT_AER(:,:) + REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PRE_ICE(:,:) + REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PCCN(:,:) ! liquid cloud condensation nuclei + REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PNICE(:,:) ! ice number concentration (cf. CCN) + REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PT(:,:) ! T at start of callpar + REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PQ(:,:) ! Q at start of callpar + REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PVFA(:,:) ! CC from VDF scheme + REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PVFL(:,:) ! Liq from VDF scheme + REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PVFI(:,:) ! Ice from VDF scheme + REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PDYNA(:,:) ! CC from Dynamics + REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PDYNL(:,:) ! Liq from Dynamics + REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PDYNI(:,:) ! Liq from Dynamics + REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PHRSW(:,:) ! Short-wave heating rate + REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PHRLW(:,:) ! Long-wave heating rate + REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PVERVEL(:,:) ! Vertical velocity + REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PAP(:,:) ! Pressure on full levels + REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PLU(:,:) ! Conv. condensate + REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PLUDE(:,:) ! Conv. detrained water + REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PSNDE(:,:) ! Conv. detrained snow + REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PMFU(:,:) ! Conv. mass flux up + REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PMFD(:,:) ! Conv. mass flux down + REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PA(:,:) ! Original Cloud fraction (t) + REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PSUPSAT(:,:) + + REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PLSM(:) ! Land fraction (0-1) + LOGICAL, POINTER, CONTIGUOUS :: LDCUM(:) ! Convection active + INTEGER, POINTER, CONTIGUOUS :: KTYPE(:) ! Convection type 0,1,2 + REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PAPH(:,:) ! Pressure on half levels + REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PEXTRA(:,:,:) ! extra fields + REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PCLV(:,:,:) + + TYPE(STATE_TYPE) :: TENDENCY_CML ! cumulative tendency used for final output + TYPE(STATE_TYPE) :: TENDENCY_TMP ! cumulative tendency used as input + TYPE(STATE_TYPE) :: TENDENCY_LOC ! local tendency from cloud scheme + + ! Output fields used for validation + REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PFSQLF(:,:) ! Flux of liquid + REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PFSQIF(:,:) ! Flux of ice + REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PFCQLNG(:,:) ! -ve corr for liq + REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PFCQNNG(:,:) ! -ve corr for ice + REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PFSQRF(:,:) ! Flux diagnostics + REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PFSQSF(:,:) ! for DDH, generic + REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PFCQRNG(:,:) ! rain + REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PFCQSNG(:,:) ! snow + REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PFSQLTUR(:,:) ! liquid flux due to VDF + REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PFSQITUR(:,:) ! ice flux due to VDF + REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PFPLSL(:,:) ! liq+rain sedim flux + REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PFPLSN(:,:) ! ice+snow sedim flux + REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PFHPSL(:,:) ! Enthalpy flux for liq + REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PFHPSN(:,:) ! Enthalpy flux for ice + REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PCOVPTOT(:,:) ! Precip fraction + REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PRAINFRAC_TOPRFZ(:) + CONTAINS + PROCEDURE :: GET_BLOCK => CLOUDSC_GLOBAL_ATLAS_STATE_BLOCK + END TYPE CLOUDSC_GLOBAL_ATLAS_STATE_BLOCK_VIEW + TYPE CLOUDSC_GLOBAL_ATLAS_STATE ! Memory state containing raw fields annd tendencies for CLOUDSC dwarf ! @@ -59,68 +120,6 @@ MODULE CLOUDSC_GLOBAL_ATLAS_STATE_MOD LOGICAL :: LDMAINCALL ! T if main call to cloudsc REAL(KIND=JPRB) :: PTSPHY ! Physics timestep - TYPE(VAR3D_PTR), DIMENSION(24) :: IN_VARS_3D_REAL64 - TYPE(VAR3D_PTR), DIMENSION(15) :: OUT_VARS_3D_REAL64 - - ! Input field variables and tendencies - REAL(C_DOUBLE), POINTER :: PLCRIT_AER(:,:,:) - REAL(C_DOUBLE), POINTER :: PICRIT_AER(:,:,:) - REAL(C_DOUBLE), POINTER :: PRE_ICE(:,:,:) - REAL(C_DOUBLE), POINTER :: PCCN(:,:,:) ! liquid cloud condensation nuclei - REAL(C_DOUBLE), POINTER :: PNICE(:,:,:) ! ice number concentration (cf. CCN) - REAL(C_DOUBLE), POINTER :: PT(:,:,:) ! T at start of callpar - REAL(C_DOUBLE), POINTER :: PQ(:,:,:) ! Q at start of callpar - REAL(C_DOUBLE), POINTER :: PVFA(:,:,:) ! CC from VDF scheme - REAL(C_DOUBLE), POINTER :: PVFL(:,:,:) ! Liq from VDF scheme - REAL(C_DOUBLE), POINTER :: PVFI(:,:,:) ! Ice from VDF scheme - REAL(C_DOUBLE), POINTER :: PDYNA(:,:,:) ! CC from Dynamics - REAL(C_DOUBLE), POINTER :: PDYNL(:,:,:) ! Liq from Dynamics - REAL(C_DOUBLE), POINTER :: PDYNI(:,:,:) ! Liq from Dynamics - REAL(C_DOUBLE), POINTER :: PHRSW(:,:,:) ! Short-wave heating rate - REAL(C_DOUBLE), POINTER :: PHRLW(:,:,:) ! Long-wave heating rate - REAL(C_DOUBLE), POINTER :: PVERVEL(:,:,:) ! Vertical velocity - REAL(C_DOUBLE), POINTER :: PAP(:,:,:) ! Pressure on full levels - REAL(C_DOUBLE), POINTER :: PLU(:,:,:) ! Conv. condensate - REAL(C_DOUBLE), POINTER :: PLUDE(:,:,:) ! Conv. detrained water - REAL(C_DOUBLE), POINTER :: PSNDE(:,:,:) ! Conv. detrained snow - REAL(C_DOUBLE), POINTER :: PMFU(:,:,:) ! Conv. mass flux up - REAL(C_DOUBLE), POINTER :: PMFD(:,:,:) ! Conv. mass flux down - REAL(C_DOUBLE), POINTER :: PA(:,:,:) ! Original Cloud fraction (t) - REAL(C_DOUBLE), POINTER :: PSUPSAT(:,:,:) - - REAL(C_DOUBLE), POINTER :: PLSM(:,:) ! Land fraction (0-1) - LOGICAL, POINTER :: LDCUM(:,:) ! Convection active - INTEGER(c_int), POINTER :: KTYPE(:,:) ! Convection type 0,1,2 - REAL(C_DOUBLE), POINTER :: PAPH(:,:,:) ! Pressure on half levels - REAL(C_DOUBLE), POINTER :: PEXTRA(:,:,:,:) ! extra fields - REAL(C_DOUBLE), POINTER :: PCLV(:,:,:,:) - - TYPE(STATE_TYPE), ALLOCATABLE :: TENDENCY_CML(:) ! cumulative tendency used for final output - TYPE(STATE_TYPE), ALLOCATABLE :: TENDENCY_TMP(:) ! cumulative tendency used as input - TYPE(STATE_TYPE), ALLOCATABLE :: TENDENCY_LOC(:) ! local tendency from cloud scheme - - ! Output fields used for validation - REAL(C_DOUBLE), POINTER :: PFSQLF(:,:,:) ! Flux of liquid - REAL(C_DOUBLE), POINTER :: PFSQIF(:,:,:) ! Flux of ice - REAL(C_DOUBLE), POINTER :: PFCQLNG(:,:,:) ! -ve corr for liq - REAL(C_DOUBLE), POINTER :: PFCQNNG(:,:,:) ! -ve corr for ice - REAL(C_DOUBLE), POINTER :: PFSQRF(:,:,:) ! Flux diagnostics - REAL(C_DOUBLE), POINTER :: PFSQSF(:,:,:) ! for DDH, generic - REAL(C_DOUBLE), POINTER :: PFCQRNG(:,:,:) ! rain - REAL(C_DOUBLE), POINTER :: PFCQSNG(:,:,:) ! snow - REAL(C_DOUBLE), POINTER :: PFSQLTUR(:,:,:) ! liquid flux due to VDF - REAL(C_DOUBLE), POINTER :: PFSQITUR(:,:,:) ! ice flux due to VDF - REAL(C_DOUBLE), POINTER :: PFPLSL(:,:,:) ! liq+rain sedim flux - REAL(C_DOUBLE), POINTER :: PFPLSN(:,:,:) ! ice+snow sedim flux - REAL(C_DOUBLE), POINTER :: PFHPSL(:,:,:) ! Enthalpy flux for liq - REAL(C_DOUBLE), POINTER :: PFHPSN(:,:,:) ! Enthalpy flux for ice - REAL(C_DOUBLE), POINTER :: PCOVPTOT(:,:,:) ! Precip fraction - REAL(C_DOUBLE), POINTER :: PRAINFRAC_TOPRFZ(:,:) - - ! Underlying data buffers for AOSOA allcoated STATE_TYPE arrays - REAL(C_DOUBLE), POINTER :: B_CML(:,:,:,:) - REAL(C_DOUBLE), POINTER :: B_TMP(:,:,:,:) - REAL(C_DOUBLE), POINTER :: B_LOC(:,:,:,:) CONTAINS PROCEDURE :: LOAD => CLOUDSC_GLOBAL_ATLAS_STATE_LOAD PROCEDURE :: VALIDATE => CLOUDSC_GLOBAL_ATLAS_STATE_VALIDATE @@ -128,13 +127,117 @@ MODULE CLOUDSC_GLOBAL_ATLAS_STATE_MOD CONTAINS - SUBROUTINE CLOUDSC_GLOBAL_ATLAS_STATE_LOAD(SELF, NPROMA, NGPTOT, NGPTOTG) + SUBROUTINE CLOUDSC_GLOBAL_ATLAS_STATE_BLOCK(SELF, FSET, IBLK) + ! Load reference input data via serialbox + CLASS(CLOUDSC_GLOBAL_ATLAS_STATE_BLOCK_VIEW), INTENT(INOUT) :: SELF + TYPE(ATLAS_FIELDSET), INTENT(INOUT) :: FSET + INTEGER, INTENT(IN) :: IBLK + + TYPE(ATLAS_FUNCTIONSPACE_BLOCKSTRUCTUREDCOLUMNS) :: FSPACE + INTEGER(KIND=JPIM) :: KLON, IVAR, B + TYPE(ATLAS_FIELD) :: FIELD + REAL(C_DOUBLE), POINTER :: TMP3D(:,:,:), TMP2D(:,:) + + ! NOTE the last six input variables need special treatment - different types + DO IVAR = 1, SIZE(IN_VAR_NAMES) - 6 + FIELD = FSET%FIELD(TRIM(IN_VAR_NAMES(IVAR))) + CALL FIELD%DATA(SELF%IN_VARS_2D_REAL64(IVAR)%PTR, IBLK) + ENDDO + SELF%PLCRIT_AER => SELF%IN_VARS_2D_REAL64(1)%PTR + SELF%PICRIT_AER => SELF%IN_VARS_2D_REAL64(2)%PTR + SELF%PRE_ICE => SELF%IN_VARS_2D_REAL64(3)%PTR + SELF%PCCN => SELF%IN_VARS_2D_REAL64(4)%PTR + SELF%PNICE => SELF%IN_VARS_2D_REAL64(5)%PTR + SELF%PT => SELF%IN_VARS_2D_REAL64(6)%PTR + SELF%PQ => SELF%IN_VARS_2D_REAL64(7)%PTR + SELF%PVFA => SELF%IN_VARS_2D_REAL64(8)%PTR + SELF%PVFL => SELF%IN_VARS_2D_REAL64(9)%PTR + SELF%PVFI => SELF%IN_VARS_2D_REAL64(10)%PTR + SELF%PDYNA => SELF%IN_VARS_2D_REAL64(11)%PTR + SELF%PDYNL => SELF%IN_VARS_2D_REAL64(12)%PTR + SELF%PDYNI => SELF%IN_VARS_2D_REAL64(13)%PTR + SELF%PHRSW => SELF%IN_VARS_2D_REAL64(14)%PTR + SELF%PHRLW => SELF%IN_VARS_2D_REAL64(15)%PTR + SELF%PVERVEL => SELF%IN_VARS_2D_REAL64(16)%PTR + SELF%PAP => SELF%IN_VARS_2D_REAL64(17)%PTR + SELF%PLU => SELF%IN_VARS_2D_REAL64(18)%PTR + SELF%PLUDE => SELF%IN_VARS_2D_REAL64(19)%PTR + SELF%PSNDE => SELF%IN_VARS_2D_REAL64(20)%PTR + SELF%PMFU => SELF%IN_VARS_2D_REAL64(21)%PTR + SELF%PMFD => SELF%IN_VARS_2D_REAL64(22)%PTR + SELF%PA => SELF%IN_VARS_2D_REAL64(23)%PTR + SELF%PSUPSAT => SELF%IN_VARS_2D_REAL64(24)%PTR + + FIELD = FSET%FIELD("PLSM") + CALL FIELD%DATA(SELF%PLSM, IBLK) + FIELD = FSET%FIELD("LDCUM") + CALL FIELD%DATA(SELF%LDCUM, IBLK) + FIELD = FSET%FIELD("KTYPE") + CALL FIELD%DATA(SELF%KTYPE, IBLK) + FIELD = FSET%FIELD("PAPH") + CALL FIELD%DATA(SELF%PAPH, IBLK) + FIELD = FSET%FIELD("PEXTRA") + CALL FIELD%DATA(SELF%PEXTRA, IBLK) + FIELD = FSET%FIELD("PCLV") + CALL FIELD%DATA(SELF%PCLV, IBLK) + + FIELD = FSET%FIELD('TENDENCY_CML') + CALL FIELD%DATA(TMP3D, IBLK) + SELF%TENDENCY_CML%T => TMP3D(:,:,1) + SELF%TENDENCY_CML%A => TMP3D(:,:,2) + SELF%TENDENCY_CML%Q => TMP3D(:,:,3) + SELF%TENDENCY_CML%CLD => TMP3D(:,:,4:) + FIELD = FSET%FIELD('TENDENCY_TMP') + CALL FIELD%DATA(TMP3D, IBLK) + SELF%TENDENCY_TMP%T => TMP3D(:,:,1) + SELF%TENDENCY_TMP%A => TMP3D(:,:,2) + SELF%TENDENCY_TMP%Q => TMP3D(:,:,3) + SELF%TENDENCY_TMP%CLD => TMP3D(:,:,4:) + FIELD = FSET%FIELD('TENDENCY_LOC') + CALL FIELD%DATA(TMP3D, IBLK) + SELF%TENDENCY_LOC%T => TMP3D(:,:,1) + SELF%TENDENCY_LOC%A => TMP3D(:,:,2) + SELF%TENDENCY_LOC%Q => TMP3D(:,:,3) + SELF%TENDENCY_LOC%CLD => TMP3D(:,:,4:) + + DO IVAR = 1, SIZE(SELF%OUT_VARS_2D_REAL64) + FIELD = FSET%FIELD(TRIM(OUT_VAR_NAMES(IVAR))) + CALL FIELD%DATA(SELF%OUT_VARS_2D_REAL64(IVAR)%PTR, IBLK) + ENDDO + SELF%PFSQLF => SELF%OUT_VARS_2D_REAL64(1)%PTR + SELF%PFSQIF => SELF%OUT_VARS_2D_REAL64(2)%PTR + SELF%PFCQLNG => SELF%OUT_VARS_2D_REAL64(3)%PTR + SELF%PFCQNNG => SELF%OUT_VARS_2D_REAL64(4)%PTR + SELF%PFSQRF => SELF%OUT_VARS_2D_REAL64(5)%PTR + SELF%PFSQSF => SELF%OUT_VARS_2D_REAL64(6)%PTR + SELF%PFCQRNG => SELF%OUT_VARS_2D_REAL64(7)%PTR + SELF%PFCQSNG => SELF%OUT_VARS_2D_REAL64(8)%PTR + SELF%PFSQLTUR => SELF%OUT_VARS_2D_REAL64(9)%PTR + SELF%PFSQITUR => SELF%OUT_VARS_2D_REAL64(10)%PTR + SELF%PFPLSL => SELF%OUT_VARS_2D_REAL64(11)%PTR + SELF%PFPLSN => SELF%OUT_VARS_2D_REAL64(12)%PTR + SELF%PFHPSL => SELF%OUT_VARS_2D_REAL64(13)%PTR + SELF%PFHPSN => SELF%OUT_VARS_2D_REAL64(14)%PTR + SELF%PCOVPTOT => SELF%OUT_VARS_2D_REAL64(15)%PTR + + FIELD = FSET%FIELD('PRAINFRAC_TOPRFZ') + CALL FIELD%DATA(SELF%PRAINFRAC_TOPRFZ, IBLK) + END SUBROUTINE CLOUDSC_GLOBAL_ATLAS_STATE_BLOCK + + SUBROUTINE CLOUDSC_GLOBAL_ATLAS_STATE_LOAD(SELF, FSET, NPROMA, NGPTOT, NGPTOTG) ! Load reference input data via serialbox CLASS(CLOUDSC_GLOBAL_ATLAS_STATE) :: SELF + TYPE(ATLAS_FIELDSET), INTENT(INOUT) :: FSET INTEGER(KIND=JPIM), INTENT(IN) :: NPROMA, NGPTOT INTEGER(KIND=JPIM), INTENT(IN), OPTIONAL :: NGPTOTG + TYPE(ATLAS_STRUCTUREDGRID) :: GRID + TYPE(ATLAS_FUNCTIONSPACE_BLOCKSTRUCTUREDCOLUMNS) :: FSPACE INTEGER(KIND=JPIM) :: KLON, IVAR, B + TYPE(VAR3D_PTR), DIMENSION(24) :: IN_VARS_3D_REAL64 + TYPE(VAR3D_PTR), DIMENSION(15) :: OUT_VARS_3D_REAL64 + REAL(C_DOUBLE), POINTER :: TMP3D(:,:,:) + REAL(C_DOUBLE), POINTER :: TMP2D(:,:) TYPE(ATLAS_FIELD) :: FIELD CALL INPUT_INITIALIZE(NAME='input') @@ -146,7 +249,7 @@ SUBROUTINE CLOUDSC_GLOBAL_ATLAS_STATE_LOAD(SELF, NPROMA, NGPTOT, NGPTOTG) GRID = ATLAS_REGULARLONLATGRID(NGPTOT, 1) FSPACE = ATLAS_FUNCTIONSPACE_BLOCKSTRUCTUREDCOLUMNS(GRID, LEVELS=SELF%KLEV, NPROMA=NPROMA, HALO=0) SELF%NBLOCKS = FSPACE%NBLKS() - FSET = ATLAS_FIELDSET(); + FSET = ATLAS_FIELDSET() DO IVAR = 1, SIZE(IN_VAR_NAMES) - 6 ! last six variables are special CALL FSET%ADD(FSPACE%CREATE_FIELD(NAME=TRIM(IN_VAR_NAMES(IVAR)), KIND=ATLAS_REAL(JPRB))) @@ -158,76 +261,21 @@ SUBROUTINE CLOUDSC_GLOBAL_ATLAS_STATE_LOAD(SELF, NPROMA, NGPTOT, NGPTOTG) CALL FSET%ADD(FSPACE%CREATE_FIELD(NAME="PEXTRA", KIND=ATLAS_REAL(JPRB), VARIABLES=MAX(1,SELF%KFLDX))) CALL FSET%ADD(FSPACE%CREATE_FIELD(NAME="PCLV", KIND=ATLAS_REAL(JPRB), VARIABLES=MAX(1,NCLV))) - DO IVAR = 1, SIZE(SELF%IN_VARS_3D_REAL64) - FIELD = FSET%FIELD(TRIM(IN_VAR_NAMES(IVAR))) - CALL FIELD%DATA(SELF%IN_VARS_3D_REAL64(IVAR)%PTR) - ENDDO - SELF%PLCRIT_AER => SELF%IN_VARS_3D_REAL64(1)%PTR - SELF%PICRIT_AER => SELF%IN_VARS_3D_REAL64(2)%PTR - SELF%PRE_ICE => SELF%IN_VARS_3D_REAL64(3)%PTR - SELF%PCCN => SELF%IN_VARS_3D_REAL64(4)%PTR - SELF%PNICE => SELF%IN_VARS_3D_REAL64(5)%PTR - SELF%PT => SELF%IN_VARS_3D_REAL64(6)%PTR - SELF%PQ => SELF%IN_VARS_3D_REAL64(7)%PTR - SELF%PVFA => SELF%IN_VARS_3D_REAL64(8)%PTR - SELF%PVFL => SELF%IN_VARS_3D_REAL64(9)%PTR - SELF%PVFI => SELF%IN_VARS_3D_REAL64(10)%PTR - SELF%PDYNA => SELF%IN_VARS_3D_REAL64(11)%PTR - SELF%PDYNL => SELF%IN_VARS_3D_REAL64(12)%PTR - SELF%PDYNI => SELF%IN_VARS_3D_REAL64(13)%PTR - SELF%PHRSW => SELF%IN_VARS_3D_REAL64(14)%PTR - SELF%PHRLW => SELF%IN_VARS_3D_REAL64(15)%PTR - SELF%PVERVEL => SELF%IN_VARS_3D_REAL64(16)%PTR - SELF%PAP => SELF%IN_VARS_3D_REAL64(17)%PTR - SELF%PLU => SELF%IN_VARS_3D_REAL64(18)%PTR - SELF%PLUDE => SELF%IN_VARS_3D_REAL64(19)%PTR - SELF%PSNDE => SELF%IN_VARS_3D_REAL64(20)%PTR - SELF%PMFU => SELF%IN_VARS_3D_REAL64(21)%PTR - SELF%PMFD => SELF%IN_VARS_3D_REAL64(22)%PTR - SELF%PA => SELF%IN_VARS_3D_REAL64(23)%PTR - SELF%PSUPSAT => SELF%IN_VARS_3D_REAL64(24)%PTR - - FIELD = FSET%FIELD("PLSM") - CALL FIELD%DATA(SELF%PLSM) - FIELD = FSET%FIELD("LDCUM") - CALL FIELD%DATA(SELF%LDCUM) - FIELD = FSET%FIELD("KTYPE") - CALL FIELD%DATA(SELF%KTYPE) - FIELD = FSET%FIELD("PAPH") - CALL FIELD%DATA(SELF%PAPH) - FIELD = FSET%FIELD("PEXTRA") - CALL FIELD%DATA(SELF%PEXTRA) - FIELD = FSET%FIELD("PCLV") - CALL FIELD%DATA(SELF%PCLV) - DO IVAR = 1, SIZE(IN_VAR_NAMES) CALL LOADVAR_ATLAS(FSET, TRIM(IN_VAR_NAMES(IVAR)), KLON, NGPTOTG) ENDDO FIELD = FSPACE%CREATE_FIELD(NAME='TENDENCY_CML', KIND=ATLAS_REAL(JPRB), VARIABLES=3+NCLV) - CALL FIELD%DATA(SELF%B_CML) CALL FSET%ADD(FIELD) FIELD = FSPACE%CREATE_FIELD(NAME='TENDENCY_TMP', KIND=ATLAS_REAL(JPRB), VARIABLES=3+NCLV) - CALL FIELD%DATA(SELF%B_TMP) CALL FSET%ADD(FIELD) FIELD = FSPACE%CREATE_FIELD(NAME='TENDENCY_LOC', KIND=ATLAS_REAL(JPRB), VARIABLES=3+NCLV) - CALL FIELD%DATA(SELF%B_LOC) CALL FSET%ADD(FIELD) ! The STATE_TYPE arrays are tricky, as the AOSOA layout needs to be expictly ! unrolled at every step, and we rely on dirty hackery to do this. - CALL LOADSTATE_ATLAS(FSET, 'TENDENCY_CML', SELF%TENDENCY_CML, KLON, NGPTOTG) - CALL LOADSTATE_ATLAS(FSET, 'TENDENCY_TMP', SELF%TENDENCY_TMP, KLON, NGPTOTG) - ALLOCATE(SELF%TENDENCY_LOC(SELF%NBLOCKS)) - !$OMP PARALLEL DO DEFAULT(SHARED), PRIVATE(B) schedule(runtime) - DO B=1, SELF%NBLOCKS - SELF%TENDENCY_LOC(B)%T => SELF%B_LOC(:,:,1,B) - SELF%TENDENCY_LOC(B)%A => SELF%B_LOC(:,:,2,B) - SELF%TENDENCY_LOC(B)%Q => SELF%B_LOC(:,:,3,B) - SELF%TENDENCY_LOC(B)%CLD => SELF%B_LOC(:,:,4:,B) - END DO - !$OMP END PARALLEL DO - + CALL LOADSTATE_ATLAS(FSET, 'TENDENCY_CML', KLON, NGPTOTG) + CALL LOADSTATE_ATLAS(FSET, 'TENDENCY_TMP', KLON, NGPTOTG) ! Output fields are simply allocated and zero'd DO IVAR = 1, SIZE(OUT_VAR_NAMES) - 2 CALL FSET%ADD(FSPACE%CREATE_FIELD(NAME=TRIM(OUT_VAR_NAMES(IVAR)), KIND=ATLAS_REAL(JPRB), LEVELS=SELF%KLEV+1)) @@ -237,35 +285,23 @@ SUBROUTINE CLOUDSC_GLOBAL_ATLAS_STATE_LOAD(SELF, NPROMA, NGPTOT, NGPTOTG) DO IVAR = 1, SIZE(OUT_VAR_NAMES) - 1 FIELD = FSET%FIELD(TRIM(OUT_VAR_NAMES(IVAR))) - CALL FIELD%DATA(SELF%OUT_VARS_3D_REAL64(IVAR)%PTR) + CALL FIELD%DATA(TMP3D) !$OMP PARALLEL DO DEFAULT(SHARED), PRIVATE(B) schedule(runtime) DO B=1, SELF%NBLOCKS - SELF%OUT_VARS_3D_REAL64(IVAR)%PTR(:,:,B) = 0.0_JPRB + TMP3D(:,:,B) = 0.0_JPRB END DO !$omp end parallel do ENDDO - - SELF%PFSQLF => SELF%OUT_VARS_3D_REAL64(1)%PTR - SELF%PFSQIF => SELF%OUT_VARS_3D_REAL64(2)%PTR - SELF%PFCQLNG => SELF%OUT_VARS_3D_REAL64(3)%PTR - SELF%PFCQNNG => SELF%OUT_VARS_3D_REAL64(4)%PTR - SELF%PFSQRF => SELF%OUT_VARS_3D_REAL64(5)%PTR - SELF%PFSQSF => SELF%OUT_VARS_3D_REAL64(6)%PTR - SELF%PFCQRNG => SELF%OUT_VARS_3D_REAL64(7)%PTR - SELF%PFCQSNG => SELF%OUT_VARS_3D_REAL64(8)%PTR - SELF%PFSQLTUR => SELF%OUT_VARS_3D_REAL64(9)%PTR - SELF%PFSQITUR => SELF%OUT_VARS_3D_REAL64(10)%PTR - SELF%PFPLSL => SELF%OUT_VARS_3D_REAL64(11)%PTR - SELF%PFPLSN => SELF%OUT_VARS_3D_REAL64(12)%PTR - SELF%PFHPSL => SELF%OUT_VARS_3D_REAL64(13)%PTR - SELF%PFHPSN => SELF%OUT_VARS_3D_REAL64(14)%PTR - SELF%PCOVPTOT => SELF%OUT_VARS_3D_REAL64(15)%PTR + ! DEBUG + !FIELD = FSET%FIELD("PAP") + !call field%data(tmp3d) + !print *, MINVAL(tmp3d), MAXVAL(tmp3d) FIELD = FSET%FIELD("PRAINFRAC_TOPRFZ") - CALL FIELD%DATA(SELF%PRAINFRAC_TOPRFZ) + CALL FIELD%DATA(TMP2D) !$OMP PARALLEL DO DEFAULT(SHARED), PRIVATE(B) schedule(runtime) DO B=1, SELF%NBLOCKS - SELF%PRAINFRAC_TOPRFZ(:,B) = 0.0_JPRB + TMP2D(:,B) = 0.0_JPRB END DO !$OMP END PARALLEL DO @@ -279,12 +315,12 @@ SUBROUTINE CLOUDSC_GLOBAL_ATLAS_STATE_LOAD(SELF, NPROMA, NGPTOT, NGPTOTG) CALL YREPHLI_LOAD_PARAMETERS() CALL INPUT_FINALIZE() - END SUBROUTINE CLOUDSC_GLOBAL_ATLAS_STATE_LOAD - SUBROUTINE CLOUDSC_GLOBAL_ATLAS_STATE_VALIDATE(SELF, NGPTOT, NGPTOTG) + SUBROUTINE CLOUDSC_GLOBAL_ATLAS_STATE_VALIDATE(SELF, FSET, NGPTOT, NGPTOTG) ! Validate the correctness of output against reference data CLASS(CLOUDSC_GLOBAL_ATLAS_STATE) :: SELF + TYPE(ATLAS_FIELDSET), INTENT(INOUT) :: FSET INTEGER(KIND=JPIM), INTENT(IN) :: NGPTOT INTEGER(KIND=JPIM), INTENT(IN), OPTIONAL :: NGPTOTG @@ -292,6 +328,7 @@ SUBROUTINE CLOUDSC_GLOBAL_ATLAS_STATE_VALIDATE(SELF, NGPTOT, NGPTOTG) CALL INPUT_INITIALIZE(NAME='reference') CALL LOAD_SCALAR('KLON', KLON) + print *, "KLON = ", KLON CALL INPUT_FINALIZE() ! Write variable validation header diff --git a/src/cloudsc_fortran_atlas/dwarf_cloudsc_atlas.F90 b/src/cloudsc_fortran_atlas/dwarf_cloudsc_atlas.F90 index 2504766c..1e2a7df2 100644 --- a/src/cloudsc_fortran_atlas/dwarf_cloudsc_atlas.F90 +++ b/src/cloudsc_fortran_atlas/dwarf_cloudsc_atlas.F90 @@ -28,6 +28,10 @@ PROGRAM DWARF_CLOUDSC INTEGER(KIND=JPIM) :: NPROMA = 32 ! NPROMA blocking factor (currently active) INTEGER(KIND=JPIM) :: NGPTOT ! Local number of grid points +REAL(c_double), pointer :: tmp3d(:,:,:) +type(atlas_fieldset) :: fset +type(atlas_field) :: field + TYPE(CLOUDSC_GLOBAL_ATLAS_STATE) :: GLOBAL_ATLAS_STATE INTEGER(KIND=JPIB) :: ENERGY, POWER @@ -69,33 +73,20 @@ PROGRAM DWARF_CLOUDSC READ(CLARG(1:LENARG),*) NPROMA ENDIF -! TODO: Create a global global memory state from serialized input data -CALL GLOBAL_ATLAS_STATE%LOAD(NPROMA, NGPTOT, NGPTOTG) +FSET = ATLAS_FIELDSET() + +! TODO: Create a global memory state from serialized input data +CALL GLOBAL_ATLAS_STATE%LOAD(FSET, NPROMA, NGPTOT, NGPTOTG) + +!FIELD = FSET%FIELD("PAP") +!call field%data(tmp3d) +!print *, MINVAL(tmp3d), MAXVAL(tmp3d) ! Call the driver to perform the parallel loop over our kernel -CALL CLOUDSC_DRIVER(NUMOMP, NPROMA, GLOBAL_ATLAS_STATE%KLEV, NGPTOT, NGPTOTG, & - & GLOBAL_ATLAS_STATE%KFLDX, GLOBAL_ATLAS_STATE%PTSPHY, & - & GLOBAL_ATLAS_STATE%PT, GLOBAL_ATLAS_STATE%PQ, & - & GLOBAL_ATLAS_STATE%TENDENCY_CML, GLOBAL_ATLAS_STATE%TENDENCY_TMP, GLOBAL_ATLAS_STATE%TENDENCY_LOC, & - & GLOBAL_ATLAS_STATE%PVFA, GLOBAL_ATLAS_STATE%PVFL, GLOBAL_ATLAS_STATE%PVFI, & - & GLOBAL_ATLAS_STATE%PDYNA, GLOBAL_ATLAS_STATE%PDYNL, GLOBAL_ATLAS_STATE%PDYNI, & - & GLOBAL_ATLAS_STATE%PHRSW, GLOBAL_ATLAS_STATE%PHRLW, & - & GLOBAL_ATLAS_STATE%PVERVEL, GLOBAL_ATLAS_STATE%PAP, GLOBAL_ATLAS_STATE%PAPH, & - & GLOBAL_ATLAS_STATE%PLSM, GLOBAL_ATLAS_STATE%LDCUM, GLOBAL_ATLAS_STATE%KTYPE, & - & GLOBAL_ATLAS_STATE%PLU, GLOBAL_ATLAS_STATE%PLUDE, GLOBAL_ATLAS_STATE%PSNDE, & - & GLOBAL_ATLAS_STATE%PMFU, GLOBAL_ATLAS_STATE%PMFD, & - & GLOBAL_ATLAS_STATE%PA, GLOBAL_ATLAS_STATE%PCLV, GLOBAL_ATLAS_STATE%PSUPSAT,& - & GLOBAL_ATLAS_STATE%PLCRIT_AER, GLOBAL_ATLAS_STATE%PICRIT_AER, GLOBAL_ATLAS_STATE%PRE_ICE, & - & GLOBAL_ATLAS_STATE%PCCN, GLOBAL_ATLAS_STATE%PNICE,& - & GLOBAL_ATLAS_STATE%PCOVPTOT, GLOBAL_ATLAS_STATE%PRAINFRAC_TOPRFZ, & - & GLOBAL_ATLAS_STATE%PFSQLF, GLOBAL_ATLAS_STATE%PFSQIF , GLOBAL_ATLAS_STATE%PFCQNNG, GLOBAL_ATLAS_STATE%PFCQLNG, & - & GLOBAL_ATLAS_STATE%PFSQRF, GLOBAL_ATLAS_STATE%PFSQSF , GLOBAL_ATLAS_STATE%PFCQRNG, GLOBAL_ATLAS_STATE%PFCQSNG, & - & GLOBAL_ATLAS_STATE%PFSQLTUR, GLOBAL_ATLAS_STATE%PFSQITUR, & - & GLOBAL_ATLAS_STATE%PFPLSL, GLOBAL_ATLAS_STATE%PFPLSN, GLOBAL_ATLAS_STATE%PFHPSL, GLOBAL_ATLAS_STATE%PFHPSN & - & ) +CALL CLOUDSC_DRIVER(FSET, NUMOMP, NGPTOT, NGPTOTG, GLOBAL_ATLAS_STATE%KFLDX, GLOBAL_ATLAS_STATE%PTSPHY) ! Validate the output against serialized reference data -!CALL GLOBAL_ATLAS_STATE%VALIDATE(NGPTOT, NGPTOTG) +CALL GLOBAL_ATLAS_STATE%VALIDATE(FSET, NGPTOT, NGPTOTG) CALL ATLAS_LIBRARY%FINALISE() diff --git a/src/cloudsc_fortran_atlas/expand_atlas_mod.F90 b/src/cloudsc_fortran_atlas/expand_atlas_mod.F90 index 1037f9c1..edcf8c1b 100644 --- a/src/cloudsc_fortran_atlas/expand_atlas_mod.F90 +++ b/src/cloudsc_fortran_atlas/expand_atlas_mod.F90 @@ -98,15 +98,13 @@ subroutine loadvar_atlas(fset, name, nlon, ngptotg) endif end subroutine loadvar_atlas - subroutine loadstate_atlas(fset, name, state, nlon, ngptotg) + subroutine loadstate_atlas(fset, name, nlon, ngptotg) ! Load into the local memory buffer and expand to global field type(atlas_fieldset), intent(inout) :: fset character(len=*) :: name - type(state_type), allocatable, intent(inout) :: state(:) integer(kind=jpim), intent(in) :: nlon integer(kind=jpim), intent(in), optional :: ngptotg - integer :: b integer(kind=jpim) :: start, end, size, nlev, nproma, ngptot, nblocks, ndim type(atlas_field) :: field type(atlas_functionspace_blockstructuredcolumns) :: fspace @@ -124,7 +122,6 @@ subroutine loadstate_atlas(fset, name, state, nlon, ngptotg) call get_offsets(start, end, size, nlon, ndim, nlev, ngptot, ngptotg) allocate(buffer(size, nlev, 3+ndim)) - if (.not. allocated(state)) allocate(state(nblocks)) call field%data(field_r3) call load_array(name//'_T', start, end, size, nlon, nlev, buffer(:,:,1)) @@ -137,15 +134,6 @@ subroutine loadstate_atlas(fset, name, state, nlon, ngptotg) call expand(buffer(:,:,3), field_r3(:,:,3,:), size, nproma, nlev, ngptot, nblocks) call expand(buffer(:,:,4:), field_r3(:,:,4:,:), size, nproma, nlev, ndim, ngptot, nblocks) deallocate(buffer) - -!$OMP PARALLEL DO DEFAULT(SHARED), PRIVATE(B) schedule(runtime) - do b=1, nblocks - state(b)%t => field_r3(:,:,1,b) - state(b)%a => field_r3(:,:,2,b) - state(b)%q => field_r3(:,:,3,b) - state(b)%cld => field_r3(:,:,4:3+ndim,b) - end do -!$OMP end parallel do end subroutine loadstate_atlas end module expand_atlas_mod diff --git a/src/cloudsc_fortran_atlas/validate_atlas_mod.F90 b/src/cloudsc_fortran_atlas/validate_atlas_mod.F90 index 32365859..1b1b43f1 100644 --- a/src/cloudsc_fortran_atlas/validate_atlas_mod.F90 +++ b/src/cloudsc_fortran_atlas/validate_atlas_mod.F90 @@ -141,7 +141,7 @@ SUBROUTINE VALIDATEVAR_ATLAS(FSET, NAME, NLON, NGPTOTG, STATE_VAR) !OMP PARALLEL DO DEFAULT(SHARED) PRIVATE(B, BSIZE) & !& REDUCTION(MIN:ZMINVAL, MAX:ZMAX_VAL_ERR, +:ZSUM_ERR_ABS) DO B=1, NBLOCKS - BSIZE = MIN(NLON, NGPTOT - (B-1)*NLON) ! Field block size + BSIZE = FSPACE%BLOCK_SIZE(B) ZMINVAL(1) = MIN(ZMINVAL(1),MINVAL(FIELD_R3(:,:,4:,B))) ZMAX_VAL_ERR(1) = MAX(ZMAX_VAL_ERR(1),MAXVAL(FIELD_R3(:,:,4:,B))) DO JM=1, NDIM From d24f63fb8811ebc58600fd7c7d5b4eefb542a769 Mon Sep 17 00:00:00 2001 From: Slavko Brdar Date: Tue, 9 May 2023 15:15:24 +0200 Subject: [PATCH 03/45] FieldSet is not thread-safe -> unpack fields prior the cloudsc loop (tnx Willem) --- .../cloudsc_driver_mod.F90 | 66 +---- .../cloudsc_global_atlas_state_mod.F90 | 260 ++++++++++++------ 2 files changed, 189 insertions(+), 137 deletions(-) diff --git a/src/cloudsc_fortran_atlas/cloudsc_driver_mod.F90 b/src/cloudsc_fortran_atlas/cloudsc_driver_mod.F90 index 8a84c043..d173c870 100644 --- a/src/cloudsc_fortran_atlas/cloudsc_driver_mod.F90 +++ b/src/cloudsc_fortran_atlas/cloudsc_driver_mod.F90 @@ -14,7 +14,7 @@ MODULE CLOUDSC_DRIVER_MOD USE CLOUDSC_MPI_MOD, ONLY: NUMPROC, IRANK USE TIMER_MOD, ONLY : PERFORMANCE_TIMER, GET_THREAD_NUM USE EC_PMON_MOD, ONLY: EC_PMON - USE CLOUDSC_GLOBAL_ATLAS_STATE_MOD, ONLY: CLOUDSC_GLOBAL_ATLAS_STATE_BLOCK_VIEW + USE CLOUDSC_GLOBAL_ATLAS_STATE_MOD, ONLY: CLOUDSC_GLOBAL_ATLAS_STATE_BLOCK_VIEW, CLOUDSC_GLOBAL_ATLAS_STATE_FIELDS USE ATLAS_MODULE USE, INTRINSIC :: ISO_C_BINDING @@ -33,60 +33,11 @@ SUBROUTINE CLOUDSC_DRIVER(FSET, NUMOMP, NGPTOT, NGPTOTG, KFLDX, PTSPHY) INTEGER(KIND=JPIM), INTENT(IN) :: NUMOMP, NGPTOT, NGPTOTG, KFLDX REAL(KIND=JPRB), INTENT(IN) :: PTSPHY ! Physics timestep + TYPE(CLOUDSC_GLOBAL_ATLAS_STATE_FIELDS) :: SFIELDS TYPE(CLOUDSC_GLOBAL_ATLAS_STATE_BLOCK_VIEW) :: FBLOCK TYPE(ATLAS_FUNCTIONSPACE_BLOCKSTRUCTUREDCOLUMNS) :: FSPACE TYPE(ATLAS_FIELD) :: FIELD INTEGER(KIND=JPIM) :: NPROMA, NLEV - REAL(KIND=JPRB), POINTER :: PT(:,:) ! T at start of callpar - REAL(KIND=JPRB), POINTER :: PQ(:,:) ! Q at start of callpar - TYPE(STATE_TYPE), POINTER :: TENDENCY_CML(:) ! cumulative tendency used for final output - TYPE(STATE_TYPE), POINTER :: TENDENCY_TMP(:) ! cumulative tendency used as input - TYPE(STATE_TYPE), POINTER :: TENDENCY_LOC(:) ! local tendency from cloud scheme - REAL(KIND=JPRB), POINTER:: PVFA(:,:) ! CC from VDF scheme - REAL(KIND=JPRB), POINTER:: PVFL(:,:) ! Liq from VDF scheme - REAL(KIND=JPRB), POINTER:: PVFI(:,:) ! Ice from VDF scheme - REAL(KIND=JPRB), POINTER:: PDYNA(:,:) ! CC from Dynamics - REAL(KIND=JPRB), POINTER:: PDYNL(:,:) ! Liq from Dynamics - REAL(KIND=JPRB), POINTER:: PDYNI(:,:) ! Liq from Dynamics - REAL(KIND=JPRB), POINTER:: PHRSW(:,:) ! Short-wave heating rate - REAL(KIND=JPRB), POINTER:: PHRLW(:,:) ! Long-wave heating rate - REAL(KIND=JPRB), POINTER:: PVERVEL(:,:) !Vertical velocity - REAL(KIND=JPRB), POINTER:: PAP(:,:) ! Pressure on full levels - REAL(KIND=JPRB), POINTER:: PAPH(:,:) ! Pressure on half levels - REAL(KIND=JPRB), POINTER:: PLSM(:) ! Land fraction (0-1) - LOGICAL, POINTER :: LDCUM(:) ! Convection active - INTEGER(KIND=JPIM), POINTER :: KTYPE(:) ! Convection type 0,1,2 - REAL(KIND=JPRB), POINTER:: PLU(:,:) ! Conv. condensate - REAL(KIND=JPRB), POINTER:: PLUDE(:,:) ! Conv. detrained water - REAL(KIND=JPRB), POINTER:: PSNDE(:,:) ! Conv. detrained snow - REAL(KIND=JPRB), POINTER:: PMFU(:,:) ! Conv. mass flux up - REAL(KIND=JPRB), POINTER:: PMFD(:,:) ! Conv. mass flux down - REAL(KIND=JPRB), POINTER:: PA(:,:) ! Original Cloud fraction (t) - REAL(KIND=JPRB), POINTER:: PCLV(:,:,:) - REAL(KIND=JPRB), POINTER:: PSUPSAT(:,:) - REAL(KIND=JPRB), POINTER:: PLCRIT_AER(:,:) - REAL(KIND=JPRB), POINTER:: PICRIT_AER(:,:) - REAL(KIND=JPRB), POINTER:: PRE_ICE(:,:) - REAL(KIND=JPRB), POINTER:: PCCN(:,:) ! liquid cloud condensation nuclei - REAL(KIND=JPRB), POINTER:: PNICE(:,:) ! ice number concentration (cf. CCN) - - REAL(KIND=JPRB), POINTER:: PCOVPTOT(:,:) ! Precip fraction - REAL(KIND=JPRB), POINTER:: PRAINFRAC_TOPRFZ(:) - ! Flux diagnostics for DDH budget - REAL(KIND=JPRB), POINTER :: PFSQLF(:,:) ! Flux of liquid - REAL(KIND=JPRB), POINTER :: PFSQIF(:,:) ! Flux of ice - REAL(KIND=JPRB), POINTER :: PFCQLNG(:,:) ! -ve corr for liq - REAL(KIND=JPRB), POINTER :: PFCQNNG(:,:) ! -ve corr for ice - REAL(KIND=JPRB), POINTER :: PFSQRF(:,:) ! Flux diagnostics - REAL(KIND=JPRB), POINTER :: PFSQSF(:,:) ! for DDH, generic - REAL(KIND=JPRB), POINTER :: PFCQRNG(:,:) ! rain - REAL(KIND=JPRB), POINTER :: PFCQSNG(:,:) ! snow - REAL(KIND=JPRB), POINTER :: PFSQLTUR(:,:) ! liquid flux due to VDF - REAL(KIND=JPRB), POINTER :: PFSQITUR(:,:) ! ice flux due to VDF - REAL(KIND=JPRB), POINTER :: PFPLSL(:,:) ! liq+rain sedim flux - REAL(KIND=JPRB), POINTER :: PFPLSN(:,:) ! ice+snow sedim flux - REAL(KIND=JPRB), POINTER :: PFHPSL(:,:) ! Enthalpy flux for liq - REAL(KIND=JPRB), POINTER :: PFHPSN(:,:) ! Enthalp flux for ice INTEGER(KIND=JPIM) :: JKGLO,IBL,ICEND,NGPBLKS @@ -107,18 +58,20 @@ SUBROUTINE CLOUDSC_DRIVER(FSET, NUMOMP, NGPTOT, NGPTOTG, KFLDX, PTSPHY) FSPACE = FIELD%FUNCTIONSPACE() NPROMA = FSPACE%BLOCK_SIZE(1) NLEV = FSPACE%LEVELS() - print *, "NPROMA, NUMOMP ", NPROMA, NUMOMP NGPBLKS = (NGPTOT / NPROMA) + MIN(MOD(NGPTOT,NPROMA), 1) -1003 format(5x,'NUMPROC=',i0,', NUMOMP=',i0,', NGPTOTG=',i0,', NPROMA=',i0,', NGPBLKS=',i0) +1003 format(5x,'NUMPROC=',i0,', NUMOMP=',i0,', NGTOT=', i0,', NGPTOTG=',i0,', NPROMA=',i0,', NGPBLKS=',i0) if (irank == 0) then - write(0,1003) NUMPROC,NUMOMP,NGPTOTG,NPROMA,NGPBLKS + write(0,1003) NUMPROC,NUMOMP,NGPTOT, NGPTOTG,NPROMA,NGPBLKS end if ! Global timer for the parallel region CALL TIMER%START(NUMOMP) - !$omp parallel default(shared) private(JKGLO,IBL,ICEND,TID,energy,power) & + + CALL SFIELDS%SETUP(FSET) + + !$omp parallel default(shared) private(JKGLO,IBL,ICEND,TID,energy,power,FBLOCK) & !$omp& num_threads(NUMOMP) ! Local timer for each thread @@ -131,7 +84,8 @@ SUBROUTINE CLOUDSC_DRIVER(FSET, NUMOMP, NGPTOT, NGPTOTG, KFLDX, PTSPHY) ICEND=MIN(NPROMA,NGPTOT-JKGLO+1) ! get block views - call FBLOCK%GET_BLOCK(FSET, IBL) + call FBLOCK%GET_BLOCK(SFIELDS, IBL) + CONTINUE !-- These were uninitialized : meaningful only when we compare error differences FBLOCK%PCOVPTOT(:,:) = 0.0_JPRB diff --git a/src/cloudsc_fortran_atlas/cloudsc_global_atlas_state_mod.F90 b/src/cloudsc_fortran_atlas/cloudsc_global_atlas_state_mod.F90 index 713e833d..ba18ac74 100644 --- a/src/cloudsc_fortran_atlas/cloudsc_global_atlas_state_mod.F90 +++ b/src/cloudsc_fortran_atlas/cloudsc_global_atlas_state_mod.F90 @@ -34,6 +34,9 @@ MODULE CLOUDSC_GLOBAL_ATLAS_STATE_MOD REAL(C_DOUBLE), POINTER :: PTR(:,:) END TYPE + !INTEGER, PARAMETER PLCRIT_AER = 1 + !INTEGER, PARAMETER PLCRIT_AER = 2 + !IN_VAR_NAMES(PLCRIT_AER) CHARACTER(LEN=10), PARAMETER, DIMENSION(30) :: IN_VAR_NAMES = (/ & "PLCRIT_AER", "PICRIT_AER", "PRE_ICE ", "PCCN ", "PNICE ", "PT ", "PQ ", & "PVFA ", "PVFL ", "PVFI ", "PDYNA ", "PDYNL ", "PDYNI ", "PHRSW ", & @@ -83,9 +86,9 @@ MODULE CLOUDSC_GLOBAL_ATLAS_STATE_MOD REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PEXTRA(:,:,:) ! extra fields REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PCLV(:,:,:) - TYPE(STATE_TYPE) :: TENDENCY_CML ! cumulative tendency used for final output - TYPE(STATE_TYPE) :: TENDENCY_TMP ! cumulative tendency used as input - TYPE(STATE_TYPE) :: TENDENCY_LOC ! local tendency from cloud scheme + TYPE(STATE_TYPE) :: TENDENCY_CML ! cumulative tendency used for final output + TYPE(STATE_TYPE) :: TENDENCY_TMP ! cumulative tendency used as input + TYPE(STATE_TYPE) :: TENDENCY_LOC ! local tendency from cloud scheme ! Output fields used for validation REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PFSQLF(:,:) ! Flux of liquid @@ -104,10 +107,72 @@ MODULE CLOUDSC_GLOBAL_ATLAS_STATE_MOD REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PFHPSN(:,:) ! Enthalpy flux for ice REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PCOVPTOT(:,:) ! Precip fraction REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PRAINFRAC_TOPRFZ(:) - CONTAINS + + CONTAINS PROCEDURE :: GET_BLOCK => CLOUDSC_GLOBAL_ATLAS_STATE_BLOCK END TYPE CLOUDSC_GLOBAL_ATLAS_STATE_BLOCK_VIEW + TYPE CLOUDSC_GLOBAL_ATLAS_STATE_FIELDS + !TYPE(atlas_field), allocatlabe :: fields(:) + + ! Input field variables and tendencies + TYPE(atlas_field) :: fPLCRIT_AER + TYPE(atlas_field) :: fPICRIT_AER + TYPE(atlas_field) :: fPRE_ICE + TYPE(atlas_field) :: fPCCN ! liquid cloud condensation nuclei + TYPE(atlas_field) :: fPNICE ! ice number concentration (cf. CCN) + TYPE(atlas_field) :: fPT ! T at start of callpar + TYPE(atlas_field) :: fPQ ! Q at start of callpar + TYPE(atlas_field) :: fPVFA ! CC from VDF scheme + TYPE(atlas_field) :: fPVFL ! Liq from VDF scheme + TYPE(atlas_field) :: fPVFI ! Ice from VDF scheme + TYPE(atlas_field) :: fPDYNA ! CC from Dynamics + TYPE(atlas_field) :: fPDYNL ! Liq from Dynamics + TYPE(atlas_field) :: fPDYNI ! Liq from Dynamics + TYPE(atlas_field) :: fPHRSW ! Short-wave heating rate + TYPE(atlas_field) :: fPHRLW ! Long-wave heating rate + TYPE(atlas_field) :: fPVERVEL ! Vertical velocity + TYPE(atlas_field) :: fPAP ! Pressure on full levels + TYPE(atlas_field) :: fPLU ! Conv. condensate + TYPE(atlas_field) :: fPLUDE ! Conv. detrained water + TYPE(atlas_field) :: fPSNDE ! Conv. detrained snow + TYPE(atlas_field) :: fPMFU ! Conv. mass flux up + TYPE(atlas_field) :: fPMFD ! Conv. mass flux down + TYPE(atlas_field) :: fPA ! Original Cloud fraction (t) + TYPE(atlas_field) :: fPSUPSAT + + TYPE(atlas_field) :: fPLSM ! Land fraction (0-1) + TYPE(atlas_field) :: fLDCUM ! Convection active + TYPE(atlas_field) :: fKTYPE ! Convection type 0,1,2 + TYPE(atlas_field) :: fPAPH ! Pressure on half levels + TYPE(atlas_field) :: fPEXTRA ! extra fields + TYPE(atlas_field) :: fPCLV + + TYPE(atlas_field) :: fTENDENCY_CML ! cumulative tendency used for final output + TYPE(atlas_field) :: fTENDENCY_TMP ! cumulative tendency used as input + TYPE(atlas_field) :: fTENDENCY_LOC ! local tendency from cloud scheme + + ! Output fields used for validation + TYPE(atlas_field) :: fPFSQLF ! Flux of liquid + TYPE(atlas_field) :: fPFSQIF ! Flux of ice + TYPE(atlas_field) :: fPFCQLNG ! -ve corr for liq + TYPE(atlas_field) :: fPFCQNNG ! -ve corr for ice + TYPE(atlas_field) :: fPFSQRF ! Flux diagnostics + TYPE(atlas_field) :: fPFSQSF ! for DDH, generic + TYPE(atlas_field) :: fPFCQRNG ! rain + TYPE(atlas_field) :: fPFCQSNG ! snow + TYPE(atlas_field) :: fPFSQLTUR ! liquid flux due to VDF + TYPE(atlas_field) :: fPFSQITUR ! ice flux due to VDF + TYPE(atlas_field) :: fPFPLSL ! liq+rain sedim flux + TYPE(atlas_field) :: fPFPLSN ! ice+snow sedim flux + TYPE(atlas_field) :: fPFHPSL ! Enthalpy flux for liq + TYPE(atlas_field) :: fPFHPSN ! Enthalpy flux for ice + TYPE(atlas_field) :: fPCOVPTOT ! Precip fraction + TYPE(atlas_field) :: fPRAINFRAC_TOPRFZ + CONTAINS + PROCEDURE :: SETUP => CLOUDSC_GLOBAL_ATLAS_SETUP_BLOCK + END TYPE CLOUDSC_GLOBAL_ATLAS_STATE_FIELDS + TYPE CLOUDSC_GLOBAL_ATLAS_STATE ! Memory state containing raw fields annd tendencies for CLOUDSC dwarf ! @@ -127,101 +192,134 @@ MODULE CLOUDSC_GLOBAL_ATLAS_STATE_MOD CONTAINS - SUBROUTINE CLOUDSC_GLOBAL_ATLAS_STATE_BLOCK(SELF, FSET, IBLK) - ! Load reference input data via serialbox - CLASS(CLOUDSC_GLOBAL_ATLAS_STATE_BLOCK_VIEW), INTENT(INOUT) :: SELF + SUBROUTINE CLOUDSC_GLOBAL_ATLAS_SETUP_BLOCK(SELF, FSET) + CLASS(CLOUDSC_GLOBAL_ATLAS_STATE_FIELDS), INTENT(INOUT) :: SELF TYPE(ATLAS_FIELDSET), INTENT(INOUT) :: FSET + + SELF%fPLCRIT_AER = FSET%FIELD("PLCRIT_AER") + SELF%fPICRIT_AER = FSET%FIELD("PICRIT_AER") + SELF%fPRE_ICE = FSET%FIELD("PRE_ICE") + SELF%fPCCN = FSET%FIELD("PCCN") + SELF%fPNICE = FSET%FIELD("PNICE") + SELF%fPT = FSET%FIELD("PT") + SELF%fPQ = FSET%FIELD("PQ") + SELF%fPVFA = FSET%FIELD("PVFA") + SELF%fPVFL = FSET%FIELD("PVFL") + SELF%fPVFI = FSET%FIELD("PVFI") + SELF%fPDYNA = FSET%FIELD("PDYNA") + SELF%fPDYNL = FSET%FIELD("PDYNL") + SELF%fPDYNI = FSET%FIELD("PDYNI") + SELF%fPHRSW = FSET%FIELD("PHRSW") + SELF%fPHRLW = FSET%FIELD("PHRLW") + SELF%fPVERVEL = FSET%FIELD("PVERVEL") + SELF%fPAP = FSET%FIELD("PAP") + SELF%fPLU = FSET%FIELD("PLU") + SELF%fPLUDE = FSET%FIELD("PLUDE") + SELF%fPSNDE = FSET%FIELD("PSNDE") + SELF%fPMFU = FSET%FIELD("PMFU") + SELF%fPMFD = FSET%FIELD("PMFD") + SELF%fPA = FSET%FIELD("PA") + SELF%fPSUPSAT = FSET%FIELD("PSUPSAT") + SELF%fPLSM = FSET%FIELD("PLSM") + SELF%fLDCUM = FSET%FIELD("LDCUM") + SELF%fKTYPE = FSET%FIELD("KTYPE") + SELF%fPAPH = FSET%FIELD("PAPH") + SELF%fPEXTRA = FSET%FIELD("PEXTRA") + SELF%fPCLV = FSET%FIELD("PCLV") + + SELF%fTENDENCY_CML = FSET%FIELD('TENDENCY_CML') + SELF%fTENDENCY_TMP = FSET%FIELD('TENDENCY_TMP') + SELF%fTENDENCY_LOC = FSET%FIELD('TENDENCY_LOC') + + SELF%fPFSQLF = FSET%FIELD("PFSQLF") + SELF%fPFSQIF = FSET%FIELD("PFSQIF") + SELF%fPFCQLNG = FSET%FIELD("PFCQLNG") + SELF%fPFCQNNG = FSET%FIELD("PFCQNNG") + SELF%fPFSQRF = FSET%FIELD("PFSQRF") + SELF%fPFSQSF = FSET%FIELD("PFSQSF") + SELF%fPFCQRNG = FSET%FIELD("PFCQRNG") + SELF%fPFCQSNG = FSET%FIELD("PFCQSNG") + SELF%fPFSQLTUR = FSET%FIELD("PFSQLTUR") + SELF%fPFSQITUR = FSET%FIELD("PFSQITUR") + SELF%fPFPLSL = FSET%FIELD("PFPLSL") + SELF%fPFPLSN = FSET%FIELD("PFPLSN") + SELF%fPFHPSL = FSET%FIELD("PFHPSL") + SELF%fPFHPSN = FSET%FIELD("PFHPSN") + SELF%fPCOVPTOT = FSET%FIELD("PCOVPTOT") + SELF%fPRAINFRAC_TOPRFZ = FSET%FIELD("PRAINFRAC_TOPRFZ") + END SUBROUTINE + + SUBROUTINE CLOUDSC_GLOBAL_ATLAS_STATE_BLOCK(SELF, FIELDS, IBLK) + CLASS(CLOUDSC_GLOBAL_ATLAS_STATE_BLOCK_VIEW), INTENT(INOUT) :: SELF + CLASS(CLOUDSC_GLOBAL_ATLAS_STATE_FIELDS), INTENT(INOUT) :: FIELDS INTEGER, INTENT(IN) :: IBLK - TYPE(ATLAS_FUNCTIONSPACE_BLOCKSTRUCTUREDCOLUMNS) :: FSPACE - INTEGER(KIND=JPIM) :: KLON, IVAR, B - TYPE(ATLAS_FIELD) :: FIELD - REAL(C_DOUBLE), POINTER :: TMP3D(:,:,:), TMP2D(:,:) + REAL(C_DOUBLE), POINTER :: TMP3D(:,:,:) ! NOTE the last six input variables need special treatment - different types - DO IVAR = 1, SIZE(IN_VAR_NAMES) - 6 - FIELD = FSET%FIELD(TRIM(IN_VAR_NAMES(IVAR))) - CALL FIELD%DATA(SELF%IN_VARS_2D_REAL64(IVAR)%PTR, IBLK) - ENDDO - SELF%PLCRIT_AER => SELF%IN_VARS_2D_REAL64(1)%PTR - SELF%PICRIT_AER => SELF%IN_VARS_2D_REAL64(2)%PTR - SELF%PRE_ICE => SELF%IN_VARS_2D_REAL64(3)%PTR - SELF%PCCN => SELF%IN_VARS_2D_REAL64(4)%PTR - SELF%PNICE => SELF%IN_VARS_2D_REAL64(5)%PTR - SELF%PT => SELF%IN_VARS_2D_REAL64(6)%PTR - SELF%PQ => SELF%IN_VARS_2D_REAL64(7)%PTR - SELF%PVFA => SELF%IN_VARS_2D_REAL64(8)%PTR - SELF%PVFL => SELF%IN_VARS_2D_REAL64(9)%PTR - SELF%PVFI => SELF%IN_VARS_2D_REAL64(10)%PTR - SELF%PDYNA => SELF%IN_VARS_2D_REAL64(11)%PTR - SELF%PDYNL => SELF%IN_VARS_2D_REAL64(12)%PTR - SELF%PDYNI => SELF%IN_VARS_2D_REAL64(13)%PTR - SELF%PHRSW => SELF%IN_VARS_2D_REAL64(14)%PTR - SELF%PHRLW => SELF%IN_VARS_2D_REAL64(15)%PTR - SELF%PVERVEL => SELF%IN_VARS_2D_REAL64(16)%PTR - SELF%PAP => SELF%IN_VARS_2D_REAL64(17)%PTR - SELF%PLU => SELF%IN_VARS_2D_REAL64(18)%PTR - SELF%PLUDE => SELF%IN_VARS_2D_REAL64(19)%PTR - SELF%PSNDE => SELF%IN_VARS_2D_REAL64(20)%PTR - SELF%PMFU => SELF%IN_VARS_2D_REAL64(21)%PTR - SELF%PMFD => SELF%IN_VARS_2D_REAL64(22)%PTR - SELF%PA => SELF%IN_VARS_2D_REAL64(23)%PTR - SELF%PSUPSAT => SELF%IN_VARS_2D_REAL64(24)%PTR - - FIELD = FSET%FIELD("PLSM") - CALL FIELD%DATA(SELF%PLSM, IBLK) - FIELD = FSET%FIELD("LDCUM") - CALL FIELD%DATA(SELF%LDCUM, IBLK) - FIELD = FSET%FIELD("KTYPE") - CALL FIELD%DATA(SELF%KTYPE, IBLK) - FIELD = FSET%FIELD("PAPH") - CALL FIELD%DATA(SELF%PAPH, IBLK) - FIELD = FSET%FIELD("PEXTRA") - CALL FIELD%DATA(SELF%PEXTRA, IBLK) - FIELD = FSET%FIELD("PCLV") - CALL FIELD%DATA(SELF%PCLV, IBLK) - - FIELD = FSET%FIELD('TENDENCY_CML') - CALL FIELD%DATA(TMP3D, IBLK) + CALL FIELDS%fPLCRIT_AER%DATA(SELF%PLCRIT_AER, IBLK) + CALL FIELDS%fPICRIT_AER%DATA(SELF%PICRIT_AER, IBLK) + CALL FIELDS%fPRE_ICE%DATA(SELF%PRE_ICE, IBLK) + CALL FIELDS%fPCCN%DATA(SELF%PCCN, IBLK) + CALL FIELDS%fPNICE%DATA(SELF%PNICE, IBLK) + CALL FIELDS%fPT%DATA(SELF%PT, IBLK) + CALL FIELDS%fPQ%DATA(SELF%PQ, IBLK) + CALL FIELDS%fPVFA%DATA(SELF%PVFA, IBLK) + CALL FIELDS%fPVFL%DATA(SELF%PVFL, IBLK) + CALL FIELDS%fPVFI%DATA(SELF%PVFI, IBLK) + CALL FIELDS%fPDYNA%DATA(SELF%PDYNA, IBLK) + CALL FIELDS%fPDYNL%DATA(SELF%PDYNL, IBLK) + CALL FIELDS%fPDYNI%DATA(SELF%PDYNI, IBLK) + CALL FIELDS%fPHRSW%DATA(SELF%PHRSW, IBLK) + CALL FIELDS%fPHRLW%DATA(SELF%PHRLW, IBLK) + CALL FIELDS%fPVERVEL%DATA(SELF%PVERVEL, IBLK) + CALL FIELDS%fPAP%DATA(SELF%PAP, IBLK) + CALL FIELDS%fPLU%DATA(SELF%PLU, IBLK) + CALL FIELDS%fPLUDE%DATA(SELF%PLUDE, IBLK) + CALL FIELDS%fPSNDE%DATA(SELF%PSNDE, IBLK) + CALL FIELDS%fPMFU%DATA(SELF%PMFU, IBLK) + CALL FIELDS%fPMFD%DATA(SELF%PMFD, IBLK) + CALL FIELDS%fPA%DATA(SELF%PA, IBLK) + CALL FIELDS%fPSUPSAT%DATA(SELF%PSUPSAT, IBLK) + CALL FIELDS%fPLSM%DATA(SELF%PLSM, IBLK) + CALL FIELDS%fLDCUM%DATA(SELF%LDCUM, IBLK) + CALL FIELDS%fKTYPE%DATA(SELF%KTYPE, IBLK) + CALL FIELDS%fPAPH%DATA(SELF%PAPH, IBLK) + CALL FIELDS%fPEXTRA%DATA(SELF%PEXTRA, IBLK) + CALL FIELDS%fPCLV%DATA(SELF%PCLV, IBLK) + + CALL FIELDS%fTENDENCY_CML%DATA(TMP3D, IBLK) SELF%TENDENCY_CML%T => TMP3D(:,:,1) SELF%TENDENCY_CML%A => TMP3D(:,:,2) SELF%TENDENCY_CML%Q => TMP3D(:,:,3) SELF%TENDENCY_CML%CLD => TMP3D(:,:,4:) - FIELD = FSET%FIELD('TENDENCY_TMP') - CALL FIELD%DATA(TMP3D, IBLK) + CALL FIELDS%fTENDENCY_TMP%DATA(TMP3D, IBLK) SELF%TENDENCY_TMP%T => TMP3D(:,:,1) SELF%TENDENCY_TMP%A => TMP3D(:,:,2) SELF%TENDENCY_TMP%Q => TMP3D(:,:,3) SELF%TENDENCY_TMP%CLD => TMP3D(:,:,4:) - FIELD = FSET%FIELD('TENDENCY_LOC') - CALL FIELD%DATA(TMP3D, IBLK) + CALL FIELDS%fTENDENCY_LOC%DATA(TMP3D, IBLK) SELF%TENDENCY_LOC%T => TMP3D(:,:,1) SELF%TENDENCY_LOC%A => TMP3D(:,:,2) SELF%TENDENCY_LOC%Q => TMP3D(:,:,3) SELF%TENDENCY_LOC%CLD => TMP3D(:,:,4:) - DO IVAR = 1, SIZE(SELF%OUT_VARS_2D_REAL64) - FIELD = FSET%FIELD(TRIM(OUT_VAR_NAMES(IVAR))) - CALL FIELD%DATA(SELF%OUT_VARS_2D_REAL64(IVAR)%PTR, IBLK) - ENDDO - SELF%PFSQLF => SELF%OUT_VARS_2D_REAL64(1)%PTR - SELF%PFSQIF => SELF%OUT_VARS_2D_REAL64(2)%PTR - SELF%PFCQLNG => SELF%OUT_VARS_2D_REAL64(3)%PTR - SELF%PFCQNNG => SELF%OUT_VARS_2D_REAL64(4)%PTR - SELF%PFSQRF => SELF%OUT_VARS_2D_REAL64(5)%PTR - SELF%PFSQSF => SELF%OUT_VARS_2D_REAL64(6)%PTR - SELF%PFCQRNG => SELF%OUT_VARS_2D_REAL64(7)%PTR - SELF%PFCQSNG => SELF%OUT_VARS_2D_REAL64(8)%PTR - SELF%PFSQLTUR => SELF%OUT_VARS_2D_REAL64(9)%PTR - SELF%PFSQITUR => SELF%OUT_VARS_2D_REAL64(10)%PTR - SELF%PFPLSL => SELF%OUT_VARS_2D_REAL64(11)%PTR - SELF%PFPLSN => SELF%OUT_VARS_2D_REAL64(12)%PTR - SELF%PFHPSL => SELF%OUT_VARS_2D_REAL64(13)%PTR - SELF%PFHPSN => SELF%OUT_VARS_2D_REAL64(14)%PTR - SELF%PCOVPTOT => SELF%OUT_VARS_2D_REAL64(15)%PTR - - FIELD = FSET%FIELD('PRAINFRAC_TOPRFZ') - CALL FIELD%DATA(SELF%PRAINFRAC_TOPRFZ, IBLK) + CALL FIELDS%fPFSQLF%DATA(SELF%PFSQLF, IBLK) + CALL FIELDS%fPFSQIF%DATA(SELF%PFSQIF, IBLK) + CALL FIELDS%fPFCQLNG%DATA(SELF%PFCQLNG, IBLK) + CALL FIELDS%fPFCQNNG%DATA(SELF%PFCQNNG, IBLK) + CALL FIELDS%fPFSQRF%DATA(SELF%PFSQRF, IBLK) + CALL FIELDS%fPFSQSF%DATA(SELF%PFSQSF, IBLK) + CALL FIELDS%fPFCQRNG%DATA(SELF%PFCQRNG, IBLK) + CALL FIELDS%fPFCQSNG%DATA(SELF%PFCQSNG, IBLK) + CALL FIELDS%fPFSQLTUR%DATA(SELF%PFSQLTUR, IBLK) + CALL FIELDS%fPFSQITUR%DATA(SELF%PFSQITUR, IBLK) + CALL FIELDS%fPFPLSL%DATA(SELF%PFPLSL, IBLK) + CALL FIELDS%fPFPLSN%DATA(SELF%PFPLSN, IBLK) + CALL FIELDS%fPFHPSL%DATA(SELF%PFHPSL, IBLK) + CALL FIELDS%fPFHPSN%DATA(SELF%PFHPSN, IBLK) + CALL FIELDS%fPCOVPTOT%DATA(SELF%PCOVPTOT, IBLK) + CALL FIELDS%fPRAINFRAC_TOPRFZ%DATA(SELF%PRAINFRAC_TOPRFZ, IBLK) END SUBROUTINE CLOUDSC_GLOBAL_ATLAS_STATE_BLOCK SUBROUTINE CLOUDSC_GLOBAL_ATLAS_STATE_LOAD(SELF, FSET, NPROMA, NGPTOT, NGPTOTG) From 64c3f8af88788fd1e2e79da74e7161ccb31dea01 Mon Sep 17 00:00:00 2001 From: Slavko Brdar Date: Tue, 16 May 2023 01:27:20 +0200 Subject: [PATCH 04/45] use FieldSet instead of array of Field, this time in a thread same manner --- .../cloudsc_driver_mod.F90 | 22 +- .../cloudsc_global_atlas_state_mod.F90 | 223 ++++-------------- 2 files changed, 54 insertions(+), 191 deletions(-) diff --git a/src/cloudsc_fortran_atlas/cloudsc_driver_mod.F90 b/src/cloudsc_fortran_atlas/cloudsc_driver_mod.F90 index d173c870..a6d11c27 100644 --- a/src/cloudsc_fortran_atlas/cloudsc_driver_mod.F90 +++ b/src/cloudsc_fortran_atlas/cloudsc_driver_mod.F90 @@ -14,7 +14,7 @@ MODULE CLOUDSC_DRIVER_MOD USE CLOUDSC_MPI_MOD, ONLY: NUMPROC, IRANK USE TIMER_MOD, ONLY : PERFORMANCE_TIMER, GET_THREAD_NUM USE EC_PMON_MOD, ONLY: EC_PMON - USE CLOUDSC_GLOBAL_ATLAS_STATE_MOD, ONLY: CLOUDSC_GLOBAL_ATLAS_STATE_BLOCK_VIEW, CLOUDSC_GLOBAL_ATLAS_STATE_FIELDS + USE CLOUDSC_GLOBAL_ATLAS_STATE_MOD, ONLY: CLOUDSC_GLOBAL_ATLAS_STATE_BLOCK_VIEW USE ATLAS_MODULE USE, INTRINSIC :: ISO_C_BINDING @@ -33,7 +33,6 @@ SUBROUTINE CLOUDSC_DRIVER(FSET, NUMOMP, NGPTOT, NGPTOTG, KFLDX, PTSPHY) INTEGER(KIND=JPIM), INTENT(IN) :: NUMOMP, NGPTOT, NGPTOTG, KFLDX REAL(KIND=JPRB), INTENT(IN) :: PTSPHY ! Physics timestep - TYPE(CLOUDSC_GLOBAL_ATLAS_STATE_FIELDS) :: SFIELDS TYPE(CLOUDSC_GLOBAL_ATLAS_STATE_BLOCK_VIEW) :: FBLOCK TYPE(ATLAS_FUNCTIONSPACE_BLOCKSTRUCTUREDCOLUMNS) :: FSPACE TYPE(ATLAS_FIELD) :: FIELD @@ -68,9 +67,6 @@ SUBROUTINE CLOUDSC_DRIVER(FSET, NUMOMP, NGPTOT, NGPTOTG, KFLDX, PTSPHY) ! Global timer for the parallel region CALL TIMER%START(NUMOMP) - - CALL SFIELDS%SETUP(FSET) - !$omp parallel default(shared) private(JKGLO,IBL,ICEND,TID,energy,power,FBLOCK) & !$omp& num_threads(NUMOMP) @@ -84,26 +80,12 @@ SUBROUTINE CLOUDSC_DRIVER(FSET, NUMOMP, NGPTOT, NGPTOTG, KFLDX, PTSPHY) ICEND=MIN(NPROMA,NGPTOT-JKGLO+1) ! get block views - call FBLOCK%GET_BLOCK(SFIELDS, IBL) - CONTINUE + call FBLOCK%GET_BLOCK(FSET, IBL) !-- These were uninitialized : meaningful only when we compare error differences FBLOCK%PCOVPTOT(:,:) = 0.0_JPRB FBLOCK%TENDENCY_LOC%cld(:,:,NCLV) = 0.0_JPRB - !--- a future plan to replace the call to CLOUDSC ------ - ! - ! type( block_state_t ) - ! real(c_double), pointer :: PT(:,:) - ! type(state_type) :: tendency_LOC - ! type(state_type) :: tendency_TMP - ! type(state_type) :: tendency_CML - ! end type - ! call extract_block( FSET, IBL, config, block_state ) - ! call FSET%FIELD("PT")%BLOCK_DATA(IBL,PT,CONFIG) - ! call FSET%FIELD("PQ")%BLOCK_DATA(IBL,PQ,CONFIG) - ! call cloudsc_atlas ( FSET, IBL, config ) - CALL CLOUDSC & & ( 1, ICEND, NPROMA, NLEV,& & PTSPHY,& diff --git a/src/cloudsc_fortran_atlas/cloudsc_global_atlas_state_mod.F90 b/src/cloudsc_fortran_atlas/cloudsc_global_atlas_state_mod.F90 index ba18ac74..0bef9bc0 100644 --- a/src/cloudsc_fortran_atlas/cloudsc_global_atlas_state_mod.F90 +++ b/src/cloudsc_fortran_atlas/cloudsc_global_atlas_state_mod.F90 @@ -112,67 +112,6 @@ MODULE CLOUDSC_GLOBAL_ATLAS_STATE_MOD PROCEDURE :: GET_BLOCK => CLOUDSC_GLOBAL_ATLAS_STATE_BLOCK END TYPE CLOUDSC_GLOBAL_ATLAS_STATE_BLOCK_VIEW - TYPE CLOUDSC_GLOBAL_ATLAS_STATE_FIELDS - !TYPE(atlas_field), allocatlabe :: fields(:) - - ! Input field variables and tendencies - TYPE(atlas_field) :: fPLCRIT_AER - TYPE(atlas_field) :: fPICRIT_AER - TYPE(atlas_field) :: fPRE_ICE - TYPE(atlas_field) :: fPCCN ! liquid cloud condensation nuclei - TYPE(atlas_field) :: fPNICE ! ice number concentration (cf. CCN) - TYPE(atlas_field) :: fPT ! T at start of callpar - TYPE(atlas_field) :: fPQ ! Q at start of callpar - TYPE(atlas_field) :: fPVFA ! CC from VDF scheme - TYPE(atlas_field) :: fPVFL ! Liq from VDF scheme - TYPE(atlas_field) :: fPVFI ! Ice from VDF scheme - TYPE(atlas_field) :: fPDYNA ! CC from Dynamics - TYPE(atlas_field) :: fPDYNL ! Liq from Dynamics - TYPE(atlas_field) :: fPDYNI ! Liq from Dynamics - TYPE(atlas_field) :: fPHRSW ! Short-wave heating rate - TYPE(atlas_field) :: fPHRLW ! Long-wave heating rate - TYPE(atlas_field) :: fPVERVEL ! Vertical velocity - TYPE(atlas_field) :: fPAP ! Pressure on full levels - TYPE(atlas_field) :: fPLU ! Conv. condensate - TYPE(atlas_field) :: fPLUDE ! Conv. detrained water - TYPE(atlas_field) :: fPSNDE ! Conv. detrained snow - TYPE(atlas_field) :: fPMFU ! Conv. mass flux up - TYPE(atlas_field) :: fPMFD ! Conv. mass flux down - TYPE(atlas_field) :: fPA ! Original Cloud fraction (t) - TYPE(atlas_field) :: fPSUPSAT - - TYPE(atlas_field) :: fPLSM ! Land fraction (0-1) - TYPE(atlas_field) :: fLDCUM ! Convection active - TYPE(atlas_field) :: fKTYPE ! Convection type 0,1,2 - TYPE(atlas_field) :: fPAPH ! Pressure on half levels - TYPE(atlas_field) :: fPEXTRA ! extra fields - TYPE(atlas_field) :: fPCLV - - TYPE(atlas_field) :: fTENDENCY_CML ! cumulative tendency used for final output - TYPE(atlas_field) :: fTENDENCY_TMP ! cumulative tendency used as input - TYPE(atlas_field) :: fTENDENCY_LOC ! local tendency from cloud scheme - - ! Output fields used for validation - TYPE(atlas_field) :: fPFSQLF ! Flux of liquid - TYPE(atlas_field) :: fPFSQIF ! Flux of ice - TYPE(atlas_field) :: fPFCQLNG ! -ve corr for liq - TYPE(atlas_field) :: fPFCQNNG ! -ve corr for ice - TYPE(atlas_field) :: fPFSQRF ! Flux diagnostics - TYPE(atlas_field) :: fPFSQSF ! for DDH, generic - TYPE(atlas_field) :: fPFCQRNG ! rain - TYPE(atlas_field) :: fPFCQSNG ! snow - TYPE(atlas_field) :: fPFSQLTUR ! liquid flux due to VDF - TYPE(atlas_field) :: fPFSQITUR ! ice flux due to VDF - TYPE(atlas_field) :: fPFPLSL ! liq+rain sedim flux - TYPE(atlas_field) :: fPFPLSN ! ice+snow sedim flux - TYPE(atlas_field) :: fPFHPSL ! Enthalpy flux for liq - TYPE(atlas_field) :: fPFHPSN ! Enthalpy flux for ice - TYPE(atlas_field) :: fPCOVPTOT ! Precip fraction - TYPE(atlas_field) :: fPRAINFRAC_TOPRFZ - CONTAINS - PROCEDURE :: SETUP => CLOUDSC_GLOBAL_ATLAS_SETUP_BLOCK - END TYPE CLOUDSC_GLOBAL_ATLAS_STATE_FIELDS - TYPE CLOUDSC_GLOBAL_ATLAS_STATE ! Memory state containing raw fields annd tendencies for CLOUDSC dwarf ! @@ -192,134 +131,76 @@ MODULE CLOUDSC_GLOBAL_ATLAS_STATE_MOD CONTAINS - SUBROUTINE CLOUDSC_GLOBAL_ATLAS_SETUP_BLOCK(SELF, FSET) - CLASS(CLOUDSC_GLOBAL_ATLAS_STATE_FIELDS), INTENT(INOUT) :: SELF - TYPE(ATLAS_FIELDSET), INTENT(INOUT) :: FSET - - SELF%fPLCRIT_AER = FSET%FIELD("PLCRIT_AER") - SELF%fPICRIT_AER = FSET%FIELD("PICRIT_AER") - SELF%fPRE_ICE = FSET%FIELD("PRE_ICE") - SELF%fPCCN = FSET%FIELD("PCCN") - SELF%fPNICE = FSET%FIELD("PNICE") - SELF%fPT = FSET%FIELD("PT") - SELF%fPQ = FSET%FIELD("PQ") - SELF%fPVFA = FSET%FIELD("PVFA") - SELF%fPVFL = FSET%FIELD("PVFL") - SELF%fPVFI = FSET%FIELD("PVFI") - SELF%fPDYNA = FSET%FIELD("PDYNA") - SELF%fPDYNL = FSET%FIELD("PDYNL") - SELF%fPDYNI = FSET%FIELD("PDYNI") - SELF%fPHRSW = FSET%FIELD("PHRSW") - SELF%fPHRLW = FSET%FIELD("PHRLW") - SELF%fPVERVEL = FSET%FIELD("PVERVEL") - SELF%fPAP = FSET%FIELD("PAP") - SELF%fPLU = FSET%FIELD("PLU") - SELF%fPLUDE = FSET%FIELD("PLUDE") - SELF%fPSNDE = FSET%FIELD("PSNDE") - SELF%fPMFU = FSET%FIELD("PMFU") - SELF%fPMFD = FSET%FIELD("PMFD") - SELF%fPA = FSET%FIELD("PA") - SELF%fPSUPSAT = FSET%FIELD("PSUPSAT") - SELF%fPLSM = FSET%FIELD("PLSM") - SELF%fLDCUM = FSET%FIELD("LDCUM") - SELF%fKTYPE = FSET%FIELD("KTYPE") - SELF%fPAPH = FSET%FIELD("PAPH") - SELF%fPEXTRA = FSET%FIELD("PEXTRA") - SELF%fPCLV = FSET%FIELD("PCLV") - - SELF%fTENDENCY_CML = FSET%FIELD('TENDENCY_CML') - SELF%fTENDENCY_TMP = FSET%FIELD('TENDENCY_TMP') - SELF%fTENDENCY_LOC = FSET%FIELD('TENDENCY_LOC') - - SELF%fPFSQLF = FSET%FIELD("PFSQLF") - SELF%fPFSQIF = FSET%FIELD("PFSQIF") - SELF%fPFCQLNG = FSET%FIELD("PFCQLNG") - SELF%fPFCQNNG = FSET%FIELD("PFCQNNG") - SELF%fPFSQRF = FSET%FIELD("PFSQRF") - SELF%fPFSQSF = FSET%FIELD("PFSQSF") - SELF%fPFCQRNG = FSET%FIELD("PFCQRNG") - SELF%fPFCQSNG = FSET%FIELD("PFCQSNG") - SELF%fPFSQLTUR = FSET%FIELD("PFSQLTUR") - SELF%fPFSQITUR = FSET%FIELD("PFSQITUR") - SELF%fPFPLSL = FSET%FIELD("PFPLSL") - SELF%fPFPLSN = FSET%FIELD("PFPLSN") - SELF%fPFHPSL = FSET%FIELD("PFHPSL") - SELF%fPFHPSN = FSET%FIELD("PFHPSN") - SELF%fPCOVPTOT = FSET%FIELD("PCOVPTOT") - SELF%fPRAINFRAC_TOPRFZ = FSET%FIELD("PRAINFRAC_TOPRFZ") - END SUBROUTINE - - SUBROUTINE CLOUDSC_GLOBAL_ATLAS_STATE_BLOCK(SELF, FIELDS, IBLK) + SUBROUTINE CLOUDSC_GLOBAL_ATLAS_STATE_BLOCK(SELF, FSET, IBLK) CLASS(CLOUDSC_GLOBAL_ATLAS_STATE_BLOCK_VIEW), INTENT(INOUT) :: SELF - CLASS(CLOUDSC_GLOBAL_ATLAS_STATE_FIELDS), INTENT(INOUT) :: FIELDS + CLASS(ATLAS_FIELDSET), INTENT(INOUT) :: FSET INTEGER, INTENT(IN) :: IBLK REAL(C_DOUBLE), POINTER :: TMP3D(:,:,:) - ! NOTE the last six input variables need special treatment - different types - CALL FIELDS%fPLCRIT_AER%DATA(SELF%PLCRIT_AER, IBLK) - CALL FIELDS%fPICRIT_AER%DATA(SELF%PICRIT_AER, IBLK) - CALL FIELDS%fPRE_ICE%DATA(SELF%PRE_ICE, IBLK) - CALL FIELDS%fPCCN%DATA(SELF%PCCN, IBLK) - CALL FIELDS%fPNICE%DATA(SELF%PNICE, IBLK) - CALL FIELDS%fPT%DATA(SELF%PT, IBLK) - CALL FIELDS%fPQ%DATA(SELF%PQ, IBLK) - CALL FIELDS%fPVFA%DATA(SELF%PVFA, IBLK) - CALL FIELDS%fPVFL%DATA(SELF%PVFL, IBLK) - CALL FIELDS%fPVFI%DATA(SELF%PVFI, IBLK) - CALL FIELDS%fPDYNA%DATA(SELF%PDYNA, IBLK) - CALL FIELDS%fPDYNL%DATA(SELF%PDYNL, IBLK) - CALL FIELDS%fPDYNI%DATA(SELF%PDYNI, IBLK) - CALL FIELDS%fPHRSW%DATA(SELF%PHRSW, IBLK) - CALL FIELDS%fPHRLW%DATA(SELF%PHRLW, IBLK) - CALL FIELDS%fPVERVEL%DATA(SELF%PVERVEL, IBLK) - CALL FIELDS%fPAP%DATA(SELF%PAP, IBLK) - CALL FIELDS%fPLU%DATA(SELF%PLU, IBLK) - CALL FIELDS%fPLUDE%DATA(SELF%PLUDE, IBLK) - CALL FIELDS%fPSNDE%DATA(SELF%PSNDE, IBLK) - CALL FIELDS%fPMFU%DATA(SELF%PMFU, IBLK) - CALL FIELDS%fPMFD%DATA(SELF%PMFD, IBLK) - CALL FIELDS%fPA%DATA(SELF%PA, IBLK) - CALL FIELDS%fPSUPSAT%DATA(SELF%PSUPSAT, IBLK) - CALL FIELDS%fPLSM%DATA(SELF%PLSM, IBLK) - CALL FIELDS%fLDCUM%DATA(SELF%LDCUM, IBLK) - CALL FIELDS%fKTYPE%DATA(SELF%KTYPE, IBLK) - CALL FIELDS%fPAPH%DATA(SELF%PAPH, IBLK) - CALL FIELDS%fPEXTRA%DATA(SELF%PEXTRA, IBLK) - CALL FIELDS%fPCLV%DATA(SELF%PCLV, IBLK) - - CALL FIELDS%fTENDENCY_CML%DATA(TMP3D, IBLK) + CALL FSET%DATA("PLCRIT_AER", SELF%PLCRIT_AER, IBLK) + CALL FSET%DATA("PICRIT_AER", SELF%PICRIT_AER, IBLK) + CALL FSET%DATA("PRE_ICE", SELF%PRE_ICE, IBLK) + CALL FSET%DATA("PCCN", SELF%PCCN, IBLK) + CALL FSET%DATA("PNICE", SELF%PNICE, IBLK) + CALL FSET%DATA("PT", SELF%PT, IBLK) + CALL FSET%DATA("PQ", SELF%PQ, IBLK) + CALL FSET%DATA("PVFA", SELF%PVFA, IBLK) + CALL FSET%DATA("PVFL", SELF%PVFL, IBLK) + CALL FSET%DATA("PVFI", SELF%PVFI, IBLK) + CALL FSET%DATA("PDYNA", SELF%PDYNA, IBLK) + CALL FSET%DATA("PDYNL", SELF%PDYNL, IBLK) + CALL FSET%DATA("PDYNI", SELF%PDYNI, IBLK) + CALL FSET%DATA("PHRSW", SELF%PHRSW, IBLK) + CALL FSET%DATA("PHRLW", SELF%PHRLW, IBLK) + CALL FSET%DATA("PVERVEL", SELF%PVERVEL, IBLK) + CALL FSET%DATA("PAP", SELF%PAP, IBLK) + CALL FSET%DATA("PLU", SELF%PLU, IBLK) + CALL FSET%DATA("PLUDE", SELF%PLUDE, IBLK) + CALL FSET%DATA("PSNDE", SELF%PSNDE, IBLK) + CALL FSET%DATA("PMFU", SELF%PMFU, IBLK) + CALL FSET%DATA("PMFD", SELF%PMFD, IBLK) + CALL FSET%DATA("PA", SELF%PA, IBLK) + CALL FSET%DATA("PSUPSAT", SELF%PSUPSAT, IBLK) + CALL FSET%DATA("PLSM", SELF%PLSM, IBLK) + CALL FSET%DATA("LDCUM", SELF%LDCUM, IBLK) + CALL FSET%DATA("KTYPE", SELF%KTYPE, IBLK) + CALL FSET%DATA("PAPH", SELF%PAPH, IBLK) + CALL FSET%DATA("PEXTRA", SELF%PEXTRA, IBLK) + CALL FSET%DATA("PCLV", SELF%PCLV, IBLK) + + CALL FSET%DATA("TENDENCY_CML", TMP3D, IBLK) SELF%TENDENCY_CML%T => TMP3D(:,:,1) SELF%TENDENCY_CML%A => TMP3D(:,:,2) SELF%TENDENCY_CML%Q => TMP3D(:,:,3) SELF%TENDENCY_CML%CLD => TMP3D(:,:,4:) - CALL FIELDS%fTENDENCY_TMP%DATA(TMP3D, IBLK) + CALL FSET%DATA("TENDENCY_TMP", TMP3D, IBLK) SELF%TENDENCY_TMP%T => TMP3D(:,:,1) SELF%TENDENCY_TMP%A => TMP3D(:,:,2) SELF%TENDENCY_TMP%Q => TMP3D(:,:,3) SELF%TENDENCY_TMP%CLD => TMP3D(:,:,4:) - CALL FIELDS%fTENDENCY_LOC%DATA(TMP3D, IBLK) + CALL FSET%DATA("TENDENCY_LOC", TMP3D, IBLK) SELF%TENDENCY_LOC%T => TMP3D(:,:,1) SELF%TENDENCY_LOC%A => TMP3D(:,:,2) SELF%TENDENCY_LOC%Q => TMP3D(:,:,3) SELF%TENDENCY_LOC%CLD => TMP3D(:,:,4:) - CALL FIELDS%fPFSQLF%DATA(SELF%PFSQLF, IBLK) - CALL FIELDS%fPFSQIF%DATA(SELF%PFSQIF, IBLK) - CALL FIELDS%fPFCQLNG%DATA(SELF%PFCQLNG, IBLK) - CALL FIELDS%fPFCQNNG%DATA(SELF%PFCQNNG, IBLK) - CALL FIELDS%fPFSQRF%DATA(SELF%PFSQRF, IBLK) - CALL FIELDS%fPFSQSF%DATA(SELF%PFSQSF, IBLK) - CALL FIELDS%fPFCQRNG%DATA(SELF%PFCQRNG, IBLK) - CALL FIELDS%fPFCQSNG%DATA(SELF%PFCQSNG, IBLK) - CALL FIELDS%fPFSQLTUR%DATA(SELF%PFSQLTUR, IBLK) - CALL FIELDS%fPFSQITUR%DATA(SELF%PFSQITUR, IBLK) - CALL FIELDS%fPFPLSL%DATA(SELF%PFPLSL, IBLK) - CALL FIELDS%fPFPLSN%DATA(SELF%PFPLSN, IBLK) - CALL FIELDS%fPFHPSL%DATA(SELF%PFHPSL, IBLK) - CALL FIELDS%fPFHPSN%DATA(SELF%PFHPSN, IBLK) - CALL FIELDS%fPCOVPTOT%DATA(SELF%PCOVPTOT, IBLK) - CALL FIELDS%fPRAINFRAC_TOPRFZ%DATA(SELF%PRAINFRAC_TOPRFZ, IBLK) + CALL FSET%DATA("PFSQLF", SELF%PFSQLF, IBLK) + CALL FSET%DATA("PFSQIF", SELF%PFSQIF, IBLK) + CALL FSET%DATA("PFCQLNG", SELF%PFCQLNG, IBLK) + CALL FSET%DATA("PFCQNNG", SELF%PFCQNNG, IBLK) + CALL FSET%DATA("PFSQRF", SELF%PFSQRF, IBLK) + CALL FSET%DATA("PFSQSF", SELF%PFSQSF, IBLK) + CALL FSET%DATA("PFCQRNG", SELF%PFCQRNG, IBLK) + CALL FSET%DATA("PFCQSNG", SELF%PFCQSNG, IBLK) + CALL FSET%DATA("PFSQLTUR", SELF%PFSQLTUR, IBLK) + CALL FSET%DATA("PFSQITUR", SELF%PFSQITUR, IBLK) + CALL FSET%DATA("PFPLSL", SELF%PFPLSL, IBLK) + CALL FSET%DATA("PFPLSN", SELF%PFPLSN, IBLK) + CALL FSET%DATA("PFHPSL", SELF%PFHPSL, IBLK) + CALL FSET%DATA("PFHPSN", SELF%PFHPSN, IBLK) + CALL FSET%DATA("PCOVPTOT", SELF%PCOVPTOT, IBLK) + CALL FSET%DATA("PRAINFRAC_TOPRFZ", SELF%PRAINFRAC_TOPRFZ, IBLK) END SUBROUTINE CLOUDSC_GLOBAL_ATLAS_STATE_BLOCK SUBROUTINE CLOUDSC_GLOBAL_ATLAS_STATE_LOAD(SELF, FSET, NPROMA, NGPTOT, NGPTOTG) From f9a1657e2a48ab0891dd9da24a8948d89bc55d2f Mon Sep 17 00:00:00 2001 From: Slavko Brdar Date: Sun, 21 May 2023 23:52:02 +0200 Subject: [PATCH 05/45] switch to index to acces a field from a FieldSet rather than using string --- .../cloudsc_global_atlas_state_mod.F90 | 103 +++++++++--------- 1 file changed, 50 insertions(+), 53 deletions(-) diff --git a/src/cloudsc_fortran_atlas/cloudsc_global_atlas_state_mod.F90 b/src/cloudsc_fortran_atlas/cloudsc_global_atlas_state_mod.F90 index 0bef9bc0..ba09384f 100644 --- a/src/cloudsc_fortran_atlas/cloudsc_global_atlas_state_mod.F90 +++ b/src/cloudsc_fortran_atlas/cloudsc_global_atlas_state_mod.F90 @@ -34,9 +34,6 @@ MODULE CLOUDSC_GLOBAL_ATLAS_STATE_MOD REAL(C_DOUBLE), POINTER :: PTR(:,:) END TYPE - !INTEGER, PARAMETER PLCRIT_AER = 1 - !INTEGER, PARAMETER PLCRIT_AER = 2 - !IN_VAR_NAMES(PLCRIT_AER) CHARACTER(LEN=10), PARAMETER, DIMENSION(30) :: IN_VAR_NAMES = (/ & "PLCRIT_AER", "PICRIT_AER", "PRE_ICE ", "PCCN ", "PNICE ", "PT ", "PQ ", & "PVFA ", "PVFL ", "PVFI ", "PDYNA ", "PDYNL ", "PDYNI ", "PHRSW ", & @@ -138,69 +135,69 @@ SUBROUTINE CLOUDSC_GLOBAL_ATLAS_STATE_BLOCK(SELF, FSET, IBLK) REAL(C_DOUBLE), POINTER :: TMP3D(:,:,:) - CALL FSET%DATA("PLCRIT_AER", SELF%PLCRIT_AER, IBLK) - CALL FSET%DATA("PICRIT_AER", SELF%PICRIT_AER, IBLK) - CALL FSET%DATA("PRE_ICE", SELF%PRE_ICE, IBLK) - CALL FSET%DATA("PCCN", SELF%PCCN, IBLK) - CALL FSET%DATA("PNICE", SELF%PNICE, IBLK) - CALL FSET%DATA("PT", SELF%PT, IBLK) - CALL FSET%DATA("PQ", SELF%PQ, IBLK) - CALL FSET%DATA("PVFA", SELF%PVFA, IBLK) - CALL FSET%DATA("PVFL", SELF%PVFL, IBLK) - CALL FSET%DATA("PVFI", SELF%PVFI, IBLK) - CALL FSET%DATA("PDYNA", SELF%PDYNA, IBLK) - CALL FSET%DATA("PDYNL", SELF%PDYNL, IBLK) - CALL FSET%DATA("PDYNI", SELF%PDYNI, IBLK) - CALL FSET%DATA("PHRSW", SELF%PHRSW, IBLK) - CALL FSET%DATA("PHRLW", SELF%PHRLW, IBLK) - CALL FSET%DATA("PVERVEL", SELF%PVERVEL, IBLK) - CALL FSET%DATA("PAP", SELF%PAP, IBLK) - CALL FSET%DATA("PLU", SELF%PLU, IBLK) - CALL FSET%DATA("PLUDE", SELF%PLUDE, IBLK) - CALL FSET%DATA("PSNDE", SELF%PSNDE, IBLK) - CALL FSET%DATA("PMFU", SELF%PMFU, IBLK) - CALL FSET%DATA("PMFD", SELF%PMFD, IBLK) - CALL FSET%DATA("PA", SELF%PA, IBLK) - CALL FSET%DATA("PSUPSAT", SELF%PSUPSAT, IBLK) - CALL FSET%DATA("PLSM", SELF%PLSM, IBLK) - CALL FSET%DATA("LDCUM", SELF%LDCUM, IBLK) - CALL FSET%DATA("KTYPE", SELF%KTYPE, IBLK) - CALL FSET%DATA("PAPH", SELF%PAPH, IBLK) - CALL FSET%DATA("PEXTRA", SELF%PEXTRA, IBLK) - CALL FSET%DATA("PCLV", SELF%PCLV, IBLK) - - CALL FSET%DATA("TENDENCY_CML", TMP3D, IBLK) + CALL FSET%DATA(1, SELF%PLCRIT_AER, IBLK) + CALL FSET%DATA(2, SELF%PICRIT_AER, IBLK) + CALL FSET%DATA(3, SELF%PRE_ICE, IBLK) + CALL FSET%DATA(4, SELF%PCCN, IBLK) + CALL FSET%DATA(5, SELF%PNICE, IBLK) + CALL FSET%DATA(6, SELF%PT, IBLK) + CALL FSET%DATA(7, SELF%PQ, IBLK) + CALL FSET%DATA(8, SELF%PVFA, IBLK) + CALL FSET%DATA(9, SELF%PVFL, IBLK) + CALL FSET%DATA(10, SELF%PVFI, IBLK) + CALL FSET%DATA(11, SELF%PDYNA, IBLK) + CALL FSET%DATA(12, SELF%PDYNL, IBLK) + CALL FSET%DATA(13, SELF%PDYNI, IBLK) + CALL FSET%DATA(14, SELF%PHRSW, IBLK) + CALL FSET%DATA(15, SELF%PHRLW, IBLK) + CALL FSET%DATA(16, SELF%PVERVEL, IBLK) + CALL FSET%DATA(17, SELF%PAP, IBLK) + CALL FSET%DATA(18, SELF%PLU, IBLK) + CALL FSET%DATA(19, SELF%PLUDE, IBLK) + CALL FSET%DATA(20, SELF%PSNDE, IBLK) + CALL FSET%DATA(21, SELF%PMFU, IBLK) + CALL FSET%DATA(22, SELF%PMFD, IBLK) + CALL FSET%DATA(23, SELF%PA, IBLK) + CALL FSET%DATA(24, SELF%PSUPSAT, IBLK) + CALL FSET%DATA(25, SELF%PLSM, IBLK) + CALL FSET%DATA(26, SELF%LDCUM, IBLK) + CALL FSET%DATA(27, SELF%KTYPE, IBLK) + CALL FSET%DATA(28, SELF%PAPH, IBLK) + CALL FSET%DATA(29, SELF%PEXTRA, IBLK) + CALL FSET%DATA(30, SELF%PCLV, IBLK) + + CALL FSET%DATA(31, TMP3D, IBLK) SELF%TENDENCY_CML%T => TMP3D(:,:,1) SELF%TENDENCY_CML%A => TMP3D(:,:,2) SELF%TENDENCY_CML%Q => TMP3D(:,:,3) SELF%TENDENCY_CML%CLD => TMP3D(:,:,4:) - CALL FSET%DATA("TENDENCY_TMP", TMP3D, IBLK) + CALL FSET%DATA(32, TMP3D, IBLK) SELF%TENDENCY_TMP%T => TMP3D(:,:,1) SELF%TENDENCY_TMP%A => TMP3D(:,:,2) SELF%TENDENCY_TMP%Q => TMP3D(:,:,3) SELF%TENDENCY_TMP%CLD => TMP3D(:,:,4:) - CALL FSET%DATA("TENDENCY_LOC", TMP3D, IBLK) + CALL FSET%DATA(33, TMP3D, IBLK) SELF%TENDENCY_LOC%T => TMP3D(:,:,1) SELF%TENDENCY_LOC%A => TMP3D(:,:,2) SELF%TENDENCY_LOC%Q => TMP3D(:,:,3) SELF%TENDENCY_LOC%CLD => TMP3D(:,:,4:) - CALL FSET%DATA("PFSQLF", SELF%PFSQLF, IBLK) - CALL FSET%DATA("PFSQIF", SELF%PFSQIF, IBLK) - CALL FSET%DATA("PFCQLNG", SELF%PFCQLNG, IBLK) - CALL FSET%DATA("PFCQNNG", SELF%PFCQNNG, IBLK) - CALL FSET%DATA("PFSQRF", SELF%PFSQRF, IBLK) - CALL FSET%DATA("PFSQSF", SELF%PFSQSF, IBLK) - CALL FSET%DATA("PFCQRNG", SELF%PFCQRNG, IBLK) - CALL FSET%DATA("PFCQSNG", SELF%PFCQSNG, IBLK) - CALL FSET%DATA("PFSQLTUR", SELF%PFSQLTUR, IBLK) - CALL FSET%DATA("PFSQITUR", SELF%PFSQITUR, IBLK) - CALL FSET%DATA("PFPLSL", SELF%PFPLSL, IBLK) - CALL FSET%DATA("PFPLSN", SELF%PFPLSN, IBLK) - CALL FSET%DATA("PFHPSL", SELF%PFHPSL, IBLK) - CALL FSET%DATA("PFHPSN", SELF%PFHPSN, IBLK) - CALL FSET%DATA("PCOVPTOT", SELF%PCOVPTOT, IBLK) - CALL FSET%DATA("PRAINFRAC_TOPRFZ", SELF%PRAINFRAC_TOPRFZ, IBLK) + CALL FSET%DATA(34, SELF%PFSQLF, IBLK) + CALL FSET%DATA(35, SELF%PFSQIF, IBLK) + CALL FSET%DATA(36, SELF%PFCQLNG, IBLK) + CALL FSET%DATA(37, SELF%PFCQNNG, IBLK) + CALL FSET%DATA(38, SELF%PFSQRF, IBLK) + CALL FSET%DATA(39, SELF%PFSQSF, IBLK) + CALL FSET%DATA(40, SELF%PFCQRNG, IBLK) + CALL FSET%DATA(41, SELF%PFCQSNG, IBLK) + CALL FSET%DATA(42, SELF%PFSQLTUR, IBLK) + CALL FSET%DATA(43, SELF%PFSQITUR, IBLK) + CALL FSET%DATA(44, SELF%PFPLSL, IBLK) + CALL FSET%DATA(45, SELF%PFPLSN, IBLK) + CALL FSET%DATA(46, SELF%PFHPSL, IBLK) + CALL FSET%DATA(47, SELF%PFHPSN, IBLK) + CALL FSET%DATA(48, SELF%PCOVPTOT, IBLK) + CALL FSET%DATA(49, SELF%PRAINFRAC_TOPRFZ, IBLK) END SUBROUTINE CLOUDSC_GLOBAL_ATLAS_STATE_BLOCK SUBROUTINE CLOUDSC_GLOBAL_ATLAS_STATE_LOAD(SELF, FSET, NPROMA, NGPTOT, NGPTOTG) From 6e730725a27c8d0c8663cd159b419a07d52e69a4 Mon Sep 17 00:00:00 2001 From: Balthasar Reuter Date: Tue, 18 Jul 2023 13:12:10 +0100 Subject: [PATCH 06/45] Fix wrong binary name in test --- src/cloudsc_fortran_atlas/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cloudsc_fortran_atlas/CMakeLists.txt b/src/cloudsc_fortran_atlas/CMakeLists.txt index dc21c6ae..87e6d137 100644 --- a/src/cloudsc_fortran_atlas/CMakeLists.txt +++ b/src/cloudsc_fortran_atlas/CMakeLists.txt @@ -44,7 +44,7 @@ endif() ecbuild_add_test( TARGET dwarf-cloudsc-fortran-atlas-serial - COMMAND bin/dwarf-cloudsc-atlas-fortran + COMMAND bin/dwarf-cloudsc-fortran-atlas ARGS 1 100 16 WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/../../.. OMP 1 From 37ae6739acd937b4f69b4bb0ee0451d42f735b15 Mon Sep 17 00:00:00 2001 From: Balthasar Reuter Date: Tue, 18 Jul 2023 13:12:38 +0100 Subject: [PATCH 07/45] Disable atlas tests if executable is not built --- src/cloudsc_fortran_atlas/CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/cloudsc_fortran_atlas/CMakeLists.txt b/src/cloudsc_fortran_atlas/CMakeLists.txt index 87e6d137..f1e61123 100644 --- a/src/cloudsc_fortran_atlas/CMakeLists.txt +++ b/src/cloudsc_fortran_atlas/CMakeLists.txt @@ -27,7 +27,6 @@ if( HAVE_CLOUDSC_FORTRAN_ATLAS ) atlas_f DEFINITIONS ${CLOUDSC_DEFINITIONS} ) -endif() # Create symlink for the input data if( HAVE_SERIALBOX ) @@ -75,3 +74,5 @@ endif() OMP 4 CONDITION HAVE_OMP AND HAVE_MPI ) + +endif() From 245899215e0b96f7d3733b1e7671424f7387a93c Mon Sep 17 00:00:00 2001 From: Balthasar Reuter Date: Tue, 18 Jul 2023 13:12:57 +0100 Subject: [PATCH 08/45] Check if MPI was already initialized --- src/common/module/cloudsc_mpi_mod.F90 | 31 ++++++++++++++++----------- 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/src/common/module/cloudsc_mpi_mod.F90 b/src/common/module/cloudsc_mpi_mod.F90 index 3d011545..215de33c 100644 --- a/src/common/module/cloudsc_mpi_mod.F90 +++ b/src/common/module/cloudsc_mpi_mod.F90 @@ -61,21 +61,28 @@ subroutine cloudsc_mpi_init(numomp) integer(kind=jpim), intent(in), optional :: numomp ! number of OpenMP threads #ifdef HAVE_MPI integer(kind=jpim) :: ierror, iprovided, irequired ! MPI status variables - - ! request threading support if multiple OpenMP threads are used - iprovided = mpi_thread_single - irequired = mpi_thread_single - if (present(numomp)) then - if (numomp > 1) then - irequired = mpi_thread_multiple + logical :: linit + + ! check if MPI has already been initialized + call mpi_initialized(linit, ierror) + if (ierror /= 0) call abor1('cloudsc_mpi: mpi_initialized failed') + + if (.not. linit) then + ! request threading support if multiple OpenMP threads are used + iprovided = mpi_thread_single + irequired = mpi_thread_single + if (present(numomp)) then + if (numomp > 1) then + irequired = mpi_thread_multiple + end if end if - end if - call mpi_init_thread(irequired, iprovided, ierror) + call mpi_init_thread(irequired, iprovided, ierror) - if (ierror /= 0) call abor1('cloudsc_mpi: mpi_init_thread failed') - if (iprovided < irequired) then - print *, "WARNING: MPI_INIT_THREAD reports insufficient threading support" + if (ierror /= 0) call abor1('cloudsc_mpi: mpi_init_thread failed') + if (iprovided < irequired) then + print *, "WARNING: MPI_INIT_THREAD reports insufficient threading support" + end if end if ! determine communicator size and local rank From c41a0c40eadb588746eac2f80fb6219cf31ae290 Mon Sep 17 00:00:00 2001 From: Balthasar Reuter Date: Tue, 18 Jul 2023 13:42:39 +0100 Subject: [PATCH 09/45] Add eckit, fckit, atlas as optional bundle dependencies --- bundle.yml | 47 +++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 45 insertions(+), 2 deletions(-) diff --git a/bundle.yml b/bundle.yml index c087d131..57230a55 100644 --- a/bundle.yml +++ b/bundle.yml @@ -5,11 +5,15 @@ name : cloudsc-bundle version : 1.0.0-develop cmake : > CMAKE_LINK_DEPENDS_NO_SHARED=ON + BUILD_field_api=OFF + BUILD_eckit=OFF + BUILD_fckit=OFF + BUILD_atlas=OFF projects : - ecbuild : - git : https://github.com/ecmwf/ecbuild + git : https://github.com/ecmwf/ecbuild version : 3.7.0 bundle : false @@ -36,13 +40,37 @@ projects : LOKI_ENABLE_TESTS=OFF LOKI_ENABLE_NO_INSTALL=ON + - eckit : + git : https://github.com/ecmwf/eckit + version : 1.24.3 + optional: true + require : ecbuild + cmake : > + ECKIT_ENABLE_TESTS=OFF + ECKIT_ENABLE_BUILD_TOOLS=OFF + + - fckit : + git : https://github.com/ecmwf/fckit + version : 0.9.0 + optional: true + require : ecbuild eckit + cmake : > + FCKIT_ENABLE_TESTS=OFF + + - atlas : + git : https://github.com/ecmwf/atlas + version : master + optional: true + require : ecbuild eckit fckit + cmake : > + ATLAS_ENABLE_TESTS=OFF + - field_api : git : ${BITBUCKET}/rdx/field_api version : master optional: true require : ecbuild cmake : > - BUILD_field_api=OFF ENABLE_FIELD_API_TESTS=OFF ENABLE_FIELD_API_FIAT_BUILD=OFF FIELD_API_UTIL_MODULE_PATH=${CMAKE_SOURCE_DIR}/cloudsc-dwarf/src/common/module @@ -111,6 +139,21 @@ options : cmake : > CLOUDSC_PYTHON_F2PY=ON + - with-atlas : + help : Build Atlas and its dependencies (eckit, fckit) and enable Atlas-based variants of CLOUDSC + cmake : > + BUILD_eckit=ON + BUILD_fckit=ON + BUILD_atlas=ON + + - with-dependency-tests : + help : Build and enable tests for CLOUDSC dependencies that are build as part of the bundle (eckit, fckit, Atlas, Loki) + cmake : > + LOKI_ENABLE_TESTS=ON + ECKIT_ENABLE_TESTS=ON + FCKIT_ENABLE_TESTS=ON + ATLAS_ENABLE_TESTS=ON + - cloudsc-prototype1 : help : Build the original operational Fortran prototype [ON|OFF] cmake : ENABLE_CLOUDSC_PROTOTYPE1={{value}} From 95aaef3654d9b42279240065763c9c462650b321 Mon Sep 17 00:00:00 2001 From: Balthasar Reuter Date: Tue, 18 Jul 2023 13:52:01 +0100 Subject: [PATCH 10/45] Add Atlas variant to Github actions --- .github/scripts/verify-targets.sh | 7 ++++++- .github/workflows/build.yml | 8 +++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/.github/scripts/verify-targets.sh b/.github/scripts/verify-targets.sh index 7040502d..634f41eb 100755 --- a/.github/scripts/verify-targets.sh +++ b/.github/scripts/verify-targets.sh @@ -17,7 +17,7 @@ fi if [[ "$gpu_flag" == "--with-gpu" ]] then - targets+=(dwarf-cloudsc-gpu-scc dwarf-cloudsc-gpu-scc-hoist dwarf-cloudsc-gpu-scc-k-caching) + targets+=(dwarf-cloudsc-gpu-scc dwarf-cloudsc-gpu-scc-hoist dwarf-cloudsc-gpu-scc-k-caching) targets+=(dwarf-cloudsc-gpu-omp-scc-hoist) if [[ "$claw_flag" == "--with-claw" ]] then @@ -52,6 +52,11 @@ then fi fi +if [[ "$atlas_flag" == "--with-atlas" ]] +then + targets+=(dwarf-cloudsc-fortran-atlas) +fi + if [[ "$pyiface_flag" == "--cloudsc-fortran-pyiface=ON" ]] then targets+=(cloudsc_pyiface.py) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5b634e3c..99aded56 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -42,6 +42,8 @@ jobs: loki_flag: ['', '--with-loki'] # Loki source-to-source translation enabled + atlas_flag: ['--with-atlas'] # Variant using Atlas-managed fields + claw_flag: [''] # Flag to enable CLAW-generated variants pyiface_flag: [''] # Flag to enable Python-interface variant @@ -57,6 +59,7 @@ jobs: gpu_flag: '--with-gpu' cuda_flag: '--with-cuda' loki_flag: '--with-loki' + atlas_flag: '--with-atlas' pyiface_flag: '' python_f2py_flag: '' - arch: github/ubuntu/nvhpc/21.9 @@ -66,6 +69,7 @@ jobs: gpu_flag: '--with-gpu' cuda_flag: '--with-cuda' loki_flag: '--with-loki' + atlas_flag: '--with-atlas' pyiface_flag: '' python_f2py_flag: '' # Add pyiface build configuration for HDF5 only @@ -76,6 +80,7 @@ jobs: gpu_flag: '' cuda_flag: '' loki_flag: '' + atlas_flag: '' pyiface_flag: '--cloudsc-fortran-pyiface=ON' python_f2py_flag: '--cloudsc-python-f2py=ON' @@ -124,7 +129,7 @@ jobs: --arch=arch/${{ matrix.arch }} ${{ matrix.prec_flag }} \ ${{ matrix.mpi_flag }} ${{ matrix.io_library_flag }} ${{ matrix.gpu_flag }} \ ${{ matrix.claw_flag}} ${{ matrix.loki_flag }} ${{ matrix.cuda_flag }} \ - ${{ matrix.pyiface_flag }} ${{ matrix.python_f2py_flag }} + ${{ matrix.atlas_flag }} ${{ matrix.pyiface_flag }} ${{ matrix.python_f2py_flag }} # Verify targets exist - name: Verify targets @@ -134,6 +139,7 @@ jobs: gpu_flag: ${{ matrix.gpu_flag }} cuda_flag: ${{ matrix.cuda_flag }} loki_flag: ${{ matrix.loki_flag }} + atlas_flag: ${{ matrix.atlas_flag }} claw_flag: ${{ matrix.claw_flag }} pyiface_flag: ${{ matrix.pyiface_flag }} python_f2py_flag: ${{ matrix.python_f2py_flag }} From 596098959e0525f563eecb36832e1ee4299b0756 Mon Sep 17 00:00:00 2001 From: Balthasar Reuter Date: Tue, 18 Jul 2023 14:58:54 +0100 Subject: [PATCH 11/45] Allow for Atlas binaries in Github CI --- .github/scripts/verify-targets.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/scripts/verify-targets.sh b/.github/scripts/verify-targets.sh index 634f41eb..7e29efa2 100755 --- a/.github/scripts/verify-targets.sh +++ b/.github/scripts/verify-targets.sh @@ -55,6 +55,9 @@ fi if [[ "$atlas_flag" == "--with-atlas" ]] then targets+=(dwarf-cloudsc-fortran-atlas) + # Atlas builds a number of binaries that end up in bin, too: + targets+=(atlas atlas-atest-mgrids atlas-gaussian-latitudes atlas-grids) + targets+=(atlas-io-list atlas-meshgen fckit) fi if [[ "$pyiface_flag" == "--cloudsc-fortran-pyiface=ON" ]] From dbaa29343852dfb64bf2b6dc71485489423a9f96 Mon Sep 17 00:00:00 2001 From: Slavko Brdar Date: Thu, 17 Aug 2023 17:19:39 +0200 Subject: [PATCH 12/45] debug leftover (tnx Balthasar) --- src/cloudsc_fortran_atlas/dwarf_cloudsc_atlas.F90 | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/cloudsc_fortran_atlas/dwarf_cloudsc_atlas.F90 b/src/cloudsc_fortran_atlas/dwarf_cloudsc_atlas.F90 index 1e2a7df2..dbbb9dc9 100644 --- a/src/cloudsc_fortran_atlas/dwarf_cloudsc_atlas.F90 +++ b/src/cloudsc_fortran_atlas/dwarf_cloudsc_atlas.F90 @@ -78,10 +78,6 @@ PROGRAM DWARF_CLOUDSC ! TODO: Create a global memory state from serialized input data CALL GLOBAL_ATLAS_STATE%LOAD(FSET, NPROMA, NGPTOT, NGPTOTG) -!FIELD = FSET%FIELD("PAP") -!call field%data(tmp3d) -!print *, MINVAL(tmp3d), MAXVAL(tmp3d) - ! Call the driver to perform the parallel loop over our kernel CALL CLOUDSC_DRIVER(FSET, NUMOMP, NGPTOT, NGPTOTG, GLOBAL_ATLAS_STATE%KFLDX, GLOBAL_ATLAS_STATE%PTSPHY) From b23deb3983cc5f9406ed69971ada129a71462e2d Mon Sep 17 00:00:00 2001 From: Slavko Brdar Date: Tue, 22 Aug 2023 11:58:51 +0200 Subject: [PATCH 13/45] Atlas lets the dwarf do the mpi initialisation --- src/cloudsc_fortran_atlas/dwarf_cloudsc_atlas.F90 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cloudsc_fortran_atlas/dwarf_cloudsc_atlas.F90 b/src/cloudsc_fortran_atlas/dwarf_cloudsc_atlas.F90 index dbbb9dc9..20755a9e 100644 --- a/src/cloudsc_fortran_atlas/dwarf_cloudsc_atlas.F90 +++ b/src/cloudsc_fortran_atlas/dwarf_cloudsc_atlas.F90 @@ -10,7 +10,7 @@ PROGRAM DWARF_CLOUDSC USE PARKIND1, ONLY: JPIM, JPIB -USE CLOUDSC_MPI_MOD, ONLY: CLOUDSC_MPI_INIT, CLOUDSC_MPI_END, NUMPROC, IRANK +USE CLOUDSC_MPI_MOD, ONLY: CLOUDSC_MPI_INIT, CLOUDSC_MPI_END, NUMPROC, IRANK, MPI_COMM_WORLD USE CLOUDSC_GLOBAL_ATLAS_STATE_MOD, ONLY: CLOUDSC_GLOBAL_ATLAS_STATE USE CLOUDSC_DRIVER_MOD, ONLY: CLOUDSC_DRIVER USE EC_PMON_MOD, ONLY: EC_PMON @@ -52,8 +52,8 @@ PROGRAM DWARF_CLOUDSC end if ! Initialize MPI environment -CALL ATLAS_LIBRARY%INITIALISE() CALL CLOUDSC_MPI_INIT(NUMOMP) +CALL ATLAS_INIT(COMM=MPI_COMM_WORLD) ! Get total number of grid points (NGPTOT) with which to run the benchmark IF (IARGS >= 2) THEN From 698bdd205c2b3442ceaf44644f1837f782be63f9 Mon Sep 17 00:00:00 2001 From: Slavko Brdar Date: Wed, 23 Aug 2023 16:09:01 +0000 Subject: [PATCH 14/45] update authors --- AUTHORS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS.md b/AUTHORS.md index 1670b2a7..650e42ac 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -2,6 +2,7 @@ - M. Ahlgrimm (ECMWF) - P. Bechtold (ECMWF) +- S. Brdar (ECMWF) - W. Deconinck (ECMWF) - R. Forbes (ECMWF) - C. Jakob (ECMWF) From 3b31fdf8ccb3ea55d7b927f5e59ef6d7cee0d369 Mon Sep 17 00:00:00 2001 From: Slavko Brdar Date: Wed, 23 Aug 2023 16:09:35 +0000 Subject: [PATCH 15/45] take newest developments from Atlas --- bundle.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bundle.yml b/bundle.yml index 57230a55..3b2ff9fa 100644 --- a/bundle.yml +++ b/bundle.yml @@ -42,7 +42,7 @@ projects : - eckit : git : https://github.com/ecmwf/eckit - version : 1.24.3 + version : develop optional: true require : ecbuild cmake : > @@ -51,7 +51,7 @@ projects : - fckit : git : https://github.com/ecmwf/fckit - version : 0.9.0 + version : develop optional: true require : ecbuild eckit cmake : > @@ -59,7 +59,7 @@ projects : - atlas : git : https://github.com/ecmwf/atlas - version : master + version : feature/MultiField optional: true require : ecbuild eckit fckit cmake : > From 1a06752979ef861351ef156eeb6be61f643531bb Mon Sep 17 00:00:00 2001 From: Slavko Brdar Date: Wed, 23 Aug 2023 16:20:12 +0000 Subject: [PATCH 16/45] bug fix in Atlas structured when the data array in the first block is less in size than nproma --- src/cloudsc_fortran_atlas/cloudsc_driver_mod.F90 | 2 +- src/cloudsc_fortran_atlas/expand_atlas_mod.F90 | 4 ++-- src/cloudsc_fortran_atlas/validate_atlas_mod.F90 | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/cloudsc_fortran_atlas/cloudsc_driver_mod.F90 b/src/cloudsc_fortran_atlas/cloudsc_driver_mod.F90 index a6d11c27..a3018f4e 100644 --- a/src/cloudsc_fortran_atlas/cloudsc_driver_mod.F90 +++ b/src/cloudsc_fortran_atlas/cloudsc_driver_mod.F90 @@ -55,7 +55,7 @@ SUBROUTINE CLOUDSC_DRIVER(FSET, NUMOMP, NGPTOT, NGPTOTG, KFLDX, PTSPHY) FIELD = FSET%FIELD("PEXTRA") FSPACE = FIELD%FUNCTIONSPACE() - NPROMA = FSPACE%BLOCK_SIZE(1) + NPROMA = FSPACE%NPROMA() NLEV = FSPACE%LEVELS() NGPBLKS = (NGPTOT / NPROMA) + MIN(MOD(NGPTOT,NPROMA), 1) diff --git a/src/cloudsc_fortran_atlas/expand_atlas_mod.F90 b/src/cloudsc_fortran_atlas/expand_atlas_mod.F90 index edcf8c1b..16cf8f38 100644 --- a/src/cloudsc_fortran_atlas/expand_atlas_mod.F90 +++ b/src/cloudsc_fortran_atlas/expand_atlas_mod.F90 @@ -51,7 +51,7 @@ subroutine loadvar_atlas(fset, name, nlon, ngptotg) fspace = field%functionspace() nlev = field%levels() - nproma = fspace%block_size(1) + nproma = fspace%nproma() ngptot = fspace%size() nblocks = fspace%nblks() @@ -115,8 +115,8 @@ subroutine loadstate_atlas(fset, name, nlon, ngptotg) field = fset%field(name) fspace = field%functionspace() nlev = field%levels() - nproma = fspace%block_size(1) ngptot = fspace%size() + nproma = fspace%nproma() nblocks = fspace%nblks() ndim = field%shape(3) - 3 diff --git a/src/cloudsc_fortran_atlas/validate_atlas_mod.F90 b/src/cloudsc_fortran_atlas/validate_atlas_mod.F90 index 1b1b43f1..3ebcebc0 100644 --- a/src/cloudsc_fortran_atlas/validate_atlas_mod.F90 +++ b/src/cloudsc_fortran_atlas/validate_atlas_mod.F90 @@ -64,7 +64,7 @@ SUBROUTINE VALIDATEVAR_ATLAS(FSET, NAME, NLON, NGPTOTG, STATE_VAR) NLEV = FIELD%LEVELS() NGPTOT = FSPACE%SIZE() NBLOCKS = FSPACE%NBLKS() - NPROMA = FSPACE%BLOCK_SIZE(1) + NPROMA = FSPACE%NPROMA() ZMINVAL(1) = +HUGE(ZMINVAL(1)) ZMAX_VAL_ERR(1) = -HUGE(ZMAX_VAL_ERR(1)) From 24b9132391900cd2d616de8fbd0a954e3814f64a Mon Sep 17 00:00:00 2001 From: Slavko Brdar Date: Wed, 23 Aug 2023 16:34:29 +0000 Subject: [PATCH 17/45] bug fix the parallel setup with Atlas structure: 1) number of point per process must be taken from Atlas decomposition; 2) Atlas grid has to get the global number of points --- .../cloudsc_global_atlas_state_mod.F90 | 11 +++++++---- src/cloudsc_fortran_atlas/dwarf_cloudsc_atlas.F90 | 7 +++---- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/cloudsc_fortran_atlas/cloudsc_global_atlas_state_mod.F90 b/src/cloudsc_fortran_atlas/cloudsc_global_atlas_state_mod.F90 index ba09384f..71fc1d0e 100644 --- a/src/cloudsc_fortran_atlas/cloudsc_global_atlas_state_mod.F90 +++ b/src/cloudsc_fortran_atlas/cloudsc_global_atlas_state_mod.F90 @@ -204,7 +204,8 @@ SUBROUTINE CLOUDSC_GLOBAL_ATLAS_STATE_LOAD(SELF, FSET, NPROMA, NGPTOT, NGPTOTG) ! Load reference input data via serialbox CLASS(CLOUDSC_GLOBAL_ATLAS_STATE) :: SELF TYPE(ATLAS_FIELDSET), INTENT(INOUT) :: FSET - INTEGER(KIND=JPIM), INTENT(IN) :: NPROMA, NGPTOT + INTEGER(KIND=JPIM), INTENT(IN) :: NPROMA + INTEGER(KIND=JPIM), INTENT(INOUT) :: NGPTOT INTEGER(KIND=JPIM), INTENT(IN), OPTIONAL :: NGPTOTG TYPE(ATLAS_STRUCTUREDGRID) :: GRID @@ -215,15 +216,17 @@ SUBROUTINE CLOUDSC_GLOBAL_ATLAS_STATE_LOAD(SELF, FSET, NPROMA, NGPTOT, NGPTOTG) REAL(C_DOUBLE), POINTER :: TMP3D(:,:,:) REAL(C_DOUBLE), POINTER :: TMP2D(:,:) TYPE(ATLAS_FIELD) :: FIELD - + TYPE(ATLAS_PARTITIONER) :: PARTITIONER CALL INPUT_INITIALIZE(NAME='input') CALL LOAD_SCALAR('KLON', KLON) CALL LOAD_SCALAR('KLEV', SELF%KLEV) CALL LOAD_SCALAR('KFLDX', SELF%KFLDX) - GRID = ATLAS_REGULARLONLATGRID(NGPTOT, 1) - FSPACE = ATLAS_FUNCTIONSPACE_BLOCKSTRUCTUREDCOLUMNS(GRID, LEVELS=SELF%KLEV, NPROMA=NPROMA, HALO=0) + GRID = ATLAS_REGULARLONLATGRID(NGPTOTG,1) + PARTITIONER = ATLAS_PARTITIONER("bands") + FSPACE = ATLAS_FUNCTIONSPACE_BLOCKSTRUCTUREDCOLUMNS(GRID, PARTITIONER, LEVELS=SELF%KLEV, NPROMA=NPROMA, HALO=0) + NGPTOT = FSPACE%SIZE() SELF%NBLOCKS = FSPACE%NBLKS() FSET = ATLAS_FIELDSET() diff --git a/src/cloudsc_fortran_atlas/dwarf_cloudsc_atlas.F90 b/src/cloudsc_fortran_atlas/dwarf_cloudsc_atlas.F90 index 20755a9e..4b2f62d8 100644 --- a/src/cloudsc_fortran_atlas/dwarf_cloudsc_atlas.F90 +++ b/src/cloudsc_fortran_atlas/dwarf_cloudsc_atlas.F90 @@ -10,7 +10,7 @@ PROGRAM DWARF_CLOUDSC USE PARKIND1, ONLY: JPIM, JPIB -USE CLOUDSC_MPI_MOD, ONLY: CLOUDSC_MPI_INIT, CLOUDSC_MPI_END, NUMPROC, IRANK, MPI_COMM_WORLD +USE CLOUDSC_MPI_MOD, ONLY: CLOUDSC_MPI_INIT, CLOUDSC_MPI_END, NUMPROC, IRANK USE CLOUDSC_GLOBAL_ATLAS_STATE_MOD, ONLY: CLOUDSC_GLOBAL_ATLAS_STATE USE CLOUDSC_DRIVER_MOD, ONLY: CLOUDSC_DRIVER USE EC_PMON_MOD, ONLY: EC_PMON @@ -53,7 +53,7 @@ PROGRAM DWARF_CLOUDSC ! Initialize MPI environment CALL CLOUDSC_MPI_INIT(NUMOMP) -CALL ATLAS_INIT(COMM=MPI_COMM_WORLD) +CALL ATLAS_LIBRARY%INITIALISE() ! Get total number of grid points (NGPTOT) with which to run the benchmark IF (IARGS >= 2) THEN @@ -84,9 +84,8 @@ PROGRAM DWARF_CLOUDSC ! Validate the output against serialized reference data CALL GLOBAL_ATLAS_STATE%VALIDATE(FSET, NGPTOT, NGPTOTG) -CALL ATLAS_LIBRARY%FINALISE() - ! Tear down MPI environment +CALL ATLAS_LIBRARY%FINALISE() CALL CLOUDSC_MPI_END() END PROGRAM DWARF_CLOUDSC From b39e918b03d6093339f73b83fcb49fcb444f77c0 Mon Sep 17 00:00:00 2001 From: Slavko Brdar Date: Wed, 23 Aug 2023 16:43:49 +0000 Subject: [PATCH 18/45] NGPTOT is controlled by Atlas (tnx Willem) --- src/cloudsc_fortran_atlas/cloudsc_driver_mod.F90 | 7 ++++--- .../cloudsc_global_atlas_state_mod.F90 | 7 +++---- src/cloudsc_fortran_atlas/dwarf_cloudsc_atlas.F90 | 15 ++++----------- 3 files changed, 11 insertions(+), 18 deletions(-) diff --git a/src/cloudsc_fortran_atlas/cloudsc_driver_mod.F90 b/src/cloudsc_fortran_atlas/cloudsc_driver_mod.F90 index a3018f4e..ba9937f4 100644 --- a/src/cloudsc_fortran_atlas/cloudsc_driver_mod.F90 +++ b/src/cloudsc_fortran_atlas/cloudsc_driver_mod.F90 @@ -24,19 +24,19 @@ MODULE CLOUDSC_DRIVER_MOD CONTAINS - SUBROUTINE CLOUDSC_DRIVER(FSET, NUMOMP, NGPTOT, NGPTOTG, KFLDX, PTSPHY) + SUBROUTINE CLOUDSC_DRIVER(FSET, NUMOMP, NGPTOTG, KFLDX, PTSPHY) ! Driver routine that performans the parallel NPROMA-blocking and ! invokes the CLOUDSC kernel TYPE(ATLAS_FIELDSET), INTENT(INOUT) :: FSET - INTEGER(KIND=JPIM), INTENT(IN) :: NUMOMP, NGPTOT, NGPTOTG, KFLDX + INTEGER(KIND=JPIM), INTENT(IN) :: NUMOMP, NGPTOTG, KFLDX REAL(KIND=JPRB), INTENT(IN) :: PTSPHY ! Physics timestep TYPE(CLOUDSC_GLOBAL_ATLAS_STATE_BLOCK_VIEW) :: FBLOCK TYPE(ATLAS_FUNCTIONSPACE_BLOCKSTRUCTUREDCOLUMNS) :: FSPACE TYPE(ATLAS_FIELD) :: FIELD - INTEGER(KIND=JPIM) :: NPROMA, NLEV + INTEGER(KIND=JPIM) :: NPROMA, NLEV, NGPTOT INTEGER(KIND=JPIM) :: JKGLO,IBL,ICEND,NGPBLKS @@ -57,6 +57,7 @@ SUBROUTINE CLOUDSC_DRIVER(FSET, NUMOMP, NGPTOT, NGPTOTG, KFLDX, PTSPHY) FSPACE = FIELD%FUNCTIONSPACE() NPROMA = FSPACE%NPROMA() NLEV = FSPACE%LEVELS() + NGPTOT = FSPACE%SIZE() NGPBLKS = (NGPTOT / NPROMA) + MIN(MOD(NGPTOT,NPROMA), 1) 1003 format(5x,'NUMPROC=',i0,', NUMOMP=',i0,', NGTOT=', i0,', NGPTOTG=',i0,', NPROMA=',i0,', NGPBLKS=',i0) diff --git a/src/cloudsc_fortran_atlas/cloudsc_global_atlas_state_mod.F90 b/src/cloudsc_fortran_atlas/cloudsc_global_atlas_state_mod.F90 index 71fc1d0e..5ccb174e 100644 --- a/src/cloudsc_fortran_atlas/cloudsc_global_atlas_state_mod.F90 +++ b/src/cloudsc_fortran_atlas/cloudsc_global_atlas_state_mod.F90 @@ -200,12 +200,11 @@ SUBROUTINE CLOUDSC_GLOBAL_ATLAS_STATE_BLOCK(SELF, FSET, IBLK) CALL FSET%DATA(49, SELF%PRAINFRAC_TOPRFZ, IBLK) END SUBROUTINE CLOUDSC_GLOBAL_ATLAS_STATE_BLOCK - SUBROUTINE CLOUDSC_GLOBAL_ATLAS_STATE_LOAD(SELF, FSET, NPROMA, NGPTOT, NGPTOTG) + SUBROUTINE CLOUDSC_GLOBAL_ATLAS_STATE_LOAD(SELF, FSET, NPROMA, NGPTOTG) ! Load reference input data via serialbox CLASS(CLOUDSC_GLOBAL_ATLAS_STATE) :: SELF TYPE(ATLAS_FIELDSET), INTENT(INOUT) :: FSET INTEGER(KIND=JPIM), INTENT(IN) :: NPROMA - INTEGER(KIND=JPIM), INTENT(INOUT) :: NGPTOT INTEGER(KIND=JPIM), INTENT(IN), OPTIONAL :: NGPTOTG TYPE(ATLAS_STRUCTUREDGRID) :: GRID @@ -217,6 +216,7 @@ SUBROUTINE CLOUDSC_GLOBAL_ATLAS_STATE_LOAD(SELF, FSET, NPROMA, NGPTOT, NGPTOTG) REAL(C_DOUBLE), POINTER :: TMP2D(:,:) TYPE(ATLAS_FIELD) :: FIELD TYPE(ATLAS_PARTITIONER) :: PARTITIONER + INTEGER(KIND=JPIM) :: NGPTOT CALL INPUT_INITIALIZE(NAME='input') CALL LOAD_SCALAR('KLON', KLON) @@ -296,11 +296,10 @@ SUBROUTINE CLOUDSC_GLOBAL_ATLAS_STATE_LOAD(SELF, FSET, NPROMA, NGPTOT, NGPTOTG) CALL INPUT_FINALIZE() END SUBROUTINE CLOUDSC_GLOBAL_ATLAS_STATE_LOAD - SUBROUTINE CLOUDSC_GLOBAL_ATLAS_STATE_VALIDATE(SELF, FSET, NGPTOT, NGPTOTG) + SUBROUTINE CLOUDSC_GLOBAL_ATLAS_STATE_VALIDATE(SELF, FSET, NGPTOTG) ! Validate the correctness of output against reference data CLASS(CLOUDSC_GLOBAL_ATLAS_STATE) :: SELF TYPE(ATLAS_FIELDSET), INTENT(INOUT) :: FSET - INTEGER(KIND=JPIM), INTENT(IN) :: NGPTOT INTEGER(KIND=JPIM), INTENT(IN), OPTIONAL :: NGPTOTG INTEGER(KIND=JPIM) :: KLON, IVAR diff --git a/src/cloudsc_fortran_atlas/dwarf_cloudsc_atlas.F90 b/src/cloudsc_fortran_atlas/dwarf_cloudsc_atlas.F90 index 4b2f62d8..dd8470cb 100644 --- a/src/cloudsc_fortran_atlas/dwarf_cloudsc_atlas.F90 +++ b/src/cloudsc_fortran_atlas/dwarf_cloudsc_atlas.F90 @@ -26,7 +26,6 @@ PROGRAM DWARF_CLOUDSC INTEGER(KIND=JPIM) :: NUMOMP = 1 ! Number of OpenMP threads for this run INTEGER(KIND=JPIM) :: NGPTOTG = 16384 ! Number of grid points (as read from command line) INTEGER(KIND=JPIM) :: NPROMA = 32 ! NPROMA blocking factor (currently active) -INTEGER(KIND=JPIM) :: NGPTOT ! Local number of grid points REAL(c_double), pointer :: tmp3d(:,:,:) type(atlas_fieldset) :: fset @@ -55,18 +54,12 @@ PROGRAM DWARF_CLOUDSC CALL CLOUDSC_MPI_INIT(NUMOMP) CALL ATLAS_LIBRARY%INITIALISE() -! Get total number of grid points (NGPTOT) with which to run the benchmark +! Get total number of grid points (NGPTOTG) with which to run the benchmark IF (IARGS >= 2) THEN CALL GET_COMMAND_ARGUMENT(2, CLARG, LENARG) READ(CLARG(1:LENARG),*) NGPTOTG END IF -! Determine local number of grid points -NGPTOT = (NGPTOTG - 1) / NUMPROC + 1 -if (IRANK == NUMPROC - 1) then - NGPTOT = NGPTOTG - (NUMPROC - 1) * NGPTOT -end if - ! Get the block size (NPROMA) for which to run the benchmark IF (IARGS >= 3) THEN CALL GET_COMMAND_ARGUMENT(3, CLARG, LENARG) @@ -76,13 +69,13 @@ PROGRAM DWARF_CLOUDSC FSET = ATLAS_FIELDSET() ! TODO: Create a global memory state from serialized input data -CALL GLOBAL_ATLAS_STATE%LOAD(FSET, NPROMA, NGPTOT, NGPTOTG) +CALL GLOBAL_ATLAS_STATE%LOAD(FSET, NPROMA, NGPTOTG) ! Call the driver to perform the parallel loop over our kernel -CALL CLOUDSC_DRIVER(FSET, NUMOMP, NGPTOT, NGPTOTG, GLOBAL_ATLAS_STATE%KFLDX, GLOBAL_ATLAS_STATE%PTSPHY) +CALL CLOUDSC_DRIVER(FSET, NUMOMP, NGPTOTG, GLOBAL_ATLAS_STATE%KFLDX, GLOBAL_ATLAS_STATE%PTSPHY) ! Validate the output against serialized reference data -CALL GLOBAL_ATLAS_STATE%VALIDATE(FSET, NGPTOT, NGPTOTG) +CALL GLOBAL_ATLAS_STATE%VALIDATE(FSET, NGPTOTG) ! Tear down MPI environment CALL ATLAS_LIBRARY%FINALISE() From 20b25e1cf5ec8677a4558a0a0acdade25c333fd6 Mon Sep 17 00:00:00 2001 From: Balthasar Reuter Date: Fri, 25 Aug 2023 21:20:19 +0100 Subject: [PATCH 19/45] Remove conflict with CUDA option of eckit --- bundle.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bundle.yml b/bundle.yml index 3b2ff9fa..b4b2204d 100644 --- a/bundle.yml +++ b/bundle.yml @@ -5,6 +5,7 @@ name : cloudsc-bundle version : 1.0.0-develop cmake : > CMAKE_LINK_DEPENDS_NO_SHARED=ON + BUILD_serialbox=OFF BUILD_field_api=OFF BUILD_eckit=OFF BUILD_fckit=OFF @@ -22,7 +23,6 @@ projects : version : v2.5.4/patched optional: true cmake : > - BUILD_serialbox=OFF SERIALBOX_BUILD_SHARED=ON SERIALBOX_ENABLE_FORTRAN=ON SERIALBOX_ENABLE_EXPERIMENTAL_FILESYSTEM=OFF @@ -102,7 +102,8 @@ options : - with-cuda : help : Enable GPU kernel variant based on CUDA-Fortran cmake : > - ENABLE_CUDA=ON + ENABLE_FIELD_API_CUDA=ON + ENABLE_CLOUDSC_CUDA=ON ENABLE_CLOUDSC_GPU_SCC_CUF=ON ENABLE_CLOUDSC_GPU_SCC_CUF_K_CACHING=ON BUILD_field_api=ON From c4b6c9a00d74b72ba659e30d162f4fc47c70aeea Mon Sep 17 00:00:00 2001 From: Balthasar Reuter Date: Fri, 25 Aug 2023 21:20:39 +0100 Subject: [PATCH 20/45] Restore Github workflow after merge --- .github/workflows/build.yml | 71 ++++++++++++++----------------------- 1 file changed, 27 insertions(+), 44 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 473f0e4d..14b2d591 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -35,19 +35,13 @@ jobs: build_flags: - '' # Plain build without any options - - '--with-gpu --with-loki' # Enable Loki and GPU variants - - '--with-gpu --with-loki --with-mpi' # Enable Loki and GPU variants with MPI + - '--with-gpu --with-loki --with-atlas' # Enable Loki, Atlas, and GPU variants + - '--with-gpu --with-loki --with-atlas --with-mpi' # Enable Loki, Atlas, and GPU variants with MPI pyiface_flag: [''] # Enable the pyiface variant - atlas_flag: ['--with-atlas'] # Variant using Atlas-managed fields - - claw_flag: [''] # Flag to enable CLAW-generated variants - python_f2py_flag: [''] # Enable the f2py variant - claw_flag: [''] - ctest_exclude_pattern: ['-scc-hoist-'] # Regex to disable CTest tests include: @@ -55,33 +49,30 @@ jobs: - arch: gnu/9.4.0 io_library_flag: '' prec_flag: '' - gpu_flag: '--with-gpu' - cuda_flag: '--with-cuda' - loki_flag: '--with-loki' - atlas_flag: '--with-atlas' - pyiface_flag: '' - python_f2py_flag: '' - - arch: github/ubuntu/nvhpc/21.9 + build_flags: '--cloudsc-fortran-pyiface=ON --cloudsc-python-f2py=ON' + + # Add nvhpc build configurations with serialbox and HDF5 + - arch: nvhpc/21.9 + nvhpc_version: 21.9 + io_library_flag: '' + build_flags: '--with-gpu --with-loki --with-cuda --with-atlas' + ctest_exclude_pattern: '-gpu-|-scc-|-loki-c|-cuda-' # GPU variants don't work on CPU runners, loki-c variant causes SIGFPE + - arch: nvhpc/21.9 + nvhpc_version: 21.9 io_library_flag: '--with-serialbox' - mpi_flag: '' - prec_flag: '' - gpu_flag: '--with-gpu' - cuda_flag: '--with-cuda' - loki_flag: '--with-loki' - atlas_flag: '--with-atlas' - pyiface_flag: '' - python_f2py_flag: '' - # Add pyiface build configuration for HDF5 only - - arch: github/ubuntu/gnu/9.4.0 + build_flags: '--with-gpu --with-loki --with-cuda --with-atlas' + ctest_exclude_pattern: '-gpu-|-scc-|-loki-c|-cuda-' # GPU variants don't work on CPU runners, loki-c variant causes SIGFPE + + - arch: nvhpc/23.5 + nvhpc_version: 23.5 io_library_flag: '' - mpi_flag: '' - prec_flag: '' - gpu_flag: '' - cuda_flag: '' - loki_flag: '' - atlas_flag: '' - pyiface_flag: '--cloudsc-fortran-pyiface=ON' - python_f2py_flag: '--cloudsc-python-f2py=ON' + build_flags: '--with-gpu --with-loki --with-cuda --with-atlas' + ctest_exclude_pattern: '-gpu-|-scc-|-loki-c|-loki-sca|-cuda-' # GPU variants don't work on CPU runners, loki-c variant causes SIGFPE + - arch: nvhpc/23.5 + nvhpc_version: 23.5 + io_library_flag: '--with-serialbox' + build_flags: '--with-gpu --with-loki --with-cuda --with-atlas' + ctest_exclude_pattern: '-gpu-|-scc-|-loki-c|-loki-sca|-cuda-' # GPU variants don't work on CPU runners, loki-c variant causes SIGFPE # Steps represent a sequence of tasks that will be executed as part of the job steps: @@ -147,23 +138,15 @@ jobs: - name: Bundle build run: | ./cloudsc-bundle build --retry-verbose \ - --arch=arch/${{ matrix.arch }} ${{ matrix.prec_flag }} \ - ${{ matrix.mpi_flag }} ${{ matrix.io_library_flag }} ${{ matrix.gpu_flag }} \ - ${{ matrix.claw_flag}} ${{ matrix.loki_flag }} ${{ matrix.cuda_flag }} \ - ${{ matrix.atlas_flag }} ${{ matrix.pyiface_flag }} ${{ matrix.python_f2py_flag }} + --arch=arch/github/ubuntu/${{ matrix.arch }} \ + ${{ matrix.prec_flag }} ${{ matrix.io_library_flag }} ${{ matrix.build_flags }} # Verify targets exist - name: Verify targets env: io_library_flag: ${{ matrix.io_library_flag }} prec_flag: ${{ matrix.prec_flag }} - gpu_flag: ${{ matrix.gpu_flag }} - cuda_flag: ${{ matrix.cuda_flag }} - loki_flag: ${{ matrix.loki_flag }} - atlas_flag: ${{ matrix.atlas_flag }} - claw_flag: ${{ matrix.claw_flag }} - pyiface_flag: ${{ matrix.pyiface_flag }} - python_f2py_flag: ${{ matrix.python_f2py_flag }} + build_flags: ${{ matrix.build_flags }} run: .github/scripts/verify-targets.sh # Run ctest From 1de5e78bbac822169a0a3b52f1b515512c091898 Mon Sep 17 00:00:00 2001 From: Balthasar Reuter Date: Fri, 25 Aug 2023 21:30:23 +0100 Subject: [PATCH 21/45] Trim down target verification --- .github/scripts/verify-targets.sh | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/.github/scripts/verify-targets.sh b/.github/scripts/verify-targets.sh index 28bd2595..b8726c29 100755 --- a/.github/scripts/verify-targets.sh +++ b/.github/scripts/verify-targets.sh @@ -52,12 +52,9 @@ then fi fi -if [[ "$atlas_flag" == "--with-atlas" ]] +if [[ "$build_flags" == *"--with-atlas"* ]] then targets+=(dwarf-cloudsc-fortran-atlas) - # Atlas builds a number of binaries that end up in bin, too: - targets+=(atlas atlas-atest-mgrids atlas-gaussian-latitudes atlas-grids) - targets+=(atlas-io-list atlas-meshgen fckit) fi if [[ "$build_flags" == *"--cloudsc-fortran-pyiface=ON"* ]] @@ -84,16 +81,4 @@ do fi done -# -# Check there aren't any other binaries -# - -if [[ ${#targets[@]} -lt $(ls build/bin | wc -l) ]] -then - exit_code=1 - echo "::error::Additional targets found in build/bin" - echo "::error::Expected targets: ${targets[@]}" - echo "::error::Found targets: $(ls -1 build/bin | tr '\n' ' ')" -fi - exit $exit_code From 4b1ab64bdc8eaab6bcbcf180e14b8ffd76123191 Mon Sep 17 00:00:00 2001 From: Slavko Brdar Date: Thu, 21 Sep 2023 23:36:51 +0200 Subject: [PATCH 22/45] make compile with Atlas --- src/cloudsc_fortran_atlas/cloudsc_global_atlas_state_mod.F90 | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/cloudsc_fortran_atlas/cloudsc_global_atlas_state_mod.F90 b/src/cloudsc_fortran_atlas/cloudsc_global_atlas_state_mod.F90 index 5ccb174e..ba2261e4 100644 --- a/src/cloudsc_fortran_atlas/cloudsc_global_atlas_state_mod.F90 +++ b/src/cloudsc_fortran_atlas/cloudsc_global_atlas_state_mod.F90 @@ -224,8 +224,7 @@ SUBROUTINE CLOUDSC_GLOBAL_ATLAS_STATE_LOAD(SELF, FSET, NPROMA, NGPTOTG) CALL LOAD_SCALAR('KFLDX', SELF%KFLDX) GRID = ATLAS_REGULARLONLATGRID(NGPTOTG,1) - PARTITIONER = ATLAS_PARTITIONER("bands") - FSPACE = ATLAS_FUNCTIONSPACE_BLOCKSTRUCTUREDCOLUMNS(GRID, PARTITIONER, LEVELS=SELF%KLEV, NPROMA=NPROMA, HALO=0) + FSPACE = ATLAS_FUNCTIONSPACE_BLOCKSTRUCTUREDCOLUMNS(GRID, LEVELS=SELF%KLEV, NPROMA=NPROMA, HALO=0) NGPTOT = FSPACE%SIZE() SELF%NBLOCKS = FSPACE%NBLKS() FSET = ATLAS_FIELDSET() From 0b8e593a382309215dbb4d264a12a091677d32c3 Mon Sep 17 00:00:00 2001 From: Balthasar Reuter Date: Fri, 22 Sep 2023 11:11:58 +0100 Subject: [PATCH 23/45] Increase output verbosity in Actions to debug OpenMP --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 14b2d591..fe0e716f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -137,7 +137,7 @@ jobs: # Build the targets - name: Bundle build run: | - ./cloudsc-bundle build --retry-verbose \ + ./cloudsc-bundle build --verbose --retry-verbose \ --arch=arch/github/ubuntu/${{ matrix.arch }} \ ${{ matrix.prec_flag }} ${{ matrix.io_library_flag }} ${{ matrix.build_flags }} From 27771ef742593a546189d4a338ec15e4c20beb1f Mon Sep 17 00:00:00 2001 From: Balthasar Reuter Date: Fri, 22 Sep 2023 11:24:22 +0100 Subject: [PATCH 24/45] Disable atlas for single-precision builds --- .github/workflows/build.yml | 23 +++++++++++++++-------- bundle.yml | 2 +- src/cloudsc_fortran_atlas/CMakeLists.txt | 2 +- 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index fe0e716f..4e4188f9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -17,7 +17,7 @@ on: jobs: # This workflow contains a single job called "build" build: - name: Test on ${{ matrix.arch }} ${{ matrix.io_library_flag }} ${{ matrix.prec_flag }} ${{ matrix.build_flags }} + name: ${{ matrix.arch }} ${{ matrix.io_library_flag }} ${{ matrix.build_flags }} # The type of runner that the job will run on runs-on: ubuntu-20.04 @@ -31,12 +31,11 @@ jobs: io_library_flag: ['', '--with-serialbox'] # Switch between Serialbox and HDF5 - prec_flag: ['', '--single-precision'] # Switch single/double precision - build_flags: - '' # Plain build without any options - '--with-gpu --with-loki --with-atlas' # Enable Loki, Atlas, and GPU variants - '--with-gpu --with-loki --with-atlas --with-mpi' # Enable Loki, Atlas, and GPU variants with MPI + - '--single-precision --with-gpu --with-loki --with-mpi' # Enable Loki, and GPU variants with MPI in a single-precision build pyiface_flag: [''] # Enable the pyiface variant @@ -48,7 +47,6 @@ jobs: # Add pyiface build configuration for double precision, non-MPI, HDF5 only - arch: gnu/9.4.0 io_library_flag: '' - prec_flag: '' build_flags: '--cloudsc-fortran-pyiface=ON --cloudsc-python-f2py=ON' # Add nvhpc build configurations with serialbox and HDF5 @@ -57,6 +55,11 @@ jobs: io_library_flag: '' build_flags: '--with-gpu --with-loki --with-cuda --with-atlas' ctest_exclude_pattern: '-gpu-|-scc-|-loki-c|-cuda-' # GPU variants don't work on CPU runners, loki-c variant causes SIGFPE + - arch: nvhpc/21.9 + nvhpc_version: 21.9 + io_library_flag: '' + build_flags: '--single-precision --with-gpu --with-loki --with-cuda' + ctest_exclude_pattern: '-gpu-|-scc-|-loki-c|-cuda-' # GPU variants don't work on CPU runners, loki-c variant causes SIGFPE - arch: nvhpc/21.9 nvhpc_version: 21.9 io_library_flag: '--with-serialbox' @@ -68,6 +71,11 @@ jobs: io_library_flag: '' build_flags: '--with-gpu --with-loki --with-cuda --with-atlas' ctest_exclude_pattern: '-gpu-|-scc-|-loki-c|-loki-sca|-cuda-' # GPU variants don't work on CPU runners, loki-c variant causes SIGFPE + - arch: nvhpc/23.5 + nvhpc_version: 23.5 + io_library_flag: '' + build_flags: '--single-precision --with-gpu --with-loki --with-cuda' + ctest_exclude_pattern: '-gpu-|-scc-|-loki-c|-loki-sca|-cuda-' # GPU variants don't work on CPU runners, loki-c variant causes SIGFPE - arch: nvhpc/23.5 nvhpc_version: 23.5 io_library_flag: '--with-serialbox' @@ -139,19 +147,18 @@ jobs: run: | ./cloudsc-bundle build --verbose --retry-verbose \ --arch=arch/github/ubuntu/${{ matrix.arch }} \ - ${{ matrix.prec_flag }} ${{ matrix.io_library_flag }} ${{ matrix.build_flags }} + ${{ matrix.io_library_flag }} ${{ matrix.build_flags }} # Verify targets exist - name: Verify targets env: io_library_flag: ${{ matrix.io_library_flag }} - prec_flag: ${{ matrix.prec_flag }} build_flags: ${{ matrix.build_flags }} run: .github/scripts/verify-targets.sh # Run ctest - name: Run CTest - if: ${{ matrix.prec_flag == '' }} + if: ${{ !contains(matrix.build_flags, '--single-precision') }} working-directory: ./build run: | source env.sh @@ -160,7 +167,7 @@ jobs: # Upload test output - name: Archive CTest output uses: actions/upload-artifact@v3 - if: ${{ matrix.prec_flag == '' }} + if: ${{ !contains(matrix.build_flags, '--single-precision') }} with: name: ctest-log path: build/ctest.log diff --git a/bundle.yml b/bundle.yml index b4b2204d..057d38e5 100644 --- a/bundle.yml +++ b/bundle.yml @@ -141,7 +141,7 @@ options : CLOUDSC_PYTHON_F2PY=ON - with-atlas : - help : Build Atlas and its dependencies (eckit, fckit) and enable Atlas-based variants of CLOUDSC + help : Build Atlas and its dependencies (eckit, fckit) and enable Atlas-based variants of CLOUDSC (incompatible with --single-precision) cmake : > BUILD_eckit=ON BUILD_fckit=ON diff --git a/src/cloudsc_fortran_atlas/CMakeLists.txt b/src/cloudsc_fortran_atlas/CMakeLists.txt index f1e61123..fe9f98fe 100644 --- a/src/cloudsc_fortran_atlas/CMakeLists.txt +++ b/src/cloudsc_fortran_atlas/CMakeLists.txt @@ -9,7 +9,7 @@ # Define this dwarf variant as an ECBuild feature ecbuild_add_option( FEATURE CLOUDSC_FORTRAN_ATLAS DESCRIPTION "Build the Fortran version CLOUDSC using Atlas and Serialbox" DEFAULT ON - CONDITION atlas_FOUND AND (Serialbox_FOUND OR HDF5_FOUND) + CONDITION atlas_FOUND AND (Serialbox_FOUND OR HDF5_FOUND) AND NOT HAVE_SINGLE_PRECISION ) if( HAVE_CLOUDSC_FORTRAN_ATLAS ) From 0e6568c6a105015991a4efac7c19b5ee806045be Mon Sep 17 00:00:00 2001 From: Balthasar Reuter Date: Fri, 22 Sep 2023 11:37:34 +0100 Subject: [PATCH 25/45] Enable OpenMP by default and add bundle option to switch off --- bundle.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/bundle.yml b/bundle.yml index 057d38e5..c925e01a 100644 --- a/bundle.yml +++ b/bundle.yml @@ -10,6 +10,7 @@ cmake : > BUILD_eckit=OFF BUILD_fckit=OFF BUILD_atlas=OFF + ENABLE_OMP=ON projects : @@ -112,6 +113,10 @@ options : help : Enable MPI-parallel kernel cmake : ENABLE_MPI=ON + - without-openmp : + help : Disable OpenMP + cmake : ENABLE_OMP=OFF + - with-loki : help : Enable Loki source-to-source transformations cmake : > From 484d58c483b9872879c82c19145f74f8bd311381 Mon Sep 17 00:00:00 2001 From: Balthasar Reuter Date: Fri, 22 Sep 2023 11:55:36 +0100 Subject: [PATCH 26/45] Remove prec_flag from verify-targets.sh --- .github/scripts/verify-targets.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/verify-targets.sh b/.github/scripts/verify-targets.sh index b8726c29..40e29cfa 100755 --- a/.github/scripts/verify-targets.sh +++ b/.github/scripts/verify-targets.sh @@ -38,7 +38,7 @@ then targets+=(dwarf-cloudsc-loki-idem dwarf-cloudsc-loki-sca) targets+=(dwarf-cloudsc-loki-scc dwarf-cloudsc-loki-scc-hoist) targets+=(dwarf-cloudsc-loki-idem-stack dwarf-cloudsc-loki-scc-stack) - if [[ "$prec_flag" != "--single-precision" ]] + if [[ "$build_flags" != *"--single-precision"* ]] then targets+=(dwarf-cloudsc-loki-c) fi From 74a384859dd5358fa43f3ad7b34809b28fa84934 Mon Sep 17 00:00:00 2001 From: Balthasar Reuter Date: Fri, 22 Sep 2023 12:50:25 +0100 Subject: [PATCH 27/45] Rename to cloudsc-c-cuda to avoid ecbuild option name clash --- src/cloudsc_cuda/CMakeLists.txt | 84 ++++++++++++++++----------------- 1 file changed, 41 insertions(+), 43 deletions(-) diff --git a/src/cloudsc_cuda/CMakeLists.txt b/src/cloudsc_cuda/CMakeLists.txt index 15565d81..3112b8b9 100644 --- a/src/cloudsc_cuda/CMakeLists.txt +++ b/src/cloudsc_cuda/CMakeLists.txt @@ -7,24 +7,23 @@ # nor does it submit to any jurisdiction. # Define this dwarf variant as an ECBuild feature -ecbuild_add_option( FEATURE CLOUDSC_CUDA - DESCRIPTION "Build the CUDA version CLOUDSC using Serialbox" DEFAULT ON +ecbuild_add_option( FEATURE CLOUDSC_C_CUDA + DESCRIPTION "Build the CUDA version of CLOUDSC C using Serialbox" DEFAULT ON CONDITION Serialbox_FOUND AND HAVE_CUDA ) if( HAVE_CLOUDSC_CUDA ) - enable_language(CUDA) enable_language(CXX) ###### SCC-CUDA #### ecbuild_add_library( - TARGET dwarf-cloudsc-cuda-lib - INSTALL_HEADERS LISTED + TARGET dwarf-cloudsc-c-cuda-lib + INSTALL_HEADERS LISTED SOURCES - cloudsc/yoecldp_c.h - cloudsc/load_state.h + cloudsc/yoecldp_c.h + cloudsc/load_state.h cloudsc/load_state.cu cloudsc/cloudsc_c.h cloudsc/cloudsc_c.cu @@ -43,30 +42,30 @@ if( HAVE_CLOUDSC_CUDA ) ) target_include_directories( - dwarf-cloudsc-cuda-lib - PUBLIC - ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES} + dwarf-cloudsc-c-cuda-lib + PUBLIC + ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES} ) if (NOT DEFINED CMAKE_CUDA_ARCHITECTURES) - target_compile_options(dwarf-cloudsc-cuda-lib PRIVATE $<$>) + target_compile_options(dwarf-cloudsc-c-cuda-lib PRIVATE $<$>) else() - target_compile_options(dwarf-cloudsc-cuda-lib PRIVATE $<$: - -gencode arch=compute_${CMAKE_CUDA_ARCHITECTURES},code=sm_${CMAKE_CUDA_ARCHITECTURES}>) + target_compile_options(dwarf-cloudsc-c-cuda-lib PRIVATE $<$: + -gencode arch=compute_${CMAKE_CUDA_ARCHITECTURES},code=sm_${CMAKE_CUDA_ARCHITECTURES}>) endif() - set_target_properties( dwarf-cloudsc-cuda-lib PROPERTIES CUDA_SEPARABLE_COMPILATION ON) - + set_target_properties( dwarf-cloudsc-c-cuda-lib PROPERTIES CUDA_SEPARABLE_COMPILATION ON) + ecbuild_add_executable( - TARGET dwarf-cloudsc-cuda + TARGET dwarf-cloudsc-c-cuda SOURCES dwarf_cloudsc.cpp - LIBS dwarf-cloudsc-cuda-lib + LIBS dwarf-cloudsc-c-cuda-lib ) - target_link_libraries(dwarf-cloudsc-cuda dwarf-cloudsc-cuda-lib) + target_link_libraries(dwarf-cloudsc-c-cuda dwarf-cloudsc-c-cuda-lib) ecbuild_add_test( - TARGET dwarf-cloudsc-cuda-serial - COMMAND bin/dwarf-cloudsc-cuda + TARGET dwarf-cloudsc-c-cuda-serial + COMMAND bin/dwarf-cloudsc-c-cuda ARGS 1 1000 128 WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/../../.. OMP 1 @@ -75,7 +74,7 @@ if( HAVE_CLOUDSC_CUDA ) ###### SCC-CUDA-HOIST #### ecbuild_add_library( - TARGET dwarf-cloudsc-cuda-hoist-lib + TARGET dwarf-cloudsc-c-cuda-hoist-lib INSTALL_HEADERS LISTED SOURCES cloudsc/yoecldp_c.h @@ -98,30 +97,30 @@ if( HAVE_CLOUDSC_CUDA ) ) target_include_directories( - dwarf-cloudsc-cuda-hoist-lib + dwarf-cloudsc-c-cuda-hoist-lib PUBLIC ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES} ) if (NOT DEFINED CMAKE_CUDA_ARCHITECTURES) - target_compile_options(dwarf-cloudsc-cuda-hoist-lib PRIVATE $<$>) + target_compile_options(dwarf-cloudsc-c-cuda-hoist-lib PRIVATE $<$>) else() - target_compile_options(dwarf-cloudsc-cuda-hoist-lib PRIVATE $<$: + target_compile_options(dwarf-cloudsc-c-cuda-hoist-lib PRIVATE $<$: -gencode arch=compute_${CMAKE_CUDA_ARCHITECTURES},code=sm_${CMAKE_CUDA_ARCHITECTURES}>) endif() - set_target_properties( dwarf-cloudsc-cuda-hoist-lib PROPERTIES CUDA_SEPARABLE_COMPILATION ON) + set_target_properties( dwarf-cloudsc-c-cuda-hoist-lib PROPERTIES CUDA_SEPARABLE_COMPILATION ON) ecbuild_add_executable( - TARGET dwarf-cloudsc-cuda-hoist + TARGET dwarf-cloudsc-c-cuda-hoist SOURCES dwarf_cloudsc.cpp - LIBS dwarf-cloudsc-cuda-hoist-lib + LIBS dwarf-cloudsc-c-cuda-hoist-lib ) - target_link_libraries(dwarf-cloudsc-cuda-hoist dwarf-cloudsc-cuda-hoist-lib) - + target_link_libraries(dwarf-cloudsc-c-cuda-hoist dwarf-cloudsc-c-cuda-hoist-lib) + ecbuild_add_test( - TARGET dwarf-cloudsc-cuda-hoist-serial - COMMAND bin/dwarf-cloudsc-cuda-hoist + TARGET dwarf-cloudsc-c-cuda-hoist-serial + COMMAND bin/dwarf-cloudsc-c-cuda-hoist ARGS 1 1000 128 WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/../../.. OMP 1 @@ -130,7 +129,7 @@ if( HAVE_CLOUDSC_CUDA ) ###### SCC-CUDA-K-CACHING #### ecbuild_add_library( - TARGET dwarf-cloudsc-cuda-k-caching-lib + TARGET dwarf-cloudsc-c-cuda-k-caching-lib INSTALL_HEADERS LISTED SOURCES cloudsc/yoecldp_c.h @@ -153,29 +152,29 @@ if( HAVE_CLOUDSC_CUDA ) ) target_include_directories( - dwarf-cloudsc-cuda-k-caching-lib + dwarf-cloudsc-c-cuda-k-caching-lib PUBLIC ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES} ) if (NOT DEFINED CMAKE_CUDA_ARCHITECTURES) - target_compile_options(dwarf-cloudsc-cuda-k-caching-lib PRIVATE $<$>) + target_compile_options(dwarf-cloudsc-c-cuda-k-caching-lib PRIVATE $<$>) else() - target_compile_options(dwarf-cloudsc-cuda-k-caching-lib PRIVATE $<$: + target_compile_options(dwarf-cloudsc-c-cuda-k-caching-lib PRIVATE $<$: -gencode arch=compute_${CMAKE_CUDA_ARCHITECTURES},code=sm_${CMAKE_CUDA_ARCHITECTURES}>) endif() - set_target_properties( dwarf-cloudsc-cuda-k-caching-lib PROPERTIES CUDA_SEPARABLE_COMPILATION ON) + set_target_properties( dwarf-cloudsc-c-cuda-k-caching-lib PROPERTIES CUDA_SEPARABLE_COMPILATION ON) ecbuild_add_executable( - TARGET dwarf-cloudsc-cuda-k-caching + TARGET dwarf-cloudsc-c-cuda-k-caching SOURCES dwarf_cloudsc.cpp - LIBS dwarf-cloudsc-cuda-k-caching-lib + LIBS dwarf-cloudsc-c-cuda-k-caching-lib ) - target_link_libraries(dwarf-cloudsc-cuda-k-caching dwarf-cloudsc-cuda-k-caching-lib) - + target_link_libraries(dwarf-cloudsc-c-cuda-k-caching dwarf-cloudsc-c-cuda-k-caching-lib) + ecbuild_add_test( - TARGET dwarf-cloudsc-cuda-k-caching-serial - COMMAND bin/dwarf-cloudsc-cuda-k-caching + TARGET dwarf-cloudsc-c-cuda-k-caching-serial + COMMAND bin/dwarf-cloudsc-c-cuda-k-caching ARGS 1 1000 128 WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/../../.. OMP 1 @@ -189,4 +188,3 @@ if( HAVE_CLOUDSC_CUDA ) else() ecbuild_info( "Serialbox and/or CUDA not found, disabling CUDA prototype(s)" ) endif() - From 6d86b95772d3340124b8b6ec959d2daf55f8892f Mon Sep 17 00:00:00 2001 From: Balthasar Reuter Date: Fri, 22 Sep 2023 13:08:26 +0100 Subject: [PATCH 28/45] Use latest releases for ecbuild/eckit/fckit/atlas --- bundle.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bundle.yml b/bundle.yml index c925e01a..3b30e8c1 100644 --- a/bundle.yml +++ b/bundle.yml @@ -16,7 +16,7 @@ projects : - ecbuild : git : https://github.com/ecmwf/ecbuild - version : 3.7.0 + version : 3.8.0 bundle : false - serialbox : @@ -43,7 +43,7 @@ projects : - eckit : git : https://github.com/ecmwf/eckit - version : develop + version : 1.24.4 optional: true require : ecbuild cmake : > @@ -52,7 +52,7 @@ projects : - fckit : git : https://github.com/ecmwf/fckit - version : develop + version : 0.11.0 optional: true require : ecbuild eckit cmake : > @@ -60,7 +60,7 @@ projects : - atlas : git : https://github.com/ecmwf/atlas - version : feature/MultiField + version : 0.34.0 optional: true require : ecbuild eckit fckit cmake : > From 6527babe7ab01f034b6a196685478887f1068923 Mon Sep 17 00:00:00 2001 From: Balthasar Reuter Date: Fri, 22 Sep 2023 13:57:43 +0100 Subject: [PATCH 29/45] Fix CUDA options in bundle --- bundle.yml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/bundle.yml b/bundle.yml index 3b30e8c1..a96bc35a 100644 --- a/bundle.yml +++ b/bundle.yml @@ -101,10 +101,9 @@ options : ENABLE_CLOUDSC_GPU_OMP_SCC_HOIST=ON - with-cuda : - help : Enable GPU kernel variant based on CUDA-Fortran + help : Enable GPU kernel variants based on CUDA and CUDA-Fortran cmake : > - ENABLE_FIELD_API_CUDA=ON - ENABLE_CLOUDSC_CUDA=ON + ENABLE_CUDA=ON ENABLE_CLOUDSC_GPU_SCC_CUF=ON ENABLE_CLOUDSC_GPU_SCC_CUF_K_CACHING=ON BUILD_field_api=ON @@ -180,10 +179,6 @@ options : help : Build the C version of CLOUDSC [ON|OFF] cmake : ENABLE_CLOUDSC_C={{value}} - - cloudsc-cuda : - help : Build the CUDA C version of CLOUDSC [ON|OFF] - cmake : ENABLE_CLOUDSC_CUDA={{value}} - - cloudsc-gpu-claw : help : Build the deprecated CLAW-based GPU version CLOUDSC [ON|OFF] cmake : ENABLE_CLOUDSC_GPU_CLAW={{value}} From 6780b0e51abab59459e86767cb7fdbb9c93a44f3 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Fri, 22 Sep 2023 15:02:46 +0200 Subject: [PATCH 30/45] Fix compilation with atlas 0.34.0 --- src/cloudsc_fortran_atlas/cloudsc_driver_mod.F90 | 3 ++- src/cloudsc_fortran_atlas/expand_atlas_mod.F90 | 6 ++++-- src/cloudsc_fortran_atlas/validate_atlas_mod.F90 | 3 ++- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/cloudsc_fortran_atlas/cloudsc_driver_mod.F90 b/src/cloudsc_fortran_atlas/cloudsc_driver_mod.F90 index ba9937f4..690f9fb7 100644 --- a/src/cloudsc_fortran_atlas/cloudsc_driver_mod.F90 +++ b/src/cloudsc_fortran_atlas/cloudsc_driver_mod.F90 @@ -55,7 +55,8 @@ SUBROUTINE CLOUDSC_DRIVER(FSET, NUMOMP, NGPTOTG, KFLDX, PTSPHY) FIELD = FSET%FIELD("PEXTRA") FSPACE = FIELD%FUNCTIONSPACE() - NPROMA = FSPACE%NPROMA() + !NPROMA = FSPACE%NPROMA() + NPROMA = FIELD%SHAPE(1) NLEV = FSPACE%LEVELS() NGPTOT = FSPACE%SIZE() diff --git a/src/cloudsc_fortran_atlas/expand_atlas_mod.F90 b/src/cloudsc_fortran_atlas/expand_atlas_mod.F90 index 16cf8f38..ec91ddf2 100644 --- a/src/cloudsc_fortran_atlas/expand_atlas_mod.F90 +++ b/src/cloudsc_fortran_atlas/expand_atlas_mod.F90 @@ -51,7 +51,8 @@ subroutine loadvar_atlas(fset, name, nlon, ngptotg) fspace = field%functionspace() nlev = field%levels() - nproma = fspace%nproma() + !nproma = fspace%nproma() + nproma = field%shape(1) ngptot = fspace%size() nblocks = fspace%nblks() @@ -116,7 +117,8 @@ subroutine loadstate_atlas(fset, name, nlon, ngptotg) fspace = field%functionspace() nlev = field%levels() ngptot = fspace%size() - nproma = fspace%nproma() + !nproma = fspace%nproma() + nproma = field%shape(1) nblocks = fspace%nblks() ndim = field%shape(3) - 3 diff --git a/src/cloudsc_fortran_atlas/validate_atlas_mod.F90 b/src/cloudsc_fortran_atlas/validate_atlas_mod.F90 index 3ebcebc0..9a9f8e6f 100644 --- a/src/cloudsc_fortran_atlas/validate_atlas_mod.F90 +++ b/src/cloudsc_fortran_atlas/validate_atlas_mod.F90 @@ -64,7 +64,8 @@ SUBROUTINE VALIDATEVAR_ATLAS(FSET, NAME, NLON, NGPTOTG, STATE_VAR) NLEV = FIELD%LEVELS() NGPTOT = FSPACE%SIZE() NBLOCKS = FSPACE%NBLKS() - NPROMA = FSPACE%NPROMA() + !NPROMA = FSPACE%NPROMA() + NPROMA = FIELD%SHAPE(1) ZMINVAL(1) = +HUGE(ZMINVAL(1)) ZMAX_VAL_ERR(1) = -HUGE(ZMAX_VAL_ERR(1)) From 32be5227d1cb22e35c4c966138610c413576cd05 Mon Sep 17 00:00:00 2001 From: Balthasar Reuter Date: Fri, 22 Sep 2023 14:16:59 +0100 Subject: [PATCH 31/45] Use project-specific CUDA option --- bundle.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bundle.yml b/bundle.yml index a96bc35a..ffe1b0fa 100644 --- a/bundle.yml +++ b/bundle.yml @@ -49,6 +49,7 @@ projects : cmake : > ECKIT_ENABLE_TESTS=OFF ECKIT_ENABLE_BUILD_TOOLS=OFF + ECKIT_ENABLE_CUDA=OFF - fckit : git : https://github.com/ecmwf/fckit @@ -65,6 +66,7 @@ projects : require : ecbuild eckit fckit cmake : > ATLAS_ENABLE_TESTS=OFF + ATLAS_ENABLE_CUDA=OFF - field_api : git : ${BITBUCKET}/rdx/field_api From 692a98c948ce5282805ad24d7d9a9a2580b6e38b Mon Sep 17 00:00:00 2001 From: Balthasar Reuter Date: Fri, 22 Sep 2023 15:38:08 +0100 Subject: [PATCH 32/45] Remove obsolete CMake workaround --- arch/toolchains/github-ubuntu-nvhpc.cmake | 6 ------ 1 file changed, 6 deletions(-) diff --git a/arch/toolchains/github-ubuntu-nvhpc.cmake b/arch/toolchains/github-ubuntu-nvhpc.cmake index be437031..deda8b6f 100644 --- a/arch/toolchains/github-ubuntu-nvhpc.cmake +++ b/arch/toolchains/github-ubuntu-nvhpc.cmake @@ -19,13 +19,7 @@ set( ECBUILD_FIND_MPI ON ) # Note: OpenMP_Fortran_FLAGS gets overwritten by the FindOpenMP module # unless its stored as a cache variable set( OpenMP_Fortran_FLAGS "-mp -mp=bind,allcores,numa" CACHE STRING "" ) - -# Note: OpenMP_C_FLAGS and OpenMP_C_LIB_NAMES have to be provided _both_ to -# keep FindOpenMP from overwriting the FLAGS variable (the cache entry alone -# doesn't have any effect here as the module uses FORCE to overwrite the -# existing value) set( OpenMP_C_FLAGS "-mp -mp=bind,allcores,numa" CACHE STRING "" ) -set( OpenMP_C_LIB_NAMES "acchost" CACHE STRING "") #################################################################### # OpenAcc FLAGS From 9931f882109bddf74dbfb877f5c1d4e7bcc8f4c7 Mon Sep 17 00:00:00 2001 From: Balthasar Reuter Date: Mon, 25 Sep 2023 16:07:56 +0100 Subject: [PATCH 33/45] Fix binary names for CUDA variants in verify-targets.sh --- .github/scripts/verify-targets.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/verify-targets.sh b/.github/scripts/verify-targets.sh index 40e29cfa..41c8caee 100755 --- a/.github/scripts/verify-targets.sh +++ b/.github/scripts/verify-targets.sh @@ -28,7 +28,7 @@ then targets+=(dwarf-cloudsc-gpu-scc-cuf dwarf-cloudsc-gpu-scc-cuf-k-caching) if [[ "$io_library_flag" == "--with-serialbox" ]] then - targets+=(dwarf-cloudsc-cuda dwarf-cloudsc-cuda-hoist dwarf-cloudsc-cuda-k-caching) + targets+=(dwarf-cloudsc-c-cuda dwarf-cloudsc-c-cuda-hoist dwarf-cloudsc-c-cuda-k-caching) fi fi fi From bba1dc3f6e7e1dbd69fc787d22cf5ded7141e87f Mon Sep 17 00:00:00 2001 From: Balthasar Reuter Date: Mon, 25 Sep 2023 17:26:48 +0100 Subject: [PATCH 34/45] Fix NVHPC builds --- .github/workflows/build.yml | 12 ++++++------ arch/github/ubuntu/nvhpc/21.9/env.sh | 4 +++- arch/github/ubuntu/nvhpc/23.5/env.sh | 4 +++- src/cloudsc_cuda/CMakeLists.txt | 2 +- 4 files changed, 13 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4e4188f9..b881a63f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -54,33 +54,33 @@ jobs: nvhpc_version: 21.9 io_library_flag: '' build_flags: '--with-gpu --with-loki --with-cuda --with-atlas' - ctest_exclude_pattern: '-gpu-|-scc-|-loki-c|-cuda-' # GPU variants don't work on CPU runners, loki-c variant causes SIGFPE + ctest_exclude_pattern: '-gpu-|-scc-|-loki-c|-cuda' # GPU variants don't work on CPU runners, loki-c variant causes SIGFPE - arch: nvhpc/21.9 nvhpc_version: 21.9 io_library_flag: '' build_flags: '--single-precision --with-gpu --with-loki --with-cuda' - ctest_exclude_pattern: '-gpu-|-scc-|-loki-c|-cuda-' # GPU variants don't work on CPU runners, loki-c variant causes SIGFPE + ctest_exclude_pattern: '-gpu-|-scc-|-loki-c|-cuda' # GPU variants don't work on CPU runners, loki-c variant causes SIGFPE - arch: nvhpc/21.9 nvhpc_version: 21.9 io_library_flag: '--with-serialbox' build_flags: '--with-gpu --with-loki --with-cuda --with-atlas' - ctest_exclude_pattern: '-gpu-|-scc-|-loki-c|-cuda-' # GPU variants don't work on CPU runners, loki-c variant causes SIGFPE + ctest_exclude_pattern: '-gpu-|-scc-|-loki-c|-cuda' # GPU variants don't work on CPU runners, loki-c variant causes SIGFPE - arch: nvhpc/23.5 nvhpc_version: 23.5 io_library_flag: '' build_flags: '--with-gpu --with-loki --with-cuda --with-atlas' - ctest_exclude_pattern: '-gpu-|-scc-|-loki-c|-loki-sca|-cuda-' # GPU variants don't work on CPU runners, loki-c variant causes SIGFPE + ctest_exclude_pattern: '-gpu-|-scc-|-loki-c|-loki-sca|-cuda' # GPU variants don't work on CPU runners, loki-c variant causes SIGFPE - arch: nvhpc/23.5 nvhpc_version: 23.5 io_library_flag: '' build_flags: '--single-precision --with-gpu --with-loki --with-cuda' - ctest_exclude_pattern: '-gpu-|-scc-|-loki-c|-loki-sca|-cuda-' # GPU variants don't work on CPU runners, loki-c variant causes SIGFPE + ctest_exclude_pattern: '-gpu-|-scc-|-loki-c|-loki-sca|-cuda' # GPU variants don't work on CPU runners, loki-c variant causes SIGFPE - arch: nvhpc/23.5 nvhpc_version: 23.5 io_library_flag: '--with-serialbox' build_flags: '--with-gpu --with-loki --with-cuda --with-atlas' - ctest_exclude_pattern: '-gpu-|-scc-|-loki-c|-loki-sca|-cuda-' # GPU variants don't work on CPU runners, loki-c variant causes SIGFPE + ctest_exclude_pattern: '-gpu-|-scc-|-loki-c|-loki-sca|-cuda' # GPU variants don't work on CPU runners, loki-c variant causes SIGFPE # Steps represent a sequence of tasks that will be executed as part of the job steps: diff --git a/arch/github/ubuntu/nvhpc/21.9/env.sh b/arch/github/ubuntu/nvhpc/21.9/env.sh index de7afa2f..6e05f756 100644 --- a/arch/github/ubuntu/nvhpc/21.9/env.sh +++ b/arch/github/ubuntu/nvhpc/21.9/env.sh @@ -11,16 +11,18 @@ ### Variables export NVHPC_INSTALL_DIR=${GITHUB_WORKSPACE}/nvhpc-install export NVHPC_VERSION=21.9 +export CUDA_VERSION=11.4 export NVHPC_DIR=${NVHPC_INSTALL_DIR}/Linux_x86_64/${NVHPC_VERSION} ### Compilers export PATH=${NVHPC_DIR}/compilers/bin:${PATH} export NVHPC_LIBRARY_PATH=${NVHPC_DIR}/compilers/lib -export LD_LIBRARY_PATH=${NVHPC_LIBRARY_PATH} +export LD_LIBRARY_PATH=${NVHPC_LIBRARY_PATH}:${LD_LIBRARY_PATH} ### MPI export MPI_HOME=${NVHPC_DIR}/comm_libs/mpi export PATH=${MPI_HOME}/bin:${PATH} +export LD_LIBRARY_PATH=${NVHPC_DIR}/cuda/${CUDA_VERSION}/targets/x86_64-linux/lib/stubs:${LD_LIBRARY_PATH} ### HDF5 export HDF5_DIR=${GITHUB_WORKSPACE}/hdf5-install diff --git a/arch/github/ubuntu/nvhpc/23.5/env.sh b/arch/github/ubuntu/nvhpc/23.5/env.sh index dfbac6e0..a2261c25 100644 --- a/arch/github/ubuntu/nvhpc/23.5/env.sh +++ b/arch/github/ubuntu/nvhpc/23.5/env.sh @@ -11,16 +11,18 @@ ### Variables export NVHPC_INSTALL_DIR=${GITHUB_WORKSPACE}/nvhpc-install export NVHPC_VERSION=23.5 +export CUDA_VERSION=12.1 export NVHPC_DIR=${NVHPC_INSTALL_DIR}/Linux_x86_64/${NVHPC_VERSION} ### Compilers export PATH=${NVHPC_DIR}/compilers/bin:${PATH} export NVHPC_LIBRARY_PATH=${NVHPC_DIR}/compilers/lib -export LD_LIBRARY_PATH=${NVHPC_LIBRARY_PATH} +export LD_LIBRARY_PATH=${NVHPC_LIBRARY_PATH}:${LD_LIBRARY_PATH} ### MPI export MPI_HOME=${NVHPC_DIR}/comm_libs/mpi export PATH=${MPI_HOME}/bin:${PATH} +export LD_LIBRARY_PATH=${NVHPC_DIR}/cuda/${CUDA_VERSION}/targets/x86_64-linux/lib/stubs:${LD_LIBRARY_PATH} ### HDF5 export HDF5_DIR=${GITHUB_WORKSPACE}/hdf5-install diff --git a/src/cloudsc_cuda/CMakeLists.txt b/src/cloudsc_cuda/CMakeLists.txt index 3112b8b9..5393d3e6 100644 --- a/src/cloudsc_cuda/CMakeLists.txt +++ b/src/cloudsc_cuda/CMakeLists.txt @@ -12,7 +12,7 @@ ecbuild_add_option( FEATURE CLOUDSC_C_CUDA CONDITION Serialbox_FOUND AND HAVE_CUDA ) -if( HAVE_CLOUDSC_CUDA ) +if( HAVE_CLOUDSC_C_CUDA ) enable_language(CUDA) enable_language(CXX) From 83785c7c34cfe84590bef1dbb753e20574c110d5 Mon Sep 17 00:00:00 2001 From: Slavko Brdar Date: Tue, 26 Sep 2023 14:40:42 +0000 Subject: [PATCH 35/45] make serialbox work in the Atlas implementation --- .../cloudsc_driver_mod.F90 | 2 +- .../cloudsc_global_atlas_state_mod.F90 | 56 ++++++++----------- 2 files changed, 25 insertions(+), 33 deletions(-) diff --git a/src/cloudsc_fortran_atlas/cloudsc_driver_mod.F90 b/src/cloudsc_fortran_atlas/cloudsc_driver_mod.F90 index 690f9fb7..17915207 100644 --- a/src/cloudsc_fortran_atlas/cloudsc_driver_mod.F90 +++ b/src/cloudsc_fortran_atlas/cloudsc_driver_mod.F90 @@ -53,7 +53,7 @@ SUBROUTINE CLOUDSC_DRIVER(FSET, NUMOMP, NGPTOTG, KFLDX, PTSPHY) POWER_TOTAL = 0_JPIB POWER_COUNT = 0_JPIB - FIELD = FSET%FIELD("PEXTRA") + FIELD = FSET%FIELD(1) FSPACE = FIELD%FUNCTIONSPACE() !NPROMA = FSPACE%NPROMA() NPROMA = FIELD%SHAPE(1) diff --git a/src/cloudsc_fortran_atlas/cloudsc_global_atlas_state_mod.F90 b/src/cloudsc_fortran_atlas/cloudsc_global_atlas_state_mod.F90 index ba2261e4..80ce6b96 100644 --- a/src/cloudsc_fortran_atlas/cloudsc_global_atlas_state_mod.F90 +++ b/src/cloudsc_fortran_atlas/cloudsc_global_atlas_state_mod.F90 @@ -34,12 +34,12 @@ MODULE CLOUDSC_GLOBAL_ATLAS_STATE_MOD REAL(C_DOUBLE), POINTER :: PTR(:,:) END TYPE - CHARACTER(LEN=10), PARAMETER, DIMENSION(30) :: IN_VAR_NAMES = (/ & + CHARACTER(LEN=10), PARAMETER, DIMENSION(29) :: IN_VAR_NAMES = (/ & "PLCRIT_AER", "PICRIT_AER", "PRE_ICE ", "PCCN ", "PNICE ", "PT ", "PQ ", & "PVFA ", "PVFL ", "PVFI ", "PDYNA ", "PDYNL ", "PDYNI ", "PHRSW ", & "PHRLW ", "PVERVEL ", "PAP ", "PLU ", "PLUDE ", "PSNDE ", "PMFU ", & "PMFD ", "PA ", "PSUPSAT ", & - "PLSM ", "LDCUM ", "KTYPE ", "PAPH ", "PEXTRA ", "PCLV " /) + "PLSM ", "LDCUM ", "KTYPE ", "PAPH ", "PCLV " /) CHARACTER(LEN=16), PARAMETER, DIMENSION(16) :: OUT_VAR_NAMES = (/ & "PFSQLF ", "PFSQIF ", "PFCQLNG ", "PFCQNNG ", "PFSQRF ", & "PFSQSF ", "PFCQRNG ", "PFCQSNG ", "PFSQLTUR ", "PFSQITUR ", & @@ -80,7 +80,6 @@ MODULE CLOUDSC_GLOBAL_ATLAS_STATE_MOD LOGICAL, POINTER, CONTIGUOUS :: LDCUM(:) ! Convection active INTEGER, POINTER, CONTIGUOUS :: KTYPE(:) ! Convection type 0,1,2 REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PAPH(:,:) ! Pressure on half levels - REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PEXTRA(:,:,:) ! extra fields REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PCLV(:,:,:) TYPE(STATE_TYPE) :: TENDENCY_CML ! cumulative tendency used for final output @@ -163,41 +162,40 @@ SUBROUTINE CLOUDSC_GLOBAL_ATLAS_STATE_BLOCK(SELF, FSET, IBLK) CALL FSET%DATA(26, SELF%LDCUM, IBLK) CALL FSET%DATA(27, SELF%KTYPE, IBLK) CALL FSET%DATA(28, SELF%PAPH, IBLK) - CALL FSET%DATA(29, SELF%PEXTRA, IBLK) - CALL FSET%DATA(30, SELF%PCLV, IBLK) + CALL FSET%DATA(29, SELF%PCLV, IBLK) - CALL FSET%DATA(31, TMP3D, IBLK) + CALL FSET%DATA(30, TMP3D, IBLK) SELF%TENDENCY_CML%T => TMP3D(:,:,1) SELF%TENDENCY_CML%A => TMP3D(:,:,2) SELF%TENDENCY_CML%Q => TMP3D(:,:,3) SELF%TENDENCY_CML%CLD => TMP3D(:,:,4:) - CALL FSET%DATA(32, TMP3D, IBLK) + CALL FSET%DATA(31, TMP3D, IBLK) SELF%TENDENCY_TMP%T => TMP3D(:,:,1) SELF%TENDENCY_TMP%A => TMP3D(:,:,2) SELF%TENDENCY_TMP%Q => TMP3D(:,:,3) SELF%TENDENCY_TMP%CLD => TMP3D(:,:,4:) - CALL FSET%DATA(33, TMP3D, IBLK) + CALL FSET%DATA(32, TMP3D, IBLK) SELF%TENDENCY_LOC%T => TMP3D(:,:,1) SELF%TENDENCY_LOC%A => TMP3D(:,:,2) SELF%TENDENCY_LOC%Q => TMP3D(:,:,3) SELF%TENDENCY_LOC%CLD => TMP3D(:,:,4:) - CALL FSET%DATA(34, SELF%PFSQLF, IBLK) - CALL FSET%DATA(35, SELF%PFSQIF, IBLK) - CALL FSET%DATA(36, SELF%PFCQLNG, IBLK) - CALL FSET%DATA(37, SELF%PFCQNNG, IBLK) - CALL FSET%DATA(38, SELF%PFSQRF, IBLK) - CALL FSET%DATA(39, SELF%PFSQSF, IBLK) - CALL FSET%DATA(40, SELF%PFCQRNG, IBLK) - CALL FSET%DATA(41, SELF%PFCQSNG, IBLK) - CALL FSET%DATA(42, SELF%PFSQLTUR, IBLK) - CALL FSET%DATA(43, SELF%PFSQITUR, IBLK) - CALL FSET%DATA(44, SELF%PFPLSL, IBLK) - CALL FSET%DATA(45, SELF%PFPLSN, IBLK) - CALL FSET%DATA(46, SELF%PFHPSL, IBLK) - CALL FSET%DATA(47, SELF%PFHPSN, IBLK) - CALL FSET%DATA(48, SELF%PCOVPTOT, IBLK) - CALL FSET%DATA(49, SELF%PRAINFRAC_TOPRFZ, IBLK) + CALL FSET%DATA(33, SELF%PFSQLF, IBLK) + CALL FSET%DATA(34, SELF%PFSQIF, IBLK) + CALL FSET%DATA(35, SELF%PFCQLNG, IBLK) + CALL FSET%DATA(36, SELF%PFCQNNG, IBLK) + CALL FSET%DATA(37, SELF%PFSQRF, IBLK) + CALL FSET%DATA(38, SELF%PFSQSF, IBLK) + CALL FSET%DATA(39, SELF%PFCQRNG, IBLK) + CALL FSET%DATA(40, SELF%PFCQSNG, IBLK) + CALL FSET%DATA(41, SELF%PFSQLTUR, IBLK) + CALL FSET%DATA(42, SELF%PFSQITUR, IBLK) + CALL FSET%DATA(43,SELF%PFPLSL, IBLK) + CALL FSET%DATA(44, SELF%PFPLSN, IBLK) + CALL FSET%DATA(45, SELF%PFHPSL, IBLK) + CALL FSET%DATA(46, SELF%PFHPSN, IBLK) + CALL FSET%DATA(47, SELF%PCOVPTOT, IBLK) + CALL FSET%DATA(48, SELF%PRAINFRAC_TOPRFZ, IBLK) END SUBROUTINE CLOUDSC_GLOBAL_ATLAS_STATE_BLOCK SUBROUTINE CLOUDSC_GLOBAL_ATLAS_STATE_LOAD(SELF, FSET, NPROMA, NGPTOTG) @@ -229,14 +227,14 @@ SUBROUTINE CLOUDSC_GLOBAL_ATLAS_STATE_LOAD(SELF, FSET, NPROMA, NGPTOTG) SELF%NBLOCKS = FSPACE%NBLKS() FSET = ATLAS_FIELDSET() - DO IVAR = 1, SIZE(IN_VAR_NAMES) - 6 ! last six variables are special + DO IVAR = 1, SIZE(IN_VAR_NAMES) - 5 ! the last variables are special CALL FSET%ADD(FSPACE%CREATE_FIELD(NAME=TRIM(IN_VAR_NAMES(IVAR)), KIND=ATLAS_REAL(JPRB))) ENDDO CALL FSET%ADD(FSPACE%CREATE_FIELD(NAME="PLSM", KIND=ATLAS_REAL(JPRB), LEVELS=0)) CALL FSET%ADD(FSPACE%CREATE_FIELD(NAME="LDCUM", KIND=ATLAS_LOGICAL(), LEVELS=0)) CALL FSET%ADD(FSPACE%CREATE_FIELD(NAME="KTYPE", KIND=ATLAS_INTEGER(JPIM), LEVELS=0)) CALL FSET%ADD(FSPACE%CREATE_FIELD(NAME="PAPH", KIND=ATLAS_REAL(JPRB), LEVELS=SELF%KLEV+1)) - CALL FSET%ADD(FSPACE%CREATE_FIELD(NAME="PEXTRA", KIND=ATLAS_REAL(JPRB), VARIABLES=MAX(1,SELF%KFLDX))) + !CALL FSET%ADD(FSPACE%CREATE_FIELD(NAME="PEXTRA", KIND=ATLAS_REAL(JPRB), VARIABLES=SELF%KFLDX)) CALL FSET%ADD(FSPACE%CREATE_FIELD(NAME="PCLV", KIND=ATLAS_REAL(JPRB), VARIABLES=MAX(1,NCLV))) DO IVAR = 1, SIZE(IN_VAR_NAMES) @@ -250,8 +248,6 @@ SUBROUTINE CLOUDSC_GLOBAL_ATLAS_STATE_LOAD(SELF, FSET, NPROMA, NGPTOTG) FIELD = FSPACE%CREATE_FIELD(NAME='TENDENCY_LOC', KIND=ATLAS_REAL(JPRB), VARIABLES=3+NCLV) CALL FSET%ADD(FIELD) - ! The STATE_TYPE arrays are tricky, as the AOSOA layout needs to be expictly - ! unrolled at every step, and we rely on dirty hackery to do this. CALL LOADSTATE_ATLAS(FSET, 'TENDENCY_CML', KLON, NGPTOTG) CALL LOADSTATE_ATLAS(FSET, 'TENDENCY_TMP', KLON, NGPTOTG) ! Output fields are simply allocated and zero'd @@ -270,10 +266,6 @@ SUBROUTINE CLOUDSC_GLOBAL_ATLAS_STATE_LOAD(SELF, FSET, NPROMA, NGPTOTG) END DO !$omp end parallel do ENDDO - ! DEBUG - !FIELD = FSET%FIELD("PAP") - !call field%data(tmp3d) - !print *, MINVAL(tmp3d), MAXVAL(tmp3d) FIELD = FSET%FIELD("PRAINFRAC_TOPRFZ") CALL FIELD%DATA(TMP2D) From 261a577920308370536e9368e33a4468b2896e57 Mon Sep 17 00:00:00 2001 From: Slavko Brdar Date: Tue, 26 Sep 2023 14:54:39 +0000 Subject: [PATCH 36/45] support single precision in expand_atlas_mod --- src/cloudsc_fortran_atlas/CMakeLists.txt | 2 +- src/cloudsc_fortran_atlas/expand_atlas_mod.F90 | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/cloudsc_fortran_atlas/CMakeLists.txt b/src/cloudsc_fortran_atlas/CMakeLists.txt index fe9f98fe..f1e61123 100644 --- a/src/cloudsc_fortran_atlas/CMakeLists.txt +++ b/src/cloudsc_fortran_atlas/CMakeLists.txt @@ -9,7 +9,7 @@ # Define this dwarf variant as an ECBuild feature ecbuild_add_option( FEATURE CLOUDSC_FORTRAN_ATLAS DESCRIPTION "Build the Fortran version CLOUDSC using Atlas and Serialbox" DEFAULT ON - CONDITION atlas_FOUND AND (Serialbox_FOUND OR HDF5_FOUND) AND NOT HAVE_SINGLE_PRECISION + CONDITION atlas_FOUND AND (Serialbox_FOUND OR HDF5_FOUND) ) if( HAVE_CLOUDSC_FORTRAN_ATLAS ) diff --git a/src/cloudsc_fortran_atlas/expand_atlas_mod.F90 b/src/cloudsc_fortran_atlas/expand_atlas_mod.F90 index ec91ddf2..435b88f1 100644 --- a/src/cloudsc_fortran_atlas/expand_atlas_mod.F90 +++ b/src/cloudsc_fortran_atlas/expand_atlas_mod.F90 @@ -19,7 +19,7 @@ module expand_atlas_mod use file_io_mod, only: input_initialize, load_scalar, load_array use expand_mod, only: get_offsets, expand - use, intrinsic :: iso_c_binding, only : c_int, c_double + use, intrinsic :: iso_c_binding, only : c_int implicit none @@ -37,7 +37,7 @@ subroutine loadvar_atlas(fset, name, nlon, ngptotg) real(kind=jprb), allocatable :: buffer_r1(:), buffer_r2(:,:), buffer_r3(:,:,:) integer(kind=jpim), allocatable :: buffer_i1(:) logical, allocatable :: buffer_l1(:) - real(c_double), pointer :: field_r1(:,:), field_r2(:,:,:), field_r3(:,:,:,:) + real(kind=jprb), pointer :: field_r1(:,:), field_r2(:,:,:), field_r3(:,:,:,:) integer(c_int), pointer :: field_i1(:,:) logical, pointer :: field_l1(:,:) type(atlas_functionspace_blockstructuredcolumns) :: fspace @@ -111,7 +111,7 @@ subroutine loadstate_atlas(fset, name, nlon, ngptotg) type(atlas_functionspace_blockstructuredcolumns) :: fspace real(kind=jprb), allocatable :: buffer(:,:,:) - real(c_double), pointer :: field_r3(:,:,:,:) + real(kind=jprb), pointer :: field_r3(:,:,:,:) field = fset%field(name) fspace = field%functionspace() From ebe59028a236079edb06707e8c3571fbe0348cd2 Mon Sep 17 00:00:00 2001 From: Slavko Brdar Date: Tue, 26 Sep 2023 14:58:13 +0000 Subject: [PATCH 37/45] add single precision tests for Atlas --- .github/workflows/build.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b881a63f..31f74c08 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -35,7 +35,7 @@ jobs: - '' # Plain build without any options - '--with-gpu --with-loki --with-atlas' # Enable Loki, Atlas, and GPU variants - '--with-gpu --with-loki --with-atlas --with-mpi' # Enable Loki, Atlas, and GPU variants with MPI - - '--single-precision --with-gpu --with-loki --with-mpi' # Enable Loki, and GPU variants with MPI in a single-precision build + - '--single-precision --with-gpu --with-loki --with-atlas --with-mpi' # Enable Loki, and GPU variants with MPI in a single-precision build pyiface_flag: [''] # Enable the pyiface variant @@ -58,7 +58,7 @@ jobs: - arch: nvhpc/21.9 nvhpc_version: 21.9 io_library_flag: '' - build_flags: '--single-precision --with-gpu --with-loki --with-cuda' + build_flags: '--single-precision --with-gpu --with-loki --with-cuda --with-atlas' ctest_exclude_pattern: '-gpu-|-scc-|-loki-c|-cuda' # GPU variants don't work on CPU runners, loki-c variant causes SIGFPE - arch: nvhpc/21.9 nvhpc_version: 21.9 @@ -74,7 +74,7 @@ jobs: - arch: nvhpc/23.5 nvhpc_version: 23.5 io_library_flag: '' - build_flags: '--single-precision --with-gpu --with-loki --with-cuda' + build_flags: '--single-precision --with-gpu --with-loki --with-cuda --with-atlas' ctest_exclude_pattern: '-gpu-|-scc-|-loki-c|-loki-sca|-cuda' # GPU variants don't work on CPU runners, loki-c variant causes SIGFPE - arch: nvhpc/23.5 nvhpc_version: 23.5 From 74c753b45dda0ec658adfca376125c46ff4c5115 Mon Sep 17 00:00:00 2001 From: Slavko Brdar Date: Tue, 26 Sep 2023 15:19:46 +0000 Subject: [PATCH 38/45] fixed for single precision in Atlas structures --- .../cloudsc_global_atlas_state_mod.F90 | 98 +++++++++---------- .../dwarf_cloudsc_atlas.F90 | 1 - .../validate_atlas_mod.F90 | 2 +- 3 files changed, 50 insertions(+), 51 deletions(-) diff --git a/src/cloudsc_fortran_atlas/cloudsc_global_atlas_state_mod.F90 b/src/cloudsc_fortran_atlas/cloudsc_global_atlas_state_mod.F90 index 80ce6b96..527a57a1 100644 --- a/src/cloudsc_fortran_atlas/cloudsc_global_atlas_state_mod.F90 +++ b/src/cloudsc_fortran_atlas/cloudsc_global_atlas_state_mod.F90 @@ -28,10 +28,10 @@ MODULE CLOUDSC_GLOBAL_ATLAS_STATE_MOD IMPLICIT NONE TYPE VAR3D_PTR - REAL(C_DOUBLE), POINTER :: PTR(:,:,:) + REAL(KIND=JPRB), POINTER :: PTR(:,:,:) END TYPE TYPE VAR2D_PTR - REAL(C_DOUBLE), POINTER :: PTR(:,:) + REAL(KIND=JPRB), POINTER :: PTR(:,:) END TYPE CHARACTER(LEN=10), PARAMETER, DIMENSION(29) :: IN_VAR_NAMES = (/ & @@ -51,58 +51,58 @@ MODULE CLOUDSC_GLOBAL_ATLAS_STATE_MOD TYPE(VAR2D_PTR), DIMENSION(15) :: OUT_VARS_2D_REAL64 ! Input field variables and tendencies - REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PLCRIT_AER(:,:) - REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PICRIT_AER(:,:) - REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PRE_ICE(:,:) - REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PCCN(:,:) ! liquid cloud condensation nuclei - REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PNICE(:,:) ! ice number concentration (cf. CCN) - REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PT(:,:) ! T at start of callpar - REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PQ(:,:) ! Q at start of callpar - REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PVFA(:,:) ! CC from VDF scheme - REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PVFL(:,:) ! Liq from VDF scheme - REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PVFI(:,:) ! Ice from VDF scheme - REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PDYNA(:,:) ! CC from Dynamics - REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PDYNL(:,:) ! Liq from Dynamics - REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PDYNI(:,:) ! Liq from Dynamics - REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PHRSW(:,:) ! Short-wave heating rate - REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PHRLW(:,:) ! Long-wave heating rate - REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PVERVEL(:,:) ! Vertical velocity - REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PAP(:,:) ! Pressure on full levels - REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PLU(:,:) ! Conv. condensate - REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PLUDE(:,:) ! Conv. detrained water - REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PSNDE(:,:) ! Conv. detrained snow - REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PMFU(:,:) ! Conv. mass flux up - REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PMFD(:,:) ! Conv. mass flux down - REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PA(:,:) ! Original Cloud fraction (t) - REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PSUPSAT(:,:) - - REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PLSM(:) ! Land fraction (0-1) + REAL(KIND=JPRB), POINTER, CONTIGUOUS :: PLCRIT_AER(:,:) + REAL(KIND=JPRB), POINTER, CONTIGUOUS :: PICRIT_AER(:,:) + REAL(KIND=JPRB), POINTER, CONTIGUOUS :: PRE_ICE(:,:) + REAL(KIND=JPRB), POINTER, CONTIGUOUS :: PCCN(:,:) ! liquid cloud condensation nuclei + REAL(KIND=JPRB), POINTER, CONTIGUOUS :: PNICE(:,:) ! ice number concentration (cf. CCN) + REAL(KIND=JPRB), POINTER, CONTIGUOUS :: PT(:,:) ! T at start of callpar + REAL(KIND=JPRB), POINTER, CONTIGUOUS :: PQ(:,:) ! Q at start of callpar + REAL(KIND=JPRB), POINTER, CONTIGUOUS :: PVFA(:,:) ! CC from VDF scheme + REAL(KIND=JPRB), POINTER, CONTIGUOUS :: PVFL(:,:) ! Liq from VDF scheme + REAL(KIND=JPRB), POINTER, CONTIGUOUS :: PVFI(:,:) ! Ice from VDF scheme + REAL(KIND=JPRB), POINTER, CONTIGUOUS :: PDYNA(:,:) ! CC from Dynamics + REAL(KIND=JPRB), POINTER, CONTIGUOUS :: PDYNL(:,:) ! Liq from Dynamics + REAL(KIND=JPRB), POINTER, CONTIGUOUS :: PDYNI(:,:) ! Liq from Dynamics + REAL(KIND=JPRB), POINTER, CONTIGUOUS :: PHRSW(:,:) ! Short-wave heating rate + REAL(KIND=JPRB), POINTER, CONTIGUOUS :: PHRLW(:,:) ! Long-wave heating rate + REAL(KIND=JPRB), POINTER, CONTIGUOUS :: PVERVEL(:,:) ! Vertical velocity + REAL(KIND=JPRB), POINTER, CONTIGUOUS :: PAP(:,:) ! Pressure on full levels + REAL(KIND=JPRB), POINTER, CONTIGUOUS :: PLU(:,:) ! Conv. condensate + REAL(KIND=JPRB), POINTER, CONTIGUOUS :: PLUDE(:,:) ! Conv. detrained water + REAL(KIND=JPRB), POINTER, CONTIGUOUS :: PSNDE(:,:) ! Conv. detrained snow + REAL(KIND=JPRB), POINTER, CONTIGUOUS :: PMFU(:,:) ! Conv. mass flux up + REAL(KIND=JPRB), POINTER, CONTIGUOUS :: PMFD(:,:) ! Conv. mass flux down + REAL(KIND=JPRB), POINTER, CONTIGUOUS :: PA(:,:) ! Original Cloud fraction (t) + REAL(KIND=JPRB), POINTER, CONTIGUOUS :: PSUPSAT(:,:) + + REAL(KIND=JPRB), POINTER, CONTIGUOUS :: PLSM(:) ! Land fraction (0-1) LOGICAL, POINTER, CONTIGUOUS :: LDCUM(:) ! Convection active INTEGER, POINTER, CONTIGUOUS :: KTYPE(:) ! Convection type 0,1,2 - REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PAPH(:,:) ! Pressure on half levels - REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PCLV(:,:,:) + REAL(KIND=JPRB), POINTER, CONTIGUOUS :: PAPH(:,:) ! Pressure on half levels + REAL(KIND=JPRB), POINTER, CONTIGUOUS :: PCLV(:,:,:) TYPE(STATE_TYPE) :: TENDENCY_CML ! cumulative tendency used for final output TYPE(STATE_TYPE) :: TENDENCY_TMP ! cumulative tendency used as input TYPE(STATE_TYPE) :: TENDENCY_LOC ! local tendency from cloud scheme ! Output fields used for validation - REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PFSQLF(:,:) ! Flux of liquid - REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PFSQIF(:,:) ! Flux of ice - REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PFCQLNG(:,:) ! -ve corr for liq - REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PFCQNNG(:,:) ! -ve corr for ice - REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PFSQRF(:,:) ! Flux diagnostics - REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PFSQSF(:,:) ! for DDH, generic - REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PFCQRNG(:,:) ! rain - REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PFCQSNG(:,:) ! snow - REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PFSQLTUR(:,:) ! liquid flux due to VDF - REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PFSQITUR(:,:) ! ice flux due to VDF - REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PFPLSL(:,:) ! liq+rain sedim flux - REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PFPLSN(:,:) ! ice+snow sedim flux - REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PFHPSL(:,:) ! Enthalpy flux for liq - REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PFHPSN(:,:) ! Enthalpy flux for ice - REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PCOVPTOT(:,:) ! Precip fraction - REAL(C_DOUBLE), POINTER, CONTIGUOUS :: PRAINFRAC_TOPRFZ(:) + REAL(KIND=JPRB), POINTER, CONTIGUOUS :: PFSQLF(:,:) ! Flux of liquid + REAL(KIND=JPRB), POINTER, CONTIGUOUS :: PFSQIF(:,:) ! Flux of ice + REAL(KIND=JPRB), POINTER, CONTIGUOUS :: PFCQLNG(:,:) ! -ve corr for liq + REAL(KIND=JPRB), POINTER, CONTIGUOUS :: PFCQNNG(:,:) ! -ve corr for ice + REAL(KIND=JPRB), POINTER, CONTIGUOUS :: PFSQRF(:,:) ! Flux diagnostics + REAL(KIND=JPRB), POINTER, CONTIGUOUS :: PFSQSF(:,:) ! for DDH, generic + REAL(KIND=JPRB), POINTER, CONTIGUOUS :: PFCQRNG(:,:) ! rain + REAL(KIND=JPRB), POINTER, CONTIGUOUS :: PFCQSNG(:,:) ! snow + REAL(KIND=JPRB), POINTER, CONTIGUOUS :: PFSQLTUR(:,:) ! liquid flux due to VDF + REAL(KIND=JPRB), POINTER, CONTIGUOUS :: PFSQITUR(:,:) ! ice flux due to VDF + REAL(KIND=JPRB), POINTER, CONTIGUOUS :: PFPLSL(:,:) ! liq+rain sedim flux + REAL(KIND=JPRB), POINTER, CONTIGUOUS :: PFPLSN(:,:) ! ice+snow sedim flux + REAL(KIND=JPRB), POINTER, CONTIGUOUS :: PFHPSL(:,:) ! Enthalpy flux for liq + REAL(KIND=JPRB), POINTER, CONTIGUOUS :: PFHPSN(:,:) ! Enthalpy flux for ice + REAL(KIND=JPRB), POINTER, CONTIGUOUS :: PCOVPTOT(:,:) ! Precip fraction + REAL(KIND=JPRB), POINTER, CONTIGUOUS :: PRAINFRAC_TOPRFZ(:) CONTAINS PROCEDURE :: GET_BLOCK => CLOUDSC_GLOBAL_ATLAS_STATE_BLOCK @@ -132,7 +132,7 @@ SUBROUTINE CLOUDSC_GLOBAL_ATLAS_STATE_BLOCK(SELF, FSET, IBLK) CLASS(ATLAS_FIELDSET), INTENT(INOUT) :: FSET INTEGER, INTENT(IN) :: IBLK - REAL(C_DOUBLE), POINTER :: TMP3D(:,:,:) + REAL(KIND=JPRB), POINTER :: TMP3D(:,:,:) CALL FSET%DATA(1, SELF%PLCRIT_AER, IBLK) CALL FSET%DATA(2, SELF%PICRIT_AER, IBLK) @@ -210,8 +210,8 @@ SUBROUTINE CLOUDSC_GLOBAL_ATLAS_STATE_LOAD(SELF, FSET, NPROMA, NGPTOTG) INTEGER(KIND=JPIM) :: KLON, IVAR, B TYPE(VAR3D_PTR), DIMENSION(24) :: IN_VARS_3D_REAL64 TYPE(VAR3D_PTR), DIMENSION(15) :: OUT_VARS_3D_REAL64 - REAL(C_DOUBLE), POINTER :: TMP3D(:,:,:) - REAL(C_DOUBLE), POINTER :: TMP2D(:,:) + REAL(KIND=JPRB), POINTER :: TMP3D(:,:,:) + REAL(KIND=JPRB), POINTER :: TMP2D(:,:) TYPE(ATLAS_FIELD) :: FIELD TYPE(ATLAS_PARTITIONER) :: PARTITIONER INTEGER(KIND=JPIM) :: NGPTOT diff --git a/src/cloudsc_fortran_atlas/dwarf_cloudsc_atlas.F90 b/src/cloudsc_fortran_atlas/dwarf_cloudsc_atlas.F90 index dd8470cb..83fbf103 100644 --- a/src/cloudsc_fortran_atlas/dwarf_cloudsc_atlas.F90 +++ b/src/cloudsc_fortran_atlas/dwarf_cloudsc_atlas.F90 @@ -27,7 +27,6 @@ PROGRAM DWARF_CLOUDSC INTEGER(KIND=JPIM) :: NGPTOTG = 16384 ! Number of grid points (as read from command line) INTEGER(KIND=JPIM) :: NPROMA = 32 ! NPROMA blocking factor (currently active) -REAL(c_double), pointer :: tmp3d(:,:,:) type(atlas_fieldset) :: fset type(atlas_field) :: field diff --git a/src/cloudsc_fortran_atlas/validate_atlas_mod.F90 b/src/cloudsc_fortran_atlas/validate_atlas_mod.F90 index 9a9f8e6f..34f5b1b9 100644 --- a/src/cloudsc_fortran_atlas/validate_atlas_mod.F90 +++ b/src/cloudsc_fortran_atlas/validate_atlas_mod.F90 @@ -44,7 +44,7 @@ SUBROUTINE VALIDATEVAR_ATLAS(FSET, NAME, NLON, NGPTOTG, STATE_VAR) CHARACTER(*), INTENT(IN), OPTIONAL :: STATE_VAR REAL(KIND=JPRB), ALLOCATABLE :: REF_R2(:,:), REF_R3(:,:,:), REF_R4(:,:,:,:) - REAL(C_DOUBLE), POINTER :: FIELD_R1(:,:), FIELD_R2(:,:,:), FIELD_R3(:,:,:,:) + REAL(KIND=JPRB), POINTER :: FIELD_R1(:,:), FIELD_R2(:,:,:), FIELD_R3(:,:,:,:) TYPE(ATLAS_FUNCTIONSPACE_BLOCKSTRUCTUREDCOLUMNS) :: FSPACE TYPE(ATLAS_FIELD) :: FIELD INTEGER :: B, BSIZE, JL, JK, JM From 71f6b327b3549cf38d5834c84a01cf51ff0ce669 Mon Sep 17 00:00:00 2001 From: Balthasar Reuter Date: Wed, 27 Sep 2023 13:13:22 +0100 Subject: [PATCH 39/45] Disable MPI autodiscovery for NVHPC@Github runners --- arch/toolchains/github-ubuntu-nvhpc.cmake | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/arch/toolchains/github-ubuntu-nvhpc.cmake b/arch/toolchains/github-ubuntu-nvhpc.cmake index deda8b6f..778f6f35 100644 --- a/arch/toolchains/github-ubuntu-nvhpc.cmake +++ b/arch/toolchains/github-ubuntu-nvhpc.cmake @@ -6,12 +6,6 @@ # granted to it by virtue of its status as an intergovernmental organisation # nor does it submit to any jurisdiction. -#################################################################### -# COMPILER -#################################################################### - -set( ECBUILD_FIND_MPI ON ) - #################################################################### # OpenMP FLAGS #################################################################### @@ -22,7 +16,7 @@ set( OpenMP_Fortran_FLAGS "-mp -mp=bind,allcores,numa" CACHE STRING "" ) set( OpenMP_C_FLAGS "-mp -mp=bind,allcores,numa" CACHE STRING "" ) #################################################################### -# OpenAcc FLAGS +# OpenACC FLAGS #################################################################### # NB: We have to add `-mp` again to avoid undefined symbols during linking From 0051b48dad800df987e24ece5608ed7d479e2c0b Mon Sep 17 00:00:00 2001 From: Slavko Brdar Date: Wed, 27 Sep 2023 14:10:24 +0000 Subject: [PATCH 40/45] fix SerialBox with Atlas implementation - SBox does not like too many opening and closing of reference input --- src/cloudsc_fortran_atlas/cloudsc_global_atlas_state_mod.F90 | 2 +- src/cloudsc_fortran_atlas/validate_atlas_mod.F90 | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/cloudsc_fortran_atlas/cloudsc_global_atlas_state_mod.F90 b/src/cloudsc_fortran_atlas/cloudsc_global_atlas_state_mod.F90 index 527a57a1..baa3a9ed 100644 --- a/src/cloudsc_fortran_atlas/cloudsc_global_atlas_state_mod.F90 +++ b/src/cloudsc_fortran_atlas/cloudsc_global_atlas_state_mod.F90 @@ -298,7 +298,6 @@ SUBROUTINE CLOUDSC_GLOBAL_ATLAS_STATE_VALIDATE(SELF, FSET, NGPTOTG) CALL INPUT_INITIALIZE(NAME='reference') CALL LOAD_SCALAR('KLON', KLON) print *, "KLON = ", KLON - CALL INPUT_FINALIZE() ! Write variable validation header IF (IRANK == 0) THEN @@ -313,6 +312,7 @@ SUBROUTINE CLOUDSC_GLOBAL_ATLAS_STATE_VALIDATE(SELF, FSET, NGPTOTG) CALL VALIDATEVAR_ATLAS(FSET, OUT_VAR_NAMES(IVAR), KLON, NGPTOTG) ENDDO CALL VALIDATESTATE_ATLAS(FSET, 'TENDENCY_LOC', KLON, NGPTOTG) + CALL INPUT_FINALIZE() END SUBROUTINE CLOUDSC_GLOBAL_ATLAS_STATE_VALIDATE diff --git a/src/cloudsc_fortran_atlas/validate_atlas_mod.F90 b/src/cloudsc_fortran_atlas/validate_atlas_mod.F90 index 34f5b1b9..0a821c32 100644 --- a/src/cloudsc_fortran_atlas/validate_atlas_mod.F90 +++ b/src/cloudsc_fortran_atlas/validate_atlas_mod.F90 @@ -17,7 +17,6 @@ MODULE VALIDATE_ATLAS_MOD USE ATLAS_FUNCTIONSPACE_BLOCKSTRUCTUREDCOLUMNS_MODULE USE, INTRINSIC :: ISO_C_BINDING USE EXPAND_MOD, ONLY: LOAD_AND_EXPAND - USE FILE_IO_MOD, ONLY: INPUT_INITIALIZE, INPUT_FINALIZE IMPLICIT NONE @@ -72,7 +71,6 @@ SUBROUTINE VALIDATEVAR_ATLAS(FSET, NAME, NLON, NGPTOTG, STATE_VAR) ZMAX_VAL_ERR(2) = 0.0_JPRB ZSUM_ERR_ABS(:) = 0.0_JPRB - CALL INPUT_INITIALIZE(NAME='reference') IF (FRANK == 2) THEN CALL LOAD_AND_EXPAND(NAME, REF_R2, NLON, NPROMA, NGPTOT, NBLOCKS, NGPTOTG) CALL FIELD%DATA(FIELD_R1) @@ -162,7 +160,6 @@ SUBROUTINE VALIDATEVAR_ATLAS(FSET, NAME, NLON, NGPTOTG, STATE_VAR) PRINT *, "FIELD RANK NOT SUPPORTED" CALL EXIT(1) ENDIF - CALL INPUT_FINALIZE() CALL CLOUDSC_MPI_REDUCE_MIN(ZMINVAL, 1, 0) CALL CLOUDSC_MPI_REDUCE_MAX(ZMAX_VAL_ERR, 2, 0) From d61e2524b88c1db75d8caa9d3d2ccc8bd8fc4413 Mon Sep 17 00:00:00 2001 From: Balthasar Reuter Date: Wed, 27 Sep 2023 16:00:54 +0100 Subject: [PATCH 41/45] Disable MPI with NVHPC on Github (became default ON due to Atlas default) --- arch/toolchains/github-ubuntu-nvhpc.cmake | 3 +++ 1 file changed, 3 insertions(+) diff --git a/arch/toolchains/github-ubuntu-nvhpc.cmake b/arch/toolchains/github-ubuntu-nvhpc.cmake index 778f6f35..f0b88671 100644 --- a/arch/toolchains/github-ubuntu-nvhpc.cmake +++ b/arch/toolchains/github-ubuntu-nvhpc.cmake @@ -6,6 +6,9 @@ # granted to it by virtue of its status as an intergovernmental organisation # nor does it submit to any jurisdiction. +# Disable MPI in Github runner with NVHPC +set( ENABLE_MPI OFF CACHE STRING "" ) + #################################################################### # OpenMP FLAGS #################################################################### From f453604963b88ab2327b693ba7e74e68954fb37c Mon Sep 17 00:00:00 2001 From: Balthasar Reuter Date: Thu, 28 Sep 2023 10:07:32 +0100 Subject: [PATCH 42/45] Test different CUDA runtime export --- arch/github/ubuntu/nvhpc/21.9/env.sh | 5 +++-- arch/github/ubuntu/nvhpc/23.5/env.sh | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/arch/github/ubuntu/nvhpc/21.9/env.sh b/arch/github/ubuntu/nvhpc/21.9/env.sh index 6e05f756..44442acd 100644 --- a/arch/github/ubuntu/nvhpc/21.9/env.sh +++ b/arch/github/ubuntu/nvhpc/21.9/env.sh @@ -11,7 +11,6 @@ ### Variables export NVHPC_INSTALL_DIR=${GITHUB_WORKSPACE}/nvhpc-install export NVHPC_VERSION=21.9 -export CUDA_VERSION=11.4 export NVHPC_DIR=${NVHPC_INSTALL_DIR}/Linux_x86_64/${NVHPC_VERSION} ### Compilers @@ -19,10 +18,12 @@ export PATH=${NVHPC_DIR}/compilers/bin:${PATH} export NVHPC_LIBRARY_PATH=${NVHPC_DIR}/compilers/lib export LD_LIBRARY_PATH=${NVHPC_LIBRARY_PATH}:${LD_LIBRARY_PATH} +### CUDA runtime +export LD_LIBRARY_PATH=${NVHPC_DIR}/cuda/lib64/stubs:${NVHPC_DIR}/cuda/lib64:${LD_LIBRARY_PATH} + ### MPI export MPI_HOME=${NVHPC_DIR}/comm_libs/mpi export PATH=${MPI_HOME}/bin:${PATH} -export LD_LIBRARY_PATH=${NVHPC_DIR}/cuda/${CUDA_VERSION}/targets/x86_64-linux/lib/stubs:${LD_LIBRARY_PATH} ### HDF5 export HDF5_DIR=${GITHUB_WORKSPACE}/hdf5-install diff --git a/arch/github/ubuntu/nvhpc/23.5/env.sh b/arch/github/ubuntu/nvhpc/23.5/env.sh index a2261c25..1ef451c7 100644 --- a/arch/github/ubuntu/nvhpc/23.5/env.sh +++ b/arch/github/ubuntu/nvhpc/23.5/env.sh @@ -11,7 +11,6 @@ ### Variables export NVHPC_INSTALL_DIR=${GITHUB_WORKSPACE}/nvhpc-install export NVHPC_VERSION=23.5 -export CUDA_VERSION=12.1 export NVHPC_DIR=${NVHPC_INSTALL_DIR}/Linux_x86_64/${NVHPC_VERSION} ### Compilers @@ -19,10 +18,12 @@ export PATH=${NVHPC_DIR}/compilers/bin:${PATH} export NVHPC_LIBRARY_PATH=${NVHPC_DIR}/compilers/lib export LD_LIBRARY_PATH=${NVHPC_LIBRARY_PATH}:${LD_LIBRARY_PATH} +### CUDA runtime +export LD_LIBRARY_PATH=${NVHPC_DIR}/cuda/lib64/stubs:${NVHPC_DIR}/cuda/lib64:${LD_LIBRARY_PATH} + ### MPI export MPI_HOME=${NVHPC_DIR}/comm_libs/mpi export PATH=${MPI_HOME}/bin:${PATH} -export LD_LIBRARY_PATH=${NVHPC_DIR}/cuda/${CUDA_VERSION}/targets/x86_64-linux/lib/stubs:${LD_LIBRARY_PATH} ### HDF5 export HDF5_DIR=${GITHUB_WORKSPACE}/hdf5-install From dab193c935300f40e924fdf75e6d197de7d79447 Mon Sep 17 00:00:00 2001 From: Balthasar Reuter Date: Thu, 28 Sep 2023 14:18:36 +0100 Subject: [PATCH 43/45] Do not fail on free-disk-space --- .github/workflows/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 31f74c08..4df1df3e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -101,6 +101,7 @@ jobs: - name: Free Disk Space (Ubuntu) uses: jlumbroso/free-disk-space@main if: contains( matrix.arch, 'nvhpc' ) + continue-on-error: true with: # this might remove tools that are actually needed, # if set to "true" but frees about 6 GB From d6a8d5e8b79c3671971bc9291be51539724eb833 Mon Sep 17 00:00:00 2001 From: Balthasar Reuter Date: Fri, 29 Sep 2023 16:12:55 +0100 Subject: [PATCH 44/45] Do not run GPU/CUDA variants in CI --- .github/workflows/build.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4df1df3e..998fea59 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -53,8 +53,8 @@ jobs: - arch: nvhpc/21.9 nvhpc_version: 21.9 io_library_flag: '' - build_flags: '--with-gpu --with-loki --with-cuda --with-atlas' - ctest_exclude_pattern: '-gpu-|-scc-|-loki-c|-cuda' # GPU variants don't work on CPU runners, loki-c variant causes SIGFPE + build_flags: '--with-loki --with-atlas' + ctest_exclude_pattern: '-loki-c' # loki-c variant causes SIGFPE - arch: nvhpc/21.9 nvhpc_version: 21.9 io_library_flag: '' @@ -69,8 +69,8 @@ jobs: - arch: nvhpc/23.5 nvhpc_version: 23.5 io_library_flag: '' - build_flags: '--with-gpu --with-loki --with-cuda --with-atlas' - ctest_exclude_pattern: '-gpu-|-scc-|-loki-c|-loki-sca|-cuda' # GPU variants don't work on CPU runners, loki-c variant causes SIGFPE + build_flags: '--with-loki --with-atlas' + ctest_exclude_pattern: '-loki-c' # loki-c variant causes SIGFPE - arch: nvhpc/23.5 nvhpc_version: 23.5 io_library_flag: '' @@ -159,7 +159,7 @@ jobs: # Run ctest - name: Run CTest - if: ${{ !contains(matrix.build_flags, '--single-precision') }} + if: ${{ !( contains(matrix.build_flags, '--single-precision') || contains(matrix.build_flags, '--with-cuda') || contains(matrix.build_flags, '--with-gpu') ) }} working-directory: ./build run: | source env.sh From fe77fb6289b966e6cea72cec89b6ac39c4036591 Mon Sep 17 00:00:00 2001 From: Balthasar Reuter Date: Wed, 4 Oct 2023 12:01:41 +0100 Subject: [PATCH 45/45] Experimenting with NVHPC build configuration --- .github/workflows/build.yml | 14 +++++++------- arch/toolchains/github-ubuntu-nvhpc.cmake | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 998fea59..e9ad5729 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -53,8 +53,8 @@ jobs: - arch: nvhpc/21.9 nvhpc_version: 21.9 io_library_flag: '' - build_flags: '--with-loki --with-atlas' - ctest_exclude_pattern: '-loki-c' # loki-c variant causes SIGFPE + build_flags: '--with-gpu --with-loki --with-cuda' + ctest_exclude_pattern: '-gpu-|-scc-|-loki-c|-cuda-' # GPU variants don't work on CPU runners, loki-c variant causes SIGFPE - arch: nvhpc/21.9 nvhpc_version: 21.9 io_library_flag: '' @@ -69,18 +69,18 @@ jobs: - arch: nvhpc/23.5 nvhpc_version: 23.5 io_library_flag: '' - build_flags: '--with-loki --with-atlas' - ctest_exclude_pattern: '-loki-c' # loki-c variant causes SIGFPE + build_flags: '--with-gpu --with-loki --with-cuda' + ctest_exclude_pattern: '-gpu-|-scc-|-loki-c|-loki-sca|-cuda-' # GPU variants don't work on CPU runners, loki-c and loki-sca variant causes SIGFPE - arch: nvhpc/23.5 nvhpc_version: 23.5 io_library_flag: '' build_flags: '--single-precision --with-gpu --with-loki --with-cuda --with-atlas' - ctest_exclude_pattern: '-gpu-|-scc-|-loki-c|-loki-sca|-cuda' # GPU variants don't work on CPU runners, loki-c variant causes SIGFPE + ctest_exclude_pattern: '-gpu-|-scc-|-loki-c|-loki-sca|-cuda' # GPU variants don't work on CPU runners, loki-c and loki-sca variant causes SIGFPE - arch: nvhpc/23.5 nvhpc_version: 23.5 io_library_flag: '--with-serialbox' build_flags: '--with-gpu --with-loki --with-cuda --with-atlas' - ctest_exclude_pattern: '-gpu-|-scc-|-loki-c|-loki-sca|-cuda' # GPU variants don't work on CPU runners, loki-c variant causes SIGFPE + ctest_exclude_pattern: '-gpu-|-scc-|-loki-c|-loki-sca|-cuda' # GPU variants don't work on CPU runners, loki-c and loki-sca variant causes SIGFPE # Steps represent a sequence of tasks that will be executed as part of the job steps: @@ -159,7 +159,7 @@ jobs: # Run ctest - name: Run CTest - if: ${{ !( contains(matrix.build_flags, '--single-precision') || contains(matrix.build_flags, '--with-cuda') || contains(matrix.build_flags, '--with-gpu') ) }} + if: ${{ !( contains(matrix.build_flags, '--single-precision') || (contains(matrix.build_flags, '--with-cuda') && contains(matrix.build_flags, '--with-atlas')) ) }} working-directory: ./build run: | source env.sh diff --git a/arch/toolchains/github-ubuntu-nvhpc.cmake b/arch/toolchains/github-ubuntu-nvhpc.cmake index f0b88671..8e5b0d9f 100644 --- a/arch/toolchains/github-ubuntu-nvhpc.cmake +++ b/arch/toolchains/github-ubuntu-nvhpc.cmake @@ -24,7 +24,7 @@ set( OpenMP_C_FLAGS "-mp -mp=bind,allcores,numa" CACHE STRING "" ) # NB: We have to add `-mp` again to avoid undefined symbols during linking # (smells like an Nvidia bug) -set( OpenACC_Fortran_FLAGS "-acc -mp" CACHE STRING "" ) +set( OpenACC_Fortran_FLAGS "-acc=host -mp" CACHE STRING "" ) # Enable this to get more detailed compiler output # set( OpenACC_Fortran_FLAGS "${OpenACC_Fortran_FLAGS} -Minfo" )