ufs-community · GeorgeGayno-NOAA · Jan 24, 2020 · Jul 18, 2019 · Jul 19, 2019 · Jul 18, 2019
diff --git a/modulefiles/chgres_cube.hera b/modulefiles/chgres_cube.hera
@@ -17,6 +17,9 @@ export NCEPLIBS=/scratch1/NCEPDEV/global/gwv/l819/lib
 module use -a $NCEPLIBS/modulefiles
 module load esmflocal/8_0_48b.netcdf47
 
+export WGRIB2API_INC="/apps/wgrib2/2.0.8/intel/18.0.3.222/lib"
+export WGRIB2_LIB="/apps/wgrib2/2.0.8/intel/18.0.3.222/lib/libwgrib2.a"
+
 export FCOMP=mpiifort
 export FFLAGS="-O3 -fp-model precise -g -traceback -r8 -i4 -qopenmp -convert big_endian -assume byterecl"
 # for debugging

diff --git a/modulefiles/chgres_cube.jet b/modulefiles/chgres_cube.jet
@@ -4,8 +4,8 @@
 
 module load intel/18.0.5.274
 module load impi/2018.4.274
-module load szip
-module load hdf5
+module load szip/2.1
+module load hdf5/1.8.9
 module load netcdf/4.2.1.1
 
 module load w3nco/v2.0.6
@@ -22,3 +22,6 @@ export FCOMP=mpiifort
 export FFLAGS="-O3 -fp-model precise -g -traceback -r8 -i4 -qopenmp -convert big_endian -assume byterecl"
 #debug
 #export FFLAGS="-O0 -g -traceback -r8 -i4 -qopenmp -convert big_endian -check bounds -warn unused -assume byterecl"
+
+export WGRIB2API_INC="/mnt/lfs3/projects/hwrfv3/Jili.Dong/wgrib2-2.0.8/grib2/lib"
+export WGRIB2_LIB="/mnt/lfs3/projects/hwrfv3/Jili.Dong/wgrib2-2.0.8/grib2/lib/libwgrib2.a"
diff --git a/modulefiles/chgres_cube.linux.gnu b/modulefiles/chgres_cube.linux.gnu
@@ -0,0 +1,24 @@
+#############################################################
+## chgres_cube component - linux.gnu
+#############################################################
+
+export IP_INCd=${NCEPLIBS}/ip/include_d
+export NEMSIO_INC=${NCEPLIBS}/nemsio/include
+export SFCIO_INC4=${NCEPLIBS}/sfcio/include_4
+export SIGIO_INC4=${NCEPLIBS}/sigio/include_4
+
+export BACIO_LIB4=${NCEPLIBS}/bacio/lib/libbacio_v2.1.0_4.a
+export IP_LIBd=${NCEPLIBS}/ip/lib/libip_v3.0.0_d.a
+export NEMSIO_LIB=${NCEPLIBS}/nemsio/lib/libnemsio_v2.2.3.a
+export SFCIO_LIB4=${NCEPLIBS}/sfcio/lib/libsfcio_v1.1.0_4.a
+export SIGIO_LIB4=${NCEPLIBS}/sigio/lib/libsigio_v2.1.0_4.a
+export SP_LIBd=${NCEPLIBS}/sp/lib/libsp_v2.0.2_d.a
+export W3NCO_LIBd=${NCEPLIBS}/w3nco/lib/libw3nco_v2.0.6_d.a
+
+export WGRIB2API_INC=${WGRIB2_DIR}/include
+export WGRIB2_LIB=${WGRIB2_DIR}/lib/libwgrib2.a
+
+export FCOMP=mpif90
+export FFLAGS="-O3 -g -fbacktrace -fdefault-real-8 -ffree-line-length-none -fopenmp -fconvert=big-endian"
+# for debugging
+#export FFLAGS="-O0 -g -fbacktrace -fdefault-real-8 -ffree-line-length-none -fopenmp -fconvert=big-endian"
diff --git a/modulefiles/chgres_cube.linux.intel b/modulefiles/chgres_cube.linux.intel
@@ -0,0 +1,24 @@
+#############################################################
+## chgres_cube component - linux.intel
+#############################################################
+
+export IP_INCd=${NCEPLIBS}/ip/include_d
+export NEMSIO_INC=${NCEPLIBS}/nemsio/include
+export SFCIO_INC4=${NCEPLIBS}/sfcio/include_4
+export SIGIO_INC4=${NCEPLIBS}/sigio/include_4
+
+export BACIO_LIB4=${NCEPLIBS}/bacio/lib/libbacio_v2.1.0_4.a
+export IP_LIBd=${NCEPLIBS}/ip/lib/libip_v3.0.0_d.a
+export NEMSIO_LIB=${NCEPLIBS}/nemsio/lib/libnemsio_v2.2.3.a
+export SFCIO_LIB4=${NCEPLIBS}/sfcio/lib/libsfcio_v1.1.0_4.a
+export SIGIO_LIB4=${NCEPLIBS}/sigio/lib/libsigio_v2.1.0_4.a
+export SP_LIBd=${NCEPLIBS}/sp/lib/libsp_v2.0.2_d.a
+export W3NCO_LIBd=${NCEPLIBS}/w3nco/lib/libw3nco_v2.0.6_d.a
+
+export WGRIB2API_INC=${WGRIB2_DIR}/include
+export WGRIB2_LIB=${WGRIB2_DIR}/lib/libwgrib2.a
+
+export FCOMP=mpif90
+export FFLAGS="-O3 -fp-model source -g -traceback -r8 -i4 -qopenmp -convert big_endian -assume byterecl"
+# for debugging
+#export FFLAGS="-O0 -g -traceback -r8 -i4 -qopenmp -convert big_endian -check bounds -warn unused -assume byterecl"
diff --git a/modulefiles/chgres_cube.odin b/modulefiles/chgres_cube.odin
@@ -0,0 +1,27 @@
+#%Module#####################################################
+## chgres build module for Odin
+#############################################################
+
+module use /oldscratch/ywang/external/modulefiles
+module load esmf/8.0.0bs30
+
+module load cray-netcdf-hdf5parallel
+module load cray-parallel-netcdf
+module load cray-hdf5-parallel
+module load w3nco/v2.0.6
+module load nemsio/v2.2.2
+module load bacio/v2.0.2
+module load sp/v2.0.2
+module load sfcio/v1.0.0
+module load sigio/v2.0.1
+
+
+export FCOMP=ftn
+export FFLAGS="-O3 -fp-model precise -g -traceback -r8 -i4 -qopenmp -convert big_endian -assume byterecl"
+export WGRIB2API_LIB="/home/larissa.reames/tmp/wgrib2-2/grib2/lib/libwgrib2_api.a"
+export WGRIB2API_INC="/home/larissa.reames/tmp/wgrib2-2/grib2/lib"
+export WGRIB2_LIB="/home/larissa.reames/tmp/wgrib2-2/grib2/lib/libwgrib2.a"
+
+
+# for debugging
+#export FFLAGS="-O0 -g -traceback -r8 -i4 -qopenmp -convert big_endian -check bounds -warn unused -assume byterecl"
diff --git a/modulefiles/chgres_cube.wcoss_cray b/modulefiles/chgres_cube.wcoss_cray
@@ -19,6 +19,9 @@ module load sfcio-intel/1.0.0
 # module load esmf/7.1.0r
 export ESMFMKFILE=/gpfs/hps3/emc/global/noscrub/George.Gayno/esmf/8_0_0_bs20/lib/esmf.mk
 
+export WGRIB2API_INC=/gpfs/hps3/emc/meso/save/Dusan.Jovic/wgrib2/include
+export WGRIB2_LIB=/gpfs/hps3/emc/meso/save/Dusan.Jovic/wgrib2/lib/libwgrib2.a
+
 export FCOMP=ftn
 export FFLAGS="-O3 -fp-model precise -g -r8 -i4 -qopenmp -convert big_endian -assume byterecl"
 # for debugging

diff --git a/modulefiles/chgres_cube.wcoss_dell_p3 b/modulefiles/chgres_cube.wcoss_dell_p3
@@ -14,6 +14,9 @@ module load bacio/2.0.2
 module load sfcio/1.0.0
 module load sigio/2.1.0
 
+export WGRIB2API_INC=/u/Wesley.Ebisuzaki/home/grib2.v2.0.8.intel/lib
+export WGRIB2_LIB=/u/Wesley.Ebisuzaki/home/grib2.v2.0.8.intel/lib/libwgrib2.a
+
 export FCOMP=mpif90
 export FFLAGS="-O3 -fp-model precise -g -traceback -r8 -i4 -qopenmp -convert big_endian -assume byterecl"
 # for debugging

diff --git a/modulefiles/fv3gfs/fre-nctools.wcoss_cray b/modulefiles/fv3gfs/fre-nctools.wcoss_cray
diff --git a/modulefiles/fv3gfs/fre-nctools.wcoss_dell_p3 b/modulefiles/fv3gfs/fre-nctools.wcoss_dell_p3
diff --git a/modulefiles/modulefile.global_emcsfc_ice_blend.wcoss b/modulefiles/modulefile.global_emcsfc_ice_blend.wcoss
diff --git a/modulefiles/modulefile.global_emcsfc_ice_blend.wcoss_cray b/modulefiles/modulefile.global_emcsfc_ice_blend.wcoss_cray
diff --git a/modulefiles/modulefile.global_emcsfc_ice_blend.wcoss_cray_userlib b/modulefiles/modulefile.global_emcsfc_ice_blend.wcoss_cray_userlib
diff --git a/modulefiles/modulefile.global_emcsfc_ice_blend.wcoss_dell_p3 b/modulefiles/modulefile.global_emcsfc_ice_blend.wcoss_dell_p3
diff --git a/modulefiles/modulefile.global_emcsfc_snow2mdl.wcoss b/modulefiles/modulefile.global_emcsfc_snow2mdl.wcoss
diff --git a/modulefiles/modulefile.global_emcsfc_snow2mdl.wcoss_cray b/modulefiles/modulefile.global_emcsfc_snow2mdl.wcoss_cray
diff --git a/modulefiles/modulefile.global_emcsfc_snow2mdl.wcoss_cray_userlib b/modulefiles/modulefile.global_emcsfc_snow2mdl.wcoss_cray_userlib
diff --git a/modulefiles/modulefile.global_emcsfc_snow2mdl.wcoss_dell_p3 b/modulefiles/modulefile.global_emcsfc_snow2mdl.wcoss_dell_p3
diff --git a/parm/varmap_tables/GFSphys_var_map.txt b/parm/varmap_tables/GFSphys_var_map.txt
@@ -0,0 +1,23 @@
+dzdt                dzdt      set_to_fill   0        D
+sphum               sphum     set_to_fill   1E-7     T
+liq_wat             liq_wat   set_to_fill   0        T
+o3mr                o3mr      set_to_fill   1E-7     T
+ice_wat             ice_wat   set_to_fill   0        T
+rainwat             rainwat   set_to_fill   0        T
+snowwat             snowwat   set_to_fill   0        T
+graupel             graupel   set_to_fill   0        T
+vtype               vtype     skip          0        S
+sotype              stype     skip          0        S
+vfrac               vfrac     skip          0        S
+fricv               uustar    set_to_fill   0        S
+sfcr                zorl      set_to_fill   0.01     S
+tprcp               tprcp     set_to_fill   0.00     S
+ffmm                ffmm      set_to_fill   0.00     S
+f10m                f10m      set_to_fill   0.00     S
+soilw               smc       stop          0        S
+soill               slc       set_to_fill   0.0      S
+soilt               stc       stop          0        S
+cnwat               cnwat     set_to_fill   0.0      S
+hice                icetk     set_to_fill   1.5      S
+weasd               weasd     set_to_fill   0.0      S
+snod                snod      set_to_fill   0.0      S       	
diff --git a/parm/varmap_tables/README b/parm/varmap_tables/README
@@ -0,0 +1,24 @@
+#######################################################
+#            Description of varmap_tables             #
+#######################################################
+
+These files, each named for the phys_suite variable set in the chgres_cube namelist, 
+control how chgres_cube, when processing grib2 files, handles variables that might
+be missing from the grib2 files. Since there are so many different version of grib2
+files, it's often uncertain what fields are available even if you know what source
+model the data is coming from. Each file contains :
+
+Line 1: number of entries in the table
+Column 1: Name the code searches for in the table. Do not change.
+Column 2: Name the code will use to save the variable in the output file. Unimplemented.
+Comumn 3: Behavior when the code can't find the variable in the input file. Options are:
+	  "skip": Don't write to output file.
+	  "set_to_fill": Set to user-specified field value (see column 4).
+	  "stop": Force an exception and stop code execution. Use this if you absolutely
+		require a field to be present.
+Column 4: If column 3 = "set_to_fill", then this value is used to fill in all points 
+	  in the input field. These values may be over-written by the code before 
+	  output depending on the variable (esp. for surface variables). Be careful
+	  with these values for surface variables. If you set this value too low 
+	  (e.g., -100000), the code may run extremely slowly due to variable replacment
+	  at "missing" points.
diff --git a/sorc/build_chgres_cube.sh b/sorc/build_chgres_cube.sh
@@ -1,7 +1,14 @@
 #! /usr/bin/env bash
 set -eux
 
-source ./machine-setup.sh > /dev/null 2>&1
+target=${target:-"NULL"}
+
+if [[ $target == "linux.gnu" || $target == "linux.intel" ]]; then
+ unset -f module
+else
+ source ./machine-setup.sh > /dev/null 2>&1
+fi
+
 cwd=`pwd`
 
 USE_PREINST_LIBS=${USE_PREINST_LIBS:-"true"}

diff --git a/sorc/chgres_cube.fd/atmosphere.F90 b/sorc/chgres_cube.fd/atmosphere.F90
@@ -47,7 +47,7 @@ module atmosphere
                                        terrain_target_grid
 
  use program_setup, only             : vcoord_file_target_grid, &
-                                       regional,                &
+                                       regional, &
                                        tracers, num_tracers,      &
                                        atm_weight_file
 
@@ -218,6 +218,7 @@ subroutine atmosphere_driver(localpet)
                          termorderflag=ESMF_TERMORDER_SRCSEQ, rc=rc)
    if(ESMF_logFoundError(rcToCheck=rc,msg=ESMF_LOGERR_PASSTHRU,line=__LINE__,file=__FILE__)) &
       call error_handler("IN FieldRegrid", rc)
+
  enddo
 
  print*,"- CALL Field_Regrid FOR VERTICAL VELOCITY."
@@ -386,7 +387,7 @@ subroutine atmosphere_driver(localpet)
 !-----------------------------------------------------------------------------------
 
  call convert_winds
-
+ 
 !-----------------------------------------------------------------------------------
 ! Write target data to file.
 !-----------------------------------------------------------------------------------
@@ -492,7 +493,7 @@ subroutine create_atm_esmf_fields
  allocate(tracers_target_grid(num_tracers))
 
  do n = 1, num_tracers
-    print*,"- CALL FieldCreate FOR TARGET GRID TRACERS ", trim(tracers(n))
+    print*,"- CALL FieldCreate FOR TARGET GRID TRACERS ", trim(tracers(n))    
     tracers_target_grid(n) = ESMF_FieldCreate(target_grid, &
                                    typekind=ESMF_TYPEKIND_R8, &
                                    staggerloc=ESMF_STAGGERLOC_CENTER, &
@@ -816,7 +817,7 @@ subroutine newpr1(localpet)
     call error_handler("IN FieldGet", rc)
 
  allocate(pi(clb(1):cub(1),clb(2):cub(2),1:levp1_target))
-
+ 
  if(idvc.eq.2) then
    do k=1,levp1_target
      ak = vcoord_target(k,1) 
@@ -938,7 +939,7 @@ subroutine newps(localpet)
                     farrayPtr=tptr, rc=rc)
  if(ESMF_logFoundError(rcToCheck=rc,msg=ESMF_LOGERR_PASSTHRU,line=__LINE__,file=__FILE__)) &
     call error_handler("IN FieldGet", rc)
-
+    
 ! Find specific humidity in the array of tracer fields.
 
  do ii = 1, num_tracers
@@ -950,7 +951,7 @@ subroutine newps(localpet)
                     farrayPtr=qptr, rc=rc)
  if(ESMF_logFoundError(rcToCheck=rc,msg=ESMF_LOGERR_PASSTHRU,line=__LINE__,file=__FILE__)) &
     call error_handler("IN FieldGet", rc)
-
+    
  print*,"- CALL FieldGet FOR SURFACE PRESSURE BEFORE ADJUSTMENT"
  call ESMF_FieldGet(ps_b4adj_target_grid, &
                     farrayPtr=psptr, rc=rc)
@@ -1046,6 +1047,7 @@ subroutine newps(localpet)
 ! Compute surface pressure over the top.
 !-----------------------------------------------------------------------------------
 
+
  if(ls.gt.0) then
    k=cub(3)
    gamma=0
@@ -1106,9 +1108,9 @@ subroutine read_vcoord_info
 
  print*
  do k = 1, levp1_target
-   print*,'VCOORD FOR LEV ', k, 'IS: ', vcoord_target(k,:)
+    print*,'VCOORD FOR LEV ', k, 'IS: ', vcoord_target(k,:)
  enddo
-
+ 
  close(14)
 
  end subroutine read_vcoord_info
@@ -1255,7 +1257,6 @@ SUBROUTINE VINTG
 
  CALL TERP3(IM,1,1,1,1,4+NT,(IM*KM1),(IM*KM2), &
             KM1,IM,IM,Z1,C1,KM2,IM,IM,Z2,C2)
-
 ! - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
 !  COPY OUTPUT WIND, TEMPERATURE, HUMIDITY AND OTHER TRACERS
 !  EXCEPT BELOW THE INPUT DOMAIN, LET TEMPERATURE INCREASE WITH A FIXED
@@ -1410,7 +1411,7 @@ SUBROUTINE TERP3(IM,IXZ1,IXQ1,IXZ2,IXQ2,NM,NXQ1,NXQ2,             &
 !     REAL(ESMF_KIND_R8) :: J2S 
 
 ! - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 
-!  FIND THE SURROUNDING INPUT INTERVAL FOR EACH OUTPUT POINT.           
+!  FIND THE SURROUNDING INPUT INTERVAL FOR EACH OUTPUT POINT.
       CALL RSEARCH(IM,KM1,IXZ1,KXZ1,Z1,KM2,IXZ2,KXZ2,Z2,1,IM,K1S) 
 
 ! - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 
@@ -1422,7 +1423,6 @@ SUBROUTINE TERP3(IM,IXZ1,IXQ1,IXZ2,IXQ2,NM,NXQ1,NXQ2,             &
 !$OMP PARALLEL DO DEFAULT(PRIVATE) SHARED(IM,IXZ1,IXQ1,IXZ2), &
 !$OMP& SHARED(IXQ2,NM,NXQ1,NXQ2,KM1,KXZ1,KXQ1,Z1,Q1,KM2,KXZ2), &
 !$OMP& SHARED(KXQ2,Z2,Q2,K1S)
-
       DO K2=1,KM2 
         DO I=1,IM 
           K1=K1S(I,K2) 
@@ -1490,6 +1490,7 @@ SUBROUTINE TERP3(IM,IXZ1,IXQ1,IXZ2,IXQ2,NM,NXQ1,NXQ2,             &
                          ONE/(Z1D-Z1C)                                  
           ENDIF 
         ENDDO 
+
 !  INTERPOLATE.                                                         
         DO N=1,NM 
           DO I=1,IM 
@@ -1608,23 +1609,24 @@ SUBROUTINE RSEARCH(IM,KM1,IXZ1,KXZ1,Z1,KM2,IXZ2,KXZ2,Z2,IXL2,KXL2,L2)
  REAL(ESMF_KIND_R8),INTENT(IN) :: Z1(1+(IM-1)*IXZ1+(KM1-1)*KXZ1) 
  REAL(ESMF_KIND_R8),INTENT(IN) :: Z2(1+(IM-1)*IXZ2+(KM2-1)*KXZ2) 
 
- INTEGER                       :: I,K2,L 
+ INTEGER                       :: I,K2,L
 
  REAL(ESMF_KIND_R8)            :: Z 
 
+
 ! - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
 !  FIND THE SURROUNDING INPUT INTERVAL FOR EACH OUTPUT POINT.          
  DO I=1,IM 
    IF (Z1(1+(I-1)*IXZ1).LE.Z1(1+(I-1)*IXZ1+(KM1-1)*KXZ1)) THEN 
 !  INPUT COORDINATE IS MONOTONICALLY ASCENDING.                        
-     DO K2=1,KM2 
-       Z=Z2(1+(I-1)*IXZ2+(K2-1)*KXZ2) 
+     DO K2=1,KM2
+       Z=Z2(1+(I-1)*IXZ2+(K2-1)*KXZ2)
        L=0 
        DO 
          IF(Z.LT.Z1(1+(I-1)*IXZ1+L*KXZ1)) EXIT 
          L=L+1 
          IF(L.EQ.KM1) EXIT 
-       ENDDO 
+       ENDDO
        L2(1+(I-1)*IXL2+(K2-1)*KXL2)=L 
      ENDDO 
    ELSE 
@@ -1636,7 +1638,7 @@ SUBROUTINE RSEARCH(IM,KM1,IXZ1,KXZ1,Z1,KM2,IXZ2,KXZ2,Z2,IXL2,KXL2,L2)
          IF(Z.GT.Z1(1+(I-1)*IXZ1+L*KXZ1)) EXIT 
          L=L+1 
          IF(L.EQ.KM1) EXIT 
-       ENDDO 
+       ENDDO
        L2(1+(I-1)*IXL2+(K2-1)*KXL2)=L 
      ENDDO 
    ENDIF 
@@ -1730,7 +1732,7 @@ subroutine compute_zh
  deallocate(pe0, pn0)
 
  end subroutine compute_zh 
-
+ 
  subroutine cleanup_target_atm_b4adj_data
 
  implicit none

diff --git a/sorc/chgres_cube.fd/chgres.F90 b/sorc/chgres_cube.fd/chgres.F90
@@ -14,8 +14,9 @@ program chgres
  use atmosphere, only          : atmosphere_driver
 
  use program_setup, only       : read_setup_namelist, &
+                                 read_varmap, &
                                  convert_atm, &
-                                 convert_sfc
+                                 convert_sfc 
 
  use model_grid, only          : define_target_grid,  &
                                  define_input_grid, &
@@ -60,14 +61,20 @@ program chgres
 !-------------------------------------------------------------------------
 
  call read_setup_namelist
+
+!-------------------------------------------------------------------------
+! Read variable mapping file (used for grib2 input data only).
+!-------------------------------------------------------------------------
+
+ call read_varmap
 
 !-------------------------------------------------------------------------
 ! Create esmf grid objects for input and target grids.
 !-------------------------------------------------------------------------
 
- call define_input_grid(localpet, npets)
-
  call define_target_grid(localpet, npets)
+
+ call define_input_grid(localpet, npets)
 
 !-------------------------------------------------------------------------
 ! Convert atmospheric fields