ice_dyn_vp: allow for bit-for-bit reproducibility under bfbflag

cicecore/cicedynB/general/ice_init.F90

@@ -419,7 +419,7 @@ subroutine input_data
       deltaminEVP = 1e-11_dbl_kind ! minimum delta for viscosities (EVP, Hunke 2001)
       deltaminVP  = 2e-9_dbl_kind  ! minimum delta for viscosities (VP, Hibler 1979)
       capping_method  = 'max'  ! method for capping of viscosities (max=Hibler 1979,sum=Kreyscher2000)
-      maxits_nonlin = 4        ! max nb of iteration for nonlinear solver
+      maxits_nonlin = 10       ! max nb of iteration for nonlinear solver
       precond = 'pgmres'       ! preconditioner for fgmres: 'ident' (identity), 'diag' (diagonal),
                                ! 'pgmres' (Jacobi-preconditioned GMRES)
       dim_fgmres = 50          ! size of fgmres Krylov subspace
@@ -431,7 +431,7 @@ subroutine input_data
       monitor_pgmres = .false. ! print pgmres residual norm
       ortho_type = 'mgs'       ! orthogonalization procedure 'cgs' or 'mgs'
       reltol_nonlin = 1e-8_dbl_kind ! nonlinear stopping criterion: reltol_nonlin*res(k=0)
-      reltol_fgmres = 1e-2_dbl_kind ! fgmres stopping criterion: reltol_fgmres*res(k)
+      reltol_fgmres = 1e-1_dbl_kind ! fgmres stopping criterion: reltol_fgmres*res(k)
       reltol_pgmres = 1e-6_dbl_kind ! pgmres stopping criterion: reltol_pgmres*res(k)
       algo_nonlin = 'picard'        ! nonlinear algorithm: 'picard' (Picard iteration), 'anderson' (Anderson acceleration)
       fpfunc_andacc = 1        ! fixed point function for Anderson acceleration:

cicecore/cicedynB/infrastructure/comm/mpi/ice_global_reductions.F90

@@ -36,7 +36,6 @@ module ice_global_reductions
    private
 
    public :: global_sum,      &
-             global_allreduce_sum, &
              global_sum_prod, &
              global_maxval,   &
              global_minval
@@ -56,12 +55,6 @@ module ice_global_reductions
                       global_sum_scalar_int
    end interface
 
-   interface global_allreduce_sum
-     module procedure global_allreduce_sum_vector_dbl!,   &
-     ! module procedure global_allreduce_sum_vector_real, & ! not yet implemented
-     ! module procedure global_allreduce_sum_vector_int     ! not yet implemented
-   end interface
-
    interface global_sum_prod
      module procedure global_sum_prod_dbl,         &
                       global_sum_prod_real,        &
@@ -707,68 +700,6 @@ function global_sum_scalar_int(scalar, dist) &
 
  end function global_sum_scalar_int
 
-!***********************************************************************
-
- function global_allreduce_sum_vector_dbl(vector, dist) &
-          result(globalSums)
-
-!  Computes the global sums of sets of scalars (elements of 'vector')
-!  distributed across a parallel machine.
-!
-!  This is actually the specific interface for the generic global_allreduce_sum
-!  function corresponding to double precision vectors.  The generic
-!  interface is identical but will handle real and integer vectors.
-
-   real (dbl_kind), dimension(:), intent(in) :: &
-      vector               ! vector whose components are to be summed
-
-   type (distrb), intent(in) :: &
-      dist                 ! block distribution
-
-   real (dbl_kind), dimension(size(vector)) :: &
-      globalSums           ! resulting array of global sums
-
-!-----------------------------------------------------------------------
-!
-!  local variables
-!
-!-----------------------------------------------------------------------
-
-   integer (int_kind) :: &
-      numProcs,     &! number of processor participating
-      numBlocks,    &! number of local blocks
-      communicator, &! communicator for this distribution
-      numElem        ! number of elements in vector
-
-   real (dbl_kind), dimension(:,:), allocatable :: &
-      work           ! temporary local array
-
-   character(len=*), parameter :: subname = '(global_allreduce_sum_vector_dbl)'
-
-!-----------------------------------------------------------------------
-!
-!  get communicator for MPI calls
-!
-!-----------------------------------------------------------------------
-
-   call ice_distributionGet(dist, &
-                            numLocalBlocks = numBlocks, &
-                            nprocs = numProcs,        &
-                            communicator = communicator)
-
-   numElem = size(vector)
-   allocate(work(1,numElem))
-   work(1,:) = vector
-   globalSums = c0
-
-   call compute_sums_dbl(work,globalSums,communicator,numProcs)
-
-   deallocate(work)
-
-!-----------------------------------------------------------------------
-
- end function global_allreduce_sum_vector_dbl
-
 !***********************************************************************
 
  function global_sum_prod_dbl (array1, array2, dist, field_loc, &

cicecore/cicedynB/infrastructure/comm/serial/ice_global_reductions.F90

@@ -37,7 +37,6 @@ module ice_global_reductions
    private
 
    public :: global_sum,      &
-             global_allreduce_sum, &
              global_sum_prod, &
              global_maxval,   &
              global_minval
@@ -57,12 +56,6 @@ module ice_global_reductions
                       global_sum_scalar_int
    end interface
 
-   interface global_allreduce_sum
-     module procedure global_allreduce_sum_vector_dbl!,   &
-     ! module procedure global_allreduce_sum_vector_real, & ! not yet implemented
-     ! module procedure global_allreduce_sum_vector_int     ! not yet implemented
-   end interface
-
    interface global_sum_prod
      module procedure global_sum_prod_dbl,         &
                       global_sum_prod_real,        &
@@ -708,68 +701,6 @@ function global_sum_scalar_int(scalar, dist) &
 
  end function global_sum_scalar_int
 
-!***********************************************************************
-
- function global_allreduce_sum_vector_dbl(vector, dist) &
-          result(globalSums)
-
-!  Computes the global sums of sets of scalars (elements of 'vector')
-!  distributed across a parallel machine.
-!
-!  This is actually the specific interface for the generic global_allreduce_sum
-!  function corresponding to double precision vectors.  The generic
-!  interface is identical but will handle real and integer vectors.
-
-   real (dbl_kind), dimension(:), intent(in) :: &
-      vector               ! vector whose components are to be summed
-
-   type (distrb), intent(in) :: &
-      dist                 ! block distribution
-
-   real (dbl_kind), dimension(size(vector)) :: &
-      globalSums           ! resulting array of global sums
-
-!-----------------------------------------------------------------------
-!
-!  local variables
-!
-!-----------------------------------------------------------------------
-
-   integer (int_kind) :: &
-      numProcs,     &! number of processor participating
-      numBlocks,    &! number of local blocks
-      communicator, &! communicator for this distribution
-      numElem        ! number of elements in vector
-
-   real (dbl_kind), dimension(:,:), allocatable :: &
-      work           ! temporary local array
-
-   character(len=*), parameter :: subname = '(global_allreduce_sum_vector_dbl)'
-
-!-----------------------------------------------------------------------
-!
-!  get communicator for MPI calls
-!
-!-----------------------------------------------------------------------
-
-   call ice_distributionGet(dist, &
-                            numLocalBlocks = numBlocks, &
-                            nprocs = numProcs,        &
-                            communicator = communicator)
-
-   numElem = size(vector)
-   allocate(work(1,numElem))
-   work(1,:) = vector
-   globalSums = c0
-
-   call compute_sums_dbl(work,globalSums,communicator,numProcs)
-
-   deallocate(work)
-
-!-----------------------------------------------------------------------
-
- end function global_allreduce_sum_vector_dbl
-
 !***********************************************************************
 
  function global_sum_prod_dbl (array1, array2, dist, field_loc, &

cicecore/drivers/unittest/sumchk/sumchk.F90

@@ -58,9 +58,6 @@ program sumchk
       integer(int_kind),parameter :: ntests3 = 3
       character(len=8)  :: errorflag3(ntests3)
       character(len=32) :: stringflag3(ntests3)
-      integer(int_kind),parameter :: ntests4 = 1
-      character(len=8)  :: errorflag4(ntests4)
-      character(len=32) :: stringflag4(ntests4)
 
       integer(int_kind) :: npes, ierr, ntask
 
@@ -100,7 +97,6 @@ program sumchk
       errorflag1 = passflag
       errorflag2 = passflag
       errorflag3 = passflag
-      errorflag4 = passflag
       npes = get_num_procs()
 
       if (my_task == master_task) then
@@ -600,63 +596,6 @@ program sumchk
          endif
       enddo
 
-      ! ---------------------------
-      ! Test Vector Reductions
-      ! ---------------------------
-
-      if (my_task == master_task) write(6,*) ' '
-
-      n = 1    ; stringflag4(n) = 'dble sum vector'
-      allocate(vec8(3))
-      allocate(sum8(3))
-
-      minval = -5.
-      maxval =  8.
-
-      vec8(1) = 1.
-
-      ! fill one gridcell with a min and max value
-      ntask = max(npes-1,1)-1
-      if (my_task == ntask) then
-         vec8(1) = minval
-      endif
-      ntask = min(npes,2)-1
-      if (my_task == ntask) then
-         vec8(1) = maxval
-      endif
-      vec8(2) = 2. * vec8(1)
-      vec8(3) = 3. * vec8(1)
-
-      ! compute correct results
-      if (npes == 1) then
-         minval = maxval
-         corval = maxval
-      else
-         corval = (npes - 2) * 1.0 + minval + maxval
-      endif
-
-      do k = 1,ntests4
-         string = stringflag4(k)
-         sum8 = -888e12
-         if (k == 1) then
-            sum8 = global_allreduce_sum(vec8, distrb_info)
-         else
-            call abort_ice(subname//' illegal k vector',file=__FILE__,line=__LINE__)
-         endif
-
-         if (my_task == master_task) then
-            write(6,'(1x,a,3g16.8)') string, sum8(1),sum8(2),sum8(3)
-         endif
-
-         if (sum8(1) /= corval .or. sum8(2) /= 2.*corval .or. sum8(3) /= 3.*corval) then
-           errorflag4(k) = failflag
-           errorflag0 = failflag
-           if (my_task == master_task) then
-              write(6,*) '**** ERROR ', sum8(1),sum8(2),sum8(3),corval
-           endif
-         endif
-      enddo
-
       ! ---------------------------
 
       if (my_task == master_task) then
@@ -670,9 +609,6 @@ program sumchk
          do k = 1,ntests3
             write(6,*) errorflag3(k),stringflag3(k)
          enddo
-         do k = 1,ntests4
-            write(6,*) errorflag4(k),stringflag4(k)
-         enddo
          write(6,*) ' '
          write(6,*) 'SUMCHK COMPLETED SUCCESSFULLY'
          if (errorflag0 == passflag) then

configuration/scripts/ice_in

@@ -167,7 +167,7 @@
     kridge          = 1
     ktransport      = 1
     ssh_stress      = 'geostrophic'
-    maxits_nonlin   = 4
+    maxits_nonlin   = 10
     precond         = 'pgmres'
     dim_fgmres       = 50
     dim_pgmres       = 5
@@ -178,7 +178,7 @@
     monitor_pgmres  = .false.
     ortho_type      = 'mgs'
     reltol_nonlin   = 1e-8
-    reltol_fgmres   = 1e-2
+    reltol_fgmres   = 1e-1
     reltol_pgmres   = 1e-6
     algo_nonlin     = 'picard'
     use_mean_vrel   = .true.

configuration/scripts/machines/env.ppp5_intel

@@ -18,6 +18,12 @@ source $ssmuse -d /fs/ssm/main/opt/intelcomp/inteloneapi-2022.1.2/intelcomp+mpi+
 # module load -s icc mpi
 setenv FOR_DUMP_CORE_FILE 1
 setenv I_MPI_DEBUG_COREDUMP 1
+# Reproducible collectives
+if (${ICE_BASEGEN} != ${ICE_SPVAL} || ${ICE_BASECOM} != ${ICE_SPVAL}) then
+  setenv I_MPI_CBWR 1
+endif
+# Stop being buggy
+setenv I_MPI_FABRICS ofi
 # NetCDF
 source $ssmuse -d main/opt/hdf5-netcdf4/serial/shared/inteloneapi-2022.1.2/01
 
@@ -32,6 +38,7 @@ setenv ICE_MACHINE_MAKE make
 setenv ICE_MACHINE_WKDIR ~/data/ppp5/cice/runs/
 setenv ICE_MACHINE_INPUTDATA /space/hall5/sitestore/eccc/cmd/e/sice500/
 setenv ICE_MACHINE_BASELINE ~/data/ppp5/cice/baselines/
+setenv ICE_MACHINE_MAXRUNLENGTH 6
 setenv ICE_MACHINE_SUBMIT qsub
 setenv ICE_MACHINE_TPNODE 80 
 setenv ICE_MACHINE_ACCT unused

configuration/scripts/machines/env.ppp6_gnu

@@ -8,7 +8,7 @@ endif
 if ("$inp" != "-nomodules") then
 
 # OpenMPI
-source /usr/mpi/gcc/openmpi-4.1.2a1/bin/mpivars.csh
+setenv PATH "/home/phb001/.local_rhel-8-icelake-64_gcc/bin:$PATH"
 
 # OpenMP
 setenv OMP_STACKSIZE 64M
@@ -21,6 +21,7 @@ setenv ICE_MACHINE_MAKE make
 setenv ICE_MACHINE_WKDIR ~/data/site6/cice/runs/
 setenv ICE_MACHINE_INPUTDATA /space/hall6/sitestore/eccc/cmd/e/sice500/
 setenv ICE_MACHINE_BASELINE ~/data/site6/cice/baselines/
+setenv ICE_MACHINE_MAXRUNLENGTH 6
 setenv ICE_MACHINE_SUBMIT qsub
 setenv ICE_MACHINE_TPNODE 80 
 setenv ICE_MACHINE_ACCT unused

configuration/scripts/machines/env.ppp6_gnu-impi

@@ -18,6 +18,12 @@ setenv I_MPI_F90 gfortran
 setenv I_MPI_FC gfortran
 setenv I_MPI_CC gcc
 setenv I_MPI_CXX g++
+# Reproducible collectives
+if (${ICE_BASEGEN} != ${ICE_SPVAL} || ${ICE_BASECOM} != ${ICE_SPVAL}) then
+  setenv I_MPI_CBWR 1
+endif
+# Stop being buggy
+setenv I_MPI_FABRICS ofi
 
 # OpenMP
 setenv OMP_STACKSIZE 64M
@@ -30,6 +36,7 @@ setenv ICE_MACHINE_MAKE make
 setenv ICE_MACHINE_WKDIR ~/data/site6/cice/runs/
 setenv ICE_MACHINE_INPUTDATA /space/hall6/sitestore/eccc/cmd/e/sice500/
 setenv ICE_MACHINE_BASELINE ~/data/site6/cice/baselines/
+setenv ICE_MACHINE_MAXRUNLENGTH 6
 setenv ICE_MACHINE_SUBMIT qsub
 setenv ICE_MACHINE_TPNODE 80 
 setenv ICE_MACHINE_ACCT unused

configuration/scripts/machines/env.ppp6_intel

@@ -18,6 +18,12 @@ source $ssmuse -d /fs/ssm/main/opt/intelcomp/inteloneapi-2022.1.2/intelcomp+mpi+
 # module load -s icc mpi
 setenv FOR_DUMP_CORE_FILE 1
 setenv I_MPI_DEBUG_COREDUMP 1
+# Reproducible collectives
+if (${ICE_BASEGEN} != ${ICE_SPVAL} || ${ICE_BASECOM} != ${ICE_SPVAL}) then
+  setenv I_MPI_CBWR 1
+endif
+# Stop being buggy
+setenv I_MPI_FABRICS ofi
 # NetCDF
 source $ssmuse -d main/opt/hdf5-netcdf4/serial/shared/inteloneapi-2022.1.2/01
 
@@ -32,6 +38,7 @@ setenv ICE_MACHINE_MAKE make
 setenv ICE_MACHINE_WKDIR ~/data/ppp6/cice/runs/
 setenv ICE_MACHINE_INPUTDATA /space/hall6/sitestore/eccc/cmd/e/sice500/
 setenv ICE_MACHINE_BASELINE ~/data/ppp6/cice/baselines/
+setenv ICE_MACHINE_MAXRUNLENGTH 6
 setenv ICE_MACHINE_SUBMIT qsub
 setenv ICE_MACHINE_TPNODE 80 
 setenv ICE_MACHINE_ACCT unused