Merge pull request #54 from ecmwf-ifs/nab_atlas_structures

New Atlas-based variant using BlockStructured FunctionSpace
ecmwf-ifs · Dec 4, 2023 · 0a8f41c · 0a8f41c
2 parents f9ddbd7 + fe77fb6
commit 0a8f41c
Show file tree

Hide file tree

Showing 18 changed files with 3,982 additions and 112 deletions.
diff --git a/.github/scripts/verify-targets.sh b/.github/scripts/verify-targets.sh
@@ -28,7 +28,7 @@ then
     targets+=(dwarf-cloudsc-gpu-scc-cuf dwarf-cloudsc-gpu-scc-cuf-k-caching)
     if [[ "$io_library_flag" == "--with-serialbox" ]]
     then
-        targets+=(dwarf-cloudsc-cuda dwarf-cloudsc-cuda-hoist dwarf-cloudsc-cuda-k-caching)
+        targets+=(dwarf-cloudsc-c-cuda dwarf-cloudsc-c-cuda-hoist dwarf-cloudsc-c-cuda-k-caching)
     fi
   fi
 fi
@@ -38,7 +38,7 @@ then
   targets+=(dwarf-cloudsc-loki-idem dwarf-cloudsc-loki-sca)
   targets+=(dwarf-cloudsc-loki-scc dwarf-cloudsc-loki-scc-hoist)
   targets+=(dwarf-cloudsc-loki-idem-stack dwarf-cloudsc-loki-scc-stack)
-  if [[ "$prec_flag" != "--single-precision" ]]
+  if [[ "$build_flags" != *"--single-precision"* ]]
   then
     targets+=(dwarf-cloudsc-loki-c)
   fi
@@ -52,6 +52,11 @@ then
   fi
 fi
 
+if [[ "$build_flags" == *"--with-atlas"* ]]
+then
+  targets+=(dwarf-cloudsc-fortran-atlas)
+fi
+
 if [[ "$build_flags" == *"--cloudsc-fortran-pyiface=ON"* ]]
 then
   targets+=(cloudsc_pyiface.py)
@@ -76,16 +81,4 @@ do
   fi
 done
 
-#
-# Check there aren't any other binaries
-#
-
-if [[ ${#targets[@]} -lt $(ls build/bin | wc -l) ]]
-then
-  exit_code=1
-  echo "::error::Additional targets found in build/bin"
-  echo "::error::Expected targets: ${targets[@]}"
-  echo "::error::Found targets: $(ls -1 build/bin | tr '\n' ' ')"
-fi
-
 exit $exit_code
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -17,7 +17,7 @@ on:
 jobs:
   # This workflow contains a single job called "build"
   build:
-    name: Test on ${{ matrix.arch }} ${{ matrix.io_library_flag }} ${{ matrix.prec_flag }} ${{ matrix.build_flags }}
+    name: ${{ matrix.arch }} ${{ matrix.io_library_flag }} ${{ matrix.build_flags }}
 
     # The type of runner that the job will run on
     runs-on: ubuntu-20.04
@@ -31,26 +31,22 @@ jobs:
 
         io_library_flag: ['', '--with-serialbox']  # Switch between Serialbox and HDF5
 
-        prec_flag: ['', '--single-precision']  # Switch single/double precision
-
         build_flags:
           - ''  # Plain build without any options
-          - '--with-gpu --with-loki' # Enable Loki and GPU variants
-          - '--with-gpu --with-loki --with-mpi' # Enable Loki and GPU variants with MPI
+          - '--with-gpu --with-loki --with-atlas' # Enable Loki, Atlas, and GPU variants
+          - '--with-gpu --with-loki --with-atlas --with-mpi' # Enable Loki, Atlas, and GPU variants with MPI
+          - '--single-precision --with-gpu --with-loki --with-atlas --with-mpi' # Enable Loki, and GPU variants with MPI in a single-precision build
 
         pyiface_flag: ['']  # Enable the pyiface variant
 
         python_f2py_flag: ['']  # Enable the f2py variant
 
-        claw_flag: ['']
-
         ctest_exclude_pattern: ['-scc-hoist-']  # Regex to disable CTest tests
 
         include:
           # Add pyiface build configuration for double precision, non-MPI, HDF5 only
           - arch: gnu/9.4.0
             io_library_flag: ''
-            prec_flag: ''
             build_flags: '--cloudsc-fortran-pyiface=ON --cloudsc-python-f2py=ON'
 
           # Add nvhpc build configurations with serialbox and HDF5
@@ -59,22 +55,32 @@ jobs:
             io_library_flag: ''
             build_flags: '--with-gpu --with-loki --with-cuda'
             ctest_exclude_pattern: '-gpu-|-scc-|-loki-c|-cuda-'  # GPU variants don't work on CPU runners, loki-c variant causes SIGFPE
+          - arch: nvhpc/21.9
+            nvhpc_version: 21.9
+            io_library_flag: ''
+            build_flags: '--single-precision --with-gpu --with-loki --with-cuda --with-atlas'
+            ctest_exclude_pattern: '-gpu-|-scc-|-loki-c|-cuda'  # GPU variants don't work on CPU runners, loki-c variant causes SIGFPE
           - arch: nvhpc/21.9
             nvhpc_version: 21.9
             io_library_flag: '--with-serialbox'
-            build_flags: '--with-gpu --with-loki --with-cuda'
-            ctest_exclude_pattern: '-gpu-|-scc-|-loki-c|-cuda-'  # GPU variants don't work on CPU runners, loki-c variant causes SIGFPE
+            build_flags: '--with-gpu --with-loki --with-cuda --with-atlas'
+            ctest_exclude_pattern: '-gpu-|-scc-|-loki-c|-cuda'  # GPU variants don't work on CPU runners, loki-c variant causes SIGFPE
 
           - arch: nvhpc/23.5
             nvhpc_version: 23.5
             io_library_flag: ''
             build_flags: '--with-gpu --with-loki --with-cuda'
-            ctest_exclude_pattern: '-gpu-|-scc-|-loki-c|-loki-sca|-cuda-'  # GPU variants don't work on CPU runners, loki-c variant causes SIGFPE
+            ctest_exclude_pattern: '-gpu-|-scc-|-loki-c|-loki-sca|-cuda-'  # GPU variants don't work on CPU runners, loki-c and loki-sca variant causes SIGFPE
+          - arch: nvhpc/23.5
+            nvhpc_version: 23.5
+            io_library_flag: ''
+            build_flags: '--single-precision --with-gpu --with-loki --with-cuda --with-atlas'
+            ctest_exclude_pattern: '-gpu-|-scc-|-loki-c|-loki-sca|-cuda'  # GPU variants don't work on CPU runners, loki-c and loki-sca variant causes SIGFPE
           - arch: nvhpc/23.5
             nvhpc_version: 23.5
             io_library_flag: '--with-serialbox'
-            build_flags: '--with-gpu --with-loki --with-cuda'
-            ctest_exclude_pattern: '-gpu-|-scc-|-loki-c|-loki-sca|-cuda-'  # GPU variants don't work on CPU runners, loki-c variant causes SIGFPE
+            build_flags: '--with-gpu --with-loki --with-cuda --with-atlas'
+            ctest_exclude_pattern: '-gpu-|-scc-|-loki-c|-loki-sca|-cuda'  # GPU variants don't work on CPU runners, loki-c and loki-sca variant causes SIGFPE
 
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
@@ -95,6 +101,7 @@ jobs:
       - name: Free Disk Space (Ubuntu)
         uses: jlumbroso/free-disk-space@main
         if: contains( matrix.arch, 'nvhpc' )
+        continue-on-error: true
         with:
           # this might remove tools that are actually needed,
           # if set to "true" but frees about 6 GB
@@ -139,21 +146,20 @@ jobs:
       # Build the targets
       - name: Bundle build
         run: |
-          ./cloudsc-bundle build --retry-verbose \
+          ./cloudsc-bundle build --verbose --retry-verbose \
           --arch=arch/github/ubuntu/${{ matrix.arch }} \
-          ${{ matrix.prec_flag }} ${{ matrix.io_library_flag }} ${{ matrix.build_flags }}
+          ${{ matrix.io_library_flag }} ${{ matrix.build_flags }}
 
       # Verify targets exist
       - name: Verify targets
         env:
           io_library_flag: ${{ matrix.io_library_flag }}
-          prec_flag: ${{ matrix.prec_flag }}
           build_flags: ${{ matrix.build_flags }}
         run: .github/scripts/verify-targets.sh
 
       # Run ctest
       - name: Run CTest
-        if: ${{ matrix.prec_flag == '' }}
+        if: ${{ !( contains(matrix.build_flags, '--single-precision') || (contains(matrix.build_flags, '--with-cuda') && contains(matrix.build_flags, '--with-atlas')) ) }}
         working-directory: ./build
         run: |
           source env.sh
@@ -162,7 +168,7 @@ jobs:
       # Upload test output
       - name: Archive CTest output
         uses: actions/upload-artifact@v3
-        if: ${{ matrix.prec_flag == '' }}
+        if: ${{ !contains(matrix.build_flags, '--single-precision') }}
         with:
           name: ctest-log
           path: build/ctest.log
diff --git a/AUTHORS.md b/AUTHORS.md
@@ -2,6 +2,7 @@
 
 - M. Ahlgrimm (ECMWF)
 - P. Bechtold (ECMWF)
+- S. Brdar (ECMWF)
 - W. Deconinck (ECMWF)
 - R. Forbes (ECMWF)
 - C. Jakob (ECMWF)

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -97,6 +97,7 @@ ecbuild_add_option( FEATURE FIELD_API
     DEFAULT ON )
 
 ecbuild_find_package( NAME loki )
+ecbuild_find_package( NAME atlas )
 
 # Add option for single-precision builds
 ecbuild_add_option( FEATURE SINGLE_PRECISION

diff --git a/arch/github/ubuntu/nvhpc/21.9/env.sh b/arch/github/ubuntu/nvhpc/21.9/env.sh
@@ -16,7 +16,10 @@ export NVHPC_DIR=${NVHPC_INSTALL_DIR}/Linux_x86_64/${NVHPC_VERSION}
 ### Compilers
 export PATH=${NVHPC_DIR}/compilers/bin:${PATH}
 export NVHPC_LIBRARY_PATH=${NVHPC_DIR}/compilers/lib
-export LD_LIBRARY_PATH=${NVHPC_LIBRARY_PATH}
+export LD_LIBRARY_PATH=${NVHPC_LIBRARY_PATH}:${LD_LIBRARY_PATH}
+
+### CUDA runtime
+export LD_LIBRARY_PATH=${NVHPC_DIR}/cuda/lib64/stubs:${NVHPC_DIR}/cuda/lib64:${LD_LIBRARY_PATH}
 
 ### MPI
 export MPI_HOME=${NVHPC_DIR}/comm_libs/mpi

diff --git a/arch/github/ubuntu/nvhpc/23.5/env.sh b/arch/github/ubuntu/nvhpc/23.5/env.sh
@@ -16,7 +16,10 @@ export NVHPC_DIR=${NVHPC_INSTALL_DIR}/Linux_x86_64/${NVHPC_VERSION}
 ### Compilers
 export PATH=${NVHPC_DIR}/compilers/bin:${PATH}
 export NVHPC_LIBRARY_PATH=${NVHPC_DIR}/compilers/lib
-export LD_LIBRARY_PATH=${NVHPC_LIBRARY_PATH}
+export LD_LIBRARY_PATH=${NVHPC_LIBRARY_PATH}:${LD_LIBRARY_PATH}
+
+### CUDA runtime
+export LD_LIBRARY_PATH=${NVHPC_DIR}/cuda/lib64/stubs:${NVHPC_DIR}/cuda/lib64:${LD_LIBRARY_PATH}
 
 ### MPI
 export MPI_HOME=${NVHPC_DIR}/comm_libs/mpi

diff --git a/arch/toolchains/github-ubuntu-nvhpc.cmake b/arch/toolchains/github-ubuntu-nvhpc.cmake
@@ -6,11 +6,8 @@
 # granted to it by virtue of its status as an intergovernmental organisation
 # nor does it submit to any jurisdiction.
 
-####################################################################
-# COMPILER
-####################################################################
-
-set( ECBUILD_FIND_MPI ON )
+# Disable MPI in Github runner with NVHPC
+set( ENABLE_MPI OFF CACHE STRING "" )
 
 ####################################################################
 # OpenMP FLAGS
@@ -19,21 +16,15 @@ set( ECBUILD_FIND_MPI ON )
 # Note: OpenMP_Fortran_FLAGS gets overwritten by the FindOpenMP module
 # unless its stored as a cache variable
 set( OpenMP_Fortran_FLAGS   "-mp -mp=bind,allcores,numa" CACHE STRING "" )
-
-# Note: OpenMP_C_FLAGS and OpenMP_C_LIB_NAMES have to be provided _both_ to
-# keep FindOpenMP from overwriting the FLAGS variable (the cache entry alone
-# doesn't have any effect here as the module uses FORCE to overwrite the
-# existing value)
 set( OpenMP_C_FLAGS         "-mp -mp=bind,allcores,numa" CACHE STRING "" )
-set( OpenMP_C_LIB_NAMES     "acchost" CACHE STRING "")
 
 ####################################################################
-# OpenAcc FLAGS
+# OpenACC FLAGS
 ####################################################################
 
 # NB: We have to add `-mp` again to avoid undefined symbols during linking
 # (smells like an Nvidia bug)
-set( OpenACC_Fortran_FLAGS "-acc -mp" CACHE STRING "" )
+set( OpenACC_Fortran_FLAGS "-acc=host -mp" CACHE STRING "" )
 # Enable this to get more detailed compiler output
 # set( OpenACC_Fortran_FLAGS "${OpenACC_Fortran_FLAGS} -Minfo" )
 

diff --git a/bundle.yml b/bundle.yml
@@ -5,20 +5,25 @@ name    : cloudsc-bundle
 version : 1.0.0-develop
 cmake : >
         CMAKE_LINK_DEPENDS_NO_SHARED=ON
+        BUILD_serialbox=OFF
+        BUILD_field_api=OFF
+        BUILD_eckit=OFF
+        BUILD_fckit=OFF
+        BUILD_atlas=OFF
+        ENABLE_OMP=ON
 
 projects :
 
     - ecbuild :
-        git     : https://github.com/ecmwf/ecbuild 
-        version : 3.7.0
+        git     : https://github.com/ecmwf/ecbuild
+        version : 3.8.0
         bundle  : false
 
     - serialbox :
         git     : https://github.com/reuterbal/serialbox
         version : v2.5.4/patched
         optional: true
         cmake   : >
-            BUILD_serialbox=OFF
             SERIALBOX_BUILD_SHARED=ON
             SERIALBOX_ENABLE_FORTRAN=ON
             SERIALBOX_ENABLE_EXPERIMENTAL_FILESYSTEM=OFF
@@ -36,13 +41,39 @@ projects :
             LOKI_ENABLE_TESTS=OFF
             LOKI_ENABLE_NO_INSTALL=ON
 
+    - eckit :
+        git     : https://github.com/ecmwf/eckit
+        version : 1.24.4
+        optional: true
+        require : ecbuild
+        cmake   : >
+            ECKIT_ENABLE_TESTS=OFF
+            ECKIT_ENABLE_BUILD_TOOLS=OFF
+            ECKIT_ENABLE_CUDA=OFF
+
+    - fckit :
+        git     : https://github.com/ecmwf/fckit
+        version : 0.11.0
+        optional: true
+        require : ecbuild eckit
+        cmake   : >
+            FCKIT_ENABLE_TESTS=OFF
+
+    - atlas :
+        git     : https://github.com/ecmwf/atlas
+        version : 0.34.0
+        optional: true
+        require : ecbuild eckit fckit
+        cmake   : >
+            ATLAS_ENABLE_TESTS=OFF
+            ATLAS_ENABLE_CUDA=OFF
+
     - field_api :
         git     : ${BITBUCKET}/rdx/field_api
         version : master
         optional: true
         require : ecbuild
         cmake   : >
-            BUILD_field_api=OFF
             ENABLE_FIELD_API_TESTS=OFF
             ENABLE_FIELD_API_FIAT_BUILD=OFF
             FIELD_API_UTIL_MODULE_PATH=${CMAKE_SOURCE_DIR}/cloudsc-dwarf/src/common/module
@@ -72,7 +103,7 @@ options :
             ENABLE_CLOUDSC_GPU_OMP_SCC_HOIST=ON
 
     - with-cuda :
-        help  : Enable GPU kernel variant based on CUDA-Fortran
+        help  : Enable GPU kernel variants based on CUDA and CUDA-Fortran
         cmake : >
             ENABLE_CUDA=ON
             ENABLE_CLOUDSC_GPU_SCC_CUF=ON
@@ -83,6 +114,10 @@ options :
         help  : Enable MPI-parallel kernel
         cmake : ENABLE_MPI=ON
 
+    - without-openmp :
+        help  : Disable OpenMP
+        cmake : ENABLE_OMP=OFF
+
     - with-loki :
         help  : Enable Loki source-to-source transformations
         cmake : >
@@ -111,6 +146,21 @@ options :
         cmake : >
             CLOUDSC_PYTHON_F2PY=ON
 
+    - with-atlas :
+        help  : Build Atlas and its dependencies (eckit, fckit) and enable Atlas-based variants of CLOUDSC (incompatible with --single-precision)
+        cmake : >
+            BUILD_eckit=ON
+            BUILD_fckit=ON
+            BUILD_atlas=ON
+
+    - with-dependency-tests :
+        help  : Build and enable tests for CLOUDSC dependencies that are build as part of the bundle (eckit, fckit, Atlas, Loki)
+        cmake : >
+            LOKI_ENABLE_TESTS=ON
+            ECKIT_ENABLE_TESTS=ON
+            FCKIT_ENABLE_TESTS=ON
+            ATLAS_ENABLE_TESTS=ON
+
     - cloudsc-prototype1 :
         help  : Build the original operational Fortran prototype [ON|OFF]
         cmake : ENABLE_CLOUDSC_PROTOTYPE1={{value}}
@@ -131,10 +181,6 @@ options :
         help  : Build the C version of CLOUDSC [ON|OFF]
         cmake : ENABLE_CLOUDSC_C={{value}}
 
-    - cloudsc-cuda :
-        help  : Build the CUDA C version of CLOUDSC [ON|OFF]
-        cmake : ENABLE_CLOUDSC_CUDA={{value}}
-
     - cloudsc-gpu-claw :
         help  : Build the deprecated CLAW-based GPU version CLOUDSC  [ON|OFF]
         cmake : ENABLE_CLOUDSC_GPU_CLAW={{value}}

diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
@@ -9,6 +9,7 @@
 add_subdirectory(prototype1)
 add_subdirectory(common)
 add_subdirectory(cloudsc_fortran)
+add_subdirectory(cloudsc_fortran_atlas)
 add_subdirectory(cloudsc_pyiface)
 add_subdirectory(cloudsc_python)
 add_subdirectory(cloudsc_c)