CI_parallel.yml

# This is a basic workflow to help you get started with Actions

name: CI PARALLEL (MPI)

########################################################################
# Controls when the action will run.
on:
  # Triggers the workflow on push or pull request events but only for the master branch
  # Comment lines 10-13 to deactivate automatic running until Metis webpage is online again.
  push:
    branches: [ master ]
  pull_request:
    branches: [ master ]

  # Allows you to run this workflow manually from the Actions tab
  workflow_dispatch:

# A workflow run is made up of one or more jobs that can run sequentially or in parallel
jobs:
  # This workflow contains a single job called "build"
  build:
    # The type of runner that the job will run on
    runs-on: ubuntu-latest

    # Define strategy matrix
    strategy:
      fail-fast: false
      matrix:
        compiler: ['gfortran','ifort']
        mode: ['RELEASE','DEBUG']
        comm: ['PARALLEL']
        enable_threads: ['NO']

    env:
      METIS_HOME:  /home/runner/work/horses3d/horses3d/metis-5.1.0/build/Linux-x86_64
    # Steps represent a sequence of tasks that will be executed as part of the job

    steps:
      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
      - uses: actions/checkout@v3

      # IF COMPILER = GFORTRAN -> INSTALL OPEN-MPI #
      - name: install openmpi
        if: (matrix.compiler == 'gfortran')
        run: |
          sudo apt-get install openmpi-bin libopenmpi-dev

      # IF COMPILER = IFORT -> INSTALL IFORT #
      # UNCOMMENT TO USE CACHED IFORT 
      #- name: cache-intel-compilers
      #  if: (matrix.compiler == 'ifort')
      #  id: cache-intel-compilers
      #  uses: actions/cache@v3
      #  with:
      #    path: /opt/intel
      #    key: intel-${{ runner.os }}-compilers-b

      - name: Intel Apt repository
      # UNCOMMENT TO USE CACHED IFORT 
      #  if: (steps.cache-intel-compilers.outputs.cache-hit != 'true')
        run: |
          wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB
          sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB
          sudo add-apt-repository "deb https://apt.repos.intel.com/oneapi all main"
          sudo apt-get update

      - name: Install Intel oneAPI
      # UNCOMMENT TO USE CACHED IFORT
      #  if: (steps.cache-intel-compilers.outputs.cache-hit != 'true')
        run: sudo apt-get install intel-oneapi-compiler-fortran intel-oneapi-compiler-dpcpp-cpp intel-oneapi-mpi intel-oneapi-mpi-devel intel-oneapi-mkl-devel ninja-build

      # - name: cache-metis
      #   id: cache-metis
      #   uses: actions/cache@v2
      #   with:
      #     path: /home/runner/metis-5.1.0
      #     key: metis-${{ runner.os }}-b

      - name: Install METIS (intel)
        # if: (steps.cache-metis.outputs.cache-hit != 'true')
        # wget "http://glaros.dtc.umn.edu/gkhome/fetch/sw/metis/metis-5.1.0.tar.gz" (OLD LINK TO METIS)
        if: (matrix.compiler == 'ifort') 
        run: |
          source /opt/intel/oneapi/setvars.sh || true
          wget "https://src.fedoraproject.org/lookaside/pkgs/metis/metis-5.1.0.tar.gz/5465e67079419a69e0116de24fce58fe/metis-5.1.0.tar.gz" 
          tar -xvf metis-5.1.0.tar.gz
          cd metis-5.1.0
          make config cc=icx
          make
          mv /home/runner/work/horses3d/horses3d/metis-5.1.0/build/Linux-x86_64/libmetis /home/runner/work/horses3d/horses3d/metis-5.1.0/build/Linux-x86_64/lib
          mv /home/runner/work/horses3d/horses3d/metis-5.1.0/build/Linux-x86_64/programs /home/runner/work/horses3d/horses3d/metis-5.1.0/build/Linux-x86_64/bin
#          export METIS_HOME=~/metis-5.1.0/build/Linux-x86_64
#          echo $METIS_HOME

      - name: Install METIS (gnu)
      # if: (steps.cache-metis.outputs.cache-hit != 'true')
      # wget "http://glaros.dtc.umn.edu/gkhome/fetch/sw/metis/metis-5.1.0.tar.gz" (OLD LINK TO METIS)
        if: (matrix.compiler == 'gfortran') 
        run: |
          wget "https://src.fedoraproject.org/lookaside/pkgs/metis/metis-5.1.0.tar.gz/5465e67079419a69e0116de24fce58fe/metis-5.1.0.tar.gz" 
          tar -xvf metis-5.1.0.tar.gz
          cd metis-5.1.0
          make config cc=gcc
          make
          mv /home/runner/work/horses3d/horses3d/metis-5.1.0/build/Linux-x86_64/libmetis /home/runner/work/horses3d/horses3d/metis-5.1.0/build/Linux-x86_64/lib
          mv /home/runner/work/horses3d/horses3d/metis-5.1.0/build/Linux-x86_64/programs /home/runner/work/horses3d/horses3d/metis-5.1.0/build/Linux-x86_64/bin

      - name: Check METIS PATH
        run: echo $METIS_HOME

      # Runs a single command using the runners shell
      ##- name: Install gfortran
      ##  run: |
      ##   sudo add-apt-repository ppa:ubuntu-toolchain-r/test
      ##   sudo apt update
      ##   sudo apt-get install gfortran-9

      ##- name: Show gfortran version
      ##  run: gfortran-9 --version
      - name: Show ${{matrix.compiler}} version
        run: |
          source /opt/intel/oneapi/setvars.sh || true
          ${{matrix.compiler}} --version

      - name: Configure
        working-directory: ./Solver
        run: ./configure

      - name: Compile horses3d
        working-directory: ./Solver
        run: |
         source /opt/intel/oneapi/setvars.sh || true
         make allclean MODE=${{matrix.mode}} COMPILER=${{matrix.compiler}} COMM=${{matrix.comm}} ENABLE_THREADS=${{matrix.enable_threads}} WITH_METIS=YES
         make MODE=${{matrix.mode}} COMPILER=${{matrix.compiler}} COMM=${{matrix.comm}} ENABLE_THREADS=${{matrix.enable_threads}} WITH_METIS=YES


########################################################################
#######                         EULER                           ########
########################################################################
#
# 2) UNIFORM FLOW
# ---------------

      - name: Build UniformFlow
        working-directory: ./Solver/test/Euler/UniformFlow/SETUP
        run: |
          source /opt/intel/oneapi/setvars.sh || true
          make MODE=${{matrix.mode}} COMPILER=${{matrix.compiler}} COMM=${{matrix.comm}} ENABLE_THREADS=${{matrix.enable_threads}} WITH_METIS=YES
        if: '!cancelled()'

      - name: Run UniformFlow
        working-directory: ./Solver/test/Euler/UniformFlow
        run: |
          source /opt/intel/oneapi/setvars.sh || true
          mpiexec -n 8 ./horses3d.ns UniformFlow.control
        if: '!cancelled()'

#
# 3) BOX AROUND CIRCLE
# --------------------

      - name: Build BoxAroundCircle
        working-directory: ./Solver/test/Euler/BoxAroundCircle/SETUP
        run: |
          source /opt/intel/oneapi/setvars.sh || true
          make MODE=${{matrix.mode}} COMPILER=${{matrix.compiler}} COMM=${{matrix.comm}} ENABLE_THREADS=${{matrix.enable_threads}} WITH_METIS=YES
        if: '!cancelled()'

      - name: Run BoxAroundCircle
        working-directory: ./Solver/test/Euler/BoxAroundCircle
        run: |
          source /opt/intel/oneapi/setvars.sh || true
          mpiexec -n 8 ./horses3d.ns BoxAroundCircle.control
        if: '!cancelled()'

#
# 4) BOX AROUND CIRCLE PIROZZOLI
# ------------------------------

      - name: Build BoxAroundCirclePirozzoli
        working-directory: ./Solver/test/Euler/BoxAroundCirclePirozzoli/SETUP
        run: |
          source /opt/intel/oneapi/setvars.sh || true
          make MODE=${{matrix.mode}} COMPILER=${{matrix.compiler}} COMM=${{matrix.comm}} ENABLE_THREADS=${{matrix.enable_threads}} WITH_METIS=YES
        if: '!cancelled()'

      - name: Run BoxAroundCirclePirozzoli
        working-directory: ./Solver/test/Euler/BoxAroundCirclePirozzoli
        run: |
          source /opt/intel/oneapi/setvars.sh || true
          mpiexec -n 8 ./horses3d.ns BoxAroundCirclePirozzoli.control
        if: '!cancelled()'

#
# 5) Inviscid TGV with KEPEC
# --------------------------

      - name: Build TaylorGreenKEPEC
        working-directory: ./Solver/test/Euler/TaylorGreenKEPEC/SETUP
        run: |
          source /opt/intel/oneapi/setvars.sh || true
          make MODE=${{matrix.mode}} COMPILER=${{matrix.compiler}} COMM=${{matrix.comm}} ENABLE_THREADS=${{matrix.enable_threads}} WITH_METIS=YES
        if: '!cancelled()'

      - name: Run TaylorGreenKEPEC
        working-directory: ./Solver/test/Euler/TaylorGreenKEPEC
        run: |
          source /opt/intel/oneapi/setvars.sh || true
          mpiexec -n 8 ./horses3d.ns TaylorGreenKEPEC.control
        if: '!cancelled()'

#
# 6) Vortex transport 2D with KEPEC
# ---------------------------------

      # - name: Build VortexTransport2DKEPEC
      #   working-directory: ./Solver/test/Euler/VortexTransport2D/SETUP
      #   run: |
      #     source /opt/intel/oneapi/setvars.sh || true
      #     make MODE=${{matrix.mode}} COMPILER=${{matrix.compiler}} COMM=${{matrix.comm}} ENABLE_THREADS=${{matrix.enable_threads}} WITH_METIS=YES

      # - name: Run VortexTransport2DKEPEC
      #   working-directory: ./Solver/test/Euler/VortexTransport2D
      #   run: |
      #     source /opt/intel/oneapi/setvars.sh || true
      #     mpiexec -n 8 ./horses3d.ns VortexTransport2D.control

#
# 7) Jacobian-Free Newton-Krylov
# ------------------------------

      # Deactivated because a mismatch in the accuracy.
      # Double Precision equality failed:
      #  Final maximum residual
      #  Expected [1.6257217794191102E-011], Got [3.4106051316484809E-013]

      # - name: Build JFNK Uniform Flow
      #   working-directory: ./Solver/test/Euler/JFNK/SETUP
      #   run: make MODE=${{matrix.mode}} COMPILER=${{matrix.compiler}} COMM=${{matrix.comm}} ENABLE_THREADS=${{matrix.enable_threads}} WITH_METIS=YES

      # - name: Run JFNK Uniform Flow
      #   working-directory: ./Solver/test/Euler/JFNK
      #   run: mpiexec -n 8 ./horses3d.ns UniformFlowJFNK.control

#
# 8) BOX AROUND CIRCLE WITH ADAPTATION
# ------------------------------------

      # - name: Build BoxAroundCircle_pAdapted
      #   working-directory: ./Solver/test/Euler/BoxAroundCircle_pAdapted/SETUP
      #   run: |
      #     source /opt/intel/oneapi/setvars.sh || true
      #     make MODE=${{matrix.mode}} COMPILER=${{matrix.compiler}} COMM=${{matrix.comm}} ENABLE_THREADS=${{matrix.enable_threads}} WITH_METIS=YES

      # - name: Run BoxAroundCircle_pAdapted
      #   working-directory: ./Solver/test/Euler/BoxAroundCircle_pAdapted
      #   run: |
      #     source /opt/intel/oneapi/setvars.sh || true
      #     mpiexec -n 8 ./horses3d.ns BoxAroundCircle_pAdapted.control

########################################################################
#######                         NAVIER-STOKES                   ########
########################################################################

#
# 0) Convergence
# ------------------------------

      - name: Build Convergence
        working-directory: ./Solver/test/NavierStokes/Convergence/SETUP
        run: |
          source /opt/intel/oneapi/setvars.sh || true
          make MODE=${{matrix.mode}} COMPILER=${{matrix.compiler}} COMM=${{matrix.comm}} ENABLE_THREADS=${{matrix.enable_threads}} WITH_METIS=YES
        if: '!cancelled()'

      - name: Run Convergence
        working-directory: ./Solver/test/NavierStokes/Convergence
        run: |
          source /opt/intel/oneapi/setvars.sh || true
          mpiexec -n 8 ./horses3d.ns Convergence.control
        if: '!cancelled()'

#
# 0) Convergence energy
# --------------------------

      - name: Build Convergence_energy
        working-directory: ./Solver/test/NavierStokes/Convergence_energy/SETUP
        run: |
          source /opt/intel/oneapi/setvars.sh || true
          make MODE=${{matrix.mode}} COMPILER=${{matrix.compiler}} COMM=${{matrix.comm}} ENABLE_THREADS=${{matrix.enable_threads}} WITH_METIS=YES
        if: '!cancelled()'

      - name: Run Convergence_energy
        working-directory: ./Solver/test/NavierStokes/Convergence_energy
        run: |
          source /opt/intel/oneapi/setvars.sh || true
          mpiexec -n 8 ./horses3d.ns Convergence.control
        if: '!cancelled()'

#
# 0) Convergence entropy
# ---------------------------------

      - name: Build Convergence_entropy
        working-directory: ./Solver/test/NavierStokes/Convergence_entropy/SETUP
        run: |
          source /opt/intel/oneapi/setvars.sh || true
          make MODE=${{matrix.mode}} COMPILER=${{matrix.compiler}} COMM=${{matrix.comm}} ENABLE_THREADS=${{matrix.enable_threads}} WITH_METIS=YES
        if: '!cancelled()'

      - name: Run Convergence_entropy
        working-directory: ./Solver/test/NavierStokes/Convergence_entropy
        run: |
          source /opt/intel/oneapi/setvars.sh || true
          mpiexec -n 8 ./horses3d.ns Convergence.control
        if: '!cancelled()'

#
# 1) CYLINDER
# -----------

      - name: Build NSCylinder
        working-directory: ./Solver/test/NavierStokes/Cylinder/SETUP
        run: |
          source /opt/intel/oneapi/setvars.sh || true
          make MODE=${{matrix.mode}} COMPILER=${{matrix.compiler}} COMM=${{matrix.comm}} ENABLE_THREADS=${{matrix.enable_threads}} WITH_METIS=YES
        if: '!cancelled()'

      - name: Run NSCylinder
        working-directory: ./Solver/test/NavierStokes/Cylinder
        run: |
          source /opt/intel/oneapi/setvars.sh || true
          mpiexec -n 8 ./horses3d.ns Cylinder.control
        if: '!cancelled()'

#
# 2) CYLINDER IP
# --------------

      - name: Build NSCylinderIP
        working-directory: ./Solver/test/NavierStokes/CylinderIP/SETUP
        run: |
          source /opt/intel/oneapi/setvars.sh || true
          make MODE=${{matrix.mode}} COMPILER=${{matrix.compiler}} COMM=${{matrix.comm}} ENABLE_THREADS=${{matrix.enable_threads}} WITH_METIS=YES
        if: '!cancelled()'

      - name: Run NSCylinderIP
        working-directory: ./Solver/test/NavierStokes/CylinderIP
        run: |
          source /opt/intel/oneapi/setvars.sh || true
          mpiexec -n 8 ./horses3d.ns CylinderIP.control
        if: '!cancelled()'

#
# 3) CYLINDER BR2
# ---------------

      - name: Build NSCylinderBR2
        working-directory: ./Solver/test/NavierStokes/CylinderBR2/SETUP
        run: |
          source /opt/intel/oneapi/setvars.sh || true
          make MODE=${{matrix.mode}} COMPILER=${{matrix.compiler}} COMM=${{matrix.comm}} ENABLE_THREADS=${{matrix.enable_threads}} WITH_METIS=YES
        if: '!cancelled()'

      - name: Run NSCylinderBR2
        working-directory: ./Solver/test/NavierStokes/CylinderBR2
        run: |
          source /opt/intel/oneapi/setvars.sh || true
          mpiexec -n 8 ./horses3d.ns CylinderBR2.control
        if: '!cancelled()'

#
# 4) CYLINDER DUCROS
# ------------------

      - name: Build NSCylinderDucros
        working-directory: ./Solver/test/NavierStokes/CylinderDucros/SETUP
        run: |
          source /opt/intel/oneapi/setvars.sh || true
          make MODE=${{matrix.mode}} COMPILER=${{matrix.compiler}} COMM=${{matrix.comm}} ENABLE_THREADS=${{matrix.enable_threads}} WITH_METIS=YES
        if: '!cancelled()'

      - name: Run NSCylinderDucros
        working-directory: ./Solver/test/NavierStokes/CylinderDucros
        run: |
          source /opt/intel/oneapi/setvars.sh || true
          mpiexec -n 8 ./horses3d.ns CylinderDucros.control
        if: '!cancelled()'

#
# 5) CYLINDER Smagorinsky
# ------------------

      - name: Build NSCylinderSmagorinsky
        working-directory: ./Solver/test/NavierStokes/CylinderSmagorinsky/SETUP
        run: |
          source /opt/intel/oneapi/setvars.sh || true
          make MODE=${{matrix.mode}} COMPILER=${{matrix.compiler}} COMM=${{matrix.comm}} ENABLE_THREADS=${{matrix.enable_threads}} WITH_METIS=YES
        if: '!cancelled()'

      - name: Run NSCylinderSmagorinsky
        working-directory: ./Solver/test/NavierStokes/CylinderSmagorinsky
        run: |
          source /opt/intel/oneapi/setvars.sh || true
          mpiexec -n 8 ./horses3d.ns CylinderSmagorinsky.control
        if: '!cancelled()'

#
# 6) CYLINDER WALE
# ------------------

      - name: Build NSCylinderWALE
        working-directory: ./Solver/test/NavierStokes/CylinderWALE/SETUP
        run: |
          source /opt/intel/oneapi/setvars.sh || true
          make MODE=${{matrix.mode}} COMPILER=${{matrix.compiler}} COMM=${{matrix.comm}} ENABLE_THREADS=${{matrix.enable_threads}} WITH_METIS=YES
        if: '!cancelled()'

      - name: Run NSCylinderWALE
        working-directory: ./Solver/test/NavierStokes/CylinderWALE
        run: |
          source /opt/intel/oneapi/setvars.sh || true
          mpiexec -n 8 ./horses3d.ns CylinderWALE.control
        if: '!cancelled()'

#
# 7) CYLINDER Vreman
# ------------------

      - name: Build NSCylinderVreman
        working-directory: ./Solver/test/NavierStokes/CylinderVreman/SETUP
        run: |
          source /opt/intel/oneapi/setvars.sh || true
          make MODE=${{matrix.mode}} COMPILER=${{matrix.compiler}} COMM=${{matrix.comm}} ENABLE_THREADS=${{matrix.enable_threads}} WITH_METIS=YES
        if: '!cancelled()'

      - name: Run NSCylinderVreman
        working-directory: ./Solver/test/NavierStokes/CylinderVreman
        run: |
          source /opt/intel/oneapi/setvars.sh || true
          mpiexec -n 8 ./horses3d.ns CylinderVreman.control
        if: '!cancelled()'


#
# 8) TAYLOR GREEN VORTEX
# ----------------------

      - name: Build TaylorGreen
        working-directory: ./Solver/test/NavierStokes/TaylorGreen/SETUP
        run: |
          source /opt/intel/oneapi/setvars.sh || true
          make MODE=${{matrix.mode}} COMPILER=${{matrix.compiler}} COMM=${{matrix.comm}} ENABLE_THREADS=${{matrix.enable_threads}} WITH_METIS=YES
        if: '!cancelled()'

      - name: Run TaylorGreen
        working-directory: ./Solver/test/NavierStokes/TaylorGreen
        run: |
          source /opt/intel/oneapi/setvars.sh || true
          mpiexec -n 8 ./horses3d.ns TaylorGreen.control
        if: '!cancelled()'

#
# 9) TAYLOR GREEN VORTEX KEP BR2
# -----------------------------

      - name: Build TaylorGreenKEP_BR2
        working-directory: ./Solver/test/NavierStokes/TaylorGreenKEP_BR2/SETUP
        run: |
          source /opt/intel/oneapi/setvars.sh || true
          make MODE=${{matrix.mode}} COMPILER=${{matrix.compiler}} COMM=${{matrix.comm}} ENABLE_THREADS=${{matrix.enable_threads}} WITH_METIS=YES
        if: '!cancelled()'

      - name: Run TaylorGreenKEP_BR2
        working-directory: ./Solver/test/NavierStokes/TaylorGreenKEP_BR2
        run: |
          source /opt/intel/oneapi/setvars.sh || true
          mpiexec -n 8 ./horses3d.ns TaylorGreenKEP_BR2.control
        if: '!cancelled()'

#
# 10) TAYLOR GREEN VORTEX KEPEC IP
# -------------------------------

      - name: Build TaylorGreenKEPEC_IP
        working-directory: ./Solver/test/NavierStokes/TaylorGreenKEPEC_IP/SETUP
        run: |
          source /opt/intel/oneapi/setvars.sh || true
          make MODE=${{matrix.mode}} COMPILER=${{matrix.compiler}} COMM=${{matrix.comm}} ENABLE_THREADS=${{matrix.enable_threads}} WITH_METIS=YES
        if: '!cancelled()'

      - name: Run TaylorGreenKEPEC_IP
        working-directory: ./Solver/test/NavierStokes/TaylorGreenKEPEC_IP
        run: |
          source /opt/intel/oneapi/setvars.sh || true
          mpiexec -n 8 ./horses3d.ns TaylorGreenKEPEC_IP.control
        if: '!cancelled()'

#
# 11) CYLINDER FAS
# ---------------

      # - name: Build NSCylinderFAS
      #   working-directory: ./Solver/test/NavierStokes/CylinderFAS/SETUP
      #   run: |
      #     source /opt/intel/oneapi/setvars.sh || true
      #     make MODE=${{matrix.mode}} COMPILER=${{matrix.compiler}} COMM=${{matrix.comm}} ENABLE_THREADS=${{matrix.enable_threads}} WITH_METIS=YES

      # - name: Run NSCylinderFAS
      #   working-directory: ./Solver/test/NavierStokes/CylinderFAS
      #   run: |
      #     source /opt/intel/oneapi/setvars.sh || true
      #     mpiexec -n 8 ./horses3d.ns CylinderFAS.control

#
# 12) CYLINDER IP+BDF2
# -------------------

      #Deactivated because a mismatch in the accuracy
      #- name: Build NSCylinderIP_BDF2
      #  working-directory: ./Solver/test/NavierStokes/CylinderIP_BDF2/SETUP
      #  run: make MODE=${{matrix.mode}} COMPILER=${{matrix.compiler}} COMM=${{matrix.comm}} ENABLE_THREADS=${{matrix.enable_threads}} WITH_METIS=YES
      #
      #- name: Run NSCylinderIP_BDF2
      #  working-directory: ./Solver/test/NavierStokes/CylinderIP_BDF2
      #  run: mpiexec -n 8 ./horses3d.ns CylinderIP_BDF2.control

#
# 13) CYLINDER DIFFERENT ORDERS
# ----------------------------
#?
      - name: Build NSCylinderDifferentOrders
        working-directory: ./Solver/test/NavierStokes/CylinderDifferentOrders/SETUP
        run: |
          source /opt/intel/oneapi/setvars.sh || true
          make MODE=${{matrix.mode}} COMPILER=${{matrix.compiler}} COMM=${{matrix.comm}} ENABLE_THREADS=${{matrix.enable_threads}} WITH_METIS=YES
        if: '!cancelled()'

      - name: Run NSCylinderDifferentOrders
        working-directory: ./Solver/test/NavierStokes/CylinderDifferentOrders
        run: |
          source /opt/intel/oneapi/setvars.sh || true
          mpiexec -n 8 ./horses3d.ns CylinderDifferentOrders.control
        if: '!cancelled()'

#
# 14) ENTROPY CONSERVING TEST
# ----------------------------

      - name: Build EntropyConservingTest
        working-directory: ./Solver/test/NavierStokes/EntropyConservingTest/SETUP
        run: |
          source /opt/intel/oneapi/setvars.sh || true
          make MODE=${{matrix.mode}} COMPILER=${{matrix.compiler}} COMM=${{matrix.comm}} ENABLE_THREADS=${{matrix.enable_threads}} WITH_METIS=YES
        if: '!cancelled()'

      - name: Run EntropyConservingTest
        working-directory: ./Solver/test/NavierStokes/EntropyConservingTest
        run: |
          source /opt/intel/oneapi/setvars.sh || true
          mpiexec -n 8 ./horses3d.ns EntropyConservingTest.control
        if: '!cancelled()'

#
# 15) ENERGY CONSERVING TEST
# ----------------------------

      - name: Build EnergyConservingTest
        working-directory: ./Solver/test/NavierStokes/EnergyConservingTest/SETUP
        run: |
          source /opt/intel/oneapi/setvars.sh || true
          make MODE=${{matrix.mode}} COMPILER=${{matrix.compiler}} COMM=${{matrix.comm}} ENABLE_THREADS=${{matrix.enable_threads}} WITH_METIS=YES
        if: '!cancelled()'

      - name: Run EnergyConservingTest
        working-directory: ./Solver/test/NavierStokes/EnergyConservingTest
        run: |
          source /opt/intel/oneapi/setvars.sh || true
          mpiexec -n 8 ./horses3d.ns EnergyConservingTest.control
        if: '!cancelled()'

#
# 16) NACA0012 Steady
# ----------------------------

      - name: Build NACA0012
        working-directory: ./Solver/test/NavierStokes/NACA0012/SETUP
        run: |
          source /opt/intel/oneapi/setvars.sh || true
          make MODE=${{matrix.mode}} COMPILER=${{matrix.compiler}} COMM=${{matrix.comm}} ENABLE_THREADS=${{matrix.enable_threads}} WITH_METIS=YES
        if: '!cancelled()'

      - name: Run NACA0012
        working-directory: ./Solver/test/NavierStokes/NACA0012
        run: |
          source /opt/intel/oneapi/setvars.sh || true
          mpiexec -n 8 ./horses3d.ns NACA0012.control
        if: '!cancelled()'

#
# 17) NACA0012 Unsteady Dual Time Stepping
# ----------------------------

      - name: Build DualTimeStepping
        working-directory: ./Solver/test/NavierStokes/DualTimeStepping/SETUP
        run: |
          source /opt/intel/oneapi/setvars.sh || true
          make MODE=${{matrix.mode}} COMPILER=${{matrix.compiler}} COMM=${{matrix.comm}} ENABLE_THREADS=${{matrix.enable_threads}} WITH_METIS=YES
        if: '!cancelled()'

      - name: Run DualTimeStepping
        working-directory: ./Solver/test/NavierStokes/DualTimeStepping
        run: |
          source /opt/intel/oneapi/setvars.sh || true
          mpiexec -n 8 ./horses3d.ns DualTimeStepping.control
        if: '!cancelled()'

#
# 18) Manufactured Solution for Spalart-Almaras
# ----------------------------

#      - name: Build ManufacturedSolutionsSA
#        working-directory: ./Solver/test/NavierStokes/ManufacturedSolutionsSA/SETUP
#        run: |
#          source /opt/intel/oneapi/setvars.sh || true
#          make MODE=${{matrix.mode}} COMPILER=${{matrix.compiler}} COMM=${{matrix.comm}} ENABLE_THREADS=${{matrix.enable_threads}} WITH_METIS=YES
#
#      - name: Run ManufacturedSolutionsSA
#        working-directory: ./Solver/test/NavierStokes/ManufacturedSolutionsSA
#        run: |
#          source /opt/intel/oneapi/setvars.sh || true
#          mpiexec -n 8 ./horses3d.nssa MSSA.control

#
# 19) Flat-Plate test case for Spalart-Almaras
# ----------------------------

#      - name: Build FlatPlateSA
#        working-directory: ./Solver/test/NavierStokes/FlatPlateSA/SETUP
#        run: |
#          source /opt/intel/oneapi/setvars.sh || true
#          make MODE=${{matrix.mode}} COMPILER=${{matrix.compiler}} COMM=${{matrix.comm}} ENABLE_THREADS=${{matrix.enable_threads}} WITH_METIS=YES
#
#      - name: Run FlatPlateSA
#        working-directory: ./Solver/test/NavierStokes/FlatPlateSA
#        run: |
#          source /opt/intel/oneapi/setvars.sh || true
#          mpiexec -n 8 ./horses3d.nssa FlatPlate.control

#
# 20) Numerical Jacobian for BIRK5
# --------------------------------

# Does not work with this setup 02/01/2023
#      - name: Build BIRK5NumJac
#        working-directory: ./Solver/test/NavierStokes/BIRK5NumJac/SETUP
#        run: |
#          source /opt/intel/oneapi/setvars.sh || true
#          make MODE=${{matrix.mode}} COMPILER=${{matrix.compiler}} COMM=${{matrix.comm}} ENABLE_THREADS=${{matrix.enable_threads}} WITH_METIS=YES
#      - name: Run BIRK5NumJac
#        working-directory: ./Solver/test/NavierStokes/BIRK5NumJac
#        run: |
#          source /opt/intel/oneapi/setvars.sh || true
#          mpiexec -n 8 ./horses3d.ns Cylinder.control
#
# 21) Forward facing step with SVV
# --------------------------------

      - name: Build ForwardFacingStepSVV
        working-directory: ./Solver/test/NavierStokes/ForwardFacingStepSVV/SETUP
        run: |
          source /opt/intel/oneapi/setvars.sh || true
          make MODE=${{matrix.mode}} COMPILER=${{matrix.compiler}} COMM=${{matrix.comm}} ENABLE_THREADS=${{matrix.enable_threads}} WITH_METIS=YES
        if: '!cancelled()'

      - name: Run ForwardFacingStepSVV
        working-directory: ./Solver/test/NavierStokes/ForwardFacingStepSVV
        run: |
          source /opt/intel/oneapi/setvars.sh || true
          mpiexec -n 8 ./horses3d.ns FFS_SVV.control
        if: '!cancelled()'

#
# 20) Forward facing step with SSPRK33 and limiter
# ------------------------------------------------

      - name: Build ForwardFacingStep_SSPRK33
        working-directory: ./Solver/test/NavierStokes/ForwardFacingStep_SSPRK33/SETUP
        run: |
          source /opt/intel/oneapi/setvars.sh || true
          make MODE=${{matrix.mode}} COMPILER=${{matrix.compiler}} COMM=${{matrix.comm}} ENABLE_THREADS=${{matrix.enable_threads}} WITH_METIS=YES
        if: '!cancelled()'

      - name: Run ForwardFacingStep_SSPRK33
        working-directory: ./Solver/test/NavierStokes/ForwardFacingStep_SSPRK33
        run: |
          source /opt/intel/oneapi/setvars.sh || true
          mpiexec -n 8 ./horses3d.ns FFS_SSPRK33.control
        if: '!cancelled()'

#
# 21) Forward facing step with SSPRK43
# ------------------------------------

      - name: Build ForwardFacingStep_SSPRK43
        working-directory: ./Solver/test/NavierStokes/ForwardFacingStep_SSPRK43/SETUP
        run: |
          source /opt/intel/oneapi/setvars.sh || true
          make MODE=${{matrix.mode}} COMPILER=${{matrix.compiler}} COMM=${{matrix.comm}} ENABLE_THREADS=${{matrix.enable_threads}} WITH_METIS=YES
        if: '!cancelled()'

      - name: Run ForwardFacingStep_SSPRK43
        working-directory: ./Solver/test/NavierStokes/ForwardFacingStep_SSPRK43
        run: |
          source /opt/intel/oneapi/setvars.sh || true
          mpiexec -n 8 ./horses3d.ns FFS_SSPRK43.control
        if: '!cancelled()'

#
# 22) Taylor-Green vortex with SVV-LES
# ------------------------------------

      - name: Build TaylorGreenSVVLES
        working-directory: ./Solver/test/NavierStokes/TaylorGreenSVVLES/SETUP
        run: |
          source /opt/intel/oneapi/setvars.sh || true
          make MODE=${{matrix.mode}} COMPILER=${{matrix.compiler}} COMM=${{matrix.comm}} ENABLE_THREADS=${{matrix.enable_threads}} WITH_METIS=YES
        if: '!cancelled()'

      - name: Run TaylorGreenSVVLES
        working-directory: ./Solver/test/NavierStokes/TaylorGreenSVVLES
        run: |
          source /opt/intel/oneapi/setvars.sh || true
          mpiexec -n 8 ./horses3d.ns TaylorGreenSVVLES.control
        if: '!cancelled()'

#
# 23) IBM CYLINDER 
# -------------------

      - name: Build IBM_Cylinder
        working-directory: ./Solver/test/NavierStokes/IBM_Cylinder/SETUP
        run: |
          source /opt/intel/oneapi/setvars.sh || true
          make MODE=${{matrix.mode}} COMPILER=${{matrix.compiler}} COMM=${{matrix.comm}} ENABLE_THREADS=${{matrix.enable_threads}} WITH_METIS=YES
        if: '!cancelled()'
     
      - name: Run IBM_Cylinder
        working-directory: ./Solver/test/NavierStokes/IBM_Cylinder
        run: |
          source /opt/intel/oneapi/setvars.sh || true
          mpiexec -n 8 ./horses3d.ns IBMCylinder.control
        if: '!cancelled()'
          
#
# 24) Mach 2 cylinder with GMM shock capturing
# --------------------------------------------

      - name: Build CylinderGMM
        working-directory: ./Solver/test/NavierStokes/CylinderGMM/SETUP
        run: |
          source /opt/intel/oneapi/setvars.sh || true
          make MODE=${{matrix.mode}} COMPILER=${{matrix.compiler}} COMM=${{matrix.comm}} ENABLE_THREADS=${{matrix.enable_threads}} WITH_METIS=YES
        if: '!cancelled()'

      - name: Run CylinderGMM
        working-directory: ./Solver/test/NavierStokes/CylinderGMM
        run: |
          source /opt/intel/oneapi/setvars.sh || true
          mpiexec -n 8 ./horses3d.ns CylinderGMM.control
        if: '!cancelled()'

#
# 25) Cylinder with Reinforcement Learning p-adaptation
# --------------------------------------------
      - name: Build Cylinder_pAdaptationRL
        working-directory: ./Solver/test/NavierStokes/Cylinder_pAdaptationRL/SETUP
        run: |
          source /opt/intel/oneapi/setvars.sh || true
          make MODE=${{matrix.mode}} COMPILER=${{matrix.compiler}} COMM=${{matrix.comm}} ENABLE_THREADS=${{matrix.enable_threads}} WITH_METIS=YES
        if: '!cancelled()'

      - name: Run Cylinder_pAdaptationRL
        working-directory: ./Solver/test/NavierStokes/Cylinder_pAdaptationRL
        run: |
          source /opt/intel/oneapi/setvars.sh || true
          mpiexec -n 8 ./horses3d.ns Cylinder_pAdaptationRL.control
        if: '!cancelled()'

########################################################################
#######                       PARTICLES                         ########
########################################################################

#
# 1) FreeFall
# ----------------------------
      # - name: Build ParticlesFreeFall
      #   working-directory: ./Solver/test/Particles/FreeFall/SETUP
      #   run: |
      #     source /opt/intel/oneapi/setvars.sh || true
      #     make MODE=${{matrix.mode}} COMPILER=${{matrix.compiler}} COMM=${{matrix.comm}} ENABLE_THREADS=${{matrix.enable_threads}} WITH_METIS=YES

      # - name: Run ParticlesFreeFall
      #   working-directory: ./Solver/test/Particles/FreeFall
      #   run: |
      #     source /opt/intel/oneapi/setvars.sh || true
      #     mpiexec -n 8 ./horses3d.ns FreeFall.control
#
# 2) RadiationHeat
# ----------------------------
      # - name: Build RadiationHeat
      #   working-directory: ./Solver/test/Particles/RadiationHeat/SETUP
      #   run: |
      #     source /opt/intel/oneapi/setvars.sh || true
      #     make MODE=${{matrix.mode}} COMPILER=${{matrix.compiler}} COMM=${{matrix.comm}} ENABLE_THREADS=${{matrix.enable_threads}} WITH_METIS=YES

      # - name: Run RadiationHeat
      #   working-directory: ./Solver/test/Particles/RadiationHeat
      #   run: |
      #     source /opt/intel/oneapi/setvars.sh || true
      #     mpiexec -n 8 ./horses3d.ns RadiationHeat.control
#
# 3) ParticleBoundaryCond
# ----------------------------
      # This case uses a random initialization that affects the solution.
      # it only works in Alderaan
      # - name: Build ParticleBoundaryCond
      #   working-directory: ./Solver/test/Particles/ParticleBoundaryCond/SETUP
      #   run: make MODE=${{matrix.mode}} COMPILER=${{matrix.compiler}} COMM=${{matrix.comm}} ENABLE_THREADS=${{matrix.enable_threads}} WITH_METIS=YES

      # - name: Run ParticleBoundaryCond
      #   working-directory: ./Solver/test/Particles/ParticleBoundaryCond
      #   run: mpiexec -n 8 ./horses3d.ns BoundaryCondition.control
#
# 4) Pouransari
# ----------------------------
      # This case uses a random initialization that affects the solution.
      # it only works in Alderaan
      # - name: Build Pouransari
      #   working-directory: ./Solver/test/Particles/Pouransari/SETUP
      #   run: make MODE=${{matrix.mode}} COMPILER=${{matrix.compiler}} COMM=${{matrix.comm}} ENABLE_THREADS=${{matrix.enable_threads}} WITH_METIS=YES

      # - name: Run Pouransari
      #   working-directory: ./Solver/test/Particles/Pouransari
      #   run: mpiexec -n 8 ./horses3d.ns Pouransari_St0041.control

# ########################################################################
# #######                       MULTIPHASE                        ########
# ########################################################################
# #
# # 1) Convergence
# # ----------------------------

#       - name: Build MultiphaseConvergence
#         working-directory: ./Solver/test/Multiphase/Convergence/SETUP
#         run: |
#           source /opt/intel/oneapi/setvars.sh || true
#           make MODE=${{matrix.mode}} COMPILER=${{matrix.compiler}} COMM=${{matrix.comm}} ENABLE_THREADS=${{matrix.enable_threads}} WITH_METIS=YES

#       - name: Run MultiphaseConvergence
#         working-directory: ./Solver/test/Multiphase/Convergence
#         run: |
#           source /opt/intel/oneapi/setvars.sh || true
#           mpiexec -n 8 ./horses3d.mu Convergence.control

# #
# # 2) RisingBubble
# # ----------------------------

#       - name: Build MultiphaseRisingBubble
#         working-directory: ./Solver/test/Multiphase/RisingBubble/SETUP
#         run: |
#           source /opt/intel/oneapi/setvars.sh || true
#           make MODE=${{matrix.mode}} COMPILER=${{matrix.compiler}} COMM=${{matrix.comm}} ENABLE_THREADS=${{matrix.enable_threads}} WITH_METIS=YES

#       - name: Run MultiphaseRisingBubble
#         working-directory: ./Solver/test/Multiphase/RisingBubble
#         run: |
#           source /opt/intel/oneapi/setvars.sh || true
#           mpiexec -n 8 ./horses3d.mu RisingBubble.control

# #
# # 3) Pipe
# # ----------------------------

#       - name: Build MultiphasePipe
#         working-directory: ./Solver/test/Multiphase/Pipe/SETUP
#         run: |
#           source /opt/intel/oneapi/setvars.sh || true
#           make MODE=${{matrix.mode}} COMPILER=${{matrix.compiler}} COMM=${{matrix.comm}} ENABLE_THREADS=${{matrix.enable_threads}} WITH_METIS=YES

#       - name: Run MultiphasePipe
#         working-directory: ./Solver/test/Multiphase/Pipe
#         run: |
#           source /opt/intel/oneapi/setvars.sh || true
#           mpiexec -n 8 ./horses3d.mu AnnularFlow.control

# #
# # 4) Entropy conserving test
# # ----------------------------

#       - name: Build MultiphaseEntropyConservingTest
#         working-directory: ./Solver/test/Multiphase/EntropyConservingTest/SETUP
#         run: |
#           source /opt/intel/oneapi/setvars.sh || true
#           make MODE=${{matrix.mode}} COMPILER=${{matrix.compiler}} COMM=${{matrix.comm}} ENABLE_THREADS=${{matrix.enable_threads}} WITH_METIS=YES

#       - name: Run MultiphaseEntropyConservingTest
#         working-directory: ./Solver/test/Multiphase/EntropyConservingTest
#         run: |
#           source /opt/intel/oneapi/setvars.sh || true
#           mpiexec -n 8 ./horses3d.mu EntropyConservingTest.control

# ########################################################################
# #######                      INCOMPRESSIBLE                     ########
# ########################################################################
# #
# # 1) Convergence
# # ----------------------------

#       - name: Build iNS-Convergence
#         working-directory: ./Solver/test/IncompressibleNS/Convergence/SETUP
#         run: |
#           source /opt/intel/oneapi/setvars.sh || true
#           make MODE=${{matrix.mode}} COMPILER=${{matrix.compiler}} COMM=${{matrix.comm}} ENABLE_THREADS=${{matrix.enable_threads}} WITH_METIS=YES

#       - name: Run iNS-Convergence
#         working-directory: ./Solver/test/IncompressibleNS/Convergence
#         run: |
#           source /opt/intel/oneapi/setvars.sh || true
#           mpiexec -n 8 ./horses3d.ins Convergence4P5.control

# #
# # 2) Kovasznay
# # ----------------------------

#       - name: Build iNS-Kovasznay
#         working-directory: ./Solver/test/IncompressibleNS/Kovasznay/SETUP
#         run: |
#           source /opt/intel/oneapi/setvars.sh || true
#           make MODE=${{matrix.mode}} COMPILER=${{matrix.compiler}} COMM=${{matrix.comm}} ENABLE_THREADS=${{matrix.enable_threads}} WITH_METIS=YES

#       - name: Run iNS-Kovasznay
#         working-directory: ./Solver/test/IncompressibleNS/Kovasznay
#         run: |
#           source /opt/intel/oneapi/setvars.sh || true
#           mpiexec -n 8 ./horses3d.ins Kovasznay.control

# #
# # 3) Lid driven cavity
# # ----------------------------

#       - name: Build iNS-LidDrivenCavity
#         working-directory: ./Solver/test/IncompressibleNS/LidDrivenCavity/SETUP
#         run: |
#           source /opt/intel/oneapi/setvars.sh || true
#           make MODE=${{matrix.mode}} COMPILER=${{matrix.compiler}} COMM=${{matrix.comm}} ENABLE_THREADS=${{matrix.enable_threads}} WITH_METIS=YES

#       - name: Run iNS-LidDrivenCavity
#         working-directory: ./Solver/test/IncompressibleNS/LidDrivenCavity
#         run: |
#           source /opt/intel/oneapi/setvars.sh || true
#           mpiexec -n 8 ./horses3d.ins LidDrivenCavity.control

# #
# # 4) Taylor Green Vortex
# # ----------------------------

#       - name: Build iNS-TaylorGreenVortex
#         working-directory: ./Solver/test/IncompressibleNS/TaylorGreen/SETUP
#         run: |
#           source /opt/intel/oneapi/setvars.sh || true
#           make MODE=${{matrix.mode}} COMPILER=${{matrix.compiler}} COMM=${{matrix.comm}} ENABLE_THREADS=${{matrix.enable_threads}} WITH_METIS=YES

#       - name: Run iNS-TaylorGreenVortex
#         working-directory: ./Solver/test/IncompressibleNS/TaylorGreen
#         run: |
#           source /opt/intel/oneapi/setvars.sh || true
#           mpiexec -n 8 ./horses3d.ins TaylorGreen.control

# #
# # 5) Rayleigh-Taylor instability
# # -------------------------------

#       - name: Build iNS-RayleighTaylor
#         working-directory: ./Solver/test/IncompressibleNS/RayleighTaylor/SETUP
#         run: |
#           source /opt/intel/oneapi/setvars.sh || true
#           make MODE=${{matrix.mode}} COMPILER=${{matrix.compiler}} COMM=${{matrix.comm}} ENABLE_THREADS=${{matrix.enable_threads}} WITH_METIS=YES

#       - name: Run iNS-RayleighTaylor
#         working-directory: ./Solver/test/IncompressibleNS/RayleighTaylor
#         run: |
#           source /opt/intel/oneapi/setvars.sh || true
#           mpiexec -n 8 ./horses3d.ins RayleighTaylor.control