diff --git a/.github/workflows/docker_build_tpls.yml b/.github/workflows/docker_build_tpls.yml index 5df5c9f1..6f4561f0 100644 --- a/.github/workflows/docker_build_tpls.yml +++ b/.github/workflows/docker_build_tpls.yml @@ -61,13 +61,20 @@ jobs: DOCKER_REPOSITORY: geosx/rockylinux8-clang17-cuda12.5 TPL_DOCKERFILE: docker/tpl-rockylinux-clang-cuda-12.Dockerfile RUNS_ON: Runner_4core_16GB - - name: Pangea 3 (AlmaLinux 8.8, gcc 9.4.0, open-mpi 4.1.2, cuda 11.5.0, openblas 0.3.10) - DOCKER_REPOSITORY: geosx/pangea3-almalinux8-gcc9.4-openmpi4.1.2-cuda11.5.0-openblas0.3.18 - TPL_DOCKERFILE: docker/TotalEnergies/Pangea3.Dockerfile - HOST_CONFIG: host-configs/TOTAL/pangea3-gcc8.4.1-openmpi-4.1.2.cmake - DOCKER_ROOT_IMAGE: 7g8efcehpff/pangea-almalinux8-gcc9.4-openmpi4.1.2-cuda11.5.0-openblas0.3.18:4 + - name: Pangea 3 (AlmaLinux 8.8, gcc 11.4.0, open-mpi 4.1.6, cuda 11.8.0, openblas 0.3.10) + DOCKER_REPOSITORY: geosx/pangea3-almalinux8-gcc11.4.0-openmpi4.1.6-cuda11.8.0-openblas0.3.18 + TPL_DOCKERFILE: docker/TotalEnergies/pangea-3/pangea3-tpl.Dockerfile + DOCKER_ROOT_IMAGE: onetechssc/pangea3:almalinux8-gcc11.4.0-openmpi4.1.6-cuda11.8.0-openblas0.3.18_v2.0 + HOST_CONFIG: host-configs/TotalEnergies/pangea-3/pangea3-gcc11.4.0-openmpi-4.1.6.cmake HOST_ARCH: ppc64le RUNS_ON: streak2 + - name: Pangea 4 (redhat 8, gcc 12.1.0, hpcx 2.20.0, mkl 2023.2.0) + DOCKER_REPOSITORY: geosx/pangea4-gcc12.1-hpcxompi2.20.0-onemkl2023.2.0 + TPL_DOCKERFILE: docker/TotalEnergies/pangea-4/pangea4-tpl.Dockerfile + DOCKER_ROOT_IMAGE: onetechssc/pangea4:gcc12.1-hpcxompi2.20.0-onemkl2023.2.0_v1.0 + HOST_CONFIG: host-configs/TotalEnergies/pangea-4/pangea4-gcc12.1-hpcxompi2.20.0-onemkl2023.2.0.cmake + INSTALL_DIR_ROOT: /workrd/SCR/NUM/GEOS_environment/p4/install/tpl + RUNS_ON: ubuntu-latest steps: - name: Checkout diff --git a/docker/Stanford/Sherlock.Dockerfile b/docker/Stanford/Sherlock.Dockerfile index ab3992f6..9eb0c2f5 100644 --- a/docker/Stanford/Sherlock.Dockerfile +++ b/docker/Stanford/Sherlock.Dockerfile @@ -1,6 +1,5 @@ # This Dockerfile aims at reproducing (some part of) the SHERLOCK environment. -# Please see TotalEnergies/Pangea2.Dockerfile for fully commented version - +# # While loading module ompi4.1.2 on Sherlock, the following are aut-loaded: # UCX/1.12.1 # LIBFABRIC/1.14.0 diff --git a/docker/TotalEnergies/Dockerfile b/docker/TotalEnergies/Dockerfile deleted file mode 100644 index 929cd081..00000000 --- a/docker/TotalEnergies/Dockerfile +++ /dev/null @@ -1,58 +0,0 @@ -# NOTE: see docker/tpl-ubuntu-gcc.Dockerfile for detailed comments -ARG TMP_DIR=/tmp -ARG SRC_DIR=$TMP_DIR/thirdPartyLibs -ARG BLD_DIR=$TMP_DIR/build - -# The docker base image can be pecan or pangea. -ARG DOCKER_ROOT_IMAGE -FROM $DOCKER_ROOT_IMAGE as tpl_toolchain_intersect_geosx_toolchain -ARG SRC_DIR - -ARG INSTALL_DIR -ENV GEOSX_TPL_DIR=$INSTALL_DIR - -RUN yum install -y \ - ca-certificates \ - curl \ - python3 \ - zlib-devel - -RUN --mount=src=.,dst=$SRC_DIR $SRC_DIR/docker/install-cmake.sh - -FROM tpl_toolchain_intersect_geosx_toolchain AS tpl_toolchain -ARG SRC_DIR -ARG BLD_DIR - -RUN yum install -y \ - make \ - bc \ - file \ - bison \ - flex \ - patch - -ARG HOST_CONFIG - -RUN --mount=src=.,dst=$SRC_DIR $SRC_DIR/docker/configure-tpl.sh -WORKDIR $BLD_DIR -RUN --mount=src=.,dst=$SRC_DIR make - -FROM tpl_toolchain_intersect_geosx_toolchain AS geosx_toolchain -ARG SRC_DIR - -COPY --from=tpl_toolchain $GEOSX_TPL_DIR $GEOSX_TPL_DIR - -RUN yum -y install \ - openssh-clients \ - ca-certificates \ - curl \ - python3 \ - texlive \ - graphviz \ - libxml2 \ - git - -RUN --mount=src=.,dst=$SRC_DIR $SRC_DIR/docker/install-ninja.sh - -RUN --mount=src=.,dst=$SRC_DIR $SRC_DIR/docker/install-sccache.sh -ENV SCCACHE=/opt/sccache/bin/sccache diff --git a/docker/TotalEnergies/Pangea3.Dockerfile b/docker/TotalEnergies/Pangea3.Dockerfile deleted file mode 100644 index 974f79dd..00000000 --- a/docker/TotalEnergies/Pangea3.Dockerfile +++ /dev/null @@ -1,38 +0,0 @@ -# Temporary local variables dedicated to the TPL build -ARG TMP_DIR=/tmp -ARG SRC_DIR=$TMP_DIR/thirdPartyLibs -ARG BLD_DIR=$TMP_DIR/build - -# The docker base image has to be pangea3-almalinux8-* -ARG DOCKER_ROOT_IMAGE - -FROM $DOCKER_ROOT_IMAGE as tpl_toolchain_intersect_geosx_toolchain -ARG SRC_DIR -ARG BLD_DIR - -# All the environment variables defined in this Dockerfile -# (GEOSX_TPL_DIR but also compiler information like CC, CXX...) -# are part of the image contract (otherwise ARG is used). -# GEOSX use them so consider modifying their names with care. -# -# The installation directory is provided as a docker build argument. -# We forward it using an environment variable. -ARG INSTALL_DIR -ENV GEOSX_TPL_DIR=$INSTALL_DIR - -# Get host config file from docker build arguments -ARG HOST_CONFIG - -FROM tpl_toolchain_intersect_geosx_toolchain AS tpl_toolchain -# We now configure the build... -RUN --mount=src=.,dst=$SRC_DIR $SRC_DIR/docker/configure-tpl.sh -# ... before we compile the TPLs! -WORKDIR $BLD_DIR -RUN --mount=src=.,dst=$SRC_DIR make - -# Extract only TPL's from previous stage -FROM tpl_toolchain_intersect_geosx_toolchain AS geosx_toolchain - -COPY --from=tpl_toolchain $GEOSX_TPL_DIR $GEOSX_TPL_DIR - -ENV SCCACHE=/opt/sccache/bin/sccache diff --git a/docker/TotalEnergies/README.md b/docker/TotalEnergies/README.md new file mode 100644 index 00000000..30eba182 --- /dev/null +++ b/docker/TotalEnergies/README.md @@ -0,0 +1,20 @@ +# TotalEnergies Dockerfiles + +## Overview + +This directory contains several Dockerfiles used for building and deploying the TPLs. +Below is a table summarizing the purpose and key characteristics of each Dockerfile. + +| Dockerfile | Description | Base Image | Key Components Installed | +|------------------------------------------|-------------------------------------------------------------------------------|-------------------------------|----------------------------------------------------------------------| +| `pangea3-gcc-openmpi-openblas.Dockerfile`| Builds a toolchain with GCC, OpenMPI, CUDA, and OpenBLAS for GEOS on Pangea 3.| `ppc64le/almalinux:8` | GCC, OpenMPI, CUDA, OpenBLAS, CMake, Python3.8 | +| `pangea3-tpl.Dockerfile` | Builds third-party libraries for GEOS on Pangea 3 environment. | `pangea3-gcc-openmpi-openblas`| GCC, OpenMPI, CUDA, OpenBLAS, CMake, Python3.8, TPLs | +| `pangea4-gcc.Dockerfile` | Builds a toolchain with GCC, CMake and Python on Pangea 4. | `spack/centos-stream` | GCC, CMake, Python3.11, Wget | +| `pangea4-gcc-hpcxompi-onemkl.Dockerfile` | Builds a toolchain with GCC, HPCXOMPI and ONEMKL on Pangea 4. | `pangea4-gcc` | GCC, CMake, Python3.11, Wget, HPCXOMPI, ONEMKL | +| `pangea4-tpl.Dockerfile` | Builds third-party libraries for GEOS on Pangea 4 environment. | `pangea4-gcc-hpcxompi-onemkl` | GCC, CMake, Python3.11, Wget, HPCXOMPI, ONEMKL, ninja, sccache, TPLs | + +## DockerHub + +The Docker images built from the ${cluster}.Dockerfile are available on DockerHub under for: + - pangea3 cluster : [7g8efcehpff/pangea-almalinux8-gcc9.4-openmpi4.1.2-cuda11.5.0-openblas0.3.18](https://hub.docker.com/r/7g8efcehpff/pangea-almalinux8-gcc9.4-openmpi4.1.2-cuda11.5.0-openblas0.3.18), will move to onetechssc/pangea3 once newer version of gcc is available. + - pangea4 cluster : [onetechssc/pangea4](https://hub.docker.com/r/onetechssc/pangea4) diff --git a/docker/TotalEnergies/Pangea3-base.Dockerfile b/docker/TotalEnergies/pangea-3/pangea3-gcc-openmpi-openblas.Dockerfile similarity index 69% rename from docker/TotalEnergies/Pangea3-base.Dockerfile rename to docker/TotalEnergies/pangea-3/pangea3-gcc-openmpi-openblas.Dockerfile index d3510d82..fdb1cb61 100644 --- a/docker/TotalEnergies/Pangea3-base.Dockerfile +++ b/docker/TotalEnergies/pangea-3/pangea3-gcc-openmpi-openblas.Dockerfile @@ -1,10 +1,48 @@ -# This Dockerfile is used to build a docker image reproducing the Pangea installation over a ppc64le architecture: -# It is not directly callable by the TPL ci but the built image is. - # syntax=docker/dockerfile:1 -FROM ppc64le/almalinux:8 -# Install other needed packages +####################################### +# Pangea 3 image : gcc - openmpi - openblas +####################################### +# +# Installs : +# - gcc = 11.4.0 +# - cmake = 3.27.9 +# - openmpi = 4.1.6 +# - openblas = 0.3.18 +# - cuda = 11.8.0 +# +####################################### +# +# Description : +# - this Dockerfile is used to build a docker image reproducing the Pangea-3 installation over a ppc64le architecture: +# - it is not directly callable by the TPL ci but the built image is. +# - the image is based on ppc64le/almalinux:8 +# - this image is deployed as onetechssc/pangea3:almalinux8-gcc11.4.0-openmpi4.1.6-cuda11.8.0-openblas0.3.18_v2.0 +# - gcc, cmake, openmpi, openblas and cuda are copied from the tarball directory of Pangea 3 +# +# Usage : +# build the image: +# - copy the tarball directory from the Pangea 3 repository to the current directory +# - podman build -f pangea3-gcc-openmpi-openblas.Dockerfile -t onetechssc/pangea3:almalinux8-gcc11.4.0-openmpi4.1.6-cuda11.8.0-openblas0.3.18_v2.0 -v /etc/pki/ca-trust/extracted/pem/:/etc/pki/ca-trust/extracted/pem/ +# run the image: +# - podman run -it --detach --privileged --name pangea3_gcc_ompi_oblas -v /etc/pki/ca-trust/extracted/pem/:/etc/pki/ca-trust/extracted/pem localhost/onetechssc/pangea3:almalinux8-gcc11.4.0-openmpi4.1.6-cuda11.8.0-openblas0.3.18_v2.0 +# - podman exec -it pangea3_gcc_ompi_oblas /bin/bash +# push the image (requires to be part of the onetechssc docker organization): +# - podman login docker.io +# - podman push pangea3:almalinux8-gcc11.4.0-openmpi4.1.6-cuda11.8.0-openblas0.3.18_v2.0 docker://docker.io/onetechssc/pangea3:almalinux8-gcc11.4.0-openmpi4.1.6-cuda11.8.0-openblas0.3.18_v2.0 +# +####################################### + +FROM ppc64le/almalinux:8 AS pangea3 + +# ------ +# LABELS +LABEL description="Pangea 3 image : gcc - cmake - openmpi - openblas - cuda" +LABEL version="2.0" +LABEL maintainer="TotalEnergies HPC Team" + +# ------ +# INSTALL BASE PACKAGES RUN dnf install -y \ # gcc deps \ libmpc-devel.ppc64le glibc-devel \ @@ -27,31 +65,32 @@ RUN dnf install -y \ ## Temporary local variables needed buy several modules ARG MODULE_PATH="/data_local/sw" - -ARG SPACK_PATH="spack/0.17.0/opt/spack/linux-rhel8-power9le" +ARG SPACK_PATH="spack/0.22.1/opt/spack/linux-rhel8-power9le" ARG COMPILER="gcc" ARG DEFAULT_COMPILER_VER="8.4.1" ARG SPACK_COMPILER_VER=$DEFAULT_COMPILER_VER -ARG MODULE_COMPILER_VER="9.4.0" +ARG MODULE_COMPILER_VER="11.4.0" ARG SPACK_COMPILER=$COMPILER-$SPACK_COMPILER_VER ## liblustre COPY ./tarball/liblustreapi.so.1 /lib64/ -## CMake +# ------ +# CMAKE ADD ./tarball/cmake-*.tgz / ### Environment variables to export -ENV PATH="/data_local/appli_local/MTS/GEOSX/cmake/3.26.4/bin:${PATH}" +ENV PATH="$MODULE_PATH/$SPACK_PATH/$SPACK_COMPILER/cmake-3.27.9-yfuovjb3tx73ymsxuw5hoxv3eqdchned/bin:${PATH}" -## gcc +# ------ +# GCC ADD ./tarball/gcc-*.tgz / ### Temporary local variables ARG GCC_VER=$COMPILER-$MODULE_COMPILER_VER -ARG GCC_DIR="$GCC_VER-xe5cqnyajaqz75up3gflln5zlj2rue5v" +ARG GCC_DIR="$GCC_VER-bbeypfg5smd3pgbsdswprcja7cxdxyqn" ### Environment variables to export ENV CPATH="$MODULE_PATH/$SPACK_PATH/$SPACK_COMPILER/$GCC_DIR/include:${CPATH}" \ @@ -68,11 +107,12 @@ $MODULE_PATH/$SPACK_PATH/$SPACK_COMPILER/$GCC_DIR/lib64:${LD_RUN_PATH}" \ FC=gfortran \ GCC_ROOT=$MODULE_PATH/$SPACK_PATH/$SPACK_COMPILER/$GCC_DIR -## ompi +# ------ +# OMPI ADD ./tarball/ompi-*.tgz / ### Temporary local variables -ARG MPI_VER="4.1.2" +ARG MPI_VER="4.1.6" ARG MPI_DIR="openmpi/$MPI_VER" ARG MPI_PREFIX="OMPI" @@ -92,14 +132,15 @@ ENV CPATH="$MODULE_PATH/$MPI_DIR/env/$SPACK_COMPILER/include:${CPATH}" \ OMPI_MCA_btl_openib_allow_ib="true" \ OMPI_MCA_btl_openib_warn_default_gid_prefix="0" -## Cuda +# ------ +# CUDA ADD ./tarball/cuda-*.tgz / ### Temporary local variables ARG LIBCUDA_VER=450.156.00 ARG CUDA_VER_MAJ="11" -ARG CUDA_VER_MIN="5" +ARG CUDA_VER_MIN="8" ARG CUDA_VER_PATCH="0" ARG CUDA_VER="$CUDA_VER_MAJ.$CUDA_VER_MIN.$CUDA_VER_PATCH" ARG CUDA_DIR="cuda/$CUDA_VER" @@ -128,7 +169,8 @@ $MODULE_PATH/$CUDA_DIR/samples/bin/ppc64le/linux/release:${PATH}" \ CUDA_VERSION="$CUDA_VER_MAJ.$CUDA_VER_MIN" \ NVHPC_CUDA_HOME="$MODULE_PATH/$CUDA_DIR" -## Openblas +# ------ +# OPENBLAS ADD ./tarball/openblas-*.tgz / ### Temporary local variables @@ -139,7 +181,7 @@ ARG BLAS_VER_MIN="3" ARG BLAS_VER_PATCH="18" ARG BLAS_DISTRIB="openblas" ARG BLAS_VER="$BLAS_VER_MAJ.$BLAS_VER_MIN.$BLAS_VER_PATCH" -ARG BLAS_DIR="$BLAS_DISTRIB-$BLAS_VER-vk36pzksytuhylqesg4cca7667np5sjp" +ARG BLAS_DIR="$BLAS_DISTRIB-$BLAS_VER-cing5yuan7hsn23qmeemon4zwih3k2hd" ### Environment variables to export ENV LD_LIBRARY_PATH="$MODULE_PATH/$SPACK_PATH/$SPACK_COMPILER/$BLAS_DIR/lib:${LD_LIBRARY_PATH}" \ @@ -147,7 +189,8 @@ ENV LD_LIBRARY_PATH="$MODULE_PATH/$SPACK_PATH/$SPACK_COMPILER/$BLAS_DIR/lib:${LD PATH="$MODULE_PATH/$SPACK_PATH/$SPACK_COMPILER/$BLAS_DIR/bin:${PATH}" \ OPENBLAS_ROOT="$MODULE_PATH/$SPACK_PATH/$SPACK_COMPILER/$BLAS_DIR" -## lsf +# ------ +# LSF ADD ./tarball/lsf-*.tgz / ### Temporary local variables @@ -174,7 +217,8 @@ ENV LD_LIBRARY_PATH="$MODULE_PATH/$LSF_DIR/lib:${LD_LIBRARY_PATH}" \ # make -C build-pangea3-gcc8.4.1-openmpi-4.1.2-release -j && \ # cd .. && rm -rf thirdPartyLibs -# Install tools needed by geos ci +# ------ +# CI TOOLS RUN dnf -y --enablerepo=powertools install \ ninja-build \ openssh-clients \ @@ -183,7 +227,7 @@ RUN dnf -y --enablerepo=powertools install \ libxml2 # build sccache from source as prebuilt binary is not available for current archi / OS couple -RUN dnf makecache --refresh && dnf -y install cargo openssl-devel +RUN dnf clean all && dnf makecache --refresh && dnf -y install cargo openssl-devel RUN cargo install sccache --locked && mkdir -p /opt/sccache/ && cp -r /root/.cargo/bin /opt/sccache/ RUN dnf remove -y cargo openssl-devel diff --git a/docker/TotalEnergies/pangea-3/pangea3-tpl.Dockerfile b/docker/TotalEnergies/pangea-3/pangea3-tpl.Dockerfile new file mode 100644 index 00000000..7ee88321 --- /dev/null +++ b/docker/TotalEnergies/pangea-3/pangea3-tpl.Dockerfile @@ -0,0 +1,81 @@ +# syntax=docker/dockerfile:1 + +####################################### +# Pangea 3 tpl image +# +# Description : +# - generic image for building geos tpls on Pangea 3 environments +# - the docker base image can be any pangea3 docker file +# +# Usage : +# build the image (from the root of the repository!): +# - podman build --format docker --progress=plain \ +# --build-arg HOST_CONFIG=host-configs/TotalEnergies/pangea-3/pangea3-gcc11.4.0-openmpi-4.1.6.cmake \ +# --build-arg DOCKER_ROOT_IMAGE=onetechssc/pangea3:almalinux8-gcc11.4.0-openmpi4.1.6-cuda11.8.0-openblas0.3.18_v2.0 \ +# --build-arg INSTALL_DIR=/workrd/SCR/NUM/GEOS_environment/p3/install/tpl/install \ +# -v /etc/pki/ca-trust/extracted/pem/:/etc/pki/ca-trust/extracted/pem \ +# -t onetechssc/geos:tpl_gcc11.4.0-openmpi4.1.6-cuda11.8.0-openblas0.3.18_v2.0 \ +# -f docker/TotalEnergies/pangea-3/pangea3-tpl.Dockerfile . +# run the image: +# - podman run -it --detach --privileged --name pangea3_tpl -v /etc/pki/ca-trust/extracted/pem/:/etc/pki/ca-trust/extracted/pem localhost/onetechssc/geos:tpl_gcc11.4.0-openmpi4.1.6-cuda11.8.0-openblas0.3.18_v2.0 +# - podman exec -it pangea3_tpl /bin/bash +# push the image (requires to be part of the onetechssc docker organization): +# - podman login docker.io +# - podman push localhost/onetechssc/geos:tpl_gcc11.4.0-openmpi4.1.6-cuda11.8.0-openblas0.3.18_v2.0 docker://docker.io/onetechssc/geos:tpl_gcc11.4.0-openmpi4.1.6-cuda11.8.0-openblas0.3.18_v2.0 +# build geos from the image (from the root of geos reopsitory!): +# - podman run --name pangea4_geos \ +# -v /etc/pki/ca-trust/extracted/pem/:/etc/pki/ca-trust/extracted/pem \ +# -v .:/tmp/geos \ +# localhost/onetechssc/geos:tpl_gcc11.4.0-openmpi4.1.6-cuda11.8.0-openblas0.3.18_v2.0 \ +# /tmp/geos/scripts/ci_build_and_test_in_container.sh \ +# --host-config host-configs/TotalEnergies/pangea-3/pangea3-gcc11.4.0-openmpi-4.1.6.cmake \ +# --repository /tmp/geos --cmake-build-type Release --install-dir /tmp/install --build-exe-only +####################################### + +# ------------------------------------- +# PANGEA3 - TPL BASE +ARG DOCKER_ROOT_IMAGE +FROM $DOCKER_ROOT_IMAGE as tpl_toolchain_intersect_geosx_toolchain +# ------ +# LABELS +LABEL description="Pangea 3 image : geos_tpl" +LABEL version="2.0" +LABEL maintainer="TotalEnergies HPC Team" +# ------ +# ARGS +# The installation directory is provided as a docker build argument +ARG INSTALL_DIR +# ------ +# ENV +# All the environment variables defined in this Dockerfile +# (GEOSX_TPL_DIR but also compiler information like CC, CXX...) +# are part of the image contract (otherwise ARG is used). +# GEOSX use them so consider modifying their names with care. +ENV GEOS_TPL_DIR=$INSTALL_DIR +ENV GEOSX_TPL_DIR=$GEOS_TPL_DIR + +# ------------------------------------- +# PANGEA3 - TPL TOOLCHAIN +FROM tpl_toolchain_intersect_geosx_toolchain AS tpl_toolchain +# ------ +# ARGS +ARG TMP_DIR=/tmp +ARG SRC_DIR=$TMP_DIR/thirdPartyLibs +ARG BLD_DIR=$TMP_DIR/build +# Get host config file from docker build arguments +ARG HOST_CONFIG +# ------ +# INSTALL +RUN --mount=src=.,dst=$SRC_DIR $SRC_DIR/docker/configure-tpl.sh +# - build TPLs +WORKDIR $BLD_DIR +RUN --mount=src=.,dst=$SRC_DIR make + +# ------------------------------------- +# PANGEA3 - GEOS TOOLCHAIN +# Extract only TPL's from previous stage +FROM tpl_toolchain_intersect_geosx_toolchain AS geosx_toolchain + +COPY --from=tpl_toolchain $GEOSX_TPL_DIR $GEOSX_TPL_DIR + +ENV SCCACHE=/opt/sccache/bin/sccache diff --git a/docker/TotalEnergies/pangea-4/pangea4-gcc-hpcxompi-onemkl.Dockerfile b/docker/TotalEnergies/pangea-4/pangea4-gcc-hpcxompi-onemkl.Dockerfile new file mode 100644 index 00000000..5085401a --- /dev/null +++ b/docker/TotalEnergies/pangea-4/pangea4-gcc-hpcxompi-onemkl.Dockerfile @@ -0,0 +1,81 @@ +####################################### +# Pangea 4 image : gcc - hpcxompi - onemkl +####################################### +# +# Installs : +# - hpcx = 2.20.0 +# - intel-oneapi-mkl = 2023.2.0 +# +####################################### +# +# Description : +# - the image is based on onetechssc/pangea4:gcc12.1_v1.0 built from pangea4-gcc.Dockerfile +# - this image is deployed as onetechssc/pangea4:gcc12.1-hpcx2.20.0-onemkl2023.2.0_v1.0 +# - oneAPI MKL is installed via spack (as done for Pangea 4) +# - hpcx is installed by extracting the tarball (as done for Pangea 4) +# - gcc wrappers are created to mimic the Cray environment (cc, CC, ftn) +# +# Usage : +# build the image: +# - podman build --format docker -t onetechssc/pangea4:gcc12.1-hpcxompi2.20.0-onemkl2023.2.0_v1.0 -f pangea4-gcc-hpcxompi-onemkl.Dockerfile -v /etc/pki/ca-trust/extracted/pem/:/etc/pki/ca-trust/extracted/pem . +# run the image: +# - podman run -it --detach --privileged --name pangea4_gcc_hpcxompi_onemkl -v /etc/pki/ca-trust/extracted/pem/:/etc/pki/ca-trust/extracted/pem localhost/onetechssc/pangea4:gcc12.1-hpcxompi2.20.0-onemkl2023.2.0_v1.0 +# - podman exec -it pangea4_gcc_hpcxompi_onemkl /bin/bash +# push the image (requires to be part of the onetechssc docker organization): +# - podman login docker.io +# - podman push localhost/onetechssc/pangea4:gcc12.1-hpcxompi2.20.0-onemkl2023.2.0_v1.0 docker://docker.io/onetechssc/pangea4:gcc12.1-hpcxompi2.20.0-onemkl2023.2.0_v1.0 +# +####################################### + +# ------------------------------------- +# PANGEA4 - GCC-HPCX-MKL +FROM docker.io/onetechssc/pangea4:gcc12.1_v1.0 AS pangea4 +# ------ +# LABELS +LABEL description="Pangea 4 image : gcc - cmake - python - hpcx - mkl" +LABEL version="1.0" +LABEL maintainer="TotalEnergies HPC Team" +# ------ +# ARGS +ARG ONEAPI_MKL_VERSION=2023.2.0 +ARG HPCX_VERSION=2.20 +ARG HPCX_FULL_NAME="hpcx-v$HPCX_VERSION-gcc-mlnx_ofed-redhat8-cuda12-x86_64" +ARG HPCX_TARBALL="$HPCX_FULL_NAME.tbz" +ARG HPCX_URL="http://www.mellanox.com/downloads/hpc/hpc-x/v$HPCX_VERSION/$HPCX_TARBALL" +ARG CRAY_WRAPPERS_DIR=/sw/cray-wrappers +# ------ +# INSTALL +# spack config (make sure you survive our wonderfull proxy) +RUN spack config --scope defaults add config:connect_timeout:120 +RUN spack config --scope defaults add config:url_fetch_method:curl +# intel-oneapi-mkl +RUN spack install intel-oneapi-mkl@$ONEAPI_MKL_VERSION %gcc@$GCC_VERSION +# hpcx not available in spack -> download and untar in /sw directory +RUN mkdir -p /sw && \ + spack load wget && \ + wget --ca-certificate=/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem $HPCX_URL -O /tmp/$HPCX_TARBALL && \ + tar -xvf /tmp/$HPCX_TARBALL -C /sw && \ + rm -f /tmp/$HPCX_TARBALL +ENV HPCX_HOME=/sw/$HPCX_FULL_NAME +# numpy with mkl +RUN spack install py-numpy %gcc@$GCC_VERSION ^python@$PYTHON_VERSION ^intel-oneapi-mkl@$ONEAPI_MKL_VERSION +# ------ +# ENV +# create wrappers for gcc +RUN mkdir -p $CRAY_WRAPPERS_DIR && \ + spack load gcc@$GCC_VERSION python@$PYTHON_VERSION cmake@$CMAKE_VERSION intel-oneapi-mkl@$ONEAPI_MKL_VERSION && \ + GCC_INSTALL_DIR=$(spack location -i gcc@$GCC_VERSION) && \ + ln -s ${GCC_INSTALL_DIR}/bin/gcc $CRAY_WRAPPERS_DIR/cc && \ + ln -s ${GCC_INSTALL_DIR}/bin/g++ $CRAY_WRAPPERS_DIR/CC && \ + ln -s ${GCC_INSTALL_DIR}/bin/gfortran $CRAY_WRAPPERS_DIR/ftn +# create env script +RUN < /root/.setup_env.sh +. /opt/spack/share/spack/setup-env.sh +spack load gcc@$GCC_VERSION python@$PYTHON_VERSION cmake@$CMAKE_VERSION intel-oneapi-mkl@$ONEAPI_MKL_VERSION py-numpy +source ${HPCX_HOME}/hpcx-init.sh +hpcx_load +export PATH=$CRAY_WRAPPERS_DIR:\$PATH +export GCC_INSTALL_DIR=$(spack location -i gcc@$GCC_VERSION) +export GOMP_ROOT=\${GCC_INSTALL_DIR}/lib64 +EOF +RUN chmod +x /root/.setup_env.sh diff --git a/docker/TotalEnergies/pangea-4/pangea4-gcc.Dockerfile b/docker/TotalEnergies/pangea-4/pangea4-gcc.Dockerfile new file mode 100644 index 00000000..920162fd --- /dev/null +++ b/docker/TotalEnergies/pangea-4/pangea4-gcc.Dockerfile @@ -0,0 +1,67 @@ +# syntax=docker/dockerfile:1.4 + +####################################### +# Pangea 4 image : gcc - cmake - python +####################################### +# +# Installs : +# - gcc = 12.1 +# - cmake = 3.27.9 +# - python = 3.11 +# +####################################### +# +# Description : +# - the image is based on spack/centos-stream:latest (CentOS Stream 8 image with spack installed) +# - this image is deployed as onetechssc/pangea4:gcc12.1_v1.0 +# - gcc, cmake, python and wget are installed via spack (as done for Pangea 4) +# +# Usage : +# build the image: +# - podman build --format docker -t onetechssc/pangea4:gcc12.1_v1.0 -f pangea4-gcc.Dockerfile -v /etc/pki/ca-trust/extracted/pem/:/etc/pki/ca-trust/extracted/pem . +# run the image: +# - podman run -it --detach --privileged --name pangea4_gcc -v /etc/pki/ca-trust/extracted/pem/:/etc/pki/ca-trust/extracted/pem localhost/onetechssc/pangea4:gcc12.1_v1.0 +# - podman exec -it pangea4_gcc /bin/bash +# push the image (requires to be part of the onetechssc docker organization): +# - podman login docker.io +# - podman push localhost/onetechssc/pangea4:gcc12.1_v1.0 docker://docker.io/onetechssc/pangea4:gcc12.1_v1.0 +# +####################################### + +# ------------------------------------- +# PANGEA4 - BASE [GCC - CMAKE - PYTHON] +# install tools via spack for linux/x86_64 architecture and redhat8 platform +FROM --platform=linux/x86_64 docker.io/spack/centos-stream:latest AS pangea4-base +# ------ +# LABELS +LABEL description="Pangea 4 image : gcc - cmake - python" +LABEL version="1.0" +LABEL maintainer="TotalEnergies HPC Team" +# ------ +# ARGS +ARG GCC_VERSION=12.1 +ARG CMAKE_VERSION=3.27.9 +ARG PYTHON_VERSION=3.11 +# ------ +# INSTALL +# spack config (make sure you survive our wonderfull proxy) +RUN spack config --scope defaults add config:connect_timeout:120 +RUN spack config --scope defaults add config:url_fetch_method:curl +# gcc +RUN spack install gcc@$GCC_VERSION +RUN . /opt/spack/share/spack/setup-env.sh &&\ + spack load gcc@$GCC_VERSION &&\ + spack compiler find +# python +RUN spack install python@$PYTHON_VERSION %gcc@$GCC_VERSION +# cmake (openssl error when using curl) +RUN spack config --scope defaults add config:url_fetch_method:urllib &&\ + spack install cmake@$CMAKE_VERSION %gcc@$GCC_VERSION &&\ + spack config --scope defaults add config:url_fetch_method:curl +# wget +RUN spack install wget %gcc@$GCC_VERSION +# ------ +# ENV +ENV GCC_VERSION=${GCC_VERSION} +ENV CMAKE_VERSION=${CMAKE_VERSION} +ENV PYTHON_VERSION=${PYTHON_VERSION} \ No newline at end of file diff --git a/docker/TotalEnergies/pangea-4/pangea4-tpl.Dockerfile b/docker/TotalEnergies/pangea-4/pangea4-tpl.Dockerfile new file mode 100644 index 00000000..332cc1d1 --- /dev/null +++ b/docker/TotalEnergies/pangea-4/pangea4-tpl.Dockerfile @@ -0,0 +1,92 @@ +####################################### +# Pangea 4 tpl image +# +# Description : +# - generic image for building geos tpls on Pangea 4 environments +# - the docker base image can be any pangea4 docker file +# - tools of the base image are expected to be sourced in the set_env.sh script +# +# Usage : +# build the image (from the root of the repository!): +# - podman build --format docker --progress=plain \ +# --build-arg HOST_CONFIG=host-configs/TotalEnergies/pangea-4/pangea4-gcc12.1-hpcxompi2.20.0-onemkl2023.2.0.cmake \ +# --build-arg DOCKER_ROOT_IMAGE=onetechssc/pangea4:gcc12.1-hpcxompi2.20.0-onemkl2023.2.0_v1.0 \ +# --build-arg INSTALL_DIR=/workrd/SCR/NUM/GEOS_environment/p4/install/tpl/install \ +# -v /etc/pki/ca-trust/extracted/pem/:/etc/pki/ca-trust/extracted/pem \ +# -t onetechssc/geos:tpl_gcc12.1-hpcxompi2.20.0-onemkl2023.2.0_v1.0 \ +# -f docker/TotalEnergies/pangea-4/pangea4-tpl.Dockerfile . +# run the image: +# - podman run -it --detach --privileged --name pangea4_tpl -v /etc/pki/ca-trust/extracted/pem/:/etc/pki/ca-trust/extracted/pem localhost/onetechssc/geos:tpl_gcc12.1-hpcxompi2.20.0-onemkl2023.2.0_v1.0 +# - podman exec -it pangea4_tpl /bin/bash +# push the image (requires to be part of the onetechssc docker organization): +# - podman login docker.io +# - podman push localhost/onetechssc/geos:tpl_gcc12.1-hpcxompi2.20.0-onemkl2023.2.0_v1.0 docker://docker.io/onetechssc/geos:tpl_gcc12.1-hpcxompi2.20.0-onemkl2023.2.0_v1.0 +# build geos from the image (from the root of geos reopsitory!): +# - podman run --name pangea4_geos \ +# -v /etc/pki/ca-trust/extracted/pem/:/etc/pki/ca-trust/extracted/pem \ +# -v .:/tmp/geos \ +# localhost/onetechssc/geos:tpl_gcc12.1-hpcxompi2.20.0-onemkl2023.2.0_v1.0 \ +# /tmp/geos/scripts/ci_build_and_test_in_container.sh \ +# --host-config host-configs/TotalEnergies/pangea-4/pangea4-gcc12.1-hpcxompi2.20.0-onemkl2023.2.0.cmake \ +# --repository /tmp/geos --cmake-build-type Release --install-dir /tmp/install --build-exe-only +####################################### + +# ------------------------------------- +# PANGEA4 - TPL BASE +ARG DOCKER_ROOT_IMAGE +FROM $DOCKER_ROOT_IMAGE AS pangea4_tpl_base +# ------ +# LABELS +LABEL description="Pangea 4 image : geos_tpl" +LABEL version="1.0" +LABEL maintainer="TotalEnergies HPC Team" +# ------ +# ARGS +ARG INSTALL_DIR +# ------ +# ENV +ENV GEOS_TPL_DIR=$INSTALL_DIR +ENV GEOSX_TPL_DIR=$GEOS_TPL_DIR +ENV GCC_PATH=\${GCC_INSTALL_DIR} + +# ------------------------------------- +# PANGEA4 - TPL BUILDER +FROM pangea4_tpl_base AS pangea4_tpl_builder +# ------ +# ARGS +ARG TMP_DIR=/tmp +ARG SRC_DIR=$TMP_DIR/src +ARG BLD_DIR=$TMP_DIR/build +ARG HOST_CONFIG +# ------ +# INSTALL +# - configure TPLs +RUN --mount=src=.,dst=$SRC_DIR source /root/.setup_env.sh && \ + $SRC_DIR/docker/configure-tpl.sh +# - build TPLs +WORKDIR $BLD_DIR +RUN --mount=src=.,dst=$SRC_DIR source /root/.setup_env.sh && \ + make + +# ------------------------------------- +# PANGEA4 - TPL TOOLCHAIN +FROM pangea4_tpl_base AS pangea4_tpl_toolchain +# ------ +# ARGS +ARG TMP_DIR=/tmp +ARG SRC_DIR=$TMP_DIR/src +# ------ +# INSTALL +# - copy TPLs install directory +COPY --from=pangea4_tpl_builder $GEOS_TPL_DIR $GEOS_TPL_DIR +# - install ninja for geos build +RUN --mount=src=.,dst=$SRC_DIR $SRC_DIR/docker/install-ninja.sh +# - install `sccache` binaries to speed up the build of `geos` +RUN --mount=src=.,dst=$SRC_DIR $SRC_DIR/docker/install-sccache.sh +# ------ +# ENV +ENV SCCACHE=/opt/sccache/bin/sccache +# ------ +# ENTRYPOINT +# set entry point for geos ci build script +ENTRYPOINT ["/bin/bash", "-c", "source /root/.setup_env.sh && exec \"$@\"", "--"] \ No newline at end of file diff --git a/docker/TotalEnergies/pecan-CPU.cmake b/docker/TotalEnergies/pecan-CPU.cmake deleted file mode 100644 index 722ea2b4..00000000 --- a/docker/TotalEnergies/pecan-CPU.cmake +++ /dev/null @@ -1,31 +0,0 @@ -set(COMPILER_HOME "/apps/gcc/8.2.0/x86_64") -set(MPI_HOME "/hrtc/apps/mpi/openmpi/4.0.1/RDHPC/gcc/8.2.0") - -set(CMAKE_C_COMPILER ${COMPILER_HOME}/bin/gcc CACHE PATH "" FORCE) -set(CMAKE_CXX_COMPILER ${COMPILER_HOME}/bin/g++ CACHE PATH "" FORCE) -set(CMAKE_Fortran_COMPILER ${COMPILER_HOME}/bin/gfortran CACHE PATH "" FORCE) -set(ENABLE_FORTRAN OFF CACHE BOOL "" FORCE) - -set(ENABLE_MPI ON CACHE PATH "" FORCE) -set(ENABLE_OPENMP ON CACHE PATH "" FORCE) - -set(MPI_C_COMPILER "${MPI_HOME}/bin/mpicc" CACHE PATH "" FORCE) -set(MPI_CXX_COMPILER "${MPI_HOME}/bin/mpicxx" CACHE PATH "" FORCE) -set(MPI_Fortran_COMPILER "${MPI_HOME}/bin/mpifort" CACHE PATH "" FORCE) -set(MPIEXEC_EXECUTABLE "${MPI_HOME}/bin/mpirun" CACHE PATH "" FORCE) -#set(MPIEXEC_EXECUTABLE /apps/slurm/x86/20.02.0/bin/srun CACHE PATH "" FORCE) -#set(MPIEXEC_NUMPROC_FLAG "-p pecan -n" CACHE STRING "") -set(ENABLE_WRAP_ALL_TESTS_WITH_MPIEXEC ON CACHE BOOL "") - -set(ENABLE_GTEST_DEATH_TESTS ON CACHE BOOL "" FORCE) -set(ENABLE_CALIPER ON CACHE BOOL "") - -set(ENABLE_MKL ON CACHE BOOL "") -set(INTEL_ROOT "/apps/intel/2019/u5/compilers_and_libraries_2019.5.281/linux" ) -set(MKL_ROOT "${INTEL_ROOT}/mkl" ) -set(MKL_INCLUDE_DIRS ${MKL_ROOT}/include CACHE STRING "") -set(MKL_LIBRARIES ${MKL_ROOT}/lib/intel64/libmkl_intel_lp64.so - ${MKL_ROOT}/lib/intel64/libmkl_intel_thread.so - ${MKL_ROOT}/lib/intel64/libmkl_core.so - ${INTEL_ROOT}/compiler/lib/intel64_lin/libiomp5.so - CACHE STRING "") diff --git a/docker/TotalEnergies/pecan-GPU.cmake b/docker/TotalEnergies/pecan-GPU.cmake deleted file mode 100644 index ef6e4457..00000000 --- a/docker/TotalEnergies/pecan-GPU.cmake +++ /dev/null @@ -1,24 +0,0 @@ -# Retrieve the compilers, standard libraries... from the CPU configuration -include(${CMAKE_CURRENT_LIST_DIR}/pecan-CPU.cmake) - -# Now let's add what's dedicated to GPU. -set(ENABLE_CUDA ON CACHE PATH "" FORCE) -set(CUDA_TOOLKIT_ROOT_DIR /hrtc/apps/cuda/11.5.119/x86_64/centos7 CACHE PATH "") -set(CMAKE_CUDA_HOST_COMPILER ${CMAKE_CXX_COMPILER} CACHE STRING "") -set(CMAKE_CUDA_COMPILER ${CUDA_TOOLKIT_ROOT_DIR}/bin/nvcc CACHE STRING "") - -set(CUDA_ARCH sm_75 CACHE STRING "") -set(CMAKE_CUDA_ARCHITECTURES 75 CACHE STRING "") -set(CMAKE_CUDA_FLAGS "-restrict -arch ${CUDA_ARCH} --expt-relaxed-constexpr --expt-extended-lambda -Werror cross-execution-space-call,reorder,deprecated-declarations" CACHE STRING "") -set(CMAKE_CUDA_FLAGS_RELEASE "-O3 -DNDEBUG -Xcompiler -DNDEBUG -Xcompiler -O3" CACHE STRING "") -set(CMAKE_CUDA_FLAGS_RELWITHDEBINFO "-g -lineinfo ${CMAKE_CUDA_FLAGS_RELEASE}" CACHE STRING "") -set(CMAKE_CUDA_FLAGS_DEBUG "-g -G -O0 -Xcompiler -O0" CACHE STRING "") - -# Current version of hypre does not build with GPU support inside of docker. -# Hypre's build system awaits to be embedded into a CUDA environment. -# This is a bit tedious to reproduce in docker environment. -# And since most recent version of hypre do build without this constraint. -# Let's wait for an upgrade on our side. -# In the mean time, if you need the GPU support for hypre, -# simply install the classical way, with some `module load cuda`. -#set(ENABLE_HYPRE_CUDA ON CACHE BOOL "" FORCE) diff --git a/docker/TotalEnergies/pecan.Dockerfile b/docker/TotalEnergies/pecan.Dockerfile deleted file mode 100644 index 3e19fbef..00000000 --- a/docker/TotalEnergies/pecan.Dockerfile +++ /dev/null @@ -1,193 +0,0 @@ -ARG GCC_VERSION=8.2.0 -ARG OPENMPI_VERSION=4.0.1 -ARG UCX_VERSION=1.3.0 -# Note that you need to define both SLURM_TARBALL and SLURM_HOME because one cannot be deduced from the other smoothly. -# It would require (heavy?) text processing for little benefit. -ARG SLURM_TARBALL=slurm-20-02-0-1.tar.gz -ARG SLURM_HOME=/apps/slurm/x86/20.02.0 - -ARG GCC_HOME=/apps/gcc/${GCC_VERSION}/x86_64 -ARG UCX_HOME=/hrtc/apps/devtools/ucx/${UCX_VERSION}/x86_64_nocuda/gcc/${GCC_VERSION} -ARG OPENMPI_HOME=/hrtc/apps/mpi/openmpi/${OPENMPI_VERSION}/RDHPC/gcc/${GCC_VERSION} -# While the installation directory is defined here, the patches and exact versions are still defined in the CUDA stage. -ARG CUDA_HOME=/hrtc/apps/cuda/11.5.119/x86_64/centos7 - -FROM centos:7.7.1908 AS shared_components - -RUN sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo && \ - sed -i s/^#.*baseurl=http/baseurl=https/g /etc/yum.repos.d/*.repo && \ - sed -i s/^mirrorlist=http/#mirrorlist=https/g /etc/yum.repos.d/*.repo - -RUN yum update -y && \ - yum install -y \ - glibc-devel - -FROM shared_components AS gcc_stage - -ARG GCC_VERSION -ARG GCC_HOME - -# FIXME wget could be replaced by curl in the contrib/download_prerequisites script. To be challenged. -RUN yum install -y \ - make \ - gcc \ - gcc-c++ \ - wget \ - bzip2 \ - zlib-devel - -WORKDIR /tmp/src -RUN curl -s https://ftp.gnu.org/gnu/gcc/gcc-${GCC_VERSION}/gcc-${GCC_VERSION}.tar.gz | tar --strip-components=1 -xzf - -RUN ./contrib/download_prerequisites -RUN ./configure \ - --prefix=${GCC_HOME} \ - --disable-multilib \ - --with-system-zlib \ - --enable-threads=posix \ - --enable-languages=c,c++,fortran -RUN make -j $(nproc) && make install-strip - -FROM shared_components AS ucx_stage - -ARG UCX_VERSION -ARG UCX_HOME -ARG GCC_HOME - -COPY --from=gcc_stage ${GCC_HOME} ${GCC_HOME} - -ENV CC=${GCC_HOME}/bin/gcc \ - CXX=${GCC_HOME}/bin/g++ \ - FC=${GCC_HOME}/bin/gfortran \ - LD_LIBRARY_PATH=${GCC_HOME}/lib64 - -RUN yum install -y \ - make \ - numactl-devel - -WORKDIR /tmp/src -RUN curl -fsSL https://github.com/openucx/ucx/releases/download/v${UCX_VERSION}/ucx-${UCX_VERSION}.tar.gz | tar --strip-components=1 -xzf - -RUN ./configure \ - --prefix=${UCX_HOME} \ - --enable-mt \ - --disable-optimizations \ - --disable-logging \ - --disable-debug \ - --disable-assertions -RUN make -j $(nproc) && make install - -FROM shared_components AS slurm_stage - -ARG SLURM_TARBALL -ARG SLURM_HOME -ARG GCC_HOME - -COPY --from=gcc_stage ${GCC_HOME} ${GCC_HOME} - -ENV CC=${GCC_HOME}/bin/gcc \ - CXX=${GCC_HOME}/bin/g++ \ - FC=${GCC_HOME}/bin/gfortran \ - LD_LIBRARY_PATH=${GCC_HOME}/lib64 - -RUN yum install -y perl python3 file make - -WORKDIR /tmp/src -RUN curl -fsSL https://github.com/SchedMD/slurm/archive/${SLURM_TARBALL} | tar --strip-components=1 -xzf - -RUN ./configure --prefix=${SLURM_HOME} -RUN make -j $(nproc) && make install - -FROM shared_components AS openmpi_stage - -ARG SLURM_HOME -ARG UCX_HOME -ARG GCC_HOME -# FIXME so we do not use SLURM? -ARG OPENMPI_VERSION -ARG OPENMPI_HOME - -COPY --from=gcc_stage ${GCC_HOME} ${GCC_HOME} -COPY --from=ucx_stage ${UCX_HOME} ${UCX_HOME} -COPY --from=slurm_stage ${SLURM_HOME} ${SLURM_HOME} - -ENV CC=${GCC_HOME}/bin/gcc \ - CXX=${GCC_HOME}/bin/g++ \ - FC=${GCC_HOME}/bin/gfortran \ - LD_LIBRARY_PATH=${GCC_HOME}/lib64 - -RUN yum install -y \ - perl \ - make \ - zlib-devel \ - numactl-devel -# FIXME deal with the devel probably too much - -WORKDIR /tmp/src -RUN curl -fsSL https://download.open-mpi.org/release/open-mpi/v${OPENMPI_VERSION%.[0-9]*}/openmpi-${OPENMPI_VERSION}.tar.gz | tar --strip-components=1 -xzf - -RUN ./configure CC=$CC FC=$FC CXX=$CXX \ - --prefix=${OPENMPI_HOME} \ - --enable-static \ - --enable-smp-locks \ - --enable-mpi-thread-multiple \ - --with-slurm \ - --with-ucx=${UCX_HOME} \ - --with-ucx-libdir=${UCX_HOME}/lib \ - --with-io-romio-flags=--with-file-system=testfs+ufs+lustre -RUN make -j $(nproc) && make install - -FROM shared_components AS cuda_stage - -ARG GCC_HOME - -COPY --from=gcc_stage ${GCC_HOME} ${GCC_HOME} - -ARG CUDA_HOME - -# FIXME Why glibc-devel?!?!? -RUN yum install -y which glibc-devel - -ENV PATH=${GCC_HOME}/bin:${PATH} \ - LD_LIBRARY_PATH=${GCC_HOME}/lib64 - -WORKDIR /tmp/src -ADD https://developer.download.nvidia.com/compute/cuda/11.5.0/local_installers/cuda_11.5.0_495.29.05_linux.run . -RUN mkdir -p ${CUDA_HOME} -RUN sh cuda_11.5.0_495.29.05_linux.run --silent --toolkit --no-man-page --installpath=${CUDA_HOME} - -FROM shared_components AS intel_stage - -ARG SLURM_HOME -ARG UCX_HOME -ARG GCC_HOME -ARG OPENMPI_HOME -ARG CUDA_HOME - -COPY --from=gcc_stage ${GCC_HOME} ${GCC_HOME} -COPY --from=ucx_stage ${UCX_HOME} ${UCX_HOME} -COPY --from=slurm_stage ${SLURM_HOME} ${SLURM_HOME} -COPY --from=openmpi_stage ${OPENMPI_HOME} ${OPENMPI_HOME} -COPY --from=cuda_stage ${CUDA_HOME} ${CUDA_HOME} - -RUN yum install -y \ - numactl-devel -# FIXME maybe only numactl-libs? - -RUN rpm --import https://yum.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB -RUN yum-config-manager \ - --add-repo https://yum.repos.intel.com/mkl/setup/intel-mkl.repo -RUN yum install -y \ - intel-mkl-2019.5-075.x86_64 && \ - yum clean all - -ARG PECAN_INTEL_HOME=/apps/intel/2019/u5 -ARG INTEL_DIR_NAME=compilers_and_libraries_2019.5.281 -RUN mkdir -p /apps/intel/2019/u5 && \ - ln -s /opt/intel/${INTEL_DIR_NAME} ${PECAN_INTEL_HOME}/${INTEL_DIR_NAME} - -# Exposing quite everything, making future modularization more complicated. -# Most likely there will be no future modularization! -ENV CC=${GCC_HOME}/bin/gcc \ - CXX=${GCC_HOME}/bin/g++ \ - FC=${GCC_HOME}/bin/gfortran \ - MPICC=${OPENMPI_HOME}/bin/mpicc \ - MPICXX=${OPENMPI_HOME}/bin/mpicxx \ - MPIFC=${OPENMPI_HOME}/bin/mpifort \ - LD_LIBRARY_PATH=${GCC_HOME}/lib64:${PECAN_INTEL_HOME}/${INTEL_DIR_NAME}/linux/mkl/lib/intel64:${PECAN_INTEL_HOME}/${INTEL_DIR_NAME}/compiler/lib/intel64:${OPENMPI_HOME}/lib diff --git a/host-configs/TOTAL/pangea3-gcc8.4.1-openmpi-4.1.2.cmake b/host-configs/TotalEnergies/pangea-3/pangea3-gcc11.4.0-openmpi-4.1.6.cmake similarity index 95% rename from host-configs/TOTAL/pangea3-gcc8.4.1-openmpi-4.1.2.cmake rename to host-configs/TotalEnergies/pangea-3/pangea3-gcc11.4.0-openmpi-4.1.6.cmake index 04d0b43d..20ba2e84 100644 --- a/host-configs/TOTAL/pangea3-gcc8.4.1-openmpi-4.1.2.cmake +++ b/host-configs/TotalEnergies/pangea-3/pangea3-gcc11.4.0-openmpi-4.1.6.cmake @@ -1,7 +1,7 @@ # hostconfig for pangea3 # # -set(CONFIG_NAME "pangea3-gcc8.4.1-ompi-4.1.2" CACHE PATH "") +set(CONFIG_NAME "pangea3-gcc11.4.0-ompi-4.1.6" CACHE PATH "") # C options set(CMAKE_C_COMPILER gcc CACHE PATH "") @@ -36,7 +36,7 @@ if (DEFINED ENV{MPI_ROOT}) set(MPIEXEC $ENV{MPI_ROOT}/bin/mpirun CACHE STRING "") set(ENABLE_WRAP_ALL_TESTS_WITH_MPIEXEC ON CACHE BOOL "") else() - message(FATAL_ERROR "You must have MPI_ROOT variable set, we advise loading module openmpi/p3/4.1.2") + message(FATAL_ERROR "You must have MPI_ROOT variable set, we advise loading module ompi/4.1.6") endif() # Cuda options @@ -56,7 +56,7 @@ if (DEFINED ENV{CUDA_ROOT}) # Uncomment this line to make nvcc output register usage for each kernel. # set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} --resource-usage" CACHE STRING "" FORCE) else() - message(FATAL_ERROR "You must have CUDA_ROOT environment variable set, we advise loading module cuda/11.0.3") + message(FATAL_ERROR "You must have CUDA_ROOT environment variable set, we advise loading module cuda/11.8.0") endif() # GTEST options @@ -90,7 +90,7 @@ if (DEFINED ENV{OPENBLAS_ROOT}) set(BLAS_LIBRARIES $ENV{OPENBLAS_ROOT}/lib/libopenblas.a) set(LAPACK_LIBRARIES $ENV{OPENBLAS_ROOT}/lib/libopenblas.a) else() - message(FATAL_ERROR "You must have OPENBLAS_ROOT environment variable set, we advise loading module openblas/0.3.18") + message(FATAL_ERROR "You must have OPENBLAS_ROOT environment variable set, we advise loading module openblas/0.3.18+openmp") endif() set(ENABLE_DOXYGEN OFF CACHE PATH "") diff --git a/host-configs/TOTAL/pangea4-base.cmake b/host-configs/TotalEnergies/pangea-4/pangea4-base.cmake similarity index 100% rename from host-configs/TOTAL/pangea4-base.cmake rename to host-configs/TotalEnergies/pangea-4/pangea4-base.cmake diff --git a/host-configs/TOTAL/pangea4-gcc12.1-hpcxompi2.17.1-onemkl2023.2.0.cmake b/host-configs/TotalEnergies/pangea-4/pangea4-gcc12.1-hpcxompi2.20.0-onemkl2023.2.0.cmake similarity index 90% rename from host-configs/TOTAL/pangea4-gcc12.1-hpcxompi2.17.1-onemkl2023.2.0.cmake rename to host-configs/TotalEnergies/pangea-4/pangea4-gcc12.1-hpcxompi2.20.0-onemkl2023.2.0.cmake index 488ca0cc..3ab0d9c8 100644 --- a/host-configs/TOTAL/pangea4-gcc12.1-hpcxompi2.17.1-onemkl2023.2.0.cmake +++ b/host-configs/TotalEnergies/pangea-4/pangea4-gcc12.1-hpcxompi2.20.0-onemkl2023.2.0.cmake @@ -20,7 +20,7 @@ # - cray-python = 3.10.10 # - craype-x86-milan = 1.0 # PrgEnv-gnu loads gcc 12 that does not support craype-x86-genoa -# - hpcx = 2.17.1 +# - hpcx = 2.20.0 # - intel-oneapi-mkl = 2023.2.0 # # Load modules this way : @@ -31,7 +31,7 @@ # ######################################## -set( CONFIG_NAME "pangea4-gcc12.1-hpcxompi2.17.1-onemkl2023.2.0" CACHE PATH "" ) +set( CONFIG_NAME "pangea4-gcc12.1-hpcxompi2.20.0-onemkl2023.2.0" CACHE PATH "" ) include(${CMAKE_CURRENT_LIST_DIR}/pangea4-base.cmake) @@ -91,7 +91,12 @@ if( NOT DEFINED ENV{MKLROOT} ) message( FATAL_ERROR "MKL is not loaded. Please load the intel-oneapi-mkl/2023.2.0 module." ) endif() +if ( NOT DEFINED ENV{GOMP_ROOT} ) + message( STATUS "GOMP_ROOT is not set. Setting it to $ENV{GCC_PATH}/lib/gcc/x86_64-redhat-linux/12" ) + set( ENV{GOMP_ROOT} $ENV{GCC_PATH}/lib/gcc/x86_64-redhat-linux/12 ) +endif() + set( MKL_INCLUDE_DIRS $ENV{MKLROOT}/include CACHE STRING "" ) set( MKL_LIBRARIES $ENV{MKLROOT}/lib/intel64/libmkl_rt.so - $ENV{GCC_PATH}/lib/gcc/x86_64-redhat-linux/12/libgomp.so + $ENV{GOMP_ROOT}/libgomp.so CACHE STRING "" ) diff --git a/host-configs/TOTAL/pangea4-gcc12.1-hpcxompi2.17.1-openblas0.3.23.cmake b/host-configs/TotalEnergies/pangea-4/pangea4-gcc12.1-hpcxompi2.20.0-openblas0.3.23.cmake similarity index 97% rename from host-configs/TOTAL/pangea4-gcc12.1-hpcxompi2.17.1-openblas0.3.23.cmake rename to host-configs/TotalEnergies/pangea-4/pangea4-gcc12.1-hpcxompi2.20.0-openblas0.3.23.cmake index 42d05975..f7c09f2e 100644 --- a/host-configs/TOTAL/pangea4-gcc12.1-hpcxompi2.17.1-openblas0.3.23.cmake +++ b/host-configs/TotalEnergies/pangea-4/pangea4-gcc12.1-hpcxompi2.20.0-openblas0.3.23.cmake @@ -20,7 +20,7 @@ # - cray-python = 3.10.10 # - craype-x86-milan = 1.0 # PrgEnv-gnu loads gcc 12 that does not support craype-x86-genoa -# - hpcx = 2.17.1 +# - hpcx = 2.20.0 # - openblas = 0.3.23 # # Load modules this way : @@ -31,7 +31,7 @@ # ######################################## -set( CONFIG_NAME "pangea4-gcc12.1-hpcxompi2.17.1-openblas0.3.23" CACHE PATH "" ) +set( CONFIG_NAME "pangea4-gcc12.1-hpcxompi2.20.0-openblas0.3.23" CACHE PATH "" ) include(${CMAKE_CURRENT_LIST_DIR}/pangea4-base.cmake) diff --git a/host-configs/TOTAL/pangea4-gcc12.1-openmpi4.1.6-onemkl2023.2.0.cmake b/host-configs/TotalEnergies/pangea-4/pangea4-gcc12.1-openmpi4.1.6-onemkl2023.2.0.cmake similarity index 92% rename from host-configs/TOTAL/pangea4-gcc12.1-openmpi4.1.6-onemkl2023.2.0.cmake rename to host-configs/TotalEnergies/pangea-4/pangea4-gcc12.1-openmpi4.1.6-onemkl2023.2.0.cmake index 5ede5b87..3a5d3a6c 100644 --- a/host-configs/TOTAL/pangea4-gcc12.1-openmpi4.1.6-onemkl2023.2.0.cmake +++ b/host-configs/TotalEnergies/pangea-4/pangea4-gcc12.1-openmpi4.1.6-onemkl2023.2.0.cmake @@ -91,7 +91,12 @@ if( NOT DEFINED ENV{MKLROOT} ) message( FATAL_ERROR "MKL is not loaded. Please load the intel-oneapi-mkl/2023.2.0 module." ) endif() +if ( NOT DEFINED ENV{GOMP_ROOT} ) + message( STATUS "GOMP_ROOT is not set. Setting it to $ENV{GCC_PATH}/lib/gcc/x86_64-redhat-linux/12" ) + set( ENV{GOMP_ROOT} $ENV{GCC_PATH}/lib/gcc/x86_64-redhat-linux/12 ) +endif() + set( MKL_INCLUDE_DIRS $ENV{MKLROOT}/include CACHE STRING "" ) set( MKL_LIBRARIES $ENV{MKLROOT}/lib/intel64/libmkl_rt.so - $ENV{GCC_PATH}/lib/gcc/x86_64-redhat-linux/12/libgomp.so + $ENV{GOMP_ROOT}/libgomp.so CACHE STRING "" ) diff --git a/host-configs/TOTAL/pangea4-gcc12.1-openmpi4.1.6-openblas0.3.23.cmake b/host-configs/TotalEnergies/pangea-4/pangea4-gcc12.1-openmpi4.1.6-openblas0.3.23.cmake similarity index 100% rename from host-configs/TOTAL/pangea4-gcc12.1-openmpi4.1.6-openblas0.3.23.cmake rename to host-configs/TotalEnergies/pangea-4/pangea4-gcc12.1-openmpi4.1.6-openblas0.3.23.cmake