Add compression to CI images

ornladios · Sep 18, 2019 · 6ad94e1 · 6ad94e1
1 parent 9460d17
commit 6ad94e1
Show file tree

Hide file tree

Showing 16 changed files with 314 additions and 107 deletions.
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -15,19 +15,19 @@ pr:
 resources:
   containers:
   - container: el7
-    image: ornladios/adios2-auto:el7
+    image: ornladios/adios2:el7
   - container: el7-gnu8-ohpc
-    image: ornladios/adios2-auto:el7-gnu8-ohpc
+    image: ornladios/adios2:el7-gnu8-ohpc
   - container: el7-gnu8-openmpi-ohpc
-    image: ornladios/adios2-auto:el7-gnu8-openmpi-ohpc
+    image: ornladios/adios2:el7-gnu8-openmpi-ohpc
   - container:  el7-intel18-ohpc
-    image: ornladios/adios2-auto:el7-intel18-ohpc
+    image: ornladios/adios2:el7-intel18-ohpc
   - container: el7-intel18-openmpi-ohpc
-    image: ornladios/adios2-auto:el7-intel18-openmpi-ohpc
+    image: ornladios/adios2:el7-intel18-openmpi-ohpc
   - container:  suse-pgi
-    image: ornladios/adios2-auto:suse-pgi
+    image: ornladios/adios2:suse-pgi
   - container:  suse-pgi-openmpi
-    image: ornladios/adios2-auto:suse-pgi-openmpi
+    image: ornladios/adios2:suse-pgi-openmpi
 
 strategy:
   matrix:

diff --git a/scripts/ci/cmake/ci-el7-gnu8-ohpc.cmake b/scripts/ci/cmake/ci-el7-gnu8-ohpc.cmake
@@ -14,8 +14,6 @@ set(ENV{CFLAGS} "-Werror -Wno-error=builtin-declaration-mismatch")
 set(ENV{CXXFLAGS} "-Werror -Wno-error=builtin-declaration-mismatch")
 set(ENV{FFLAGS} "-Werror -Wno-error=builtin-declaration-mismatch")
 
-set(ENV{CMAKE_PREFIX_PATH} "/opt/libfabric/1.6.0:$ENV{CMAKE_PREFIX_PATH}")
-
 set(dashboard_cache "
 ADIOS2_USE_BZip2:BOOL=ON
 ADIOS2_USE_Blosc:BOOL=ON
@@ -24,7 +22,9 @@ ADIOS2_USE_Fortran:BOOL=ON
 ADIOS2_USE_HDF5:BOOL=ON
 ADIOS2_USE_MPI:BOOL=OFF
 ADIOS2_USE_Python:BOOL=ON
+ADIOS2_USE_SZ:BOOL=ON
 ADIOS2_USE_ZeroMQ:STRING=ON
+ADIOS2_USE_ZFP:BOOL=ON
 
 PYTHON_EXECUTABLE:FILEPATH=/usr/bin/python3.4
 ")

diff --git a/scripts/ci/cmake/ci-el7-gnu8-openmpi-ohpc.cmake b/scripts/ci/cmake/ci-el7-gnu8-openmpi-ohpc.cmake
@@ -29,7 +29,9 @@ ADIOS2_USE_Fortran:BOOL=ON
 ADIOS2_USE_HDF5:BOOL=ON
 ADIOS2_USE_MPI:BOOL=ON
 ADIOS2_USE_Python:BOOL=ON
+ADIOS2_USE_SZ:BOOL=ON
 ADIOS2_USE_ZeroMQ:STRING=ON
+ADIOS2_USE_ZFP:BOOL=ON
 
 MPIEXEC_MAX_NUMPROCS:STRING=${N2CPUS}
 PYTHON_EXECUTABLE:FILEPATH=/usr/bin/python3.4

diff --git a/scripts/ci/cmake/ci-el7-intel18-ohpc.cmake b/scripts/ci/cmake/ci-el7-intel18-ohpc.cmake
@@ -24,7 +24,9 @@ ADIOS2_USE_Fortran:BOOL=ON
 ADIOS2_USE_HDF5:BOOL=ON
 ADIOS2_USE_MPI:BOOL=OFF
 ADIOS2_USE_Python:BOOL=ON
+ADIOS2_USE_SZ:BOOL=ON
 ADIOS2_USE_ZeroMQ:STRING=ON
+ADIOS2_USE_ZFP:STRING=ON
 
 PYTHON_EXECUTABLE:FILEPATH=/usr/bin/python2.7
 ")

diff --git a/scripts/ci/cmake/ci-el7-intel18-openmpi-ohpc.cmake b/scripts/ci/cmake/ci-el7-intel18-openmpi-ohpc.cmake
@@ -29,7 +29,9 @@ ADIOS2_USE_Fortran:BOOL=ON
 ADIOS2_USE_HDF5:BOOL=ON
 ADIOS2_USE_MPI:BOOL=ON
 ADIOS2_USE_Python:BOOL=ON
+ADIOS2_USE_SZ:BOOL=ON
 ADIOS2_USE_ZeroMQ:STRING=ON
+ADIOS2_USE_ZFP:STRING=ON
 
 MPIEXEC_MAX_NUMPROCS:STRING=${N2CPUS}
 PYTHON_EXECUTABLE:FILEPATH=/usr/bin/python2.7

diff --git a/scripts/ci/cmake/ci-el7.cmake b/scripts/ci/cmake/ci-el7.cmake
@@ -7,17 +7,16 @@ set(ENV{CFLAGS} -Werror)
 set(ENV{CXXFLAGS} -Werror)
 set(ENV{FFLAGS} -Werror)
 
-set(ENV{CMAKE_PREFIX_PATH} "/opt/libfabric/1.6.0:$ENV{CMAKE_PREFIX_PATH}")
-
 set(dashboard_cache "
 ADIOS2_USE_BZip2:STRING=ON
 ADIOS2_USE_DataMan:STRING=ON
 ADIOS2_USE_Fortran:STRING=ON
 ADIOS2_USE_HDF5:STRING=ON
 ADIOS2_USE_MPI:STRING=OFF
 ADIOS2_USE_Python:STRING=ON
-ADIOS2_USE_ZFP:STRING=OFF
+ADIOS2_USE_SZ:STRING=ON
 ADIOS2_USE_ZeroMQ:STRING=ON
+ADIOS2_USE_ZFP:STRING=ON
 ")
 
 set(CTEST_CMAKE_GENERATOR "Unix Makefiles")

diff --git a/scripts/ci/cmake/ci-suse-pgi-openmpi.cmake b/scripts/ci/cmake/ci-suse-pgi-openmpi.cmake
@@ -17,8 +17,6 @@ set(ENV{FC}  pgfortran)
 set(ENV{CXXFLAGS} --brief_diagnostics)
 #set(ENV{FFLAGS} "-warn errors")
 
-set(ENV{CMAKE_PREFIX_PATH} "/opt/libfabric/1.6.0:$ENV{CMAKE_PREFIX_PATH}")
-
 set(dashboard_cache "
 ADIOS2_USE_BZip2:BOOL=ON
 ADIOS2_USE_Blosc:BOOL=OFF
@@ -27,7 +25,9 @@ ADIOS2_USE_Fortran:BOOL=ON
 ADIOS2_USE_HDF5:BOOL=OFF
 ADIOS2_USE_MPI:BOOL=ON
 ADIOS2_USE_Python:BOOL=OFF
+ADIOS2_USE_SZ:BOOL=ON
 ADIOS2_USE_ZeroMQ:STRING=ON
+ADIOS2_USE_ZFP:BOOL=ON
 
 MPIEXEC_MAX_NUMPROCS:STRING=${N2CPUS}
 ")

diff --git a/scripts/ci/cmake/ci-suse-pgi.cmake b/scripts/ci/cmake/ci-suse-pgi.cmake
@@ -12,8 +12,6 @@ set(ENV{FC}  pgfortran)
 set(ENV{CXXFLAGS} --brief_diagnostics)
 #set(ENV{FFLAGS} "-warn errors")
 
-set(ENV{CMAKE_PREFIX_PATH} "/opt/libfabric/1.6.0:$ENV{CMAKE_PREFIX_PATH}")
-
 set(dashboard_cache "
 ADIOS2_USE_BZip2:BOOL=ON
 ADIOS2_USE_Blosc:BOOL=OFF
@@ -22,7 +20,9 @@ ADIOS2_USE_Fortran:BOOL=ON
 ADIOS2_USE_HDF5:BOOL=ON
 ADIOS2_USE_MPI:BOOL=OFF
 ADIOS2_USE_Python:BOOL=OFF
+ADIOS2_USE_SZ:BOOL=ON
 ADIOS2_USE_ZeroMQ:STRING=ON
+ADIOS2_USE_ZFP:BOOL=ON
 ")
 
 set(CTEST_CMAKE_GENERATOR "Unix Makefiles")

diff --git a/scripts/ci/images/el7-gnu8-ohpc/Dockerfile b/scripts/ci/images/el7-gnu8-ohpc/Dockerfile
@@ -5,7 +5,7 @@ RUN yum -y install make curl file git valgrind vim bison flex sudo gdb \
         pkgconfig bison flex pkgconfig \
         zlib-devel bzip2-devel libpng-devel libfabric-devel libffi-devel
 RUN yum -y install epel-release
-RUN yum -y install zeromq-devel blosc-devel python34-devel
+RUN yum -y install zeromq-devel blosc-devel python34-devel libzstd libzstd-devel
 
 # Install OpenHPC packages
 RUN yum -y install https://github.com/openhpc/ohpc/releases/download/v1.3.GA/ohpc-release-1.3-1.el7.x86_64.rpm
@@ -14,8 +14,41 @@ RUN yum -y install lmod-ohpc \
 
 # Install the CMake binary
 WORKDIR /opt/cmake
-RUN curl -L https://github.com/Kitware/CMake/releases/download/v3.15.0/cmake-3.15.0-Linux-x86_64.tar.gz | \
+RUN curl -L https://github.com/Kitware/CMake/releases/download/v3.15.3/cmake-3.15.3-Linux-x86_64.tar.gz | \
     tar -C /opt/cmake --strip-components=1 -xzv
+ENV PATH /opt/cmake/bin:${PATH}
+
+# Install ZFP
+WORKDIR /opt/zfp
+RUN curl -L https://github.com/LLNL/zfp/releases/download/0.5.5/zfp-0.5.5.tar.gz | tar -xvz && \
+    mkdir build && \
+    cd build && \
+    source /etc/profile && \
+    module load gnu8 && \
+    export CC=gcc CXX=g++ FC=gfortran && \
+    cmake -DBUILD_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/opt/zfp/0.5.5 ../zfp-0.5.5 && \
+    make -j$(grep -c '^processor' /proc/cpuinfo) install && \
+    cd .. && \
+    rm -rf zfp-0.5.5 build
+ENV PATH /opt/zfp/0.5.5/bin:${PATH}
+ENV LD_LIBRARY_PATH /opt/zfp/0.5.5/lib64:${LD_LIBRARY_PATH}
+ENV CMAKE_PREFIX_PATH /opt/zfp/0.5.5:${CMAKE_PREFIX_PATH}
+
+# Install SZ
+WORKDIR /opt/sz
+RUN curl -L https://github.com/disheng222/SZ/archive/v2.1.6.0.tar.gz | tar -xvz && \
+    mkdir build && \
+    cd build && \
+    source /etc/profile && \
+    module load gnu8 && \
+    export CC=gcc CXX=g++ FC=gfortran && \
+    cmake -DBUILD_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/opt/sz/2.1.6.0 ../SZ-2.1.6.0 && \
+    make -j$(grep -c '^processor' /proc/cpuinfo) install && \
+    cd .. && \
+    rm -rf SZ-2.1.6.0 build
+ENV PATH /opt/sz/2.1.6.0/bin:${PATH}
+ENV LD_LIBRARY_PATH /opt/sz/2.1.6.0/lib64:${LD_LIBRARY_PATH}
+ENV CMAKE_PREFIX_PATH /opt/sz/2.1.6.0:${CMAKE_PREFIX_PATH}
 
 # Install git-lfs
 RUN curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.rpm.sh | bash && \

diff --git a/scripts/ci/images/el7-gnu8-openmpi-ohpc/Dockerfile b/scripts/ci/images/el7-gnu8-openmpi-ohpc/Dockerfile
@@ -5,32 +5,68 @@ RUN yum -y install make curl file git valgrind vim bison flex sudo gdb \
         pkgconfig bison flex pkgconfig bzip2 bzip2-libs \
         zlib-devel bzip2-devel libpng-devel libfabric-devel libffi-devel
 RUN yum -y install epel-release
-RUN yum -y install zeromq-devel blosc-devel python34-devel
+RUN yum -y install zeromq-devel blosc-devel python34-devel libzstd libzstd-devel
 
 # Install OpenHPC packages
 RUN yum -y install https://github.com/openhpc/ohpc/releases/download/v1.3.GA/ohpc-release-1.3-1.el7.x86_64.rpm
 RUN yum -y install lmod-ohpc \
         gnu8-compilers-ohpc openmpi3-gnu8-ohpc \
         python34-numpy-gnu8-ohpc python34-mpi4py-gnu8-openmpi3-ohpc
 
+# Install the CMake binary
+WORKDIR /opt/cmake
+RUN curl -L https://github.com/Kitware/CMake/releases/download/v3.15.3/cmake-3.15.3-Linux-x86_64.tar.gz | \
+    tar -C /opt/cmake --strip-components=1 -xzv
+ENV PATH /opt/cmake/bin:${PATH}
+
 # Install HDF5 1.10.4 (the current 1.10.5 has a parallel close bug affecting
 # the tests
 WORKDIR /opt/hdf5
-RUN curl -O https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.10/hdf5-1.10.4/src/hdf5-1.10.4.tar.bz2 && \
-  tar -xvf hdf5-1.10.4.tar.bz2 && \
-  cd hdf5-1.10.4 && \
-  source /etc/profile && \
-  module load gnu8 openmpi3 && \
-  export CC=mpicc CXX=mpicxx FC=mpif90 && \
-  ./configure --prefix=/opt/hdf5/1.10.4 --enable-shared --disable-static --enable-parallel && \
-  make -j$(grep -c '^processor' /proc/cpuinfo) install && \
-  cd .. && \
-  rm -rf hdf5-1.10.4.tar.bz2 hdf5-1.10.4
+RUN curl -L https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.10/hdf5-1.10.4/src/hdf5-1.10.4.tar.bz2 | \
+        tar -xvj && \
+    cd hdf5-1.10.4 && \
+    source /etc/profile && \
+    module load gnu8 openmpi3 && \
+    export CC=mpicc CXX=mpicxx FC=mpif90 && \
+    ./configure --prefix=/opt/hdf5/1.10.4 --enable-shared --disable-static --enable-parallel && \
+    make -j$(grep -c '^processor' /proc/cpuinfo) install && \
+    cd .. && \
+    rm -rf hdf5-1.10.4
+ENV PATH /opt/hdf5/1.10.4/bin:${PATH}
+ENV LD_LIBRARY_PATH /opt/hdf5/1.10.4/lib:${LD_LIBRARY_PATH}
+ENV CMAKE_PREFIX_PATH /opt/hdf5/1.10.4:${CMAKE_PREFIX_PATH}
 
-# Install the CMake binary
-WORKDIR /opt/cmake
-RUN curl -L https://github.com/Kitware/CMake/releases/download/v3.15.2/cmake-3.15.2-Linux-x86_64.tar.gz | \
-    tar -C /opt/cmake --strip-components=1 -xzv
+# Install ZFP
+WORKDIR /opt/zfp
+RUN curl -L https://github.com/LLNL/zfp/releases/download/0.5.5/zfp-0.5.5.tar.gz | tar -xvz && \
+    mkdir build && \
+    cd build && \
+    source /etc/profile && \
+    module load gnu8 && \
+    export CC=gcc CXX=g++ FC=gfortran && \
+    cmake -DBUILD_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/opt/zfp/0.5.5 ../zfp-0.5.5 && \
+    make -j$(grep -c '^processor' /proc/cpuinfo) install && \
+    cd .. && \
+    rm -rf zfp-0.5.5 build
+ENV PATH /opt/zfp/0.5.5/bin:${PATH}
+ENV LD_LIBRARY_PATH /opt/zfp/0.5.5/lib64:${LD_LIBRARY_PATH}
+ENV CMAKE_PREFIX_PATH /opt/zfp/0.5.5:${CMAKE_PREFIX_PATH}
+
+# Install SZ
+WORKDIR /opt/sz
+RUN curl -L https://github.com/disheng222/SZ/archive/v2.1.6.0.tar.gz | tar -xvz && \
+    mkdir build && \
+    cd build && \
+    source /etc/profile && \
+    module load gnu8 && \
+    export CC=gcc CXX=g++ FC=gfortran && \
+    cmake -DBUILD_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/opt/sz/2.1.6.0 ../SZ-2.1.6.0 && \
+    make -j$(grep -c '^processor' /proc/cpuinfo) install && \
+    cd .. && \
+    rm -rf SZ-2.1.6.0 build
+ENV PATH /opt/sz/2.1.6.0/bin:${PATH}
+ENV LD_LIBRARY_PATH /opt/sz/2.1.6.0/lib64:${LD_LIBRARY_PATH}
+ENV CMAKE_PREFIX_PATH /opt/sz/2.1.6.0:${CMAKE_PREFIX_PATH}
 
 # Install git-lfs
 RUN curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.rpm.sh | bash && \

diff --git a/scripts/ci/images/el7-intel18-ohpc/Dockerfile b/scripts/ci/images/el7-intel18-ohpc/Dockerfile
@@ -2,9 +2,10 @@ FROM centos:centos7
 
 # Install core dev packages
 RUN yum -y install make curl file git gcc gcc-g++ valgrind vim \
-        gdb zlib zlib-devel bzip2 bzip2-libs bzip2-devel python-devel
+        gdb zlib zlib-devel bzip2 bzip2-libs bzip2-devel python-devel \
+        libfabric libfabric-devel
 RUN yum -y install epel-release
-RUN yum -y install blosc-devel zeromq-devel
+RUN yum -y install blosc-devel zeromq-devel libzstd libzstd-devel
 
 # Install Intel C++ compiler
 WORKDIR /tmp
@@ -28,6 +29,39 @@ RUN yum clean all
 WORKDIR /opt/cmake
 RUN curl -L https://cmake.org/files/v3.15/cmake-3.15.2-Linux-x86_64.tar.gz | \
     tar --strip-components=1 -xz
+ENV PATH /opt/cmake/bin:${PATH}
+
+# Install ZFP
+WORKDIR /opt/zfp
+RUN curl -L https://github.com/LLNL/zfp/releases/download/0.5.5/zfp-0.5.5.tar.gz | tar -xvz && \
+    mkdir build && \
+    cd build && \
+    source /etc/profile && \
+    module load intel && \
+    export CC=icc CXX=icpc FC=ifort && \
+    cmake -DBUILD_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/opt/zfp/0.5.5 ../zfp-0.5.5 && \
+    make -j$(grep -c '^processor' /proc/cpuinfo) install && \
+    cd .. && \
+    rm -rf zfp-0.5.5 build
+ENV PATH /opt/zfp/0.5.5/bin:${PATH}
+ENV LD_LIBRARY_PATH /opt/zfp/0.5.5/lib64:${LD_LIBRARY_PATH}
+ENV CMAKE_PREFIX_PATH /opt/zfp/0.5.5:${CMAKE_PREFIX_PATH}
+
+# Install SZ
+WORKDIR /opt/sz
+RUN curl -L https://github.com/disheng222/SZ/archive/v2.1.6.0.tar.gz | tar -xvz && \
+    mkdir build && \
+    cd build && \
+    source /etc/profile && \
+    module load intel && \
+    export CC=icc CXX=icpc FC=ifort && \
+    cmake -DBUILD_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/opt/sz/2.1.6.0 ../SZ-2.1.6.0 && \
+    make -j$(grep -c '^processor' /proc/cpuinfo) install && \
+    cd .. && \
+    rm -rf SZ-2.1.6.0 build
+ENV PATH /opt/sz/2.1.6.0/bin:${PATH}
+ENV LD_LIBRARY_PATH /opt/sz/2.1.6.0/lib64:${LD_LIBRARY_PATH}
+ENV CMAKE_PREFIX_PATH /opt/sz/2.1.6.0:${CMAKE_PREFIX_PATH}
 
 # Misc cleanup of unneeded files
 RUN rm -rfv \