Build and install IO500 in client image

.gitignore

@@ -0,0 +1,19 @@
+# Local .terraform directories
+**/.terraform/
+**/.terraform/*
+
+# .tfstate files
+*.tfstate
+*.tfstate.*
+
+# Ignore Terraform lock files
+.terraform.lock.hcl
+
+# Ignore Terraform plan file
+tfplan
+
+# Ignore other files
+id_rsa*
+hosts
+*.flag
+keys.txt

images/daos-client-image.json

@@ -1,6 +1,7 @@
 {
     "variables": {
-        "DAOS_VERSION": "1.2.0"
+        "DAOS_VERSION": "1.2.0",
+        "IO500_INSTALL_DIR": "/usr/local"
     },
     "builders": [
     {
@@ -36,6 +37,18 @@
       "execute_command": "echo 'packer' | sudo -S env {{ .Vars }} {{ .Path }}",
       "environment_vars": "DAOS_VERSION={{user `DAOS_VERSION`}}",
       "script": "./scripts/install-client.sh"
+    },
+    {
+      "type": "shell",
+      "execute_command": "echo 'packer' | sudo -S env {{ .Vars }} {{ .Path }}",
+      "environment_vars": "IO500_INSTALL_DIR={{user `IO500_INSTALL_DIR`}}",
+      "script": "./scripts/mfu_install.sh"
+    },
+    {
+      "type": "shell",
+      "execute_command": "echo 'packer' | sudo -S env {{ .Vars }} {{ .Path }}",
+      "environment_vars": "IO500_INSTALL_DIR={{user `IO500_INSTALL_DIR`}}",
+      "script": "./scripts/io500_install.sh"
     }
   ]
 }

images/scripts/install.sh

@@ -38,7 +38,7 @@ EOF
 yum install -y daos-server
 
 # Upgrade SPDK to work around the GCP NVMe bug with number of qpairs
-yum install -y wget
+yum install -y wget ca-certificates
 wget https://packages.daos.io/v1.2/CentOS7/spdk/x86_64/spdk-20.01.2-2.el7.x86_64.rpm
 wget https://packages.daos.io/v1.2/CentOS7/spdk/x86_64/spdk-tools-20.01.2-2.el7.noarch.rpm
 rpm -Uvh ./spdk-20.01.2-2.el7.x86_64.rpm ./spdk-tools-20.01.2-2.el7.noarch.rpm

images/scripts/io500_install.sh

@@ -0,0 +1,114 @@
+#!/bin/bash
+prepare_build()
+{
+  ./prepare.sh
+}
+
+mkdir -p $IO500_INSTALL_DIR
+cd $IO500_INSTALL_DIR
+root=$IO500_INSTALL_DIR
+
+MY_DAOS_INSTALL_PATH=${HOME}/daos/install
+MY_MFU_INSTALL_PATH=${root}/mpifileutils/install
+MY_IO500_PATH=${root}/io500
+git clone https://github.com/IO500/io500.git -b io500-sc20 "${MY_IO500_PATH}"
+cd "${MY_IO500_PATH}"
+
+cat << EOF > io500_prepare.patch
+diff --git a/prepare.sh b/prepare.sh
+index de354ee..a2964d7 100755
+--- a/prepare.sh
++++ b/prepare.sh
+@@ -7,8 +7,8 @@ echo It will also attempt to build the benchmarks
+ echo It will output OK at the end if builds succeed
+ echo
+
+-IOR_HASH=bd76b45ef9db
+-PFIND_HASH=9d77056adce6
++IOR_HASH=
++PFIND_HASH=mfu_integration
+
+ INSTALL_DIR=\$PWD
+ BIN=\$INSTALL_DIR/bin
+@@ -59,14 +59,14 @@ function get_ior {
+
+ function get_pfind {
+   echo "Preparing parallel find"
+-  git_co https://github.com/VI4IO/pfind.git pfind \$PFIND_HASH
++  git_co https://github.com/mchaarawi/pfind pfind \$PFIND_HASH
+ }
+
+ ###### BUILD FUNCTIONS
+ function build_ior {
+   pushd \$BUILD/ior
+   ./bootstrap
+-  ./configure --prefix=\$INSTALL_DIR
++  ./configure --prefix=\$INSTALL_DIR --with-daos=${MY_DAOS_INSTALL_PATH}
+   cd src
+   \$MAKE clean
+   \$MAKE install
+EOF
+
+git apply io500_prepare.patch
+
+cat << EOF > io500_Makefile.patch
+diff --git a/Makefile b/Makefile
+index 2975471..5dce307 100644
+--- a/Makefile
++++ b/Makefile
+@@ -1,10 +1,13 @@
+ CC = mpicc
+ CFLAGS += -std=gnu99 -Wall -Wempty-body -Werror -Wstrict-prototypes -Werror=maybe-uninitialized -Warray-bounds
++CFLAGS += -I${MY_DAOS_INSTALL_PATH}/include -I${MY_MFU_INSTALL_PATH}/include
+
+ IORCFLAGS = \$(shell grep CFLAGS ./build/ior/Makefile | cut -d "=" -f 2-)
+ CFLAGS += -g3 -lefence -I./include/ -I./src/ -I./build/pfind/src/ -I./build/ior/src/
+ IORLIBS = \$(shell grep LIBS ./build/ior/Makefile | cut -d "=" -f 2-)
+ LDFLAGS += -lm \$(IORCFLAGS) \$(IORLIBS) # -lgpfs # may need some additional flags as provided to IOR
++LDFLAGS += -L${MY_DAOS_INSTALL_PATH}/lib64 -ldaos -ldaos_common -ldfs -lgurt -luuid
++LDFLAGS += -L${MY_MFU_INSTALL_PATH}/lib64 -lmfu_dfind -lmfu
+
+ VERSION_GIT=\$(shell git describe --always --abbrev=12)
+ VERSION_TREE=\$(shell git diff src | wc -l | sed -e 's/   *//g' -e 's/^0//' | sed "s/\([0-9]\)/-\1/")
+EOF
+
+git apply io500_Makefile.patch
+
+cat << 'EOF' > io500_stonewall.patch
+diff --git a/src/phase_find.c b/src/phase_find.c
+index e282b25..f2bb69c 100644
+--- a/src/phase_find.c
++++ b/src/phase_find.c
+@@ -61,6 +61,7 @@ static double run(void){
+     int rank;
+     MPI_Comm_rank(of.pfind_com, & rank);
+
++    of.pfind_o->stonewall = 300;
+     // pfind supports stonewalling timer -s, but ignore for now
+     pfind_find_results_t * res = pfind_find(of.pfind_o);
+     if(! res){
+EOF
+
+git apply io500_stonewall.patch
+export I_MPI_OFI_LIBRARY_INTERNAL=0
+export I_MPI_OFI_PROVIDER="tcp;ofi_rxm"
+source /opt/intel/oneapi/setvars.sh
+# This is expected to error.  After, compile.sh is edited with the correct paths
+# and the build can be run again.
+prepare_build || true
+
+sed -i "/^DAOS=/c\DAOS=${MY_DAOS_INSTALL_PATH}" ${MY_IO500_PATH}/build/pfind/compile.sh
+sed -i "/^MFU=/c\MFU=${MY_MFU_INSTALL_PATH}" ${MY_IO500_PATH}/build/pfind/compile.sh
+prepare_build || true
+
+cd build/ior
+git checkout a90d414a304b53c64d331d09104cc8df8bda0226
+make install
+cd ../../
+make clean
+make
+
+wget https://raw.githubusercontent.com/mchaarawi/io500/main/config-full.ini
+sed -i 's/ --dfs.svcl=$DAOS_SVCL//g' config-full.ini
+
+echo "Complete!"

images/scripts/mfu_install.sh

@@ -0,0 +1,92 @@
+#!/bin/bash
+mkdir -p $IO500_INSTALL_DIR
+cd $IO500_INSTALL_DIR
+root=$IO500_INSTALL_DIR
+
+mkdir -p tools
+cd tools
+wget https://github.com/Kitware/CMake/releases/download/v3.19.8/cmake-3.19.8-Linux-x86_64.sh
+chmod +x cmake-3.19.8-Linux-x86_64.sh
+./cmake-3.19.8-Linux-x86_64.sh --skip-license
+cd $root
+
+PATH=$root/tools/bin:$PATH
+echo $PATH
+
+#load Intel MPI
+export I_MPI_OFI_LIBRARY_INTERNAL=0
+export I_MPI_OFI_PROVIDER="tcp;ofi_rxm"
+source /opt/intel/oneapi/setvars.sh
+
+mfu=$root/mpifileutils
+installdir=$mfu/install
+deps=$mfu/deps
+
+mkdir -p $mfu
+mkdir -p $installdir
+mkdir -p $deps
+
+cd $deps
+  wget https://github.com/hpc/libcircle/releases/download/v0.3/libcircle-0.3.0.tar.gz
+  wget https://github.com/llnl/lwgrp/releases/download/v1.0.2/lwgrp-1.0.2.tar.gz
+  wget https://github.com/llnl/dtcmp/releases/download/v1.1.0/dtcmp-1.1.0.tar.gz
+
+  tar -zxf libcircle-0.3.0.tar.gz
+  cd libcircle-0.3.0
+    ./configure --prefix=$installdir
+
+  # Navigate to libcircle source directory
+
+  # Generate patch file
+cat << 'EOF' > libcircle_opt.patch
+--- a/libcircle/token.c
++++ b/libcircle/token.c
+@@ -1307,6 +1307,12 @@
+
+         LOG(CIRCLE_LOG_DBG, "Sending work request to %d...", source);
+
++        /* first always ask rank 0 for work */
++        int temp;
++        MPI_Comm_rank(comm, &temp);
++        if (st->local_work_requested < 10 && temp != 0 && temp < 512)
++            source = 0;
++
+         /* increment number of work requests for profiling */
+         st->local_work_requested++;
+
+EOF
+    # Apply the patch
+    patch -p1 < libcircle_opt.patch
+    make install
+  cd ..
+
+  tar -zxf lwgrp-1.0.2.tar.gz
+  cd lwgrp-1.0.2
+    ./configure --prefix=$installdir
+    make install
+  cd ..
+
+  tar -zxf dtcmp-1.1.0.tar.gz
+  cd dtcmp-1.1.0
+    ./configure --prefix=$installdir --with-lwgrp=$installdir
+    make install
+  cd $root
+
+sudo yum -y install libarchive-devel bzip2-devel openssl-devel jq
+
+MY_DAOS_INSTALL_PATH=${HOME}/daos/install
+MY_MFU_INSTALL_PATH=$installdir
+MY_MFU_SOURCE_PATH=$installdir/source
+MY_MFU_BUILD_PATH=$mfu/build
+
+git clone https://github.com/mchaarawi/mpifileutils -b pfind_integration "${MY_MFU_SOURCE_PATH}"
+mkdir -p "${MY_MFU_BUILD_PATH}"
+cd "${MY_MFU_BUILD_PATH}"
+export CFLAGS="-I${MY_DAOS_INSTALL_PATH}/include -I${MY_DAOS_INSTALL_PATH}/include/gurt/"
+export LDFLAGS="-L${MY_DAOS_INSTALL_PATH}/lib64/ -luuid -ldaos -ldfs -ldaos_common -lgurt -lpthread"
+cmake "${MY_MFU_SOURCE_PATH}" \
+  -DENABLE_XATTRS=OFF \
+  -DWITH_DTCMP_PREFIX=${MY_MFU_INSTALL_PATH} \
+  -DWITH_LibCircle_PREFIX=${MY_MFU_INSTALL_PATH} \
+  -DCMAKE_INSTALL_PREFIX=${MY_MFU_INSTALL_PATH} &&
+make -j8 install