diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4446d97 --- /dev/null +++ b/.gitignore @@ -0,0 +1,19 @@ +# Local .terraform directories +**/.terraform/ +**/.terraform/* + +# .tfstate files +*.tfstate +*.tfstate.* + +# Ignore Terraform lock files +.terraform.lock.hcl + +# Ignore Terraform plan file +tfplan + +# Ignore other files +id_rsa* +hosts +*.flag +keys.txt diff --git a/images/daos-client-image.json b/images/daos-client-image.json index 5676f1f..b6fa41d 100644 --- a/images/daos-client-image.json +++ b/images/daos-client-image.json @@ -1,6 +1,7 @@ { "variables": { - "DAOS_VERSION": "1.2.0" + "DAOS_VERSION": "1.2.0", + "IO500_INSTALL_DIR": "/usr/local" }, "builders": [ { @@ -36,6 +37,18 @@ "execute_command": "echo 'packer' | sudo -S env {{ .Vars }} {{ .Path }}", "environment_vars": "DAOS_VERSION={{user `DAOS_VERSION`}}", "script": "./scripts/install-client.sh" + }, + { + "type": "shell", + "execute_command": "echo 'packer' | sudo -S env {{ .Vars }} {{ .Path }}", + "environment_vars": "IO500_INSTALL_DIR={{user `IO500_INSTALL_DIR`}}", + "script": "./scripts/mfu_install.sh" + }, + { + "type": "shell", + "execute_command": "echo 'packer' | sudo -S env {{ .Vars }} {{ .Path }}", + "environment_vars": "IO500_INSTALL_DIR={{user `IO500_INSTALL_DIR`}}", + "script": "./scripts/io500_install.sh" } ] } diff --git a/images/scripts/install.sh b/images/scripts/install.sh index 044fd8d..9e5429f 100644 --- a/images/scripts/install.sh +++ b/images/scripts/install.sh @@ -38,7 +38,7 @@ EOF yum install -y daos-server # Upgrade SPDK to work around the GCP NVMe bug with number of qpairs -yum install -y wget +yum install -y wget ca-certificates wget https://packages.daos.io/v1.2/CentOS7/spdk/x86_64/spdk-20.01.2-2.el7.x86_64.rpm wget https://packages.daos.io/v1.2/CentOS7/spdk/x86_64/spdk-tools-20.01.2-2.el7.noarch.rpm rpm -Uvh ./spdk-20.01.2-2.el7.x86_64.rpm ./spdk-tools-20.01.2-2.el7.noarch.rpm diff --git a/images/scripts/io500_install.sh b/images/scripts/io500_install.sh new file mode 100644 index 0000000..45f8b1c --- /dev/null +++ b/images/scripts/io500_install.sh @@ -0,0 +1,114 @@ +#!/bin/bash +prepare_build() +{ + ./prepare.sh +} + +mkdir -p $IO500_INSTALL_DIR +cd $IO500_INSTALL_DIR +root=$IO500_INSTALL_DIR + +MY_DAOS_INSTALL_PATH=${HOME}/daos/install +MY_MFU_INSTALL_PATH=${root}/mpifileutils/install +MY_IO500_PATH=${root}/io500 +git clone https://github.com/IO500/io500.git -b io500-sc20 "${MY_IO500_PATH}" +cd "${MY_IO500_PATH}" + +cat << EOF > io500_prepare.patch +diff --git a/prepare.sh b/prepare.sh +index de354ee..a2964d7 100755 +--- a/prepare.sh ++++ b/prepare.sh +@@ -7,8 +7,8 @@ echo It will also attempt to build the benchmarks + echo It will output OK at the end if builds succeed + echo + +-IOR_HASH=bd76b45ef9db +-PFIND_HASH=9d77056adce6 ++IOR_HASH= ++PFIND_HASH=mfu_integration + + INSTALL_DIR=\$PWD + BIN=\$INSTALL_DIR/bin +@@ -59,14 +59,14 @@ function get_ior { + + function get_pfind { + echo "Preparing parallel find" +- git_co https://github.com/VI4IO/pfind.git pfind \$PFIND_HASH ++ git_co https://github.com/mchaarawi/pfind pfind \$PFIND_HASH + } + + ###### BUILD FUNCTIONS + function build_ior { + pushd \$BUILD/ior + ./bootstrap +- ./configure --prefix=\$INSTALL_DIR ++ ./configure --prefix=\$INSTALL_DIR --with-daos=${MY_DAOS_INSTALL_PATH} + cd src + \$MAKE clean + \$MAKE install +EOF + +git apply io500_prepare.patch + +cat << EOF > io500_Makefile.patch +diff --git a/Makefile b/Makefile +index 2975471..5dce307 100644 +--- a/Makefile ++++ b/Makefile +@@ -1,10 +1,13 @@ + CC = mpicc + CFLAGS += -std=gnu99 -Wall -Wempty-body -Werror -Wstrict-prototypes -Werror=maybe-uninitialized -Warray-bounds ++CFLAGS += -I${MY_DAOS_INSTALL_PATH}/include -I${MY_MFU_INSTALL_PATH}/include + + IORCFLAGS = \$(shell grep CFLAGS ./build/ior/Makefile | cut -d "=" -f 2-) + CFLAGS += -g3 -lefence -I./include/ -I./src/ -I./build/pfind/src/ -I./build/ior/src/ + IORLIBS = \$(shell grep LIBS ./build/ior/Makefile | cut -d "=" -f 2-) + LDFLAGS += -lm \$(IORCFLAGS) \$(IORLIBS) # -lgpfs # may need some additional flags as provided to IOR ++LDFLAGS += -L${MY_DAOS_INSTALL_PATH}/lib64 -ldaos -ldaos_common -ldfs -lgurt -luuid ++LDFLAGS += -L${MY_MFU_INSTALL_PATH}/lib64 -lmfu_dfind -lmfu + + VERSION_GIT=\$(shell git describe --always --abbrev=12) + VERSION_TREE=\$(shell git diff src | wc -l | sed -e 's/ *//g' -e 's/^0//' | sed "s/\([0-9]\)/-\1/") +EOF + +git apply io500_Makefile.patch + +cat << 'EOF' > io500_stonewall.patch +diff --git a/src/phase_find.c b/src/phase_find.c +index e282b25..f2bb69c 100644 +--- a/src/phase_find.c ++++ b/src/phase_find.c +@@ -61,6 +61,7 @@ static double run(void){ + int rank; + MPI_Comm_rank(of.pfind_com, & rank); + ++ of.pfind_o->stonewall = 300; + // pfind supports stonewalling timer -s, but ignore for now + pfind_find_results_t * res = pfind_find(of.pfind_o); + if(! res){ +EOF + +git apply io500_stonewall.patch +export I_MPI_OFI_LIBRARY_INTERNAL=0 +export I_MPI_OFI_PROVIDER="tcp;ofi_rxm" +source /opt/intel/oneapi/setvars.sh +# This is expected to error. After, compile.sh is edited with the correct paths +# and the build can be run again. +prepare_build || true + +sed -i "/^DAOS=/c\DAOS=${MY_DAOS_INSTALL_PATH}" ${MY_IO500_PATH}/build/pfind/compile.sh +sed -i "/^MFU=/c\MFU=${MY_MFU_INSTALL_PATH}" ${MY_IO500_PATH}/build/pfind/compile.sh +prepare_build || true + +cd build/ior +git checkout a90d414a304b53c64d331d09104cc8df8bda0226 +make install +cd ../../ +make clean +make + +wget https://raw.githubusercontent.com/mchaarawi/io500/main/config-full.ini +sed -i 's/ --dfs.svcl=$DAOS_SVCL//g' config-full.ini + +echo "Complete!" diff --git a/images/scripts/mfu_install.sh b/images/scripts/mfu_install.sh new file mode 100644 index 0000000..b6eb837 --- /dev/null +++ b/images/scripts/mfu_install.sh @@ -0,0 +1,92 @@ +#!/bin/bash +mkdir -p $IO500_INSTALL_DIR +cd $IO500_INSTALL_DIR +root=$IO500_INSTALL_DIR + +mkdir -p tools +cd tools +wget https://github.com/Kitware/CMake/releases/download/v3.19.8/cmake-3.19.8-Linux-x86_64.sh +chmod +x cmake-3.19.8-Linux-x86_64.sh +./cmake-3.19.8-Linux-x86_64.sh --skip-license +cd $root + +PATH=$root/tools/bin:$PATH +echo $PATH + +#load Intel MPI +export I_MPI_OFI_LIBRARY_INTERNAL=0 +export I_MPI_OFI_PROVIDER="tcp;ofi_rxm" +source /opt/intel/oneapi/setvars.sh + +mfu=$root/mpifileutils +installdir=$mfu/install +deps=$mfu/deps + +mkdir -p $mfu +mkdir -p $installdir +mkdir -p $deps + +cd $deps + wget https://github.com/hpc/libcircle/releases/download/v0.3/libcircle-0.3.0.tar.gz + wget https://github.com/llnl/lwgrp/releases/download/v1.0.2/lwgrp-1.0.2.tar.gz + wget https://github.com/llnl/dtcmp/releases/download/v1.1.0/dtcmp-1.1.0.tar.gz + + tar -zxf libcircle-0.3.0.tar.gz + cd libcircle-0.3.0 + ./configure --prefix=$installdir + + # Navigate to libcircle source directory + + # Generate patch file +cat << 'EOF' > libcircle_opt.patch +--- a/libcircle/token.c ++++ b/libcircle/token.c +@@ -1307,6 +1307,12 @@ + + LOG(CIRCLE_LOG_DBG, "Sending work request to %d...", source); + ++ /* first always ask rank 0 for work */ ++ int temp; ++ MPI_Comm_rank(comm, &temp); ++ if (st->local_work_requested < 10 && temp != 0 && temp < 512) ++ source = 0; ++ + /* increment number of work requests for profiling */ + st->local_work_requested++; + +EOF + # Apply the patch + patch -p1 < libcircle_opt.patch + make install + cd .. + + tar -zxf lwgrp-1.0.2.tar.gz + cd lwgrp-1.0.2 + ./configure --prefix=$installdir + make install + cd .. + + tar -zxf dtcmp-1.1.0.tar.gz + cd dtcmp-1.1.0 + ./configure --prefix=$installdir --with-lwgrp=$installdir + make install + cd $root + +sudo yum -y install libarchive-devel bzip2-devel openssl-devel jq + +MY_DAOS_INSTALL_PATH=${HOME}/daos/install +MY_MFU_INSTALL_PATH=$installdir +MY_MFU_SOURCE_PATH=$installdir/source +MY_MFU_BUILD_PATH=$mfu/build + +git clone https://github.com/mchaarawi/mpifileutils -b pfind_integration "${MY_MFU_SOURCE_PATH}" +mkdir -p "${MY_MFU_BUILD_PATH}" +cd "${MY_MFU_BUILD_PATH}" +export CFLAGS="-I${MY_DAOS_INSTALL_PATH}/include -I${MY_DAOS_INSTALL_PATH}/include/gurt/" +export LDFLAGS="-L${MY_DAOS_INSTALL_PATH}/lib64/ -luuid -ldaos -ldfs -ldaos_common -lgurt -lpthread" +cmake "${MY_MFU_SOURCE_PATH}" \ + -DENABLE_XATTRS=OFF \ + -DWITH_DTCMP_PREFIX=${MY_MFU_INSTALL_PATH} \ + -DWITH_LibCircle_PREFIX=${MY_MFU_INSTALL_PATH} \ + -DCMAKE_INSTALL_PREFIX=${MY_MFU_INSTALL_PATH} && +make -j8 install