diff --git a/.circleci/config.yml b/.circleci/config.yml index ce66f1619b..d95cac9fa1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -15,8 +15,8 @@ defaults: &defaults name: Build command: bash source/scripts/ci/circle/run.sh build - run: - name: Test - command: bash source/scripts/ci/circle/run.sh test + name: Sanitize + command: bash source/scripts/ci/circle/run.sh memcheck version: 2 @@ -25,10 +25,10 @@ jobs: <<: *defaults docker: - image: ornladios/adios2:ci-fedora-ubsan - #"fedora-asan": - # <<: *defaults - # docker: - # - image: ornladios/adios2:ci-fedora-asan + "fedora-asan": + <<: *defaults + docker: + - image: ornladios/adios2:ci-fedora-asan "fedora-msan": <<: *defaults docker: @@ -43,6 +43,6 @@ workflows: sanitizers: jobs: - "fedora-ubsan" - #- "fedora-asan" + - "fedora-asan" - "fedora-msan" - "fedora-tsan" diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 00d7c3a934..51e6383630 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -13,10 +13,12 @@ on: jobs: linux: runs-on: ubuntu-latest - container: ${{ matrix.container }} - env: - GH_YML_JOBNAME: ${{ matrix.jobname }} - GH_YML_OS: Linux + container: + image: ${{ matrix.container }} + options: --shm-size=1g + env: + GH_YML_JOBNAME: ${{ matrix.jobname }} + GH_YML_OS: Linux strategy: fail-fast: false @@ -25,6 +27,7 @@ jobs: el7, el7-gnu8-ohpc, el7-gnu8-openmpi-ohpc, + el7-gnu8-openmpi-ohpc-static, suse-pgi, suse-pgi-openmpi, debian-sid, @@ -36,10 +39,12 @@ jobs: container: ornladios/adios2:ci-el7-gnu8-ohpc - jobname: el7-gnu8-openmpi-ohpc container: ornladios/adios2:ci-el7-gnu8-openmpi-ohpc + - jobname: el7-gnu8-openmpi-ohpc-static + container: ornladios/adios2:ci-el7-gnu8-openmpi-ohpc - jobname: suse-pgi - container: ornladios/adios2:ci-suse-pgi + container: ornladios/adios2:ci-suse-nvhpcsdk - jobname: suse-pgi-openmpi - container: ornladios/adios2:ci-suse-pgi-openmpi + container: ornladios/adios2:ci-suse-nvhpcsdk-openmpi - jobname: debian-sid container: ornladios/adios2:ci-debian-sid - jobname: debian-sid-openmpi @@ -49,7 +54,6 @@ jobs: - uses: actions/checkout@v2 with: ref: ${{ github.event.pull_request.head.sha }} - entrypoint: /usr/bin/sh -c - name: Setup run: scripts/ci/gh-actions/linux-setup.sh - name: Update @@ -65,19 +69,17 @@ jobs: runs-on: ubuntu-latest container: image: ${{ matrix.container }} - options: --privileged - env: - GH_YML_JOBNAME: ${{ matrix.jobname }} - GH_YML_OS: Linux + options: --privileged --shm-size=1g + env: + GH_YML_JOBNAME: ${{ matrix.jobname }} + GH_YML_OS: Linux strategy: fail-fast: false matrix: jobname: [ power8-el7-xl, - power8-el7-xl-smpi, - power8-el7-pgi, - power8-el7-pgi-smpi ] + power8-el7-xl-smpi ] include: - jobname: power8-el7-xl container: ornladios/adios2:ci-x86_64-power8-el7-xl @@ -85,12 +87,6 @@ jobs: - jobname: power8-el7-xl-smpi container: ornladios/adios2:ci-x86_64-power8-el7-xl-smpi arch: ppc64le - - jobname: power8-el7-pgi - container: ornladios/adios2:ci-x86_64-power8-el7-pgi - arch: ppc64le - - jobname: power8-el7-pgi-smpi - container: ornladios/adios2:ci-x86_64-power8-el7-pgi-smpi - arch: ppc64le steps: - name: Emulation Setup @@ -112,3 +108,43 @@ jobs: run: scripts/ci/gh-actions/run.sh build - name: Test run: scripts/ci/gh-actions/run.sh test + + linux_spack: + runs-on: ubuntu-latest + container: + image: ${{ matrix.container }} + options: --shm-size=1g + env: + GH_YML_JOBNAME: ${{ matrix.jobname }} + GH_YML_OS: Linux + + strategy: + fail-fast: false + matrix: + jobname: [ + el7-spack, + ubuntu1804-spack ] + include: + - jobname: el7-spack + container: ornladios/adios2:ci-el7-spack + - jobname: ubuntu1804-spack + container: ornladios/adios2:ci-ubuntu1804-spack + + defaults: + run: + shell: bash -c "source /opt/spack/share/spack/setup-env.sh && spack env activate adios2-ci && bash {0}" + + steps: + - uses: actions/checkout@v2 + with: + ref: ${{ github.event.pull_request.head.sha }} + env: + PATH: /opt/spack/var/spack/environments/adios2-ci/.spack-env/view/bin:/usr/bin:/bin + - name: Update + run: scripts/ci/gh-actions/run.sh update + - name: Configure + run: scripts/ci/gh-actions/run.sh configure + - name: Build + run: scripts/ci/gh-actions/run.sh build + - name: Test + run: scripts/ci/gh-actions/run.sh test diff --git a/.github/workflows/formatting.yml b/.github/workflows/formatting.yml new file mode 100644 index 0000000000..ffd13ae4c6 --- /dev/null +++ b/.github/workflows/formatting.yml @@ -0,0 +1,26 @@ +name: Formatting + +on: + push: + branches: + - master + - release* + pull_request: + branches: + - master + - release* + +jobs: + format: + runs-on: ubuntu-latest + container: + image: ornladios/adios2:ci-formatting + + steps: + - uses: actions/checkout@v2 + with: + ref: ${{ github.event.pull_request.head.sha }} + - name: CXX + run: scripts/ci/scripts/run-clang-format.sh + - name: Python + run: scripts/ci/scripts/run-flake8.sh diff --git a/.gitlab/gitlab-ci-gitlabdotcom.yml b/.gitlab/gitlab-ci-gitlabdotcom.yml new file mode 100644 index 0000000000..eba4958546 --- /dev/null +++ b/.gitlab/gitlab-ci-gitlabdotcom.yml @@ -0,0 +1,6 @@ +sync-github-prs: + tags: linux + only: + - schedules + script: + - scripts/ci/scripts/github-prs-to-gitlab.sh ornladios/adios2 code.ornl.gov ecpcitest/adios2 diff --git a/.gitlab/gitlab-ci-olcf.yml b/.gitlab/gitlab-ci-olcf.yml index aaf43fe5c1..ac593185a1 100644 --- a/.gitlab/gitlab-ci-olcf.yml +++ b/.gitlab/gitlab-ci-olcf.yml @@ -1,15 +1,3 @@ -sync-github-prs: - tags: [nobatch] - only: - - schedules - variables: - GIT_STRATEGY: clone - script: - - module load python/3.7.0 - - pip3 install --user PyGithub - - module load git - - bash scripts/ci/gitlab-ci/pr-sync.sh - .all-steps: except: - schedules diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index f5876f2922..0000000000 --- a/.travis.yml +++ /dev/null @@ -1,13 +0,0 @@ -sudo: false -language: cpp -dist: xenial -env: - matrix: - - BUILD_MATRIX_ENTRY=format - - BUILD_MATRIX_ENTRY=docker-ubuntu1804 - - BUILD_MATRIX_ENTRY=docker-ubuntu1910 - - BUILD_MATRIX_ENTRY=docker-centos7 -# - BUILD_MATRIX_ENTRY=docker-centos8 -script: - - git reset --hard ${TRAVIS_PULL_REQUEST_SHA} - - ${TRAVIS_BUILD_DIR}/scripts/travis/run.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index 9d145a534e..5d327a8ba3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -88,14 +88,12 @@ mark_as_advanced(ADIOS2_LIBRARY_SUFFIX ADIOS2_EXECUTABLE_SUFFIX) # Use meta-compile features if available, otherwise use specific language # features -if(CMAKE_VERSION VERSION_LESS 3.9 OR - CMAKE_CXX_COMPILER_ID MATCHES "^(GNU|Intel|Clang|AppleClang|MSVC)$") +if(CMAKE_CXX_COMPILER_ID MATCHES "^(GNU|Intel|Clang|AppleClang|MSVC)$") set(ADIOS2_CXX11_FEATURES cxx_auto_type cxx_nullptr) else() set(ADIOS2_CXX11_FEATURES cxx_std_11) endif() -if(CMAKE_VERSION VERSION_LESS 3.9 OR - CMAKE_C_COMPILER_ID MATCHES "^(GNU|Intel|Clang|AppleClang|MSVC)$") +if(CMAKE_C_COMPILER_ID MATCHES "^(GNU|Intel|Clang|AppleClang|MSVC)$") set(ADIOS2_C99_FEATURES c_restrict) else() set(ADIOS2_C99_FEATURES c_std_99) @@ -236,7 +234,7 @@ option(ADIOS2_BUILD_TESTING "Build the ADIOS2 testing tree" cmake_dependent_option(ADIOS2_RUN_MPI_MPMD_TESTS "Enable the parallel MPMD tests" ON "ADIOS2_BUILD_TESTING;ADIOS2_HAVE_MPI" OFF) -mark_as_advanced(ADIOS2_RUN_MPMD_TESTS) +mark_as_advanced(ADIOS2_RUN_MPI_MPMD_TESTS) include(CTest) set(BUILD_TESTING ${ADIOS2_BUILD_TESTING}) diff --git a/CTestCustom.cmake.in b/CTestCustom.cmake.in index e65c614747..a7a3f4f9d7 100644 --- a/CTestCustom.cmake.in +++ b/CTestCustom.cmake.in @@ -13,6 +13,9 @@ list(APPEND CTEST_CUSTOM_WARNING_EXCEPTION "warning: Using '.*' in statically linked applications requires at runtime the shared libraries from the glibc version used for linking" "PGC.*compilation completed with warnings" "Warning: stand-alone `data16' prefix" + "Warning: Type mismatch between actual argument.*" + "Warning: Rank mismatch between actual argument.*" + "zfp/types.h.*typedef" ) list(APPEND CTEST_CUSTOM_COVERAGE_EXCLUDE ".*/thirdparty/.*" diff --git a/ReadMe.md b/ReadMe.md index cc6cae0618..3e0d303a19 100644 --- a/ReadMe.md +++ b/ReadMe.md @@ -12,50 +12,59 @@ [![Coverity Scan Build Status](https://scan.coverity.com/projects/11116/badge.svg)](https://scan.coverity.com/projects/ornladios-adios2) -# ADIOS 2 : The Adaptable Input Output System version 2 -This is ADIOS 2 : The Adaptable Input/Output (I/O) System, ADIOS 2 is developed as part of the United States Department of Energy's Exascale Computing Program. +# ADIOS2 : The Adaptable Input Output System version 2 -ADIOS 2 is a framework designed for scientific data I/O to publish and subscribe (put/get) data when and where required. +This is ADIOS2: The Adaptable Input/Output (I/O) System. -ADIOS 2 transports data as groups of self-describing variables and attributes across different media types (such as files, wide-area-networks, and remote direct memory access) using a common application programming interface for all transport modes. ADIOS 2 can be used on supercomputers, cloud systems, and personal computers. +ADIOS2 is developed as part of the United States Department of Energy's Exascale Computing Project. +It is a framework for scientific data I/O to publish and subscribe to data when and where required. -ADIOS 2 focuses on: +ADIOS2 transports data as groups of self-describing variables and attributes across different media types (such as files, wide-area-networks, and remote direct memory access) using a common application programming interface for all transport modes. +ADIOS2 can be used on supercomputers, cloud systems, and personal computers. + +ADIOS2 focuses on: 1. **Performance** I/O scalability in high performance computing (HPC) applications. 2. **Adaptability** unified interfaces to allow for several modes of transport (files, memory-to-memory) 3. **Ease of Use** two-level application programming interface (APIs) - * Full APIs for HPC applications: C++11, Fortran 90, C 99, Python 2 and 3 - * Simplified High-Level APIs for data analysis: Python 2 and 3, C++11, Matlab - -In addition, ADIOS 2 APIs are based on: +* Full APIs for HPC applications: C++11, Fortran 90, C 99, Python 2 and 3 +* Simplified High-Level APIs for data analysis: Python 2 and 3, C++11, Matlab + +In addition, ADIOS2 APIs are based on: -* **MPI** ADIOS 2 is MPI-based, it can be used in non-MPI serial code. Non-MPI 100% serial build is optional. +* **MPI** Although ADIOS2 is MPI-based, it can also be used in non-MPI serial code. -* **Data Groups** ADIOS 2 favors a deferred/prefetch/grouped variables transport mode by default to maximize data-per-request ratios. Sync mode, one variable at a time, is treated as the special case. +* **Data Groups** ADIOS2 favors a deferred/prefetch/grouped variables transport mode by default to maximize data-per-request ratios. +Sync mode, one variable at a time, is treated as the special case. -* **Data Steps** ADIOS 2 follows the actual production/consumption of data using an I/O “steps” abstraction removing the need to manage extra indexing information. +* **Data Steps** ADIOS2 follows the actual production/consumption of data using an I/O “steps” abstraction removing the need to manage extra indexing information. -* **Data Engines** ADIOS 2 Engine abstraction allows for reusing the APIs for different transport modes removing the need for drastic code changes. +* **Data Engines** ADIOS2 Engine abstraction allows for reusing the APIs for different transport modes removing the need for drastic code changes. ## Documentation -Please find [The ADIOS 2 User Guide at readthedocs](https://adios2.readthedocs.io) +Documentation is hosted at [readthedocs](https://adios2.readthedocs.io). + +## Citing + +If you find ADIOS2 useful, please cite our [SoftwareX paper](https://doi.org/10.1016/j.softx.2020.100561), which also gives a high-level overview to the motivation and goals of ADIOS; complementing the documentation. ## Getting ADIOS2 -* From source: [Install ADIOS 2 documentation](https://adios2.readthedocs.io/en/latest/setting_up/setting_up.html#) requires CMake v3.6 or above. For a cmake configuration example see [scripts/runconf/runconf.sh](https://github.com/ornladios/ADIOS2/blob/master/scripts/runconf/runconf.sh) +* From source: [Install ADIOS2 documentation](https://adios2.readthedocs.io/en/latest/setting_up/setting_up.html#). +For a `cmake` configuration example see [scripts/runconf/runconf.sh](https://github.com/ornladios/ADIOS2/blob/master/scripts/runconf/runconf.sh) * Conda packages: - * [https://anaconda.org/williamfgc](https://anaconda.org/williamfgc) - * [https://anaconda.org/conda-forge/adios2](https://anaconda.org/conda-forge/adios2) +* [https://anaconda.org/williamfgc](https://anaconda.org/williamfgc) +* [https://anaconda.org/conda-forge/adios2](https://anaconda.org/conda-forge/adios2) * Spack: [adios2 package](https://spack.readthedocs.io/en/latest/package_list.html#adios2) * Docker images: - * Ubuntu 18.04: under [scripts/docker/images/ubuntu18.04/Dockerfile](https://github.com/ornladios/ADIOS2/tree/master/scripts/docker/images/ubuntu18.04/Dockerfile) +* Ubuntu 18.04: under [scripts/docker/images/ubuntu18.04/Dockerfile](https://github.com/ornladios/ADIOS2/tree/master/scripts/docker/images/ubuntu18.04/Dockerfile) Once ADIOS 2 is installed refer to: @@ -71,15 +80,15 @@ Once ADIOS 2 is installed refer to: ## Reporting Bugs -If you found a bug, please open an [issue on ADIOS2 github repository](https://github.com/ornladios/ADIOS2/issues) +If you find a bug, please open an [issue on ADIOS2 github repository](https://github.com/ornladios/ADIOS2/issues) ## Contributing -We invite the community to contribute, see [Contributor's Guide to ADIOS 2](Contributing.md) for instructions on how to contribute. ADIOS 2 will always be free and open-source. +See the [Contributor's Guide to ADIOS 2](Contributing.md) for instructions on how to contribute. ## License -ADIOS >= 2.0 is licensed under the Apache License v2.0. See the accompanying -[Copyright.txt](Copyright.txt) for more details. +ADIOS2 is licensed under the Apache License v2.0. +See the accompanying [Copyright.txt](Copyright.txt) for more details. ## Directory layout @@ -92,7 +101,7 @@ ADIOS >= 2.0 is licensed under the Apache License v2.0. See the accompanying * scripts - Project maintenance and development scripts * source - Internal source code for private components - * adios2 - source directory for the ADIOS2 library to be installed under install-dir/lib/libadios2. - * utils - source directory for the binary utilities, to be installed under install-dir/bin +* adios2 - source directory for the ADIOS2 library to be installed under install-dir/lib/libadios2. +* utils - source directory for the binary utilities, to be installed under install-dir/bin * testing - Tests using [gtest](https://github.com/google/googletest) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index fc2bca5fc0..45a689f6ad 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -13,8 +13,10 @@ resources: containers: - container: el7-intel-ohpc image: ornladios/adios2:ci-el7-intel-ohpc + options: --shm-size=1g - container: el7-intel-openmpi-ohpc image: ornladios/adios2:ci-el7-intel-openmpi-ohpc + options: --shm-size=1g jobs: - job: windows diff --git a/bindings/C/adios2/c/adios2_c_attribute.cpp b/bindings/C/adios2/c/adios2_c_attribute.cpp index fa34cb6293..fd4d4afbeb 100644 --- a/bindings/C/adios2/c/adios2_c_attribute.cpp +++ b/bindings/C/adios2/c/adios2_c_attribute.cpp @@ -49,12 +49,12 @@ adios2_error adios2_attribute_type(adios2_type *type, reinterpret_cast(attribute); auto type_s = attributeBase->m_Type; - if (type_s == adios2::helper::GetType()) + if (type_s == adios2::helper::GetDataType()) { *type = adios2_type_string; } #define make_case(T) \ - else if (type_s == adios2::helper::GetType::Type>()) \ + else if (type_s == adios2::helper::GetDataType::Type>()) \ { \ *type = T; \ } @@ -84,7 +84,7 @@ adios2_error adios2_attribute_type_string(char *type, size_t *size, const adios2::core::AttributeBase *attributeBase = reinterpret_cast(attribute); - return String2CAPI(attributeBase->m_Type, type, size); + return String2CAPI(ToString(attributeBase->m_Type), type, size); } catch (...) { @@ -146,13 +146,13 @@ adios2_error adios2_attribute_data(void *data, size_t *size, const adios2::core::AttributeBase *attributeBase = reinterpret_cast(attribute); - const std::string type(attributeBase->m_Type); + const adios2::DataType type(attributeBase->m_Type); - if (type == "") + if (type == adios2::DataType::None) { // not supported } - else if (type == adios2::helper::GetType()) + else if (type == adios2::helper::GetDataType()) { const adios2::core::Attribute *attributeCpp = dynamic_cast *>( @@ -177,7 +177,7 @@ adios2_error adios2_attribute_data(void *data, size_t *size, } } #define declare_template_instantiation(T) \ - else if (type == adios2::helper::GetType()) \ + else if (type == adios2::helper::GetDataType()) \ { \ const adios2::core::Attribute *attributeCpp = \ dynamic_cast *>(attributeBase); \ diff --git a/bindings/C/adios2/c/adios2_c_engine.cpp b/bindings/C/adios2/c/adios2_c_engine.cpp index c5f0741197..d12a2e8a36 100644 --- a/bindings/C/adios2/c/adios2_c_engine.cpp +++ b/bindings/C/adios2/c/adios2_c_engine.cpp @@ -11,7 +11,7 @@ #include "adios2_c_engine.h" #include "adios2/core/Engine.h" -#include "adios2/helper/adiosFunctions.h" //GetType +#include "adios2/helper/adiosFunctions.h" //GetDataType #include "adios2_c_internal.h" namespace @@ -241,17 +241,17 @@ adios2_error adios2_put(adios2_engine *engine, adios2_variable *variable, adios2::core::VariableBase *variableBase = reinterpret_cast(variable); - const std::string type(variableBase->m_Type); + const adios2::DataType type(variableBase->m_Type); const adios2::Mode modeCpp = adios2_ToMode( mode, "only adios2_mode_deferred or adios2_mode_sync are valid, " "in call to adios2_put"); - if (type == "compound") + if (type == adios2::DataType::Compound) { // not supported } - else if (type == adios2::helper::GetType()) + else if (type == adios2::helper::GetDataType()) { const std::string dataStr(reinterpret_cast(data)); engineCpp->Put(*dynamic_cast *>( @@ -259,7 +259,7 @@ adios2_error adios2_put(adios2_engine *engine, adios2_variable *variable, dataStr, modeCpp); } #define declare_template_instantiation(T) \ - else if (type == adios2::helper::GetType()) \ + else if (type == adios2::helper::GetDataType()) \ { \ engineCpp->Put( \ *dynamic_cast *>(variableBase), \ @@ -302,20 +302,20 @@ adios2_error adios2_put_by_name(adios2_engine *engine, mode, "only adios2_mode_deferred or adios2_mode_sync are valid, " "in call to adios2_put_by_name"); - const std::string type( + const adios2::DataType type( engineCpp->m_IO.InquireVariableType(variable_name)); - if (type == "compound") + if (type == adios2::DataType::Compound) { // not supported } - else if (type == adios2::helper::GetType()) + else if (type == adios2::helper::GetDataType()) { const std::string dataStr(reinterpret_cast(data)); engineCpp->Put(variable_name, dataStr, modeCpp); } #define declare_template_instantiation(T) \ - else if (type == adios2::helper::GetType()) \ + else if (type == adios2::helper::GetDataType()) \ { \ engineCpp->Put(variable_name, reinterpret_cast(data), \ modeCpp); \ @@ -379,13 +379,13 @@ adios2_error adios2_get(adios2_engine *engine, adios2_variable *variable, adios2::core::VariableBase *variableBase = reinterpret_cast(variable); - const std::string type(variableBase->m_Type); + const adios2::DataType type(variableBase->m_Type); - if (type == "compound") + if (type == adios2::DataType::Compound) { // not supported } - else if (type == adios2::helper::GetType()) + else if (type == adios2::helper::GetDataType()) { std::string dataStr; engineCpp->Get(*dynamic_cast *>( @@ -394,7 +394,7 @@ adios2_error adios2_get(adios2_engine *engine, adios2_variable *variable, dataStr.copy(reinterpret_cast(values), dataStr.size()); } #define declare_template_instantiation(T) \ - else if (type == adios2::helper::GetType()) \ + else if (type == adios2::helper::GetDataType()) \ { \ const adios2::Mode modeCpp = adios2_ToMode( \ mode, "only adios2_mode_deferred or adios2_mode_sync are valid, " \ @@ -438,21 +438,21 @@ adios2_error adios2_get_by_name(adios2_engine *engine, const adios2::Mode modeCpp = adios2_ToMode( mode, "only adios2_mode_deferred or adios2_mode_sync are valid, " "in call to adios2_get_by_name"); - const std::string type( + const adios2::DataType type( engineCpp->m_IO.InquireVariableType(variable_name)); - if (type == "compound") + if (type == adios2::DataType::Compound) { // not supported } - else if (type == adios2::helper::GetType()) + else if (type == adios2::helper::GetDataType()) { std::string dataStr; engineCpp->Get(variable_name, dataStr); dataStr.copy(reinterpret_cast(data), dataStr.size()); } #define declare_template_instantiation(T) \ - else if (type == adios2::helper::GetType()) \ + else if (type == adios2::helper::GetDataType()) \ { \ engineCpp->Get(variable_name, reinterpret_cast(data), modeCpp); \ } diff --git a/bindings/C/adios2/c/adios2_c_io.cpp b/bindings/C/adios2/c/adios2_c_io.cpp index ece3089301..f6c2f6a370 100644 --- a/bindings/C/adios2/c/adios2_c_io.cpp +++ b/bindings/C/adios2/c/adios2_c_io.cpp @@ -14,7 +14,7 @@ #include #include "adios2/core/IO.h" -#include "adios2/helper/adiosFunctions.h" //GetType +#include "adios2/helper/adiosFunctions.h" //GetDataType #include "adios2_c_internal.h" #ifdef __cplusplus @@ -250,7 +250,7 @@ adios2_variable *adios2_inquire_variable(adios2_io *io, const char *name) io, "for adios2_io, in call to adios2_inquire_variable"); adios2::core::IO &ioCpp = *reinterpret_cast(io); - const auto &dataMap = ioCpp.GetVariablesDataMap(); + const auto &dataMap = ioCpp.GetVariables(); auto itVariable = dataMap.find(name); if (itVariable == dataMap.end()) // not found @@ -258,15 +258,15 @@ adios2_variable *adios2_inquire_variable(adios2_io *io, const char *name) return variable; } - const std::string type(ioCpp.InquireVariableType(name)); + const adios2::DataType type(ioCpp.InquireVariableType(name)); adios2::core::VariableBase *variableCpp = nullptr; - if (type == "compound") + if (type == adios2::DataType::Compound) { // not supported } #define declare_template_instantiation(T) \ - else if (type == adios2::helper::GetType()) \ + else if (type == adios2::helper::GetDataType()) \ { \ variableCpp = ioCpp.InquireVariable(name); \ } @@ -292,7 +292,7 @@ adios2_error adios2_inquire_all_variables(adios2_variable ***variables, io, "for adios2_io, in call to adios2_inquire_all_variables"); adios2::core::IO &ioCpp = *reinterpret_cast(io); - const auto &dataMap = ioCpp.GetVariablesDataMap(); + const auto &dataMap = ioCpp.GetVariables(); *size = dataMap.size(); adios2_variable **list = @@ -308,15 +308,15 @@ adios2_error adios2_inquire_all_variables(adios2_variable ***variables, for (auto &name : names) { auto it = dataMap.find(name); - const std::string type(it->second.first); + const adios2::DataType type(it->second->m_Type); adios2::core::VariableBase *variable = nullptr; - if (type == "compound") + if (type == adios2::DataType::Compound) { // not supported } #define declare_template_instantiation(T) \ - else if (type == adios2::helper::GetType()) \ + else if (type == adios2::helper::GetDataType()) \ { \ variable = ioCpp.InquireVariable(name); \ list[n] = reinterpret_cast(variable); \ @@ -498,7 +498,7 @@ adios2_attribute *adios2_inquire_attribute(adios2_io *io, const char *name) io, "for adios2_io, in call to adios2_inquire_attribute"); adios2::core::IO &ioCpp = *reinterpret_cast(io); - const auto &dataMap = ioCpp.GetAttributesDataMap(); + const auto &dataMap = ioCpp.GetAttributes(); auto itAttribute = dataMap.find(name); if (itAttribute == dataMap.end()) // not found @@ -506,15 +506,15 @@ adios2_attribute *adios2_inquire_attribute(adios2_io *io, const char *name) return attribute; } - const std::string type(itAttribute->second.first); + const adios2::DataType type(itAttribute->second->m_Type); adios2::core::AttributeBase *attributeCpp = nullptr; - if (type == "compound") + if (type == adios2::DataType::Compound) { // not supported } #define declare_template_instantiation(T) \ - else if (type == adios2::helper::GetType()) \ + else if (type == adios2::helper::GetDataType()) \ { \ attributeCpp = ioCpp.InquireAttribute(name); \ } @@ -551,7 +551,7 @@ adios2_error adios2_inquire_all_attributes(adios2_attribute ***attributes, io, "for adios2_io, in call to adios2_inquire_all_attributes"); adios2::core::IO &ioCpp = *reinterpret_cast(io); - const auto &dataMap = ioCpp.GetAttributesDataMap(); + const auto &dataMap = ioCpp.GetAttributes(); *size = dataMap.size(); adios2_attribute **list = @@ -567,15 +567,15 @@ adios2_error adios2_inquire_all_attributes(adios2_attribute ***attributes, for (auto &name : names) { auto it = dataMap.find(name); - const std::string type(it->second.first); + const adios2::DataType type(it->second->m_Type); adios2::core::AttributeBase *attribute = nullptr; - if (type == "compound") + if (type == adios2::DataType::Compound) { // not supported } #define declare_template_instantiation(T) \ - else if (type == adios2::helper::GetType()) \ + else if (type == adios2::helper::GetDataType()) \ { \ attribute = ioCpp.InquireAttribute(name); \ list[n] = reinterpret_cast(attribute); \ diff --git a/bindings/C/adios2/c/adios2_c_io_mpi.cpp b/bindings/C/adios2/c/adios2_c_io_mpi.cpp index af4d78b96b..58344cbe45 100644 --- a/bindings/C/adios2/c/adios2_c_io_mpi.cpp +++ b/bindings/C/adios2/c/adios2_c_io_mpi.cpp @@ -11,7 +11,7 @@ #include #include "adios2/core/IO.h" -#include "adios2/helper/adiosFunctions.h" //GetType +#include "adios2/helper/adiosFunctions.h" //GetDataType #include "adios2_c_internal.h" #include "adios2/helper/adiosCommMPI.h" diff --git a/bindings/C/adios2/c/adios2_c_types.h b/bindings/C/adios2/c/adios2_c_types.h index ebfc102a54..e5c951d706 100644 --- a/bindings/C/adios2/c/adios2_c_types.h +++ b/bindings/C/adios2/c/adios2_c_types.h @@ -138,7 +138,7 @@ typedef enum adios2_shapeid_local_array = 4 } adios2_shapeid; -static size_t adios2_string_array_element_max_size = 4096; +static const size_t adios2_string_array_element_max_size = 4096; static const uint64_t adios2_local_value_dim = ULLONG_MAX - 2; diff --git a/bindings/C/adios2/c/adios2_c_variable.cpp b/bindings/C/adios2/c/adios2_c_variable.cpp index 77817948c5..196c3fceb7 100644 --- a/bindings/C/adios2/c/adios2_c_variable.cpp +++ b/bindings/C/adios2/c/adios2_c_variable.cpp @@ -219,13 +219,13 @@ adios2_error adios2_variable_type(adios2_type *type, const adios2::core::VariableBase *variableBase = reinterpret_cast(variable); - const std::string typeCpp = variableBase->m_Type; - if (typeCpp == adios2::helper::GetType()) + const adios2::DataType typeCpp = variableBase->m_Type; + if (typeCpp == adios2::helper::GetDataType()) { *type = adios2_type_string; } #define make_case(T) \ - else if (typeCpp == adios2::helper::GetType::Type>()) \ + else if (typeCpp == adios2::helper::GetDataType::Type>()) \ { \ *type = T; \ } @@ -254,7 +254,7 @@ adios2_error adios2_variable_type_string(char *type, size_t *size, const adios2::core::VariableBase *variableBase = reinterpret_cast(variable); - return String2CAPI(variableBase->m_Type, type, size); + return String2CAPI(ToString(variableBase->m_Type), type, size); } catch (...) { @@ -319,13 +319,13 @@ adios2_error adios2_variable_shape(size_t *shape, const adios2::core::VariableBase *variableBase = reinterpret_cast(variable); - const std::string typeCpp = variableBase->m_Type; - if (typeCpp == "compound") + const adios2::DataType typeCpp = variableBase->m_Type; + if (typeCpp == adios2::DataType::Compound) { // not supported } #define declare_template_instantiation(T) \ - else if (typeCpp == adios2::helper::GetType()) \ + else if (typeCpp == adios2::helper::GetDataType()) \ { \ const adios2::core::Variable *variable = \ dynamic_cast *>(variableBase); \ @@ -382,13 +382,13 @@ adios2_error adios2_variable_count(size_t *count, const adios2::core::VariableBase *variableBase = reinterpret_cast(variable); - const std::string typeCpp = variableBase->m_Type; - if (typeCpp == "compound") + const adios2::DataType typeCpp = variableBase->m_Type; + if (typeCpp == adios2::DataType::Compound) { // not supported } #define declare_template_instantiation(T) \ - else if (typeCpp == adios2::helper::GetType()) \ + else if (typeCpp == adios2::helper::GetDataType()) \ { \ const adios2::core::Variable *variable = \ dynamic_cast *>(variableBase); \ @@ -458,14 +458,14 @@ adios2_error adios2_selection_size(size_t *size, const adios2::core::VariableBase *variableBase = reinterpret_cast(variable); - const std::string typeCpp = variableBase->m_Type; + const adios2::DataType typeCpp = variableBase->m_Type; - if (typeCpp == "compound") + if (typeCpp == adios2::DataType::Compound) { // not supported } #define declare_template_instantiation(T) \ - else if (typeCpp == adios2::helper::GetType()) \ + else if (typeCpp == adios2::helper::GetDataType()) \ { \ const adios2::core::Variable *variable = \ dynamic_cast *>(variableBase); \ @@ -552,14 +552,14 @@ adios2_error adios2_variable_min(void *min, const adios2_variable *variable) const adios2::core::VariableBase *variableBase = reinterpret_cast(variable); - const std::string type(variableBase->m_Type); + const adios2::DataType type(variableBase->m_Type); - if (type == "compound") + if (type == adios2::DataType::Compound) { // not supported } #define declare_template_instantiation(T) \ - else if (type == adios2::helper::GetType()) \ + else if (type == adios2::helper::GetDataType()) \ { \ T *minT = reinterpret_cast(min); \ const adios2::core::Variable *variableT = \ @@ -589,14 +589,14 @@ adios2_error adios2_variable_max(void *max, const adios2_variable *variable) const adios2::core::VariableBase *variableBase = reinterpret_cast(variable); - const std::string type(variableBase->m_Type); + const adios2::DataType type(variableBase->m_Type); - if (type == "compound") + if (type == adios2::DataType::Compound) { // not supported } #define declare_template_instantiation(T) \ - else if (type == adios2::helper::GetType()) \ + else if (type == adios2::helper::GetDataType()) \ { \ T *maxT = reinterpret_cast(max); \ const adios2::core::Variable *variableT = \ diff --git a/bindings/CXX11/CMakeLists.txt b/bindings/CXX11/CMakeLists.txt index 606161274e..6ac821bb52 100644 --- a/bindings/CXX11/CMakeLists.txt +++ b/bindings/CXX11/CMakeLists.txt @@ -16,6 +16,8 @@ add_library(adios2_cxx11 adios2/cxx11/Types.tcc adios2/cxx11/Variable.cpp adios2/cxx11/fstream/ADIOS2fstream.cpp + adios2/cxx11/Group.cpp + adios2/cxx11/Group.tcc ) set_property(TARGET adios2_cxx11 PROPERTY EXPORT_NAME cxx11) set_property(TARGET adios2_cxx11 PROPERTY OUTPUT_NAME adios2${ADIOS2_LIBRARY_SUFFIX}_cxx11) @@ -78,6 +80,7 @@ install( FILES adios2/cxx11/ADIOS.h adios2/cxx11/ADIOS.inl adios2/cxx11/IO.h + adios2/cxx11/Group.h adios2/cxx11/Variable.h adios2/cxx11/Attribute.h adios2/cxx11/Engine.h diff --git a/bindings/CXX11/adios2.h b/bindings/CXX11/adios2.h index 73dd03d75a..4e5d97de3a 100644 --- a/bindings/CXX11/adios2.h +++ b/bindings/CXX11/adios2.h @@ -27,6 +27,7 @@ constexpr bool DebugOFF = false; #include "adios2/cxx11/ADIOS.h" #include "adios2/cxx11/Attribute.h" #include "adios2/cxx11/Engine.h" +#include "adios2/cxx11/Group.h" #include "adios2/cxx11/IO.h" #include "adios2/cxx11/Operator.h" #include "adios2/cxx11/Query.h" diff --git a/bindings/CXX11/adios2/cxx11/ADIOS.h b/bindings/CXX11/adios2/cxx11/ADIOS.h index 47ee295af7..777b280bfb 100644 --- a/bindings/CXX11/adios2/cxx11/ADIOS.h +++ b/bindings/CXX11/adios2/cxx11/ADIOS.h @@ -62,6 +62,20 @@ class ADIOS */ ADIOS(const std::string &configFile, MPI_Comm comm, const bool debugMode = true); +#else + // Client code that does not enable ADIOS2_USE_MPI may accidentally + // try to pass a MPI_Comm instance to our constructor. If the type + // the MPI library uses to implement MPI_Comm happens to be convertible + // to one of the types offered by our serial constructors (e.g. 'bool'), + // the invocation will appear to succeed but will ignore the + // communicator passed. Add explicitly deleted constructors that have + // better conversions for common MPI_Comm types. + + // openmpi uses 'ompi_communicator_t*' for MPI_Comm + ADIOS(void *define_ADIOS2_USE_MPI_to_use_MPI_constructor) = delete; + + // mpich uses 'int' for MPI_Comm + ADIOS(int define_ADIOS2_USE_MPI_to_use_MPI_constructor) = delete; #endif /** diff --git a/bindings/CXX11/adios2/cxx11/Attribute.cpp b/bindings/CXX11/adios2/cxx11/Attribute.cpp index 10dba9dae0..23a31754e1 100644 --- a/bindings/CXX11/adios2/cxx11/Attribute.cpp +++ b/bindings/CXX11/adios2/cxx11/Attribute.cpp @@ -44,7 +44,7 @@ namespace adios2 { \ helper::CheckForNullptr(m_Attribute, \ "in call to Attribute::Type()"); \ - return m_Attribute->m_Type; \ + return ToString(m_Attribute->m_Type); \ } \ \ template <> \ diff --git a/bindings/CXX11/adios2/cxx11/Engine.cpp b/bindings/CXX11/adios2/cxx11/Engine.cpp index 96a2db955a..6e2ff6d83e 100644 --- a/bindings/CXX11/adios2/cxx11/Engine.cpp +++ b/bindings/CXX11/adios2/cxx11/Engine.cpp @@ -194,11 +194,25 @@ ADIOS2_FOREACH_PRIMITIVE_TYPE_1ARG(declare_template_instantiation) Engine::AllStepsBlocksInfo(const Variable variable) const; \ \ template std::vector::Info> Engine::BlocksInfo( \ - const Variable variable, const size_t step) const; + const Variable variable, const size_t step) const; \ + \ + template std::vector Engine::GetAbsoluteSteps( \ + const Variable variable) const; ADIOS2_FOREACH_TYPE_1ARG(declare_template_instantiation) #undef declare_template_instantiation +size_t Engine::DebugGetDataBufferSize() const +{ + helper::CheckForNullptr(m_Engine, + "in call to Engine::DebugGetDataBufferSize"); + if (m_Engine->m_EngineType == "NULL") + { + return 0; + } + return m_Engine->DebugGetDataBufferSize(); +} + std::string ToString(const Engine &engine) { return std::string("Engine(Name: \"" + engine.Name() + "\", Type: \"" + diff --git a/bindings/CXX11/adios2/cxx11/Engine.h b/bindings/CXX11/adios2/cxx11/Engine.h index 0a7b28aafb..0bc8682ce1 100644 --- a/bindings/CXX11/adios2/cxx11/Engine.h +++ b/bindings/CXX11/adios2/cxx11/Engine.h @@ -395,6 +395,14 @@ class Engine std::vector::Info> BlocksInfo(const Variable variable, const size_t step) const; + /** + * Get the absolute steps of a variable in a file. This is for + * information purposes only, because absolute steps cannot be used + * in any ADIOS2 calls. + */ + template + std::vector GetAbsoluteSteps(const Variable variable) const; + /** * Inspect total number of available steps, use for file engines in read * mode only @@ -419,6 +427,9 @@ class Engine */ void LockReaderSelections(); + /* Debug function for adios2 testing framework */ + size_t DebugGetDataBufferSize() const; + private: Engine(core::Engine *engine); core::Engine *m_Engine = nullptr; diff --git a/bindings/CXX11/adios2/cxx11/Engine.tcc b/bindings/CXX11/adios2/cxx11/Engine.tcc index f2cc387dc6..161ac64611 100644 --- a/bindings/CXX11/adios2/cxx11/Engine.tcc +++ b/bindings/CXX11/adios2/cxx11/Engine.tcc @@ -323,6 +323,24 @@ Engine::BlocksInfo(const Variable variable, const size_t step) const return ToBlocksInfo(blocksInfo); } +template +std::vector Engine::GetAbsoluteSteps(const Variable variable) const +{ + using IOType = typename TypeInfo::IOType; + adios2::helper::CheckForNullptr( + m_Engine, "for Engine in call to Engine::GetAbsoluteSteps"); + if (m_Engine->m_EngineType == "NULL") + { + return std::vector(); + } + + adios2::helper::CheckForNullptr( + variable.m_Variable, + "for variable in call to Engine::GetAbsoluteSteps"); + + return m_Engine->GetAbsoluteSteps(*variable.m_Variable); +} + } // end namespace adios2 #endif /* ADIOS2_BINDINGS_CXX11_CXX11_ENGINE_TCC_ */ diff --git a/bindings/CXX11/adios2/cxx11/Group.cpp b/bindings/CXX11/adios2/cxx11/Group.cpp new file mode 100644 index 0000000000..160ff4933e --- /dev/null +++ b/bindings/CXX11/adios2/cxx11/Group.cpp @@ -0,0 +1,75 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * Group.cpp : + * + * Created on: August 25, 2020 + * Author: Dmitry Ganyushin ganyushindi@ornl.gov + */ + +#include "Group.h" +#include "Group.tcc" +#include "adios2/core/Group.h" + +namespace adios2 +{ +Group::Group(core::Group *group) : m_Group(group) {} + +Group Group::InquireGroup(std::string group_name) +{ + auto m = m_Group->InquireGroup(group_name); + return Group(m); +} +void Group::PrintTree() +{ + m_Group->PrintTree(); + return; +} + +void Group::BuildTree() +{ + m_Group->BuildTree(); + return; +} +std::vector Group::AvailableVariables() +{ + return m_Group->AvailableVariables(); +} +std::vector Group::AvailableAttributes() +{ + return m_Group->AvailableAttributes(); +} +std::vector Group::AvailableGroups() +{ + return m_Group->AvailableGroups(); +} + +std::map> &Group::getTreeMap() +{ + return m_Group->getTreeMap(); +} + +std::string Group::InquirePath() { return m_Group->InquirePath(); } + +void Group::setPath(std::string path) { m_Group->setPath(path); } +DataType Group::VariableType(const std::string &name) const +{ + helper::CheckForNullptr(m_Group, "in call to IO::VariableType"); + return m_Group->InquireVariableType(name); +} + +DataType Group::AttributeType(const std::string &name) const +{ + helper::CheckForNullptr(m_Group, "in call to IO::AttributeType"); + return m_Group->InquireAttributeType(name); +} +Group::~Group(){}; +// Explicit declaration of the public template methods +// Limits the types +#define declare_template_instantiation(T) \ + template Variable Group::InquireVariable(const std::string &); + +ADIOS2_FOREACH_TYPE_1ARG(declare_template_instantiation) +#undef declare_template_instantiation +} // end of namespace diff --git a/bindings/CXX11/adios2/cxx11/Group.h b/bindings/CXX11/adios2/cxx11/Group.h new file mode 100644 index 0000000000..fa07e8546c --- /dev/null +++ b/bindings/CXX11/adios2/cxx11/Group.h @@ -0,0 +1,145 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * Group.h : + * + * Created on: August 25, 2020 + * Author: Dmitry Ganyushin ganyushindi@ornl.gov + */ +#ifndef ADIOS2_BINDINGS_CXX11_CXX11_GROUP_H_ + +#define ADIOS2_BINDINGS_CXX11_CXX11_GROUP_H_ + +#include "Attribute.h" +#include "Variable.h" +#include +#include +#include +#include +#include + +#if ADIOS2_USE_MPI +#include +#endif + +#include "Group.h" +#include "adios2/common/ADIOSMacros.h" +#include "adios2/common/ADIOSTypes.h" + +namespace adios2 +{ +class IO; + +namespace core +{ +class Group; // private implementation +} +class Group +{ + friend class IO; + +private: + Group(core::Group *group); + core::Group *m_Group = nullptr; + +public: + ~Group(); + /** + * @brief Builds map that represents tree structure from m_Variable and + * m_Attributes from IO class + * @param + */ + void BuildTree(); + /** + * @brief Prints map that represents tree structure + * @param + */ + void PrintTree(); + /** + * @brief returns available groups on the path set + * @param + * @return vector of strings + */ + std::vector AvailableGroups(); + /** + * @brief returns available variables on the path set + * @param + * @return vector of strings + */ + std::vector AvailableVariables(); + /** + * @brief returns available attributes on the path set + * @param + * @return vector of strings + */ + std::vector AvailableAttributes(); + /** + * @brief returns the current path + * @param + * @return current path as a string + */ + std::string InquirePath(); + /** + * @brief set the path, points to a particular node on the tree + * @param next possible path extension + */ + void setPath(std::string path); + /** + * @brief returns a new group object + * @param name of the group + * @return new group object + */ + Group InquireGroup(std::string group_name); + /** + * @brief returns a reference to the map representing the tree stucture + * @param delimiter symbol + */ + std::map> &getTreeMap(); + /** + * @brief Gets an existing variable of primitive type by name. A wrapper for + * the corresponding function of the IO class + * @param name of variable to be retrieved + * @return pointer to an existing variable in current IO, nullptr if not + * found + */ + template + Variable InquireVariable(const std::string &name); + /** + * Gets an existing attribute of primitive type by name. A wrapper for + * the corresponding function of the IO class + * @param name of attribute to be retrieved + * @return pointer to an existing attribute in current IO, nullptr if not + * found + */ + template + Attribute InquireAttribute(const std::string &name, + const std::string &variableName = "", + const std::string separator = "/"); + + /** + * Inspects variable type. This function can be used in conjunction with + * MACROS in an else if (type == adios2::GetType() ) {} loop + * @param name unique variable name identifier in current IO + * @return type as in adios2::GetType() (e.g. "double", "float"), + * empty std::string if variable not found + */ + DataType VariableType(const std::string &name) const; + + /** + * Inspects attribute type. This function can be used in conjunction with + * MACROS in an else if (type == adios2::GetType() ) {} loop + * @param name unique attribute name identifier in current IO + * @return type as in adios2::GetType() (e.g. "double", "float"), empty + * std::string if attribute not found + */ + DataType AttributeType(const std::string &name) const; +}; +// Explicit declaration of the public template methods +// Limits the types +#define declare_template_instantiation(T) \ + extern template Variable Group::InquireVariable(const std::string &); +ADIOS2_FOREACH_TYPE_1ARG(declare_template_instantiation) +#undef declare_template_instantiation +} +#endif // ADIOS2_BINDINGS_CXX11_CXX11_GROUP_H_ diff --git a/bindings/CXX11/adios2/cxx11/Group.tcc b/bindings/CXX11/adios2/cxx11/Group.tcc new file mode 100644 index 0000000000..57966c0532 --- /dev/null +++ b/bindings/CXX11/adios2/cxx11/Group.tcc @@ -0,0 +1,42 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * Group.tcc : + * + * Created on: August 25, 2020 + * Author: Dmitry Ganyushin ganyushindi@ornl.gov + */ +#ifndef ADIOS2_BINDINGS_CXX11_CXX11_GROUP_TCC_ +#define ADIOS2_BINDINGS_CXX11_CXX11_GROUP_TCC_ + +#include "Group.h" + +#include "adios2/core/Group.h" + +namespace adios2 +{ +template +Variable Group::InquireVariable(const std::string &name) +{ + helper::CheckForNullptr(m_Group, "for variable name " + name + + ", in call to Group::InquireVariable"); + return Variable( + m_Group->InquireVariable::IOType>(name)); +} + +template +Attribute Group::InquireAttribute(const std::string &name, + const std::string &variableName, + const std::string separator) +{ + using IOType = typename TypeInfo::IOType; + helper::CheckForNullptr(m_Group, "for attribute name " + name + + ", in call to IO::InquireAttribute"); + return Attribute( + m_Group->InquireAttribute(name, variableName, separator)); +} + +} // end namespace adios2 + +#endif /* ADIOS2_BINDINGS_CXX11_CXX11_GROUP_TCC_ */ diff --git a/bindings/CXX11/adios2/cxx11/IO.cpp b/bindings/CXX11/adios2/cxx11/IO.cpp index 16b05d51d5..4c63287587 100644 --- a/bindings/CXX11/adios2/cxx11/IO.cpp +++ b/bindings/CXX11/adios2/cxx11/IO.cpp @@ -109,17 +109,27 @@ Engine IO::Open(const std::string &name, const Mode mode) "for engine " + name + ", in call to IO::Open"); return Engine(&m_IO->Open(name, mode)); } - +Group IO::InquireGroup(const std::string &path, char delimiter) +{ + return Group(&m_IO->CreateGroup(path, delimiter)); +}; void IO::FlushAll() { helper::CheckForNullptr(m_IO, "in call to IO::FlushAll"); m_IO->FlushAll(); } -std::map IO::AvailableVariables() +std::map IO::AvailableVariables(bool namesOnly) { helper::CheckForNullptr(m_IO, "in call to IO::AvailableVariables"); - return m_IO->GetAvailableVariables(); + if (namesOnly) + { + return m_IO->GetAvailableVariables({"name"}); + } + else + { + return m_IO->GetAvailableVariables(); + } } std::map @@ -133,13 +143,13 @@ IO::AvailableAttributes(const std::string &variableName, std::string IO::VariableType(const std::string &name) const { helper::CheckForNullptr(m_IO, "in call to IO::VariableType"); - return m_IO->InquireVariableType(name); + return ToString(m_IO->InquireVariableType(name)); } std::string IO::AttributeType(const std::string &name) const { helper::CheckForNullptr(m_IO, "in call to IO::AttributeType"); - return m_IO->InquireAttributeType(name); + return ToString(m_IO->InquireAttributeType(name)); } size_t IO::AddOperation(const Operator op, const Params ¶meters) diff --git a/bindings/CXX11/adios2/cxx11/IO.h b/bindings/CXX11/adios2/cxx11/IO.h index 62ce81ccda..3e4dee8ebc 100644 --- a/bindings/CXX11/adios2/cxx11/IO.h +++ b/bindings/CXX11/adios2/cxx11/IO.h @@ -13,9 +13,9 @@ #include "Attribute.h" #include "Engine.h" +#include "Group.h" #include "Operator.h" #include "Variable.h" - #if ADIOS2_USE_MPI #include #endif @@ -252,10 +252,17 @@ class IO * Open an Engine to start heavy-weight input/output operations. * @param name unique engine identifier * @param mode adios2::Mode::Write, adios2::Mode::Read, or - * adios2::Mode::Append (not yet support) + * adios2::Mode::Append (BP4 only) * @return engine object */ Engine Open(const std::string &name, const Mode mode); + /** + * Return a Group object for hierarchical reading. + * @param name starting path + * @param a delimiter to separate groups in a string representation + * @return Group object + */ + Group InquireGroup(const std::string &path, char delimiter = '/'); #if ADIOS2_USE_MPI /** @@ -265,7 +272,7 @@ class IO * MPI Collective function as it calls MPI_Comm_dup * @param name unique engine identifier within IO * @param mode adios2::Mode::Write, adios2::Mode::Read, or - * adios2::Mode::Append (not yet support) + * adios2::Mode::Append (BP4 only) * @param comm new communicator other than ADIOS object's communicator * @return engine object */ @@ -276,16 +283,19 @@ class IO void FlushAll(); /** - * Returns a map with variable information - * @return map: - * 
-     * key: variable name
-     * value: Params
-     * 		string key: variable info key
-     *      string value: variable info value
-     *
+ * Returns a map with variable information. + * - key: variable name + * - value: Params is a map + * - key: "Type", "Shape", "AvailableStepsCount", "Min", "Max", + * "SingleValue" + * - value: variable info value as string + * + * @param namesOnly: returns a map with the variable names but with no + * Parameters. Use this if you only need the list of variable names + * and call VariableType() and InquireVariable() on the names individually. + * @return map> */ - std::map AvailableVariables(); + std::map AvailableVariables(bool namesOnly = false); /** * Returns a map with available attributes information associated to a @@ -322,8 +332,8 @@ class IO * Inspects attribute type. This function can be used in conjunction with * MACROS in an else if (type == adios2::GetType() ) {} loop * @param name unique attribute name identifier in current IO - * @return type as in adios2::GetType() (e.g. "double", "float"), empty - * std::string if attribute not found + * @return type as in adios2::GetType() (e.g. "double", "float"), + * empty std::string if attribute not found */ std::string AttributeType(const std::string &name) const; diff --git a/bindings/CXX11/adios2/cxx11/Types.tcc b/bindings/CXX11/adios2/cxx11/Types.tcc index ff90edd72f..2ea59fd1d8 100644 --- a/bindings/CXX11/adios2/cxx11/Types.tcc +++ b/bindings/CXX11/adios2/cxx11/Types.tcc @@ -21,7 +21,7 @@ namespace adios2 template std::string GetType() noexcept { - return helper::GetType::IOType>(); + return ToString(helper::GetDataType::IOType>()); } } // end namespace adios2 diff --git a/bindings/CXX11/adios2/cxx11/Variable.cpp b/bindings/CXX11/adios2/cxx11/Variable.cpp index 56bdd78e77..fdbccb5253 100644 --- a/bindings/CXX11/adios2/cxx11/Variable.cpp +++ b/bindings/CXX11/adios2/cxx11/Variable.cpp @@ -90,7 +90,7 @@ namespace adios2 std::string Variable::Type() const \ { \ helper::CheckForNullptr(m_Variable, "in call to Variable::Type"); \ - return m_Variable->m_Type; \ + return ToString(m_Variable->m_Type); \ } \ template <> \ size_t Variable::Sizeof() const \ diff --git a/bindings/CXX11/adios2/cxx11/Variable.h b/bindings/CXX11/adios2/cxx11/Variable.h index 6b0dbb1177..c1928f47c8 100644 --- a/bindings/CXX11/adios2/cxx11/Variable.h +++ b/bindings/CXX11/adios2/cxx11/Variable.h @@ -22,7 +22,7 @@ namespace adios2 // forward declare class IO; // friend class Engine; // friend - +class Group; // friend namespace core { @@ -131,6 +131,7 @@ class Variable friend class IO; friend class Engine; + friend class Group; public: /** diff --git a/bindings/Fortran/CMakeLists.txt b/bindings/Fortran/CMakeLists.txt index 5ad3fe7c95..a9fd9b825f 100644 --- a/bindings/Fortran/CMakeLists.txt +++ b/bindings/Fortran/CMakeLists.txt @@ -11,6 +11,14 @@ FortranCInterface_VERIFY(CXX QUIET) # Check whether the compiler supports Fortran submodule constructs we need. adios2_check_fortran_submodules(ADIOS2_HAVE_FORTRAN_SUBMODULES) +# Leaving this workaround in place but commented out just in case we need to +# re-enable it in the future +# +# Cray submodules have naming issues so just don't use them for now +#if(CMAKE_Fortran_COMPILER_ID MATCHES "Cray") +# set(ADIOS2_HAVE_FORTRAN_SUBMODULES 0 CACHE INTERNAL "" FORCE) +#endif() + if(ADIOS2_USE_Fortran_flag_argument_mismatch) set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fallow-argument-mismatch") endif() @@ -29,12 +37,12 @@ add_library(adios2_fortran modules/adios2_parameters_mod.f90 modules/adios2_adios_mod.f90 modules/adios2_adios_init_mod.F90 - modules/adios2_adios_init_mod_serial.F90 + modules/adios2_adios_init_serial_smod.F90 modules/adios2_attribute_mod.f90 modules/adios2_attribute_data_mod.f90 modules/adios2_io_mod.f90 modules/adios2_io_open_mod.F90 - modules/adios2_io_open_mod_serial.F90 + modules/adios2_io_open_serial_smod.F90 modules/adios2_io_define_variable_mod.f90 modules/adios2_io_define_attribute_mod.f90 modules/adios2_engine_mod.f90 @@ -79,8 +87,8 @@ if(ADIOS2_HAVE_MPI) target_compile_definitions(adios2_fortran PRIVATE "$<$:ADIOS2_HAVE_MPI_F>") set(adios2_fortran_mpi_srcs - modules/adios2_adios_init_mod_mpi.F90 - modules/adios2_io_open_mod_mpi.F90 + modules/adios2_adios_init_mpi_smod.F90 + modules/adios2_io_open_mpi_smod.F90 f2c/adios2_f2c_adios_mpi.cpp f2c/adios2_f2c_io_mpi.cpp ) @@ -135,6 +143,9 @@ install( COMPONENT adios2_fortran-development FILES_MATCHING PATTERN "adios2*.mod" + PATTERN "adios2*.smod" + PATTERN "ADIOS2*.mod" + PATTERN "ADIOS2*.smod" PATTERN "CMakeFiles" EXCLUDE ) diff --git a/bindings/Fortran/f2c/adios2_f2c_engine.cpp b/bindings/Fortran/f2c/adios2_f2c_engine.cpp index 95dae53cd7..eecd7eb049 100644 --- a/bindings/Fortran/f2c/adios2_f2c_engine.cpp +++ b/bindings/Fortran/f2c/adios2_f2c_engine.cpp @@ -53,7 +53,7 @@ void FC_GLOBAL(adios2_steps_f2c, ADIOS2_STEPS_F2C)(int64_t *steps, { *steps = -1; size_t stepsC; - *ierr = static_cast(adios2_current_step(&stepsC, *engine)); + *ierr = static_cast(adios2_steps(&stepsC, *engine)); if (*ierr == static_cast(adios2_error_none)) { diff --git a/bindings/Fortran/modules/adios2_adios_init_mod.F90 b/bindings/Fortran/modules/adios2_adios_init_mod.F90 index 7ab08ddf98..010b6e8461 100644 --- a/bindings/Fortran/modules/adios2_adios_init_mod.F90 +++ b/bindings/Fortran/modules/adios2_adios_init_mod.F90 @@ -67,9 +67,9 @@ module subroutine adios2_init_config_debug_mpi(adios, config_file, comm, ierr) #else - use adios2_adios_init_mod_serial + use adios2_adios_init_serial_mod # ifdef ADIOS2_HAVE_MPI_F - use adios2_adios_init_mod_mpi + use adios2_adios_init_mpi_mod # endif #endif diff --git a/bindings/Fortran/modules/adios2_adios_init_mod_mpi.F90 b/bindings/Fortran/modules/adios2_adios_init_mpi_smod.F90 similarity index 79% rename from bindings/Fortran/modules/adios2_adios_init_mod_mpi.F90 rename to bindings/Fortran/modules/adios2_adios_init_mpi_smod.F90 index 5132ad17ac..2726908404 100644 --- a/bindings/Fortran/modules/adios2_adios_init_mod_mpi.F90 +++ b/bindings/Fortran/modules/adios2_adios_init_mpi_smod.F90 @@ -7,15 +7,15 @@ ! #ifdef ADIOS2_HAVE_FORTRAN_SUBMODULES -# define ADIOS2_MODULE_PROCEDURE module & +# define ADIOS2_MODULE_PROCEDURE module #else # define ADIOS2_MODULE_PROCEDURE #endif #ifdef ADIOS2_HAVE_FORTRAN_SUBMODULES -submodule ( adios2_adios_init_mod ) mpi +submodule ( adios2_adios_init_mod ) adios2_adios_init_mpi_smod #else -module adios2_adios_init_mod_mpi +module adios2_adios_init_mpi_mod #endif use adios2_parameters_mod @@ -33,8 +33,8 @@ module adios2_adios_init_mod_mpi contains - ADIOS2_MODULE_PROCEDURE - subroutine adios2_init_mpi(adios, comm, adios2_debug_mode, ierr) + ADIOS2_MODULE_PROCEDURE subroutine adios2_init_mpi( & + adios, comm, adios2_debug_mode, ierr) type(adios2_adios), intent(out) :: adios integer, intent(in) :: comm logical, intent(in) :: adios2_debug_mode @@ -44,8 +44,7 @@ subroutine adios2_init_mpi(adios, comm, adios2_debug_mode, ierr) end subroutine - ADIOS2_MODULE_PROCEDURE - subroutine adios2_init_debug_mpi(adios, comm, ierr) + ADIOS2_MODULE_PROCEDURE subroutine adios2_init_debug_mpi(adios, comm, ierr) type(adios2_adios), intent(out) :: adios integer, intent(in) :: comm integer, intent(out) :: ierr @@ -54,9 +53,8 @@ subroutine adios2_init_debug_mpi(adios, comm, ierr) end subroutine - ADIOS2_MODULE_PROCEDURE - subroutine adios2_init_config_mpi(adios, config_file, comm, adios2_debug_mode, & - ierr) + ADIOS2_MODULE_PROCEDURE subroutine adios2_init_config_mpi( & + adios, config_file, comm, adios2_debug_mode, ierr) type(adios2_adios), intent(out) :: adios character*(*), intent(in) :: config_file integer, intent(in) :: comm @@ -73,8 +71,8 @@ subroutine adios2_init_config_mpi(adios, config_file, comm, adios2_debug_mode, & end subroutine - ADIOS2_MODULE_PROCEDURE - subroutine adios2_init_config_debug_mpi(adios, config_file, comm, ierr) + ADIOS2_MODULE_PROCEDURE subroutine adios2_init_config_debug_mpi( & + adios, config_file, comm, ierr) type(adios2_adios), intent(out) :: adios character*(*), intent(in) :: config_file integer, intent(in) :: comm diff --git a/bindings/Fortran/modules/adios2_adios_init_mod_serial.F90 b/bindings/Fortran/modules/adios2_adios_init_serial_smod.F90 similarity index 77% rename from bindings/Fortran/modules/adios2_adios_init_mod_serial.F90 rename to bindings/Fortran/modules/adios2_adios_init_serial_smod.F90 index f92fe15d3f..8c95476ed8 100644 --- a/bindings/Fortran/modules/adios2_adios_init_mod_serial.F90 +++ b/bindings/Fortran/modules/adios2_adios_init_serial_smod.F90 @@ -7,15 +7,15 @@ ! #ifdef ADIOS2_HAVE_FORTRAN_SUBMODULES -# define ADIOS2_MODULE_PROCEDURE module & +# define ADIOS2_MODULE_PROCEDURE module #else # define ADIOS2_MODULE_PROCEDURE #endif #ifdef ADIOS2_HAVE_FORTRAN_SUBMODULES -submodule ( adios2_adios_init_mod ) serial +submodule ( adios2_adios_init_mod ) adios2_adios_init_serial_smod #else -module adios2_adios_init_mod_serial +module adios2_adios_init_serial_mod #endif use adios2_parameters_mod @@ -33,8 +33,8 @@ module adios2_adios_init_mod_serial contains - ADIOS2_MODULE_PROCEDURE - subroutine adios2_init_serial(adios, adios2_debug_mode, ierr) + ADIOS2_MODULE_PROCEDURE subroutine adios2_init_serial( & + adios, adios2_debug_mode, ierr) type(adios2_adios), intent(out) :: adios logical, intent(in) :: adios2_debug_mode integer, intent(out) :: ierr @@ -43,8 +43,7 @@ subroutine adios2_init_serial(adios, adios2_debug_mode, ierr) end subroutine - ADIOS2_MODULE_PROCEDURE - subroutine adios2_init_debug_serial(adios, ierr) + ADIOS2_MODULE_PROCEDURE subroutine adios2_init_debug_serial(adios, ierr) type(adios2_adios), intent(out) :: adios integer, intent(out) :: ierr @@ -52,8 +51,8 @@ subroutine adios2_init_debug_serial(adios, ierr) end subroutine - ADIOS2_MODULE_PROCEDURE - subroutine adios2_init_config_serial(adios, config_file, adios2_debug_mode, ierr) + ADIOS2_MODULE_PROCEDURE subroutine adios2_init_config_serial( & + adios, config_file, adios2_debug_mode, ierr) type(adios2_adios), intent(out) :: adios character*(*), intent(in) :: config_file logical, intent(in) :: adios2_debug_mode @@ -67,8 +66,8 @@ subroutine adios2_init_config_serial(adios, config_file, adios2_debug_mode, ierr end subroutine - ADIOS2_MODULE_PROCEDURE - subroutine adios2_init_config_debug_serial(adios, config_file, ierr) + ADIOS2_MODULE_PROCEDURE subroutine adios2_init_config_debug_serial( & + adios, config_file, ierr) type(adios2_adios), intent(out) :: adios character*(*), intent(in) :: config_file integer, intent(out) :: ierr diff --git a/bindings/Fortran/modules/adios2_io_open_mod.F90 b/bindings/Fortran/modules/adios2_io_open_mod.F90 index 56475a6e73..f464fb39f1 100644 --- a/bindings/Fortran/modules/adios2_io_open_mod.F90 +++ b/bindings/Fortran/modules/adios2_io_open_mod.F90 @@ -38,9 +38,9 @@ module subroutine adios2_open_new_comm(engine, io, name, adios2_mode, comm, ierr #else - use adios2_io_open_mod_serial + use adios2_io_open_serial_mod # ifdef ADIOS2_HAVE_MPI_F - use adios2_io_open_mod_mpi + use adios2_io_open_mpi_mod # endif #endif diff --git a/bindings/Fortran/modules/adios2_io_open_mod_mpi.F90 b/bindings/Fortran/modules/adios2_io_open_mpi_smod.F90 similarity index 84% rename from bindings/Fortran/modules/adios2_io_open_mod_mpi.F90 rename to bindings/Fortran/modules/adios2_io_open_mpi_smod.F90 index 94b2eb917e..333a6a5b6d 100644 --- a/bindings/Fortran/modules/adios2_io_open_mod_mpi.F90 +++ b/bindings/Fortran/modules/adios2_io_open_mpi_smod.F90 @@ -7,15 +7,15 @@ ! #ifdef ADIOS2_HAVE_FORTRAN_SUBMODULES -# define ADIOS2_MODULE_PROCEDURE module & +# define ADIOS2_MODULE_PROCEDURE module #else # define ADIOS2_MODULE_PROCEDURE #endif #ifdef ADIOS2_HAVE_FORTRAN_SUBMODULES -submodule ( adios2_io_open_mod ) mpi +submodule ( adios2_io_open_mod ) adios2_io_open_mpi_smod #else -module adios2_io_open_mod_mpi +module adios2_io_open_mpi_mod #endif use adios2_parameters_mod @@ -29,8 +29,8 @@ module adios2_io_open_mod_mpi contains - ADIOS2_MODULE_PROCEDURE - subroutine adios2_open_new_comm(engine, io, name, adios2_mode, comm, ierr) + ADIOS2_MODULE_PROCEDURE subroutine adios2_open_new_comm( & + engine, io, name, adios2_mode, comm, ierr) type(adios2_engine), intent(out) :: engine type(adios2_io), intent(in) :: io character*(*), intent(in) :: name diff --git a/bindings/Fortran/modules/adios2_io_open_mod_serial.F90 b/bindings/Fortran/modules/adios2_io_open_serial_smod.F90 similarity index 83% rename from bindings/Fortran/modules/adios2_io_open_mod_serial.F90 rename to bindings/Fortran/modules/adios2_io_open_serial_smod.F90 index 6aacc908dd..5e3f785e51 100644 --- a/bindings/Fortran/modules/adios2_io_open_mod_serial.F90 +++ b/bindings/Fortran/modules/adios2_io_open_serial_smod.F90 @@ -7,15 +7,15 @@ ! #ifdef ADIOS2_HAVE_FORTRAN_SUBMODULES -# define ADIOS2_MODULE_PROCEDURE module & +# define ADIOS2_MODULE_PROCEDURE module #else # define ADIOS2_MODULE_PROCEDURE #endif #ifdef ADIOS2_HAVE_FORTRAN_SUBMODULES -submodule ( adios2_io_open_mod ) serial +submodule ( adios2_io_open_mod ) adios2_io_open_serial_smod #else -module adios2_io_open_mod_serial +module adios2_io_open_serial_mod #endif use adios2_parameters_mod @@ -29,8 +29,8 @@ module adios2_io_open_mod_serial contains - ADIOS2_MODULE_PROCEDURE - subroutine adios2_open_old_comm(engine, io, name, adios2_mode, ierr) + ADIOS2_MODULE_PROCEDURE subroutine adios2_open_old_comm( & + engine, io, name, adios2_mode, ierr) type(adios2_engine), intent(out) :: engine type(adios2_io), intent(in) :: io character*(*), intent(in) :: name diff --git a/bindings/Python/py11Attribute.cpp b/bindings/Python/py11Attribute.cpp index b00e7c4528..d368ac8f10 100644 --- a/bindings/Python/py11Attribute.cpp +++ b/bindings/Python/py11Attribute.cpp @@ -36,17 +36,17 @@ std::string Attribute::Name() const std::string Attribute::Type() const { helper::CheckForNullptr(m_Attribute, "in call to Attribute::Type"); - return m_Attribute->m_Type; + return ToString(m_Attribute->m_Type); } std::vector Attribute::DataString() { helper::CheckForNullptr(m_Attribute, "in call to Attribute::DataStrings"); - const std::string type = m_Attribute->m_Type; + const adios2::DataType type = m_Attribute->m_Type; std::vector data; - if (type == helper::GetType()) + if (type == helper::GetDataType()) { const core::Attribute *attribute = dynamic_cast *>(m_Attribute); @@ -73,14 +73,14 @@ std::vector Attribute::DataString() pybind11::array Attribute::Data() { helper::CheckForNullptr(m_Attribute, "in call to Attribute::Data"); - const std::string type = m_Attribute->m_Type; + const adios2::DataType type = m_Attribute->m_Type; - if (type == "compound") + if (type == adios2::DataType::Compound) { // not supported } #define declare_type(T) \ - else if (type == helper::GetType()) \ + else if (type == helper::GetDataType()) \ { \ pybind11::array pyArray(pybind11::dtype::of(), \ m_Attribute->m_Elements); \ diff --git a/bindings/Python/py11Engine.cpp b/bindings/Python/py11Engine.cpp index 549e16f201..4af46371e7 100644 --- a/bindings/Python/py11Engine.cpp +++ b/bindings/Python/py11Engine.cpp @@ -67,14 +67,15 @@ void Engine::Put(Variable variable, const pybind11::array &array, helper::CheckForNullptr(variable.m_VariableBase, "for variable, in call to Engine::Put numpy array"); - const std::string type = variable.Type(); + const adios2::DataType type = + helper::GetDataTypeFromString(variable.Type()); - if (type == "compound") + if (type == adios2::DataType::Compound) { // not supported } #define declare_type(T) \ - else if (type == helper::GetType()) \ + else if (type == helper::GetDataType()) \ { \ m_Engine->Put( \ *dynamic_cast *>(variable.m_VariableBase), \ @@ -103,7 +104,8 @@ void Engine::Put(Variable variable, const std::string &string) helper::CheckForNullptr(variable.m_VariableBase, "for variable, in call to Engine::Put string"); - if (variable.Type() != helper::GetType()) + if (helper::GetDataTypeFromString(variable.Type()) != + helper::GetDataType()) { throw std::invalid_argument( "ERROR: variable " + variable.Name() + @@ -138,14 +140,15 @@ void Engine::Get(Variable variable, pybind11::array &array, const Mode launch) variable.m_VariableBase, "for variable, in call to Engine::Get a numpy array"); - const std::string type = variable.Type(); + const adios2::DataType type = + helper::GetDataTypeFromString(variable.Type()); - if (type == "compound") + if (type == adios2::DataType::Compound) { // not supported } #define declare_type(T) \ - else if (type == helper::GetType()) \ + else if (type == helper::GetDataType()) \ { \ m_Engine->Get( \ *dynamic_cast *>(variable.m_VariableBase), \ @@ -164,21 +167,23 @@ void Engine::Get(Variable variable, pybind11::array &array, const Mode launch) } } -void Engine::Get(Variable variable, std::string &string, const Mode launch) +std::string Engine::Get(Variable variable, const Mode launch) { + std::string string; helper::CheckForNullptr(m_Engine, "for engine, in call to Engine::Get a numpy array"); if (m_Engine->m_EngineType == "NULL") { - return; + return ""; } helper::CheckForNullptr(variable.m_VariableBase, "for variable, in call to Engine::Get a string"); - const std::string type = variable.Type(); + const adios2::DataType type = + helper::GetDataTypeFromString(variable.Type()); - if (type == helper::GetType()) + if (type == helper::GetDataType()) { m_Engine->Get(*dynamic_cast *>( variable.m_VariableBase), @@ -190,8 +195,8 @@ void Engine::Get(Variable variable, std::string &string, const Mode launch) " of type " + variable.Type() + " is not string, in call to Engine::Get"); } + return string; } - void Engine::PerformGets() { helper::CheckForNullptr(m_Engine, "in call to Engine::PerformGets"); @@ -287,7 +292,7 @@ Engine::BlocksInfo(std::string &var_name, const size_t step) const std::vector> rv; // Grab the specified variable object and get its type string - std::string var_type = m_Engine->GetIO().InquireVariableType(var_name); + adios2::DataType var_type = m_Engine->GetIO().InquireVariableType(var_name); // Use the macro incantation to call the right instantiation of // core::BlocksInfo<>() Note that we are flatting the Dims type items, and @@ -296,7 +301,7 @@ Engine::BlocksInfo(std::string &var_name, const size_t step) const { } #define GET_BLOCKS_INFO(T) \ - else if (var_type == helper::GetType()) \ + else if (var_type == helper::GetDataType()) \ { \ auto variable = m_Engine->GetIO().InquireVariable(var_name); \ auto infoVec = m_Engine->BlocksInfo(*variable, step); \ diff --git a/bindings/Python/py11Engine.h b/bindings/Python/py11Engine.h index 81b2ca101e..b268e78861 100644 --- a/bindings/Python/py11Engine.h +++ b/bindings/Python/py11Engine.h @@ -55,8 +55,8 @@ class Engine void Get(Variable variable, pybind11::array &array, const Mode launch = Mode::Deferred); - void Get(Variable variable, std::string &string, - const Mode launch = Mode::Deferred); + std::string Get(Variable variable, const Mode launch = Mode::Deferred); + void PerformGets(); void EndStep(); diff --git a/bindings/Python/py11File.cpp b/bindings/Python/py11File.cpp index ca78ab348c..096241a3f9 100644 --- a/bindings/Python/py11File.cpp +++ b/bindings/Python/py11File.cpp @@ -208,9 +208,9 @@ pybind11::array File::Read(const std::string &name, const size_t blockID) pybind11::array File::Read(const std::string &name, const Dims &start, const Dims &count, const size_t blockID) { - const std::string type = m_Stream->m_IO->InquireVariableType(name); + const DataType type = m_Stream->m_IO->InquireVariableType(name); - if (type == helper::GetType()) + if (type == helper::GetDataType()) { const std::string value = m_Stream->Read(name, blockID).front(); @@ -226,13 +226,13 @@ pybind11::array File::Read(const std::string &name, const Dims &start, const Dims &count, const size_t stepStart, const size_t stepCount, const size_t blockID) { - const std::string type = m_Stream->m_IO->InquireVariableType(name); + const DataType type = m_Stream->m_IO->InquireVariableType(name); - if (type.empty()) + if (type == DataType::None) { } #define declare_type(T) \ - else if (type == helper::GetType()) \ + else if (type == helper::GetDataType()) \ { \ return DoRead(name, start, count, stepStart, stepCount, blockID); \ } @@ -251,14 +251,14 @@ pybind11::array File::ReadAttribute(const std::string &name, const std::string &variableName, const std::string separator) { - const std::string type = + const DataType type = m_Stream->m_IO->InquireAttributeType(name, variableName, separator); - if (type.empty()) + if (type == DataType::None) { } #define declare_type(T) \ - else if (type == helper::GetType()) \ + else if (type == helper::GetDataType()) \ { \ core::Attribute *attribute = m_Stream->m_IO->InquireAttribute( \ name, variableName, separator); \ diff --git a/bindings/Python/py11IO.cpp b/bindings/Python/py11IO.cpp index 98bead103c..72feffc6c9 100644 --- a/bindings/Python/py11IO.cpp +++ b/bindings/Python/py11IO.cpp @@ -11,7 +11,7 @@ #include "py11IO.h" #include "adios2/common/ADIOSMacros.h" -#include "adios2/helper/adiosFunctions.h" //GetType +#include "adios2/helper/adiosFunctions.h" //GetDataType #include "py11types.h" @@ -111,14 +111,14 @@ Variable IO::InquireVariable(const std::string &name) helper::CheckForNullptr(m_IO, "for variable " + name + ", in call to IO::InquireVariable"); - const std::string type(m_IO->InquireVariableType(name)); + const DataType type(m_IO->InquireVariableType(name)); core::VariableBase *variable = nullptr; - if (type == "unknown") + if (type == DataType::None) { } #define declare_template_instantiation(T) \ - else if (type == helper::GetType()) \ + else if (type == helper::GetDataType()) \ { \ variable = m_IO->InquireVariable(name); \ } @@ -193,13 +193,13 @@ Attribute IO::InquireAttribute(const std::string &name) ", in call to IO::InquireAttribute"); core::AttributeBase *attribute = nullptr; - const std::string type(m_IO->InquireAttributeType(name)); + const DataType type(m_IO->InquireAttributeType(name)); - if (type == "unknown") + if (type == DataType::None) { } #define declare_template_instantiation(T) \ - else if (type == helper::GetType()) \ + else if (type == helper::GetDataType()) \ { \ attribute = m_IO->InquireAttribute(name); \ } @@ -264,14 +264,14 @@ std::string IO::VariableType(const std::string &name) const { helper::CheckForNullptr(m_IO, "for variable " + name + " in call to IO::VariableType"); - return m_IO->InquireVariableType(name); + return ToString(m_IO->InquireVariableType(name)); } std::string IO::AttributeType(const std::string &name) const { helper::CheckForNullptr(m_IO, "for attribute " + name + " in call to IO::AttributeType"); - return m_IO->InquireAttributeType(name); + return ToString(m_IO->InquireAttributeType(name)); } std::string IO::EngineType() const diff --git a/bindings/Python/py11Variable.cpp b/bindings/Python/py11Variable.cpp index 1dc6d6fdd6..051be77cda 100644 --- a/bindings/Python/py11Variable.cpp +++ b/bindings/Python/py11Variable.cpp @@ -56,15 +56,15 @@ size_t Variable::SelectionSize() const helper::CheckForNullptr(m_VariableBase, "in call to Variable::SelectionSize"); - const std::string typeCpp = m_VariableBase->m_Type; + const adios2::DataType typeCpp = m_VariableBase->m_Type; size_t size = 0; - if (typeCpp == "compound") + if (typeCpp == adios2::DataType::Compound) { // not supported } #define declare_template_instantiation(T) \ - else if (typeCpp == adios2::helper::GetType()) \ + else if (typeCpp == adios2::helper::GetDataType()) \ { \ const adios2::core::Variable *variable = \ dynamic_cast *>(m_VariableBase); \ @@ -106,7 +106,7 @@ std::string Variable::Name() const std::string Variable::Type() const { helper::CheckForNullptr(m_VariableBase, "in call to Variable::Type"); - return m_VariableBase->m_Type; + return ToString(m_VariableBase->m_Type); } size_t Variable::Sizeof() const @@ -125,15 +125,15 @@ Dims Variable::Shape(const size_t step) const { helper::CheckForNullptr(m_VariableBase, "in call to Variable::Shape"); - const std::string typeCpp = m_VariableBase->m_Type; + const adios2::DataType typeCpp = m_VariableBase->m_Type; Dims shape; - if (typeCpp == "compound") + if (typeCpp == adios2::DataType::Compound) { // not supported } #define declare_template_instantiation(T) \ - else if (typeCpp == adios2::helper::GetType()) \ + else if (typeCpp == adios2::helper::GetDataType()) \ { \ const adios2::core::Variable *variable = \ dynamic_cast *>(m_VariableBase); \ @@ -155,15 +155,15 @@ Dims Variable::Count() const { helper::CheckForNullptr(m_VariableBase, "in call to Variable::Count"); - const std::string typeCpp = m_VariableBase->m_Type; + const adios2::DataType typeCpp = m_VariableBase->m_Type; Dims count; - if (typeCpp == "compound") + if (typeCpp == adios2::DataType::Compound) { // not supported } #define declare_template_instantiation(T) \ - else if (typeCpp == adios2::helper::GetType()) \ + else if (typeCpp == adios2::helper::GetDataType()) \ { \ const adios2::core::Variable *variable = \ dynamic_cast *>(m_VariableBase); \ diff --git a/bindings/Python/py11glue.cpp b/bindings/Python/py11glue.cpp index 9a83846264..97d007a555 100644 --- a/bindings/Python/py11glue.cpp +++ b/bindings/Python/py11glue.cpp @@ -462,11 +462,10 @@ PYBIND11_MODULE(ADIOS2_PYTHON_MODULE_NAME, m) pybind11::arg("launch") = adios2::Mode::Deferred) .def("Get", - (void (adios2::py11::Engine::*)(adios2::py11::Variable, - std::string &, - const adios2::Mode launch)) & + (std::string(adios2::py11::Engine::*)(adios2::py11::Variable, + const adios2::Mode launch)) & adios2::py11::Engine::Get, - pybind11::arg("variable"), pybind11::arg("string"), + pybind11::arg("variable"), pybind11::arg("launch") = adios2::Mode::Deferred) .def("PerformGets", &adios2::py11::Engine::PerformGets) diff --git a/cmake/DetectOptions.cmake b/cmake/DetectOptions.cmake index 60f0266166..ca449feeee 100644 --- a/cmake/DetectOptions.cmake +++ b/cmake/DetectOptions.cmake @@ -142,6 +142,28 @@ endif() if(CMAKE_Fortran_COMPILER_LOADED) set(ADIOS2_HAVE_Fortran TRUE) list(APPEND mpi_find_components Fortran) + + include(CheckFortranSourceCompiles) + check_fortran_source_compiles(" + program testargs + integer :: n + character(len=256) :: v + n = command_argument_count() + call get_command_argument(0, v) + end program testargs" + ADIOS2_HAVE_FORTRAN_F03_ARGS + SRC_EXT F90 + ) + check_fortran_source_compiles(" + program testargs + integer :: n + character(len=256) :: v + n = iargc() + call getarg(0, v) + end program testargs" + ADIOS2_HAVE_FORTRAN_GNU_ARGS + SRC_EXT F90 + ) endif() # MPI @@ -166,37 +188,29 @@ if(ZeroMQ_FOUND) endif() # DataMan -# DataMan currently breaks the PGI compiler -if(NOT (CMAKE_CXX_COMPILER_ID STREQUAL "PGI") AND NOT MSVC) - if(ZeroMQ_FOUND) - if(ADIOS2_USE_DataMan STREQUAL AUTO) - set(ADIOS2_HAVE_DataMan TRUE) - elseif(ADIOS2_USE_DataMan) - set(ADIOS2_HAVE_DataMan TRUE) - endif() +if(ZeroMQ_FOUND) + if(ADIOS2_USE_DataMan STREQUAL AUTO) + set(ADIOS2_HAVE_DataMan TRUE) + elseif(ADIOS2_USE_DataMan) + set(ADIOS2_HAVE_DataMan TRUE) endif() endif() # SSC -# SSC currently breaks the PGI compiler -if(NOT (CMAKE_CXX_COMPILER_ID STREQUAL "PGI") AND NOT MSVC) - if(ADIOS2_HAVE_MPI) - if(ADIOS2_USE_SSC STREQUAL AUTO) - set(ADIOS2_HAVE_SSC TRUE) - elseif(ADIOS2_USE_SSC) - set(ADIOS2_HAVE_SSC TRUE) - endif() +if(ADIOS2_HAVE_MPI) + if(ADIOS2_USE_SSC STREQUAL AUTO) + set(ADIOS2_HAVE_SSC TRUE) + elseif(ADIOS2_USE_SSC) + set(ADIOS2_HAVE_SSC TRUE) endif() endif() # Table -if(NOT (CMAKE_CXX_COMPILER_ID STREQUAL "PGI") AND NOT MSVC) - if(ZeroMQ_FOUND) - if(ADIOS2_USE_Table STREQUAL AUTO) - set(ADIOS2_HAVE_Table TRUE) - elseif(ADIOS2_USE_Table) - set(ADIOS2_HAVE_Table TRUE) - endif() +if(ZeroMQ_FOUND) + if(ADIOS2_USE_Table STREQUAL AUTO) + set(ADIOS2_HAVE_Table TRUE) + elseif(ADIOS2_USE_Table) + set(ADIOS2_HAVE_Table TRUE) endif() endif() @@ -258,12 +272,12 @@ if(NOT SHARED_LIBS_SUPPORTED) endif() if(ADIOS2_USE_Python STREQUAL AUTO) - find_package(Python COMPONENTS Interpreter Development NumPy) + find_package(Python 3 COMPONENTS Interpreter Development NumPy) if(Python_FOUND AND ADIOS2_HAVE_MPI) find_package(PythonModule COMPONENTS mpi4py mpi4py/mpi4py.h) endif() elseif(ADIOS2_USE_Python) - find_package(Python REQUIRED COMPONENTS Interpreter Development NumPy) + find_package(Python 3 REQUIRED COMPONENTS Interpreter Development NumPy) if(ADIOS2_HAVE_MPI) find_package(PythonModule REQUIRED COMPONENTS mpi4py mpi4py/mpi4py.h) endif() @@ -313,11 +327,6 @@ if(ADIOS2_USE_SST AND NOT MSVC) set(ADIOS2_SST_HAVE_CRAY_DRC TRUE) endif() endif() - find_package(NVStream) - if(NVStream_FOUND) - find_package(Boost OPTIONAL_COMPONENTS thread log filesystem system) - set(ADIOS2_SST_HAVE_NVStream TRUE) - endif() endif() #SysV IPC diff --git a/cmake/FindHDF5.cmake b/cmake/FindHDF5.cmake index c91af92726..7582085e00 100644 --- a/cmake/FindHDF5.cmake +++ b/cmake/FindHDF5.cmake @@ -4,7 +4,7 @@ #------------------------------------------------------------------------------# # This module is already included in new versions of CMake -if(CMAKE_VERSION VERSION_LESS 3.10) +if(CMAKE_VERSION VERSION_LESS 3.19) include(${CMAKE_CURRENT_LIST_DIR}/upstream/FindHDF5.cmake) else() include(${CMAKE_ROOT}/Modules/FindHDF5.cmake) diff --git a/cmake/FindMGARD.cmake b/cmake/FindMGARD.cmake index fc2052a556..ca1e7ee287 100644 --- a/cmake/FindMGARD.cmake +++ b/cmake/FindMGARD.cmake @@ -37,19 +37,19 @@ if(NOT MGARD_FOUND) ) endif() - find_path(MGARD_INCLUDE_DIR mgard.h ${MGARD_INCLUDE_OPTS}) + find_path(MGARD_INCLUDE_DIR mgard_api.h ${MGARD_INCLUDE_OPTS}) find_library(MGARD_LIBRARY NAMES mgard ${MGARD_LIBRARY_OPTS}) find_library(ZLIB_LIBRARY NAMES zlib z ${ZLIB_LIBRARY_OPTS}) - find_library(BLOSC_LIBRARY NAMES blosc ${BLOSC_LIBRARY_OPTS}) + find_library(ZSTD_LIBRARY NAMES zstd ${ZSTD_LIBRARY_OPTS}) include(FindPackageHandleStandardArgs) find_package_handle_standard_args(MGARD FOUND_VAR MGARD_FOUND - REQUIRED_VARS MGARD_LIBRARY ZLIB_LIBRARY BLOSC_LIBRARY MGARD_INCLUDE_DIR + REQUIRED_VARS MGARD_LIBRARY ZLIB_LIBRARY ZSTD_LIBRARY MGARD_INCLUDE_DIR ) if(MGARD_FOUND) set(MGARD_INCLUDE_DIRS ${MGARD_INCLUDE_DIR}) - set(MGARD_LIBRARIES ${MGARD_LIBRARY} ${ZLIB_LIBRARY} ${BLOSC_LIBRARY}) + set(MGARD_LIBRARIES ${MGARD_LIBRARY} ${ZLIB_LIBRARY} ${ZSTD_LIBRARY}) if(MGARD_FOUND AND NOT TARGET MGARD::MGARD) add_library(MGARD::MGARD UNKNOWN IMPORTED) set_target_properties(MGARD::MGARD PROPERTIES diff --git a/cmake/FindMPI.cmake b/cmake/FindMPI.cmake index e4dcc0cb22..fada5ae0fb 100644 --- a/cmake/FindMPI.cmake +++ b/cmake/FindMPI.cmake @@ -4,7 +4,7 @@ #------------------------------------------------------------------------------# # This module is already included in new versions of CMake -if(CMAKE_VERSION VERSION_LESS 3.17.2) +if(CMAKE_VERSION VERSION_LESS 3.19) include(${CMAKE_CURRENT_LIST_DIR}/upstream/FindMPI.cmake) else() include(${CMAKE_ROOT}/Modules/FindMPI.cmake) diff --git a/cmake/FindNVStream.cmake b/cmake/FindNVStream.cmake deleted file mode 100644 index bb1aaf0956..0000000000 --- a/cmake/FindNVStream.cmake +++ /dev/null @@ -1,60 +0,0 @@ -#------------------------------------------------------------------------------# -# Distributed under the OSI-approved Apache License, Version 2.0. See -# accompanying file Copyright.txt for details. -#------------------------------------------------------------------------------# -# -# FindNVStream -# ----------- -# -# Try to find the NVStream library -# -# This module defines the following variables: -# -# NVStream_FOUND - System has NVStream -# NVStream_INCLUDE_DIRS - The NVStream include directory -# NVStream_LIBRARIES - Link these to use NVStream -# -# and the following imported targets: -# NVStream::NVStream - The core NVStream library -# -# You can also set the following variable to help guide the search: -# NVStream_ROOT - The install prefix for NVStream containing the -# include and lib folders -# Note: this can be set as a CMake variable or an -# environment variable. If specified as a CMake -# variable, it will override any setting specified -# as an environment variable. - -if(CMAKE_VERSION VERSION_LESS 3.12) - if((NOT NVStream_ROOT) AND (NOT (ENV{NVStream_ROOT} STREQUAL ""))) - set(NVStream_ROOT "$ENV{NVStream_ROOT}") - endif() - if(NVStream_ROOT) - set(NVStream_INCLUDE_OPTS HINTS ${NVStream_ROOT}/include NO_DEFAULT_PATHS) - set(NVStream_LIBRARY_OPTS - HINTS ${NVStream_ROOT}/lib ${NVStream_ROOT}/lib64 - NO_DEFAULT_PATHS - ) - endif() -endif() - -find_path(NVStream_INCLUDE_DIR nvs/store.h ${NVStream_INCLUDE_OPTS}) -find_library(NVStream_LIBRARY libyuma.a ${NVStream_LIBRARY_OPTS}) - -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(NVStream - FOUND_VAR NVStream_FOUND - REQUIRED_VARS NVStream_LIBRARY NVStream_INCLUDE_DIR -) - -if(NVStream_FOUND) - set(NVStream_INCLUDE_DIRS ${NVStream_INCLUDE_DIR}) - set(NVStream_LIBRARIES ${NVStream_LIBRARY}) - if(NOT TARGET NVStream::NVStream) - add_library(NVStream::NVStream UNKNOWN IMPORTED) - set_target_properties(NVStream::NVStream PROPERTIES - IMPORTED_LOCATION "${NVStream_LIBRARY}" - INTERFACE_INCLUDE_DIRECTORIES "${NVStream_INCLUDE_DIR}" - ) - endif() -endif() diff --git a/cmake/FindPkgConfig.cmake b/cmake/FindPkgConfig.cmake index 8b7936bc18..2d5711bfd7 100644 --- a/cmake/FindPkgConfig.cmake +++ b/cmake/FindPkgConfig.cmake @@ -4,4 +4,8 @@ #------------------------------------------------------------------------------# # This module is already included in new versions of CMake -include(${CMAKE_CURRENT_LIST_DIR}/upstream/FindPkgConfig.cmake) +if(CMAKE_VERSION VERSION_LESS 3.19) + include(${CMAKE_CURRENT_LIST_DIR}/upstream/FindPkgConfig.cmake) +else() + include(${CMAKE_ROOT}/Modules/FindPkgConfig.cmake) +endif() diff --git a/cmake/install/post/generate-adios2-config.sh.in b/cmake/install/post/generate-adios2-config.sh.in index fb8488cf0c..bf7537d5c1 100755 --- a/cmake/install/post/generate-adios2-config.sh.in +++ b/cmake/install/post/generate-adios2-config.sh.in @@ -103,7 +103,7 @@ fi for variant in $variants do - echo "Extacting ADIOS flags for C bindings (${variant})" + echo "Extracting ADIOS flags for C bindings (${variant})" echo " Without ADIOS" make_target_flags ${variant}_without_C without_C_CFLAGS=$(head -1 ${variant}_without_C.flags) @@ -119,7 +119,7 @@ do echo ADIOS2_C_CFLAGS_${variant}=\"${ADIOS2_C_CFLAGS}\" >> adios2.flags echo ADIOS2_C_LDFLAGS_${variant}=\"${ADIOS2_C_LDFLAGS}\" >> adios2.flags - echo "Extacting ADIOS flags for C++ bindings (${variant})" + echo "Extracting ADIOS flags for C++ bindings (${variant})" echo " Without ADIOS" make_target_flags ${variant}_without_CXX without_CXX_CXXFLAGS=$(head -1 ${variant}_without_CXX.flags) @@ -137,7 +137,7 @@ do if [ @ADIOS2_CONFIG_FORTRAN@ -eq 1 ] then - echo "Extacting ADIOS flags for Fortran bindings (${variant})" + echo "Extracting ADIOS flags for Fortran bindings (${variant})" echo " Without ADIOS" make_target_flags ${variant}_without_Fortran without_Fortran_FFLAGS=$(head -1 ${variant}_without_Fortran.flags) diff --git a/cmake/upstream/FindHDF5.cmake b/cmake/upstream/FindHDF5.cmake index 12536cbfc0..be68cb3550 100644 --- a/cmake/upstream/FindHDF5.cmake +++ b/cmake/upstream/FindHDF5.cmake @@ -1,114 +1,149 @@ # Distributed under the OSI-approved BSD 3-Clause License. See accompanying # file Copyright.txt or https://cmake.org/licensing for details. -#.rst: -# FindHDF5 -# -------- -# -# Find HDF5, a library for reading and writing self describing array data. -# -# -# -# This module invokes the HDF5 wrapper compiler that should be installed -# alongside HDF5. Depending upon the HDF5 Configuration, the wrapper -# compiler is called either h5cc or h5pcc. If this succeeds, the module -# will then call the compiler with the -show argument to see what flags -# are used when compiling an HDF5 client application. -# -# The module will optionally accept the COMPONENTS argument. If no -# COMPONENTS are specified, then the find module will default to finding -# only the HDF5 C library. If one or more COMPONENTS are specified, the -# module will attempt to find the language bindings for the specified -# components. The only valid components are C, CXX, Fortran, HL, and -# Fortran_HL. If the COMPONENTS argument is not given, the module will -# attempt to find only the C bindings. -# -# This module will read the variable -# HDF5_USE_STATIC_LIBRARIES to determine whether or not to prefer a -# static link to a dynamic link for HDF5 and all of it's dependencies. -# To use this feature, make sure that the HDF5_USE_STATIC_LIBRARIES -# variable is set before the call to find_package. -# -# To provide the module with a hint about where to find your HDF5 -# installation, you can set the environment variable HDF5_ROOT. The -# Find module will then look in this path when searching for HDF5 -# executables, paths, and libraries. -# -# Both the serial and parallel HDF5 wrappers are considered and the first -# directory to contain either one will be used. In the event that both appear -# in the same directory the serial version is preferentially selected. This -# behavior can be reversed by setting the variable HDF5_PREFER_PARALLEL to -# true. -# -# In addition to finding the includes and libraries required to compile -# an HDF5 client application, this module also makes an effort to find -# tools that come with the HDF5 distribution that may be useful for -# regression testing. -# -# This module will define the following variables: -# -# :: -# -# HDF5_FOUND - true if HDF5 was found on the system -# HDF5_VERSION - HDF5 version in format Major.Minor.Release -# HDF5_INCLUDE_DIRS - Location of the hdf5 includes -# HDF5_INCLUDE_DIR - Location of the hdf5 includes (deprecated) -# HDF5_DEFINITIONS - Required compiler definitions for HDF5 -# HDF5_LIBRARIES - Required libraries for all requested bindings -# HDF5_HL_LIBRARIES - Required libraries for the HDF5 high level API for all -# bindings, if the HL component is enabled -# -# Available components are: C CXX Fortran and HL. For each enabled language -# binding, a corresponding HDF5_${LANG}_LIBRARIES variable, and potentially -# HDF5_${LANG}_DEFINITIONS, will be defined. -# If the HL component is enabled, then an HDF5_${LANG}_HL_LIBRARIES will -# also be defined. With all components enabled, the following variables will be defined: -# -# :: -# -# HDF5_C_DEFINITIONS -- Required compiler definitions for HDF5 C bindings -# HDF5_CXX_DEFINITIONS -- Required compiler definitions for HDF5 C++ bindings -# HDF5_Fortran_DEFINITIONS -- Required compiler definitions for HDF5 Fortran bindings -# HDF5_C_INCLUDE_DIRS -- Required include directories for HDF5 C bindings -# HDF5_CXX_INCLUDE_DIRS -- Required include directories for HDF5 C++ bindings -# HDF5_Fortran_INCLUDE_DIRS -- Required include directories for HDF5 Fortran bindings -# HDF5_C_LIBRARIES - Required libraries for the HDF5 C bindings -# HDF5_CXX_LIBRARIES - Required libraries for the HDF5 C++ bindings -# HDF5_Fortran_LIBRARIES - Required libraries for the HDF5 Fortran bindings -# HDF5_C_HL_LIBRARIES - Required libraries for the high level C bindings -# HDF5_CXX_HL_LIBRARIES - Required libraries for the high level C++ bindings -# HDF5_Fortran_HL_LIBRARIES - Required libraries for the high level Fortran -# bindings. -# -# HDF5_IS_PARALLEL - Whether or not HDF5 was found with parallel IO support -# HDF5_C_COMPILER_EXECUTABLE - the path to the HDF5 C wrapper compiler -# HDF5_CXX_COMPILER_EXECUTABLE - the path to the HDF5 C++ wrapper compiler -# HDF5_Fortran_COMPILER_EXECUTABLE - the path to the HDF5 Fortran wrapper compiler -# HDF5_C_COMPILER_EXECUTABLE_NO_INTERROGATE - path to the primary C compiler -# which is also the HDF5 wrapper -# HDF5_CXX_COMPILER_EXECUTABLE_NO_INTERROGATE - path to the primary C++ -# compiler which is also -# the HDF5 wrapper -# HDF5_Fortran_COMPILER_EXECUTABLE_NO_INTERROGATE - path to the primary -# Fortran compiler which -# is also the HDF5 wrapper -# HDF5_DIFF_EXECUTABLE - the path to the HDF5 dataset comparison tool -# -# The following variable can be set to guide the search for HDF5 libraries and includes: -# -# ``HDF5_ROOT`` -# Specify the path to the HDF5 installation to use. -# -# ``HDF5_FIND_DEBUG`` -# Set to a true value to get some extra debugging output. -# -# ``HDF5_NO_FIND_PACKAGE_CONFIG_FILE`` -# Set to a true value to skip trying to find ``hdf5-config.cmake``. - -# This module is maintained by Will Dicharry . - -include(${CMAKE_ROOT}/Modules/SelectLibraryConfigurations.cmake) -include(${CMAKE_ROOT}/Modules/FindPackageHandleStandardArgs.cmake) +#[=======================================================================[.rst: +FindHDF5 +-------- + +Find Hierarchical Data Format (HDF5), a library for reading and writing +self describing array data. + + +This module invokes the ``HDF5`` wrapper compiler that should be installed +alongside ``HDF5``. Depending upon the ``HDF5`` Configuration, the wrapper +compiler is called either ``h5cc`` or ``h5pcc``. If this succeeds, the module +will then call the compiler with the show argument to see what flags +are used when compiling an ``HDF5`` client application. + +The module will optionally accept the ``COMPONENTS`` argument. If no +``COMPONENTS`` are specified, then the find module will default to finding +only the ``HDF5`` C library. If one or more ``COMPONENTS`` are specified, the +module will attempt to find the language bindings for the specified +components. The valid components are ``C``, ``CXX``, ``Fortran``, ``HL``. +``HL`` refers to the "high-level" HDF5 functions for C and Fortran. +If the ``COMPONENTS`` argument is not given, the module will +attempt to find only the C bindings. +For example, to use Fortran HDF5 and HDF5-HL functions, do: +``find_package(HDF5 COMPONENTS Fortran HL)``. + +This module will read the variable +``HDF5_USE_STATIC_LIBRARIES`` to determine whether or not to prefer a +static link to a dynamic link for ``HDF5`` and all of it's dependencies. +To use this feature, make sure that the ``HDF5_USE_STATIC_LIBRARIES`` +variable is set before the call to find_package. + +Both the serial and parallel ``HDF5`` wrappers are considered and the first +directory to contain either one will be used. In the event that both appear +in the same directory the serial version is preferentially selected. This +behavior can be reversed by setting the variable ``HDF5_PREFER_PARALLEL`` to +``TRUE``. + +In addition to finding the includes and libraries required to compile +an ``HDF5`` client application, this module also makes an effort to find +tools that come with the ``HDF5`` distribution that may be useful for +regression testing. + +Result Variables +^^^^^^^^^^^^^^^^ + +This module will set the following variables in your project: + +``HDF5_FOUND`` + HDF5 was found on the system +``HDF5_VERSION`` + HDF5 library version +``HDF5_INCLUDE_DIRS`` + Location of the HDF5 header files +``HDF5_DEFINITIONS`` + Required compiler definitions for HDF5 +``HDF5_LIBRARIES`` + Required libraries for all requested bindings +``HDF5_HL_LIBRARIES`` + Required libraries for the HDF5 high level API for all bindings, + if the ``HL`` component is enabled + +Available components are: ``C`` ``CXX`` ``Fortran`` and ``HL``. +For each enabled language binding, a corresponding ``HDF5_${LANG}_LIBRARIES`` +variable, and potentially ``HDF5_${LANG}_DEFINITIONS``, will be defined. +If the ``HL`` component is enabled, then an ``HDF5_${LANG}_HL_LIBRARIES`` will +also be defined. With all components enabled, the following variables will be defined: + +``HDF5_C_DEFINITIONS`` + Required compiler definitions for HDF5 C bindings +``HDF5_CXX_DEFINITIONS`` + Required compiler definitions for HDF5 C++ bindings +``HDF5_Fortran_DEFINITIONS`` + Required compiler definitions for HDF5 Fortran bindings +``HDF5_C_INCLUDE_DIRS`` + Required include directories for HDF5 C bindings +``HDF5_CXX_INCLUDE_DIRS`` + Required include directories for HDF5 C++ bindings +``HDF5_Fortran_INCLUDE_DIRS`` + Required include directories for HDF5 Fortran bindings +``HDF5_C_LIBRARIES`` + Required libraries for the HDF5 C bindings +``HDF5_CXX_LIBRARIES`` + Required libraries for the HDF5 C++ bindings +``HDF5_Fortran_LIBRARIES`` + Required libraries for the HDF5 Fortran bindings +``HDF5_C_HL_LIBRARIES`` + Required libraries for the high level C bindings +``HDF5_CXX_HL_LIBRARIES`` + Required libraries for the high level C++ bindings +``HDF5_Fortran_HL_LIBRARIES`` + Required libraries for the high level Fortran bindings. + +``HDF5_IS_PARALLEL`` + HDF5 library has parallel IO support +``HDF5_C_COMPILER_EXECUTABLE`` + path to the HDF5 C wrapper compiler +``HDF5_CXX_COMPILER_EXECUTABLE`` + path to the HDF5 C++ wrapper compiler +``HDF5_Fortran_COMPILER_EXECUTABLE`` + path to the HDF5 Fortran wrapper compiler +``HDF5_C_COMPILER_EXECUTABLE_NO_INTERROGATE`` + path to the primary C compiler which is also the HDF5 wrapper +``HDF5_CXX_COMPILER_EXECUTABLE_NO_INTERROGATE`` + path to the primary C++ compiler which is also the HDF5 wrapper +``HDF5_Fortran_COMPILER_EXECUTABLE_NO_INTERROGATE`` + path to the primary Fortran compiler which is also the HDF5 wrapper +``HDF5_DIFF_EXECUTABLE`` + path to the HDF5 dataset comparison tool + +With all components enabled, the following targets will be defined: + +:: + + ``hdf5::hdf5`` + ``hdf5::hdf5_hl_cpp`` + ``hdf5::hdf5_fortran`` + ``hdf5::hdf5_hl`` + ``hdf5::hdf5_hl_cpp`` + ``hdf5::hdf5_hl_fortran`` + ``hdf5::h5diff`` + +Hints +^^^^^ + +The following variables can be set to guide the search for HDF5 libraries and includes: + +``HDF5_PREFER_PARALLEL`` + set ``true`` to prefer parallel HDF5 (by default, serial is preferred) + +``HDF5_FIND_DEBUG`` + Set ``true`` to get extra debugging output. + +``HDF5_NO_FIND_PACKAGE_CONFIG_FILE`` + Set ``true`` to skip trying to find ``hdf5-config.cmake``. +#]=======================================================================] + +include(SelectLibraryConfigurations) +include(FindPackageHandleStandardArgs) + +# We haven't found HDF5 yet. Clear its state in case it is set in the parent +# scope somewhere else. We can't rely on it because different components may +# have been requested for this call. +set(HDF5_FOUND OFF) # List of the valid HDF5 components set(HDF5_VALID_LANGUAGE_BINDINGS C CXX Fortran) @@ -119,28 +154,30 @@ if(NOT HDF5_FIND_COMPONENTS) else() set(HDF5_LANGUAGE_BINDINGS) # add the extra specified components, ensuring that they are valid. - set(FIND_HL OFF) - foreach(component IN LISTS HDF5_FIND_COMPONENTS) - list(FIND HDF5_VALID_LANGUAGE_BINDINGS ${component} component_location) - if(NOT component_location EQUAL -1) - list(APPEND HDF5_LANGUAGE_BINDINGS ${component}) - elseif(component STREQUAL "HL") - set(FIND_HL ON) - elseif(component STREQUAL "Fortran_HL") # only for compatibility + set(HDF5_FIND_HL OFF) + foreach(_component IN LISTS HDF5_FIND_COMPONENTS) + list(FIND HDF5_VALID_LANGUAGE_BINDINGS ${_component} _component_location) + if(NOT _component_location EQUAL -1) + list(APPEND HDF5_LANGUAGE_BINDINGS ${_component}) + elseif(_component STREQUAL "HL") + set(HDF5_FIND_HL ON) + elseif(_component STREQUAL "Fortran_HL") # only for compatibility list(APPEND HDF5_LANGUAGE_BINDINGS Fortran) - set(FIND_HL ON) - set(HDF5_FIND_REQUIRED_Fortran_HL False) - set(HDF5_FIND_REQUIRED_Fortran True) - set(HDF5_FIND_REQUIRED_HL True) + set(HDF5_FIND_HL ON) + set(HDF5_FIND_REQUIRED_Fortran_HL FALSE) + set(HDF5_FIND_REQUIRED_Fortran TRUE) + set(HDF5_FIND_REQUIRED_HL TRUE) else() - message(FATAL_ERROR "${component} is not a valid HDF5 component.") + message(FATAL_ERROR "${_component} is not a valid HDF5 component.") endif() endforeach() + unset(_component) + unset(_component_location) if(NOT HDF5_LANGUAGE_BINDINGS) - get_property(__langs GLOBAL PROPERTY ENABLED_LANGUAGES) - foreach(__lang IN LISTS __langs) - if(__lang MATCHES "^(C|CXX|Fortran)$") - list(APPEND HDF5_LANGUAGE_BINDINGS ${__lang}) + get_property(_langs GLOBAL PROPERTY ENABLED_LANGUAGES) + foreach(_lang IN LISTS _langs) + if(_lang MATCHES "^(C|CXX|Fortran)$") + list(APPEND HDF5_LANGUAGE_BINDINGS ${_lang}) endif() endforeach() endif() @@ -175,9 +212,7 @@ macro(_HDF5_remove_duplicates_from_beginning _list_name) endif() endmacro() - # Test first if the current compilers automatically wrap HDF5 - function(_HDF5_test_regular_compiler_C success version is_parallel) set(scratch_directory ${CMAKE_CURRENT_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/hdf5) @@ -186,7 +221,6 @@ function(_HDF5_test_regular_compiler_C success version is_parallel) set(test_file ${scratch_directory}/cmake_hdf5_test.c) file(WRITE ${test_file} "#include \n" - "#include \n" "const char* info_ver = \"INFO\" \":\" H5_VERSION;\n" "#ifdef H5_HAVE_PARALLEL\n" "const char* info_parallel = \"INFO\" \":\" \"PARALLEL\";\n" @@ -214,7 +248,7 @@ function(_HDF5_test_regular_compiler_C success version is_parallel) ) set(${version} ${CMAKE_MATCH_1}) if(CMAKE_MATCH_3) - set(${version} ${${version}}.${CMAKE_MATCH_3}) + set(${version} ${HDF5_C_VERSION}.${CMAKE_MATCH_3}) endif() set(${version} ${${version}} PARENT_SCOPE) @@ -262,7 +296,7 @@ function(_HDF5_test_regular_compiler_CXX success version is_parallel) ) set(${version} ${CMAKE_MATCH_1}) if(CMAKE_MATCH_3) - set(${version} ${${version}}.${CMAKE_MATCH_3}) + set(${version} ${HDF5_CXX_VERSION}.${CMAKE_MATCH_3}) endif() set(${version} ${${version}} PARENT_SCOPE) @@ -306,99 +340,111 @@ endfunction() # Invoke the HDF5 wrapper compiler. The compiler return value is stored to the # return_value argument, the text output is stored to the output variable. -macro( _HDF5_invoke_compiler language output return_value version is_parallel) - set(${version}) - if(HDF5_USE_STATIC_LIBRARIES) - set(lib_type_args -noshlib) - else() - set(lib_type_args -shlib) - endif() - set(scratch_dir ${CMAKE_CURRENT_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/hdf5) - if("${language}" STREQUAL "C") - set(test_file ${scratch_dir}/cmake_hdf5_test.c) - elseif("${language}" STREQUAL "CXX") - set(test_file ${scratch_dir}/cmake_hdf5_test.cxx) - elseif("${language}" STREQUAL "Fortran") - set(test_file ${scratch_dir}/cmake_hdf5_test.f90) - endif() - exec_program( ${HDF5_${language}_COMPILER_EXECUTABLE} - ARGS -show ${lib_type_args} ${test_file} - OUTPUT_VARIABLE ${output} - RETURN_VALUE ${return_value} +function( _HDF5_invoke_compiler language output_var return_value_var version_var is_parallel_var) + set(is_parallel FALSE) + if(HDF5_USE_STATIC_LIBRARIES) + set(lib_type_args -noshlib) + else() + set(lib_type_args -shlib) + endif() + set(scratch_dir ${CMAKE_CURRENT_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/hdf5) + if("${language}" STREQUAL "C") + set(test_file ${scratch_dir}/cmake_hdf5_test.c) + elseif("${language}" STREQUAL "CXX") + set(test_file ${scratch_dir}/cmake_hdf5_test.cxx) + elseif("${language}" STREQUAL "Fortran") + set(test_file ${scratch_dir}/cmake_hdf5_test.f90) + endif() + # Verify that the compiler wrapper can actually compile: sometimes the compiler + # wrapper exists, but not the compiler. E.g. Miniconda / Anaconda Python + execute_process( + COMMAND ${HDF5_${language}_COMPILER_EXECUTABLE} ${test_file} + WORKING_DIRECTORY ${scratch_dir} + RESULT_VARIABLE return_value ) - if(NOT ${${return_value}} EQUAL 0) - message(STATUS - "Unable to determine HDF5 ${language} flags from HDF5 wrapper.") + if(return_value) + message(STATUS + "HDF5 ${language} compiler wrapper is unable to compile a minimal HDF5 program.") + else() + execute_process( + COMMAND ${HDF5_${language}_COMPILER_EXECUTABLE} -show ${lib_type_args} ${test_file} + WORKING_DIRECTORY ${scratch_dir} + OUTPUT_VARIABLE output + ERROR_VARIABLE output + RESULT_VARIABLE return_value + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + if(return_value) + message(STATUS + "Unable to determine HDF5 ${language} flags from HDF5 wrapper.") endif() - exec_program( ${HDF5_${language}_COMPILER_EXECUTABLE} - ARGS -showconfig - OUTPUT_VARIABLE config_output - RETURN_VALUE config_return - ) - if(NOT ${return_value} EQUAL 0) - message( STATUS - "Unable to determine HDF5 ${language} version from HDF5 wrapper.") + execute_process( + COMMAND ${HDF5_${language}_COMPILER_EXECUTABLE} -showconfig + OUTPUT_VARIABLE config_output + ERROR_VARIABLE config_output + RESULT_VARIABLE return_value + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + if(return_value) + message(STATUS + "Unable to determine HDF5 ${language} version_var from HDF5 wrapper.") endif() - string(REGEX MATCH "HDF5 Version: ([a-zA-Z0-9\\.\\-]*)" version_match "${config_output}") - if(version_match) - string(REPLACE "HDF5 Version: " "" ${version} "${version_match}") - string(REPLACE "-patch" "." ${version} "${${version}}") + string(REGEX MATCH "HDF5 Version: ([a-zA-Z0-9\\.\\-]*)" version "${config_output}") + if(version) + string(REPLACE "HDF5 Version: " "" version "${version}") + string(REPLACE "-patch" "." version "${version}") endif() if(config_output MATCHES "Parallel HDF5: yes") - set(${is_parallel} TRUE) - else() - set(${is_parallel} FALSE) + set(is_parallel TRUE) endif() -endmacro() + endif() + foreach(var output return_value version is_parallel) + set(${${var}_var} ${${var}} PARENT_SCOPE) + endforeach() +endfunction() # Parse a compile line for definitions, includes, library paths, and libraries. -macro( _HDF5_parse_compile_line - compile_line_var - include_paths - definitions - library_paths - libraries - libraries_hl) - - if(UNIX) - separate_arguments(_HDF5_COMPILE_ARGS UNIX_COMMAND "${${compile_line_var}}") - else() - separate_arguments(_HDF5_COMPILE_ARGS WINDOWS_COMMAND "${${compile_line_var}}") - endif() +function(_HDF5_parse_compile_line compile_line_var include_paths definitions + library_paths libraries libraries_hl) - foreach(arg IN LISTS _HDF5_COMPILE_ARGS) - if("${arg}" MATCHES "^-I(.*)$") + separate_arguments(_compile_args NATIVE_COMMAND "${${compile_line_var}}") + + foreach(_arg IN LISTS _compile_args) + if("${_arg}" MATCHES "^-I(.*)$") # include directory - list(APPEND ${include_paths} "${CMAKE_MATCH_1}") - elseif("${arg}" MATCHES "^-D(.*)$") + list(APPEND include_paths "${CMAKE_MATCH_1}") + elseif("${_arg}" MATCHES "^-D(.*)$") # compile definition - list(APPEND ${definitions} "-D${CMAKE_MATCH_1}") - elseif("${arg}" MATCHES "^-L(.*)$") + list(APPEND definitions "-D${CMAKE_MATCH_1}") + elseif("${_arg}" MATCHES "^-L(.*)$") # library search path - list(APPEND ${library_paths} "${CMAKE_MATCH_1}") - elseif("${arg}" MATCHES "^-l(hdf5.*hl.*)$") + list(APPEND library_paths "${CMAKE_MATCH_1}") + elseif("${_arg}" MATCHES "^-l(hdf5.*hl.*)$") # library name (hl) - list(APPEND ${libraries_hl} "${CMAKE_MATCH_1}") - elseif("${arg}" MATCHES "^-l(.*)$") + list(APPEND libraries_hl "${CMAKE_MATCH_1}") + elseif("${_arg}" MATCHES "^-l(.*)$") # library name - list(APPEND ${libraries} "${CMAKE_MATCH_1}") - elseif("${arg}" MATCHES "^(.:)?[/\\].*\\.(a|so|dylib|sl|lib)$") + list(APPEND libraries "${CMAKE_MATCH_1}") + elseif("${_arg}" MATCHES "^(.:)?[/\\].*\\.(a|so|dylib|sl|lib)$") # library file - if(NOT EXISTS "${arg}") + if(NOT EXISTS "${_arg}") continue() endif() - get_filename_component(_HDF5_LPATH "${arg}" DIRECTORY) - get_filename_component(_HDF5_LNAME "${arg}" NAME_WE) - string(REGEX REPLACE "^lib" "" _HDF5_LNAME "${_HDF5_LNAME}") - list(APPEND ${library_paths} "${_HDF5_LPATH}") - if(_HDF5_LNAME MATCHES "hdf5.*hl") - list(APPEND ${libraries_hl} "${_HDF5_LNAME}") + get_filename_component(_lpath "${_arg}" DIRECTORY) + get_filename_component(_lname "${_arg}" NAME_WE) + string(REGEX REPLACE "^lib" "" _lname "${_lname}") + list(APPEND library_paths "${_lpath}") + if(_lname MATCHES "hdf5.*hl") + list(APPEND libraries_hl "${_lname}") else() - list(APPEND ${libraries} "${_HDF5_LNAME}") + list(APPEND libraries "${_lname}") endif() endif() endforeach() -endmacro() + foreach(var include_paths definitions library_paths libraries libraries_hl) + set(${${var}_var} ${${var}} PARENT_SCOPE) + endforeach() +endfunction() # Select a preferred imported configuration from a target function(_HDF5_select_imported_config target imported_conf) @@ -483,7 +529,7 @@ if(NOT HDF5_FOUND AND NOT HDF5_NO_FIND_PACKAGE_CONFIG_FILE) #if we detect that occurrence clear the suffix if(_suffix AND NOT TARGET ${HDF5_${_lang}_TARGET}${_suffix}) if(NOT TARGET ${HDF5_${_lang}_TARGET}) - #cant find this component with or without the suffix + #can't find this component with or without the suffix #so bail out, and let the following locate HDF5 set(HDF5_FOUND FALSE) break() @@ -508,10 +554,10 @@ if(NOT HDF5_FOUND AND NOT HDF5_NO_FIND_PACKAGE_CONFIG_FILE) set(HDF5_${_lang}_LIBRARY ${_hdf5_lang_location}) list(APPEND HDF5_LIBRARIES ${HDF5_${_lang}_TARGET}${_suffix}) set(HDF5_${_lang}_LIBRARIES ${HDF5_${_lang}_TARGET}${_suffix}) - set(HDF5_${_lang}_FOUND True) + set(HDF5_${_lang}_FOUND TRUE) endif() - if(FIND_HL) - get_target_property(__lang_hl_location ${HDF5_${_lang}_HL_TARGET}${_suffix} IMPORTED_IMPLIB_${_hdf5_imported_conf} ) + if(HDF5_FIND_HL) + get_target_property(_lang_hl_location ${HDF5_${_lang}_HL_TARGET}${_suffix} IMPORTED_IMPLIB_${_hdf5_imported_conf} ) if (NOT _hdf5_lang_hl_location) get_target_property(_hdf5_lang_hl_location ${HDF5_${_lang}_HL_TARGET}${_suffix} LOCATION_${_hdf5_imported_conf}) if (NOT _hdf5_hl_lang_location) @@ -522,7 +568,7 @@ if(NOT HDF5_FOUND AND NOT HDF5_NO_FIND_PACKAGE_CONFIG_FILE) set(HDF5_${_lang}_HL_LIBRARY ${_hdf5_lang_hl_location}) list(APPEND HDF5_HL_LIBRARIES ${HDF5_${_lang}_HL_TARGET}${_suffix}) set(HDF5_${_lang}_HL_LIBRARIES ${HDF5_${_lang}_HL_TARGET}${_suffix}) - set(HDF5_HL_FOUND True) + set(HDF5_HL_FOUND TRUE) endif() unset(_hdf5_lang_hl_location) endif() @@ -533,172 +579,177 @@ if(NOT HDF5_FOUND AND NOT HDF5_NO_FIND_PACKAGE_CONFIG_FILE) endif() if(NOT HDF5_FOUND) - set(_HDF5_NEED_TO_SEARCH False) - set(HDF5_COMPILER_NO_INTERROGATE True) + set(_HDF5_NEED_TO_SEARCH FALSE) + set(HDF5_COMPILER_NO_INTERROGATE TRUE) # Only search for languages we've enabled - foreach(__lang IN LISTS HDF5_LANGUAGE_BINDINGS) + foreach(_lang IN LISTS HDF5_LANGUAGE_BINDINGS) # First check to see if our regular compiler is one of wrappers - if(__lang STREQUAL "C") + if(_lang STREQUAL "C") _HDF5_test_regular_compiler_C( - HDF5_${__lang}_COMPILER_NO_INTERROGATE - HDF5_${__lang}_VERSION - HDF5_${__lang}_IS_PARALLEL) - elseif(__lang STREQUAL "CXX") + HDF5_${_lang}_COMPILER_NO_INTERROGATE + HDF5_${_lang}_VERSION + HDF5_${_lang}_IS_PARALLEL) + elseif(_lang STREQUAL "CXX") _HDF5_test_regular_compiler_CXX( - HDF5_${__lang}_COMPILER_NO_INTERROGATE - HDF5_${__lang}_VERSION - HDF5_${__lang}_IS_PARALLEL) - elseif(__lang STREQUAL "Fortran") + HDF5_${_lang}_COMPILER_NO_INTERROGATE + HDF5_${_lang}_VERSION + HDF5_${_lang}_IS_PARALLEL) + elseif(_lang STREQUAL "Fortran") _HDF5_test_regular_compiler_Fortran( - HDF5_${__lang}_COMPILER_NO_INTERROGATE - HDF5_${__lang}_IS_PARALLEL) + HDF5_${_lang}_COMPILER_NO_INTERROGATE + HDF5_${_lang}_IS_PARALLEL) else() continue() endif() - if(HDF5_${__lang}_COMPILER_NO_INTERROGATE) - message(STATUS "HDF5: Using hdf5 compiler wrapper for all ${__lang} compiling") - set(HDF5_${__lang}_FOUND True) - set(HDF5_${__lang}_COMPILER_EXECUTABLE_NO_INTERROGATE - "${CMAKE_${__lang}_COMPILER}" - CACHE FILEPATH "HDF5 ${__lang} compiler wrapper") - set(HDF5_${__lang}_DEFINITIONS) - set(HDF5_${__lang}_INCLUDE_DIRS) - set(HDF5_${__lang}_LIBRARIES) - set(HDF5_${__lang}_HL_LIBRARIES) - - mark_as_advanced(HDF5_${__lang}_COMPILER_EXECUTABLE_NO_INTERROGATE) - - set(HDF5_${__lang}_FOUND True) - set(HDF5_HL_FOUND True) + if(HDF5_${_lang}_COMPILER_NO_INTERROGATE) + if(HDF5_FIND_DEBUG) + message(STATUS "HDF5: Using hdf5 compiler wrapper for all ${_lang} compiling") + endif() + set(HDF5_${_lang}_FOUND TRUE) + set(HDF5_${_lang}_COMPILER_EXECUTABLE_NO_INTERROGATE + "${CMAKE_${_lang}_COMPILER}" + CACHE FILEPATH "HDF5 ${_lang} compiler wrapper") + set(HDF5_${_lang}_DEFINITIONS) + set(HDF5_${_lang}_INCLUDE_DIRS) + set(HDF5_${_lang}_LIBRARIES) + set(HDF5_${_lang}_HL_LIBRARIES) + + mark_as_advanced(HDF5_${_lang}_COMPILER_EXECUTABLE_NO_INTERROGATE) + + set(HDF5_${_lang}_FOUND TRUE) + set(HDF5_HL_FOUND TRUE) else() - set(HDF5_COMPILER_NO_INTERROGATE False) + set(HDF5_COMPILER_NO_INTERROGATE FALSE) # If this language isn't using the wrapper, then try to seed the # search options with the wrapper - find_program(HDF5_${__lang}_COMPILER_EXECUTABLE - NAMES ${HDF5_${__lang}_COMPILER_NAMES} NAMES_PER_DIR + find_program(HDF5_${_lang}_COMPILER_EXECUTABLE + NAMES ${HDF5_${_lang}_COMPILER_NAMES} NAMES_PER_DIR HINTS ${HDF5_ROOT} PATH_SUFFIXES bin Bin - DOC "HDF5 ${__lang} Wrapper compiler. Used only to detect HDF5 compile flags." + DOC "HDF5 ${_lang} Wrapper compiler. Used only to detect HDF5 compile flags." ${_HDF5_SEARCH_OPTS} ) - mark_as_advanced( HDF5_${__lang}_COMPILER_EXECUTABLE ) - unset(HDF5_${__lang}_COMPILER_NAMES) - - if(HDF5_${__lang}_COMPILER_EXECUTABLE) - _HDF5_invoke_compiler(${__lang} HDF5_${__lang}_COMPILE_LINE - HDF5_${__lang}_RETURN_VALUE HDF5_${__lang}_VERSION HDF5_${__lang}_IS_PARALLEL) - if(HDF5_${__lang}_RETURN_VALUE EQUAL 0) - message(STATUS "HDF5: Using hdf5 compiler wrapper to determine ${__lang} configuration") - _HDF5_parse_compile_line( HDF5_${__lang}_COMPILE_LINE - HDF5_${__lang}_INCLUDE_DIRS - HDF5_${__lang}_DEFINITIONS - HDF5_${__lang}_LIBRARY_DIRS - HDF5_${__lang}_LIBRARY_NAMES - HDF5_${__lang}_HL_LIBRARY_NAMES + mark_as_advanced( HDF5_${_lang}_COMPILER_EXECUTABLE ) + unset(HDF5_${_lang}_COMPILER_NAMES) + + if(HDF5_${_lang}_COMPILER_EXECUTABLE) + _HDF5_invoke_compiler(${_lang} HDF5_${_lang}_COMPILE_LINE + HDF5_${_lang}_RETURN_VALUE HDF5_${_lang}_VERSION HDF5_${_lang}_IS_PARALLEL) + if(HDF5_${_lang}_RETURN_VALUE EQUAL 0) + if(HDF5_FIND_DEBUG) + message(STATUS "HDF5: Using hdf5 compiler wrapper to determine ${_lang} configuration") + endif() + _HDF5_parse_compile_line( HDF5_${_lang}_COMPILE_LINE + HDF5_${_lang}_INCLUDE_DIRS + HDF5_${_lang}_DEFINITIONS + HDF5_${_lang}_LIBRARY_DIRS + HDF5_${_lang}_LIBRARY_NAMES + HDF5_${_lang}_HL_LIBRARY_NAMES ) - set(HDF5_${__lang}_LIBRARIES) + set(HDF5_${_lang}_LIBRARIES) - foreach(L IN LISTS HDF5_${__lang}_LIBRARY_NAMES) + foreach(_lib IN LISTS HDF5_${_lang}_LIBRARY_NAMES) set(_HDF5_SEARCH_NAMES_LOCAL) - if("x${L}" MATCHES "hdf5") + if("x${_lib}" MATCHES "hdf5") # hdf5 library set(_HDF5_SEARCH_OPTS_LOCAL ${_HDF5_SEARCH_OPTS}) if(HDF5_USE_STATIC_LIBRARIES) if(WIN32) - set(_HDF5_SEARCH_NAMES_LOCAL lib${L}) + set(_HDF5_SEARCH_NAMES_LOCAL lib${_lib}) else() - set(_HDF5_SEARCH_NAMES_LOCAL lib${L}.a) + set(_HDF5_SEARCH_NAMES_LOCAL lib${_lib}.a) endif() endif() else() # external library set(_HDF5_SEARCH_OPTS_LOCAL) endif() - find_library(HDF5_${__lang}_LIBRARY_${L} - NAMES ${_HDF5_SEARCH_NAMES_LOCAL} ${L} NAMES_PER_DIR - HINTS ${HDF5_${__lang}_LIBRARY_DIRS} + find_library(HDF5_${_lang}_LIBRARY_${_lib} + NAMES ${_HDF5_SEARCH_NAMES_LOCAL} ${_lib} NAMES_PER_DIR + HINTS ${HDF5_${_lang}_LIBRARY_DIRS} ${HDF5_ROOT} ${_HDF5_SEARCH_OPTS_LOCAL} ) unset(_HDF5_SEARCH_OPTS_LOCAL) unset(_HDF5_SEARCH_NAMES_LOCAL) - if(HDF5_${__lang}_LIBRARY_${L}) - list(APPEND HDF5_${__lang}_LIBRARIES ${HDF5_${__lang}_LIBRARY_${L}}) + if(HDF5_${_lang}_LIBRARY_${_lib}) + list(APPEND HDF5_${_lang}_LIBRARIES ${HDF5_${_lang}_LIBRARY_${_lib}}) else() - list(APPEND HDF5_${__lang}_LIBRARIES ${L}) + list(APPEND HDF5_${_lang}_LIBRARIES ${_lib}) endif() endforeach() - if(FIND_HL) - set(HDF5_${__lang}_HL_LIBRARIES) - foreach(L IN LISTS HDF5_${__lang}_HL_LIBRARY_NAMES) + if(HDF5_FIND_HL) + set(HDF5_${_lang}_HL_LIBRARIES) + foreach(_lib IN LISTS HDF5_${_lang}_HL_LIBRARY_NAMES) set(_HDF5_SEARCH_NAMES_LOCAL) - if("x${L}" MATCHES "hdf5") + if("x${_lib}" MATCHES "hdf5") # hdf5 library set(_HDF5_SEARCH_OPTS_LOCAL ${_HDF5_SEARCH_OPTS}) if(HDF5_USE_STATIC_LIBRARIES) if(WIN32) - set(_HDF5_SEARCH_NAMES_LOCAL lib${L}) + set(_HDF5_SEARCH_NAMES_LOCAL lib${_lib}) else() - set(_HDF5_SEARCH_NAMES_LOCAL lib${L}.a) + set(_HDF5_SEARCH_NAMES_LOCAL lib${_lib}.a) endif() endif() else() # external library set(_HDF5_SEARCH_OPTS_LOCAL) endif() - find_library(HDF5_${__lang}_LIBRARY_${L} - NAMES ${_HDF5_SEARCH_NAMES_LOCAL} ${L} NAMES_PER_DIR - HINTS ${HDF5_${__lang}_LIBRARY_DIRS} + find_library(HDF5_${_lang}_LIBRARY_${_lib} + NAMES ${_HDF5_SEARCH_NAMES_LOCAL} ${_lib} NAMES_PER_DIR + HINTS ${HDF5_${_lang}_LIBRARY_DIRS} ${HDF5_ROOT} ${_HDF5_SEARCH_OPTS_LOCAL} ) unset(_HDF5_SEARCH_OPTS_LOCAL) unset(_HDF5_SEARCH_NAMES_LOCAL) - if(HDF5_${__lang}_LIBRARY_${L}) - list(APPEND HDF5_${__lang}_HL_LIBRARIES ${HDF5_${__lang}_LIBRARY_${L}}) + if(HDF5_${_lang}_LIBRARY_${_lib}) + list(APPEND HDF5_${_lang}_HL_LIBRARIES ${HDF5_${_lang}_LIBRARY_${_lib}}) else() - list(APPEND HDF5_${__lang}_HL_LIBRARIES ${L}) + list(APPEND HDF5_${_lang}_HL_LIBRARIES ${_lib}) endif() endforeach() - set(HDF5_HL_FOUND True) + set(HDF5_HL_FOUND TRUE) endif() - set(HDF5_${__lang}_FOUND True) - _HDF5_remove_duplicates_from_beginning(HDF5_${__lang}_DEFINITIONS) - _HDF5_remove_duplicates_from_beginning(HDF5_${__lang}_INCLUDE_DIRS) - _HDF5_remove_duplicates_from_beginning(HDF5_${__lang}_LIBRARIES) - _HDF5_remove_duplicates_from_beginning(HDF5_${__lang}_HL_LIBRARIES) + set(HDF5_${_lang}_FOUND TRUE) + _HDF5_remove_duplicates_from_beginning(HDF5_${_lang}_DEFINITIONS) + _HDF5_remove_duplicates_from_beginning(HDF5_${_lang}_INCLUDE_DIRS) + _HDF5_remove_duplicates_from_beginning(HDF5_${_lang}_LIBRARIES) + _HDF5_remove_duplicates_from_beginning(HDF5_${_lang}_HL_LIBRARIES) else() - set(_HDF5_NEED_TO_SEARCH True) + set(_HDF5_NEED_TO_SEARCH TRUE) endif() else() - set(_HDF5_NEED_TO_SEARCH True) + set(_HDF5_NEED_TO_SEARCH TRUE) endif() endif() - if(HDF5_${__lang}_VERSION) + if(HDF5_${_lang}_VERSION) if(NOT HDF5_VERSION) - set(HDF5_VERSION ${HDF5_${__lang}_VERSION}) - elseif(NOT HDF5_VERSION VERSION_EQUAL HDF5_${__lang}_VERSION) - message(WARNING "HDF5 Version found for language ${__lang}, ${HDF5_${__lang}_VERSION} is different than previously found version ${HDF5_VERSION}") + set(HDF5_VERSION ${HDF5_${_lang}_VERSION}) + elseif(NOT HDF5_VERSION VERSION_EQUAL HDF5_${_lang}_VERSION) + message(WARNING "HDF5 Version found for language ${_lang}, ${HDF5_${_lang}_VERSION} is different than previously found version ${HDF5_VERSION}") endif() endif() - if(DEFINED HDF5_${__lang}_IS_PARALLEL) + if(DEFINED HDF5_${_lang}_IS_PARALLEL) if(NOT DEFINED HDF5_IS_PARALLEL) - set(HDF5_IS_PARALLEL ${HDF5_${__lang}_IS_PARALLEL}) - elseif(NOT HDF5_IS_PARALLEL AND HDF5_${__lang}_IS_PARALLEL) - message(WARNING "HDF5 found for language ${__lang} is parallel but previously found language is not parallel.") - elseif(HDF5_IS_PARALLEL AND NOT HDF5_${__lang}_IS_PARALLEL) - message(WARNING "HDF5 found for language ${__lang} is not parallel but previously found language is parallel.") + set(HDF5_IS_PARALLEL ${HDF5_${_lang}_IS_PARALLEL}) + elseif(NOT HDF5_IS_PARALLEL AND HDF5_${_lang}_IS_PARALLEL) + message(WARNING "HDF5 found for language ${_lang} is parallel but previously found language is not parallel.") + elseif(HDF5_IS_PARALLEL AND NOT HDF5_${_lang}_IS_PARALLEL) + message(WARNING "HDF5 found for language ${_lang} is not parallel but previously found language is parallel.") endif() endif() endforeach() + unset(_lib) else() - set(_HDF5_NEED_TO_SEARCH True) + set(_HDF5_NEED_TO_SEARCH TRUE) endif() if(NOT HDF5_FOUND AND HDF5_COMPILER_NO_INTERROGATE) # No arguments necessary, all languages can use the compiler wrappers - set(HDF5_FOUND True) + set(HDF5_FOUND TRUE) set(HDF5_METHOD "Included by compiler wrappers") set(HDF5_REQUIRED_VARS HDF5_METHOD) elseif(NOT HDF5_FOUND AND NOT _HDF5_NEED_TO_SEARCH) @@ -707,14 +758,14 @@ elseif(NOT HDF5_FOUND AND NOT _HDF5_NEED_TO_SEARCH) set(HDF5_INCLUDE_DIRS) set(HDF5_LIBRARIES) set(HDF5_HL_LIBRARIES) - foreach(__lang IN LISTS HDF5_LANGUAGE_BINDINGS) - if(HDF5_${__lang}_FOUND) - if(NOT HDF5_${__lang}_COMPILER_NO_INTERROGATE) - list(APPEND HDF5_DEFINITIONS ${HDF5_${__lang}_DEFINITIONS}) - list(APPEND HDF5_INCLUDE_DIRS ${HDF5_${__lang}_INCLUDE_DIRS}) - list(APPEND HDF5_LIBRARIES ${HDF5_${__lang}_LIBRARIES}) - if(FIND_HL) - list(APPEND HDF5_HL_LIBRARIES ${HDF5_${__lang}_HL_LIBRARIES}) + foreach(_lang IN LISTS HDF5_LANGUAGE_BINDINGS) + if(HDF5_${_lang}_FOUND) + if(NOT HDF5_${_lang}_COMPILER_NO_INTERROGATE) + list(APPEND HDF5_DEFINITIONS ${HDF5_${_lang}_DEFINITIONS}) + list(APPEND HDF5_INCLUDE_DIRS ${HDF5_${_lang}_INCLUDE_DIRS}) + list(APPEND HDF5_LIBRARIES ${HDF5_${_lang}_LIBRARIES}) + if(HDF5_FIND_HL) + list(APPEND HDF5_HL_LIBRARIES ${HDF5_${_lang}_HL_LIBRARIES}) endif() endif() endif() @@ -723,9 +774,9 @@ elseif(NOT HDF5_FOUND AND NOT _HDF5_NEED_TO_SEARCH) _HDF5_remove_duplicates_from_beginning(HDF5_INCLUDE_DIRS) _HDF5_remove_duplicates_from_beginning(HDF5_LIBRARIES) _HDF5_remove_duplicates_from_beginning(HDF5_HL_LIBRARIES) - set(HDF5_FOUND True) + set(HDF5_FOUND TRUE) set(HDF5_REQUIRED_VARS HDF5_LIBRARIES) - if(FIND_HL) + if(HDF5_FIND_HL) list(APPEND HDF5_REQUIRED_VARS HDF5_HL_LIBRARIES) endif() endif() @@ -741,7 +792,7 @@ mark_as_advanced( HDF5_DIFF_EXECUTABLE ) if( NOT HDF5_FOUND ) # seed the initial lists of libraries to find with items we know we need set(HDF5_C_LIBRARY_NAMES hdf5) - set(HDF5_C_HL_LIBRARY_NAMES hdf5_hl) + set(HDF5_C_HL_LIBRARY_NAMES hdf5_hl ${HDF5_C_LIBRARY_NAMES} ) set(HDF5_CXX_LIBRARY_NAMES hdf5_cpp ${HDF5_C_LIBRARY_NAMES}) set(HDF5_CXX_HL_LIBRARY_NAMES hdf5_hl_cpp ${HDF5_C_HL_LIBRARY_NAMES} ${HDF5_CXX_LIBRARY_NAMES}) @@ -749,33 +800,44 @@ if( NOT HDF5_FOUND ) set(HDF5_Fortran_LIBRARY_NAMES hdf5_fortran ${HDF5_C_LIBRARY_NAMES}) set(HDF5_Fortran_HL_LIBRARY_NAMES hdf5hl_fortran ${HDF5_C_HL_LIBRARY_NAMES} ${HDF5_Fortran_LIBRARY_NAMES}) - foreach(__lang IN LISTS HDF5_LANGUAGE_BINDINGS) + # suffixes as seen on Linux, MSYS2, ... + set(_lib_suffixes hdf5) + if(NOT HDF5_PREFER_PARALLEL) + list(APPEND _lib_suffixes hdf5/serial) + endif() + if(HDF5_USE_STATIC_LIBRARIES) + set(_inc_suffixes include/static) + else() + set(_inc_suffixes include/shared) + endif() + + foreach(_lang IN LISTS HDF5_LANGUAGE_BINDINGS) # find the HDF5 include directories - if("${__lang}" STREQUAL "Fortran") - set(HDF5_INCLUDE_FILENAME hdf5.mod) - elseif("${__lang}" STREQUAL "CXX") + if("${_lang}" STREQUAL "Fortran") + set(HDF5_INCLUDE_FILENAME hdf5.mod HDF5.mod) + elseif("${_lang}" STREQUAL "CXX") set(HDF5_INCLUDE_FILENAME H5Cpp.h) else() set(HDF5_INCLUDE_FILENAME hdf5.h) endif() - find_path(HDF5_${__lang}_INCLUDE_DIR ${HDF5_INCLUDE_FILENAME} + find_path(HDF5_${_lang}_INCLUDE_DIR ${HDF5_INCLUDE_FILENAME} HINTS ${HDF5_ROOT} PATHS $ENV{HOME}/.local/include - PATH_SUFFIXES include Include + PATH_SUFFIXES include Include ${_inc_suffixes} ${_lib_suffixes} ${_HDF5_SEARCH_OPTS} ) - mark_as_advanced(HDF5_${__lang}_INCLUDE_DIR) + mark_as_advanced(HDF5_${_lang}_INCLUDE_DIR) # set the _DIRS variable as this is what the user will normally use - set(HDF5_${__lang}_INCLUDE_DIRS ${HDF5_${__lang}_INCLUDE_DIR}) - list(APPEND HDF5_INCLUDE_DIRS ${HDF5_${__lang}_INCLUDE_DIR}) + set(HDF5_${_lang}_INCLUDE_DIRS ${HDF5_${_lang}_INCLUDE_DIR}) + list(APPEND HDF5_INCLUDE_DIRS ${HDF5_${_lang}_INCLUDE_DIR}) # find the HDF5 libraries - foreach(LIB IN LISTS HDF5_${__lang}_LIBRARY_NAMES) + foreach(LIB IN LISTS HDF5_${_lang}_LIBRARY_NAMES) if(HDF5_USE_STATIC_LIBRARIES) # According to bug 1643 on the CMake bug tracker, this is the # preferred method for searching for a static library. - # See https://gitlab.kitware.com/cmake/cmake/issues/1643. We search + # See https://gitlab.kitware.com/cmake/cmake/-/issues/1643. We search # first for the full static library name, but fall back to a # generic search on the name if the static search fails. set( THIS_LIBRARY_SEARCH_DEBUG @@ -791,62 +853,64 @@ if( NOT HDF5_FOUND ) endif() find_library(HDF5_${LIB}_LIBRARY_DEBUG NAMES ${THIS_LIBRARY_SEARCH_DEBUG} - HINTS ${HDF5_ROOT} PATH_SUFFIXES lib Lib + HINTS ${HDF5_ROOT} PATH_SUFFIXES lib Lib ${_lib_suffixes} ${_HDF5_SEARCH_OPTS} ) - find_library( HDF5_${LIB}_LIBRARY_RELEASE + find_library(HDF5_${LIB}_LIBRARY_RELEASE NAMES ${THIS_LIBRARY_SEARCH_RELEASE} - HINTS ${HDF5_ROOT} PATH_SUFFIXES lib Lib + HINTS ${HDF5_ROOT} PATH_SUFFIXES lib Lib ${_lib_suffixes} ${_HDF5_SEARCH_OPTS} ) + select_library_configurations( HDF5_${LIB} ) - list(APPEND HDF5_${__lang}_LIBRARIES ${HDF5_${LIB}_LIBRARY}) + list(APPEND HDF5_${_lang}_LIBRARIES ${HDF5_${LIB}_LIBRARY}) endforeach() - if(HDF5_${__lang}_LIBRARIES) - set(HDF5_${__lang}_FOUND True) + if(HDF5_${_lang}_LIBRARIES) + set(HDF5_${_lang}_FOUND TRUE) endif() # Append the libraries for this language binding to the list of all # required libraries. - list(APPEND HDF5_LIBRARIES ${HDF5_${__lang}_LIBRARIES}) + list(APPEND HDF5_LIBRARIES ${HDF5_${_lang}_LIBRARIES}) - if(FIND_HL) - foreach(LIB IN LISTS HDF5_${__lang}_HL_LIBRARY_NAMES) + if(HDF5_FIND_HL) + foreach(LIB IN LISTS HDF5_${_lang}_HL_LIBRARY_NAMES) if(HDF5_USE_STATIC_LIBRARIES) # According to bug 1643 on the CMake bug tracker, this is the # preferred method for searching for a static library. - # See https://gitlab.kitware.com/cmake/cmake/issues/1643. We search + # See https://gitlab.kitware.com/cmake/cmake/-/issues/1643. We search # first for the full static library name, but fall back to a # generic search on the name if the static search fails. set( THIS_LIBRARY_SEARCH_DEBUG lib${LIB}d.a lib${LIB}_debug.a lib${LIB}d lib${LIB}_D lib${LIB}_debug lib${LIB}d-static.a lib${LIB}_debug-static.a lib${LIB}d-static lib${LIB}_D-static lib${LIB}_debug-static ) - set( THIS_LIBRARY_SEARCH_RELEASE lib${LIB}.a ${LIB} lib${LIB}-static.a lib${LIB}-static) + set( THIS_LIBRARY_SEARCH_RELEASE lib${LIB}.a lib${LIB} lib${LIB}-static.a lib${LIB}-static) else() set( THIS_LIBRARY_SEARCH_DEBUG ${LIB}d ${LIB}_D ${LIB}_debug ${LIB}d-shared ${LIB}_D-shared ${LIB}_debug-shared) set( THIS_LIBRARY_SEARCH_RELEASE ${LIB} ${LIB}-shared) endif() find_library(HDF5_${LIB}_LIBRARY_DEBUG NAMES ${THIS_LIBRARY_SEARCH_DEBUG} - HINTS ${HDF5_ROOT} PATH_SUFFIXES lib Lib + HINTS ${HDF5_ROOT} PATH_SUFFIXES lib Lib ${_lib_suffixes} ${_HDF5_SEARCH_OPTS} ) - find_library( HDF5_${LIB}_LIBRARY_RELEASE + find_library(HDF5_${LIB}_LIBRARY_RELEASE NAMES ${THIS_LIBRARY_SEARCH_RELEASE} - HINTS ${HDF5_ROOT} PATH_SUFFIXES lib Lib + HINTS ${HDF5_ROOT} PATH_SUFFIXES lib Lib ${_lib_suffixes} ${_HDF5_SEARCH_OPTS} ) + select_library_configurations( HDF5_${LIB} ) - list(APPEND HDF5_${__lang}_HL_LIBRARIES ${HDF5_${LIB}_LIBRARY}) + list(APPEND HDF5_${_lang}_HL_LIBRARIES ${HDF5_${LIB}_LIBRARY}) endforeach() # Append the libraries for this language binding to the list of all # required libraries. - list(APPEND HDF5_HL_LIBRARIES ${HDF5_${__lang}_HL_LIBRARIES}) + list(APPEND HDF5_HL_LIBRARIES ${HDF5_${_lang}_HL_LIBRARIES}) endif() endforeach() - if(FIND_HL AND HDF5_HL_LIBRARIES) - set(HDF5_HL_FOUND True) + if(HDF5_FIND_HL AND HDF5_HL_LIBRARIES) + set(HDF5_HL_FOUND TRUE) endif() _HDF5_remove_duplicates_from_beginning(HDF5_DEFINITIONS) @@ -883,12 +947,14 @@ if( NOT HDF5_FOUND ) endif() endforeach() endforeach() + unset(_hdr) + unset(_dir) set( HDF5_IS_PARALLEL ${HDF5_IS_PARALLEL} CACHE BOOL "HDF5 library compiled with parallel IO support" ) mark_as_advanced( HDF5_IS_PARALLEL ) set(HDF5_REQUIRED_VARS HDF5_LIBRARIES HDF5_INCLUDE_DIRS) - if(FIND_HL) + if(HDF5_FIND_HL) list(APPEND HDF5_REQUIRED_VARS HDF5_HL_LIBRARIES) endif() endif() @@ -920,19 +986,153 @@ if( HDF5_FOUND AND NOT HDF5_DIR) mark_as_advanced(HDF5_DIR) endif() +if (HDF5_FOUND) + if (NOT TARGET HDF5::HDF5) + add_library(HDF5::HDF5 INTERFACE IMPORTED) + string(REPLACE "-D" "" _hdf5_definitions "${HDF5_DEFINITIONS}") + set_target_properties(HDF5::HDF5 PROPERTIES + INTERFACE_LINK_LIBRARIES "${HDF5_LIBRARIES}" + INTERFACE_INCLUDE_DIRECTORIES "${HDF5_INCLUDE_DIRS}" + INTERFACE_COMPILE_DEFINITIONS "${_hdf5_definitions}") + unset(_hdf5_definitions) + endif () + + foreach (hdf5_lang IN LISTS HDF5_LANGUAGE_BINDINGS) + if (hdf5_lang STREQUAL "C") + set(hdf5_target_name "hdf5") + elseif (hdf5_lang STREQUAL "CXX") + set(hdf5_target_name "hdf5_cpp") + elseif (hdf5_lang STREQUAL "Fortran") + set(hdf5_target_name "hdf5_fortran") + else () + continue () + endif () + + if (NOT TARGET "hdf5::${hdf5_target_name}") + if (HDF5_COMPILER_NO_INTERROGATE) + add_library("hdf5::${hdf5_target_name}" INTERFACE IMPORTED) + string(REPLACE "-D" "" _hdf5_definitions "${HDF5_${hdf5_lang}_DEFINITIONS}") + set_target_properties("hdf5::${hdf5_target_name}" PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${HDF5_${hdf5_lang}_INCLUDE_DIRS}" + INTERFACE_COMPILE_DEFINITIONS "${_hdf5_definitions}") + else() + if (DEFINED "HDF5_${hdf5_target_name}_LIBRARY") + set(_hdf5_location "${HDF5_${hdf5_target_name}_LIBRARY}") + elseif (DEFINED "HDF5_${hdf5_lang}_LIBRARY") + set(_hdf5_location "${HDF5_${hdf5_lang}_LIBRARY}") + elseif (DEFINED "HDF5_${hdf5_lang}_LIBRARY_${hdf5_target_name}") + set(_hdf5_location "${HDF5_${hdf5_lang}_LIBRARY_${hdf5_target_name}}") + else () + # Error if we still don't have the location. + message(SEND_ERROR + "HDF5 was found, but a different variable was set which contains " + "its location.") + endif () + add_library("hdf5::${hdf5_target_name}" UNKNOWN IMPORTED) + string(REPLACE "-D" "" _hdf5_definitions "${HDF5_${hdf5_lang}_DEFINITIONS}") + if (NOT HDF5_${hdf5_lang}_INCLUDE_DIRS) + set(HDF5_${hdf5_lang}_INCLUDE_DIRS ${HDF5_INCLUDE_DIRS}) + endif () + set_target_properties("hdf5::${hdf5_target_name}" PROPERTIES + IMPORTED_LOCATION "${_hdf5_location}" + IMPORTED_IMPLIB "${_hdf5_location}" + INTERFACE_INCLUDE_DIRECTORIES "${HDF5_${hdf5_lang}_INCLUDE_DIRS}" + INTERFACE_COMPILE_DEFINITIONS "${_hdf5_definitions}") + if (_hdf5_libtype STREQUAL "SHARED") + set_property(TARGET "hdf5::${hdf5_target_name}" APPEND + PROPERTY + INTERFACE_COMPILE_DEFINITIONS H5_BUILT_AS_DYNAMIC_LIB) + elseif (_hdf5_libtype STREQUAL "STATIC") + set_property(TARGET "hdf5::${hdf5_target_name}" APPEND + PROPERTY + INTERFACE_COMPILE_DEFINITIONS H5_BUILT_AS_STATIC_LIB) + endif () + unset(_hdf5_definitions) + unset(_hdf5_libtype) + unset(_hdf5_location) + endif () + endif () + + if (NOT HDF5_FIND_HL) + continue () + endif () + + if (hdf5_lang STREQUAL "C") + set(hdf5_target_name "hdf5_hl") + elseif (hdf5_lang STREQUAL "CXX") + set(hdf5_target_name "hdf5_hl_cpp") + elseif (hdf5_lang STREQUAL "Fortran") + set(hdf5_target_name "hdf5_hl_fortran") + else () + continue () + endif () + + if (NOT TARGET "hdf5::${hdf5_target_name}") + if (HDF5_COMPILER_NO_INTERROGATE) + add_library("hdf5::${hdf5_target_name}" INTERFACE IMPORTED) + string(REPLACE "-D" "" _hdf5_definitions "${HDF5_${hdf5_lang}_HL_DEFINITIONS}") + set_target_properties("hdf5::${hdf5_target_name}" PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${HDF5_${hdf5_lang}_HL_INCLUDE_DIRS}" + INTERFACE_COMPILE_DEFINITIONS "${_hdf5_definitions}") + else() + if (DEFINED "HDF5_${hdf5_target_name}_LIBRARY") + set(_hdf5_location "${HDF5_${hdf5_target_name}_LIBRARY}") + elseif (DEFINED "HDF5_${hdf5_lang}_HL_LIBRARY") + set(_hdf5_location "${HDF5_${hdf5_lang}_HL_LIBRARY}") + elseif (DEFINED "HDF5_${hdf5_lang}_LIBRARY_${hdf5_target_name}") + set(_hdf5_location "${HDF5_${hdf5_lang}_LIBRARY_${hdf5_target_name}}") + else () + # Error if we still don't have the location. + message(SEND_ERROR + "HDF5 was found, but a different variable was set which contains " + "its location.") + endif () + add_library("hdf5::${hdf5_target_name}" UNKNOWN IMPORTED) + string(REPLACE "-D" "" _hdf5_definitions "${HDF5_${hdf5_lang}_HL_DEFINITIONS}") + set_target_properties("hdf5::${hdf5_target_name}" PROPERTIES + IMPORTED_LOCATION "${_hdf5_location}" + IMPORTED_IMPLIB "${_hdf5_location}" + INTERFACE_INCLUDE_DIRECTORIES "${HDF5_${hdf5_lang}_HL_INCLUDE_DIRS}" + INTERFACE_COMPILE_DEFINITIONS "${_hdf5_definitions}") + if (_hdf5_libtype STREQUAL "SHARED") + set_property(TARGET "hdf5::${hdf5_target_name}" APPEND + PROPERTY + INTERFACE_COMPILE_DEFINITIONS H5_BUILT_AS_DYNAMIC_LIB) + elseif (_hdf5_libtype STREQUAL "STATIC") + set_property(TARGET "hdf5::${hdf5_target_name}" APPEND + PROPERTY + INTERFACE_COMPILE_DEFINITIONS H5_BUILT_AS_STATIC_LIB) + endif () + unset(_hdf5_definitions) + unset(_hdf5_libtype) + unset(_hdf5_location) + endif () + endif () + endforeach () + unset(hdf5_lang) + + if (HDF5_DIFF_EXECUTABLE AND NOT TARGET hdf5::h5diff) + add_executable(hdf5::h5diff IMPORTED) + set_target_properties(hdf5::h5diff PROPERTIES + IMPORTED_LOCATION "${HDF5_DIFF_EXECUTABLE}") + endif () +endif () + if (HDF5_FIND_DEBUG) message(STATUS "HDF5_DIR: ${HDF5_DIR}") message(STATUS "HDF5_DEFINITIONS: ${HDF5_DEFINITIONS}") message(STATUS "HDF5_INCLUDE_DIRS: ${HDF5_INCLUDE_DIRS}") message(STATUS "HDF5_LIBRARIES: ${HDF5_LIBRARIES}") message(STATUS "HDF5_HL_LIBRARIES: ${HDF5_HL_LIBRARIES}") - foreach(__lang IN LISTS HDF5_LANGUAGE_BINDINGS) - message(STATUS "HDF5_${__lang}_DEFINITIONS: ${HDF5_${__lang}_DEFINITIONS}") - message(STATUS "HDF5_${__lang}_INCLUDE_DIR: ${HDF5_${__lang}_INCLUDE_DIR}") - message(STATUS "HDF5_${__lang}_INCLUDE_DIRS: ${HDF5_${__lang}_INCLUDE_DIRS}") - message(STATUS "HDF5_${__lang}_LIBRARY: ${HDF5_${__lang}_LIBRARY}") - message(STATUS "HDF5_${__lang}_LIBRARIES: ${HDF5_${__lang}_LIBRARIES}") - message(STATUS "HDF5_${__lang}_HL_LIBRARY: ${HDF5_${__lang}_HL_LIBRARY}") - message(STATUS "HDF5_${__lang}_HL_LIBRARIES: ${HDF5_${__lang}_HL_LIBRARIES}") + foreach(_lang IN LISTS HDF5_LANGUAGE_BINDINGS) + message(STATUS "HDF5_${_lang}_DEFINITIONS: ${HDF5_${_lang}_DEFINITIONS}") + message(STATUS "HDF5_${_lang}_INCLUDE_DIR: ${HDF5_${_lang}_INCLUDE_DIR}") + message(STATUS "HDF5_${_lang}_INCLUDE_DIRS: ${HDF5_${_lang}_INCLUDE_DIRS}") + message(STATUS "HDF5_${_lang}_LIBRARY: ${HDF5_${_lang}_LIBRARY}") + message(STATUS "HDF5_${_lang}_LIBRARIES: ${HDF5_${_lang}_LIBRARIES}") + message(STATUS "HDF5_${_lang}_HL_LIBRARY: ${HDF5_${_lang}_HL_LIBRARY}") + message(STATUS "HDF5_${_lang}_HL_LIBRARIES: ${HDF5_${_lang}_HL_LIBRARIES}") endforeach() endif() +unset(_lang) +unset(_HDF5_NEED_TO_SEARCH) diff --git a/cmake/upstream/FindMPI.cmake b/cmake/upstream/FindMPI.cmake index ab99c45e4e..bfb0dc0a1b 100644 --- a/cmake/upstream/FindMPI.cmake +++ b/cmake/upstream/FindMPI.cmake @@ -760,7 +760,7 @@ function (_MPI_interrogate_compiler LANG) # Save the explicitly given link directories set(MPI_LINK_DIRECTORIES_LEFTOVER "${MPI_LINK_DIRECTORIES_WORK}") - # An MPI compiler wrapper could have its MPI libraries in the implictly + # An MPI compiler wrapper could have its MPI libraries in the implicitly # linked directories of the compiler itself. if(DEFINED CMAKE_${LANG}_IMPLICIT_LINK_DIRECTORIES) list(APPEND MPI_LINK_DIRECTORIES_WORK "${CMAKE_${LANG}_IMPLICIT_LINK_DIRECTORIES}") @@ -867,11 +867,11 @@ function(_MPI_guess_settings LANG) # We first attempt to locate the msmpi.lib. Should be find it, we'll assume that the MPI present is indeed # Microsoft MPI. if("${CMAKE_SIZEOF_VOID_P}" EQUAL 8) - set(MPI_MSMPI_LIB_PATH "$ENV{MSMPI_LIB64}") - set(MPI_MSMPI_INC_PATH_EXTRA "$ENV{MSMPI_INC}/x64") + file(TO_CMAKE_PATH "$ENV{MSMPI_LIB64}" MPI_MSMPI_LIB_PATH) + file(TO_CMAKE_PATH "$ENV{MSMPI_INC}/x64" MPI_MSMPI_INC_PATH_EXTRA) else() - set(MPI_MSMPI_LIB_PATH "$ENV{MSMPI_LIB32}") - set(MPI_MSMPI_INC_PATH_EXTRA "$ENV{MSMPI_INC}/x86") + file(TO_CMAKE_PATH "$ENV{MSMPI_LIB32}" MPI_MSMPI_LIB_PATH) + file(TO_CMAKE_PATH "$ENV{MSMPI_INC}/x86" MPI_MSMPI_INC_PATH_EXTRA) endif() find_library(MPI_msmpi_LIBRARY @@ -1153,15 +1153,19 @@ macro(_MPI_create_imported_target LANG) add_library(MPI::MPI_${LANG} INTERFACE IMPORTED) endif() - # When this is consumed for compiling CUDA, use '-Xcompiler' to wrap '-pthread'. - string(REPLACE "-pthread" "$<$:SHELL:-Xcompiler >-pthread" + # When this is consumed for compiling CUDA, use '-Xcompiler' to wrap '-pthread' and '-fexceptions'. + string(REPLACE "-pthread" "$<$:SHELL:-Xcompiler >-pthread" _MPI_${LANG}_COMPILE_OPTIONS "${MPI_${LANG}_COMPILE_OPTIONS}") + string(REPLACE "-fexceptions" "$<$:SHELL:-Xcompiler >-fexceptions" + _MPI_${LANG}_COMPILE_OPTIONS "${_MPI_${LANG}_COMPILE_OPTIONS}") set_property(TARGET MPI::MPI_${LANG} PROPERTY INTERFACE_COMPILE_OPTIONS "${_MPI_${LANG}_COMPILE_OPTIONS}") unset(_MPI_${LANG}_COMPILE_OPTIONS) set_property(TARGET MPI::MPI_${LANG} PROPERTY INTERFACE_COMPILE_DEFINITIONS "${MPI_${LANG}_COMPILE_DEFINITIONS}") if(MPI_${LANG}_LINK_FLAGS) + string(REPLACE "-pthread" "$<$:-Xlinker >-pthread" + _MPI_${LANG}_LINK_FLAGS "${MPI_${LANG}_LINK_FLAGS}") set_property(TARGET MPI::MPI_${LANG} PROPERTY INTERFACE_LINK_OPTIONS "SHELL:${MPI_${LANG}_LINK_FLAGS}") endif() # If the compiler links MPI implicitly, no libraries will be found as they're contained within @@ -1691,12 +1695,11 @@ foreach(LANG IN ITEMS C CXX Fortran) list(APPEND MPI_${LANG}_REQUIRED_VARS "MPI_${LANG}_WORKS") endif() endif() - if(CMAKE_VERSION VERSION_GREATER_EQUAL "3.17.0") - set(_FPHSA_NAME_MISMATCHED NAME_MISMATCHED) - endif() - find_package_handle_standard_args(MPI_${LANG} ${_FPHSA_NAME_MISMATCHED} + set(FPHSA_NAME_MISMATCHED TRUE) + find_package_handle_standard_args(MPI_${LANG} REQUIRED_VARS ${MPI_${LANG}_REQUIRED_VARS} VERSION_VAR MPI_${LANG}_VERSION) + unset(FPHSA_NAME_MISMATCHED) if(DEFINED MPI_${LANG}_VERSION) if(NOT _MPI_MIN_VERSION OR _MPI_MIN_VERSION VERSION_GREATER MPI_${LANG}_VERSION) @@ -1717,9 +1720,13 @@ if(MPICXX IN_LIST MPI_FIND_COMPONENTS) list(APPEND _MPI_REQ_VARS "MPI_MPICXX_FOUND") endif() +if((CMAKE_VERSION VERSION_GREATER_EQUAL "3.16") AND _MPI_FAIL_REASON) + set(_MPI_FAIL_REASON_ARG REASON_FAILURE_MESSAGE "${_MPI_FAIL_REASON}") +endif() find_package_handle_standard_args(MPI REQUIRED_VARS ${_MPI_REQ_VARS} VERSION_VAR ${_MPI_MIN_VERSION} + ${_MPI_FAIL_REASON_ARG} HANDLE_COMPONENTS) #============================================================================= diff --git a/cmake/upstream/FindMPI/test_mpi.cxx b/cmake/upstream/FindMPI/test_mpi.cxx deleted file mode 100644 index 144dc4e591..0000000000 --- a/cmake/upstream/FindMPI/test_mpi.cxx +++ /dev/null @@ -1,23 +0,0 @@ -#include -#include - -#if defined(MPI_VERSION) && defined(MPI_SUBVERSION) -const char mpiver_str[] = { 'I', 'N', - 'F', 'O', - ':', 'M', - 'P', 'I', - '-', 'V', - 'E', 'R', - '[', ('0' + MPI_VERSION), - '.', ('0' + MPI_SUBVERSION), - ']', '\0' }; -#endif - -int main(int argc, char* argv[]) -{ -#if defined(MPI_VERSION) && defined(MPI_SUBVERSION) - std::puts(mpiver_str); -#endif - MPI::MPI_Init(&argc, &argv); - MPI::MPI_Finalize(); -} diff --git a/cmake/upstream/FindMPI/test_mpi_c.cxx b/cmake/upstream/FindMPI/test_mpi_c.cxx deleted file mode 100644 index 1597b6041a..0000000000 --- a/cmake/upstream/FindMPI/test_mpi_c.cxx +++ /dev/null @@ -1,23 +0,0 @@ -#include -#include - -#if defined(MPI_VERSION) && defined(MPI_SUBVERSION) -const char mpiver_str[] = { 'I', 'N', - 'F', 'O', - ':', 'M', - 'P', 'I', - '-', 'V', - 'E', 'R', - '[', ('0' + MPI_VERSION), - '.', ('0' + MPI_SUBVERSION), - ']', '\0' }; -#endif - -int main(int argc, char* argv[]) -{ -#if defined(MPI_VERSION) && defined(MPI_SUBVERSION) - std::puts(mpiver_str); -#endif - MPI_Init(&argc, &argv); - MPI_Finalize(); -} diff --git a/cmake/upstream/FindPkgConfig.cmake b/cmake/upstream/FindPkgConfig.cmake index 97087702ed..0b1693b438 100644 --- a/cmake/upstream/FindPkgConfig.cmake +++ b/cmake/upstream/FindPkgConfig.cmake @@ -9,15 +9,21 @@ A ``pkg-config`` module for CMake. Finds the ``pkg-config`` executable and adds the :command:`pkg_get_variable`, :command:`pkg_check_modules` and :command:`pkg_search_module` commands. The -following variables will also be set:: +following variables will also be set: - PKG_CONFIG_FOUND ... if pkg-config executable was found - PKG_CONFIG_EXECUTABLE ... pathname of the pkg-config program - PKG_CONFIG_VERSION_STRING ... the version of the pkg-config program found - (since CMake 2.8.8) +``PKG_CONFIG_FOUND`` + if pkg-config executable was found +``PKG_CONFIG_EXECUTABLE`` + pathname of the pkg-config program +``PKG_CONFIG_VERSION_STRING`` + version of pkg-config (since CMake 2.8.8) #]========================================] +cmake_policy(PUSH) +cmake_policy(SET CMP0054 NEW) # if() quoted variables not dereferenced +cmake_policy(SET CMP0057 NEW) # if IN_LIST + ### Common stuff #### set(PKG_CONFIG_VERSION 1) @@ -25,14 +31,28 @@ set(PKG_CONFIG_VERSION 1) if((NOT PKG_CONFIG_EXECUTABLE) AND (NOT "$ENV{PKG_CONFIG}" STREQUAL "")) set(PKG_CONFIG_EXECUTABLE "$ENV{PKG_CONFIG}" CACHE FILEPATH "pkg-config executable") endif() -find_program(PKG_CONFIG_EXECUTABLE NAMES pkg-config DOC "pkg-config executable") + +set(PKG_CONFIG_NAMES "pkg-config") +if(CMAKE_HOST_WIN32) + list(PREPEND PKG_CONFIG_NAMES "pkg-config.bat") +endif() + +find_program(PKG_CONFIG_EXECUTABLE NAMES ${PKG_CONFIG_NAMES} DOC "pkg-config executable") mark_as_advanced(PKG_CONFIG_EXECUTABLE) if (PKG_CONFIG_EXECUTABLE) execute_process(COMMAND ${PKG_CONFIG_EXECUTABLE} --version - OUTPUT_VARIABLE PKG_CONFIG_VERSION_STRING - ERROR_QUIET - OUTPUT_STRIP_TRAILING_WHITESPACE) + OUTPUT_VARIABLE PKG_CONFIG_VERSION_STRING OUTPUT_STRIP_TRAILING_WHITESPACE + ERROR_VARIABLE _PKG_CONFIG_VERSION_ERROR ERROR_STRIP_TRAILING_WHITESPACE + RESULT_VARIABLE _PKG_CONFIG_VERSION_RESULT + ) + + if (NOT _PKG_CONFIG_VERSION_RESULT EQUAL 0) + string(REPLACE "\n" "\n " _PKG_CONFIG_VERSION_ERROR " ${_PKG_CONFIG_VERSION_ERROR}") + set(PKG_CONFIG_EXECUTABLE "") + unset(PKG_CONFIG_VERSION_STRING) + endif () + unset(_PKG_CONFIG_VERSION_RESULT) endif () include(FindPackageHandleStandardArgs) @@ -82,24 +102,8 @@ macro(_pkgconfig_invoke _pkglist _prefix _varname _regexp) endif() endmacro() -#[========================================[.rst: -.. command:: pkg_get_variable - - Retrieves the value of a pkg-config variable ``varName`` and stores it in the - result variable ``resultVar`` in the calling scope. :: - - pkg_get_variable( ) - - If ``pkg-config`` returns multiple values for the specified variable, - ``resultVar`` will contain a :ref:`;-list `. - - For example: - - .. code-block:: cmake - - pkg_get_variable(GI_GIRDIR gobject-introspection-1.0 girdir) -#]========================================] -function (pkg_get_variable result pkg variable) +# Internal version of pkg_get_variable; expects PKG_CONFIG_PATH to already be set +function (_pkg_get_variable result pkg variable) _pkgconfig_invoke("${pkg}" "prefix" "result" "" "--variable=${variable}") set("${result}" "${prefix_result}" @@ -207,7 +211,13 @@ function(_pkg_find_libs _prefix _no_cmake_path _no_cmake_environment_path) endif() unset(_search_paths) + unset(_next_is_framework) foreach (flag IN LISTS ${_prefix}_LDFLAGS) + if (_next_is_framework) + list(APPEND _libs "-framework ${flag}") + unset(_next_is_framework) + continue() + endif () if (flag MATCHES "^-L(.*)") list(APPEND _search_paths ${CMAKE_MATCH_1}) continue() @@ -215,6 +225,9 @@ function(_pkg_find_libs _prefix _no_cmake_path _no_cmake_environment_path) if (flag MATCHES "^-l(.*)") set(_pkg_search "${CMAKE_MATCH_1}") else() + if (flag STREQUAL "-framework") + set(_next_is_framework TRUE) + endif () continue() endif() @@ -227,10 +240,11 @@ function(_pkg_find_libs _prefix _no_cmake_path _no_cmake_environment_path) find_library(pkgcfg_lib_${_prefix}_${_pkg_search} NAMES ${_pkg_search} ${_find_opts}) + mark_as_advanced(pkgcfg_lib_${_prefix}_${_pkg_search}) if(pkgcfg_lib_${_prefix}_${_pkg_search}) list(APPEND _libs "${pkgcfg_lib_${_prefix}_${_pkg_search}}") else() - list(APPEND _libs "${_pkg_search}") + list(APPEND _libs ${_pkg_search}) endif() endforeach() @@ -241,7 +255,7 @@ endfunction() function(_pkg_create_imp_target _prefix _imp_target_global) # only create the target if it is linkable, i.e. no executables if (NOT TARGET PkgConfig::${_prefix} - AND ( ${_prefix}_INCLUDE_DIRS OR ${_prefix}_LINK_LIBRARIES OR ${_prefix}_CFLAGS_OTHER )) + AND ( ${_prefix}_INCLUDE_DIRS OR ${_prefix}_LINK_LIBRARIES OR ${_prefix}_LDFLAGS_OTHER OR ${_prefix}_CFLAGS_OTHER )) if(${_imp_target_global}) set(_global_opt "GLOBAL") else() @@ -257,6 +271,10 @@ function(_pkg_create_imp_target _prefix _imp_target_global) set_property(TARGET PkgConfig::${_prefix} PROPERTY INTERFACE_LINK_LIBRARIES "${${_prefix}_LINK_LIBRARIES}") endif() + if(${_prefix}_LDFLAGS_OTHER) + set_property(TARGET PkgConfig::${_prefix} PROPERTY + INTERFACE_LINK_OPTIONS "${${_prefix}_LDFLAGS_OTHER}") + endif() if(${_prefix}_CFLAGS_OTHER) set_property(TARGET PkgConfig::${_prefix} PROPERTY INTERFACE_COMPILE_OPTIONS "${${_prefix}_CFLAGS_OTHER}") @@ -273,6 +291,156 @@ macro(_pkg_recalculate _prefix _no_cmake_path _no_cmake_environment_path _imp_ta endif() endmacro() +### +macro(_pkg_set_path_internal) + set(_extra_paths) + + if(NOT _no_cmake_path) + _pkgconfig_add_extra_path(_extra_paths CMAKE_PREFIX_PATH) + _pkgconfig_add_extra_path(_extra_paths CMAKE_FRAMEWORK_PATH) + _pkgconfig_add_extra_path(_extra_paths CMAKE_APPBUNDLE_PATH) + endif() + + if(NOT _no_cmake_environment_path) + _pkgconfig_add_extra_path(_extra_paths ENV CMAKE_PREFIX_PATH) + _pkgconfig_add_extra_path(_extra_paths ENV CMAKE_FRAMEWORK_PATH) + _pkgconfig_add_extra_path(_extra_paths ENV CMAKE_APPBUNDLE_PATH) + endif() + + if(NOT _extra_paths STREQUAL "") + # Save the PKG_CONFIG_PATH environment variable, and add paths + # from the CMAKE_PREFIX_PATH variables + set(_pkgconfig_path_old "$ENV{PKG_CONFIG_PATH}") + set(_pkgconfig_path "${_pkgconfig_path_old}") + if(NOT _pkgconfig_path STREQUAL "") + file(TO_CMAKE_PATH "${_pkgconfig_path}" _pkgconfig_path) + endif() + + # Create a list of the possible pkgconfig subfolder (depending on + # the system + set(_lib_dirs) + if(NOT DEFINED CMAKE_SYSTEM_NAME + OR (CMAKE_SYSTEM_NAME MATCHES "^(Linux|kFreeBSD|GNU)$" + AND NOT CMAKE_CROSSCOMPILING)) + if(EXISTS "/etc/debian_version") # is this a debian system ? + if(CMAKE_LIBRARY_ARCHITECTURE) + list(APPEND _lib_dirs "lib/${CMAKE_LIBRARY_ARCHITECTURE}/pkgconfig") + endif() + else() + # not debian, check the FIND_LIBRARY_USE_LIB32_PATHS and FIND_LIBRARY_USE_LIB64_PATHS properties + get_property(uselib32 GLOBAL PROPERTY FIND_LIBRARY_USE_LIB32_PATHS) + if(uselib32 AND CMAKE_SIZEOF_VOID_P EQUAL 4) + list(APPEND _lib_dirs "lib32/pkgconfig") + endif() + get_property(uselib64 GLOBAL PROPERTY FIND_LIBRARY_USE_LIB64_PATHS) + if(uselib64 AND CMAKE_SIZEOF_VOID_P EQUAL 8) + list(APPEND _lib_dirs "lib64/pkgconfig") + endif() + get_property(uselibx32 GLOBAL PROPERTY FIND_LIBRARY_USE_LIBX32_PATHS) + if(uselibx32 AND CMAKE_INTERNAL_PLATFORM_ABI STREQUAL "ELF X32") + list(APPEND _lib_dirs "libx32/pkgconfig") + endif() + endif() + endif() + if(CMAKE_SYSTEM_NAME STREQUAL "FreeBSD" AND NOT CMAKE_CROSSCOMPILING) + list(APPEND _lib_dirs "libdata/pkgconfig") + endif() + list(APPEND _lib_dirs "lib/pkgconfig") + list(APPEND _lib_dirs "share/pkgconfig") + + # Check if directories exist and eventually append them to the + # pkgconfig path list + foreach(_prefix_dir ${_extra_paths}) + foreach(_lib_dir ${_lib_dirs}) + if(EXISTS "${_prefix_dir}/${_lib_dir}") + list(APPEND _pkgconfig_path "${_prefix_dir}/${_lib_dir}") + list(REMOVE_DUPLICATES _pkgconfig_path) + endif() + endforeach() + endforeach() + + # Prepare and set the environment variable + if(NOT _pkgconfig_path STREQUAL "") + # remove empty values from the list + list(REMOVE_ITEM _pkgconfig_path "") + file(TO_NATIVE_PATH "${_pkgconfig_path}" _pkgconfig_path) + if(CMAKE_HOST_UNIX) + string(REPLACE ";" ":" _pkgconfig_path "${_pkgconfig_path}") + string(REPLACE "\\ " " " _pkgconfig_path "${_pkgconfig_path}") + endif() + set(ENV{PKG_CONFIG_PATH} "${_pkgconfig_path}") + endif() + + # Unset variables + unset(_lib_dirs) + unset(_pkgconfig_path) + endif() +endmacro() + +macro(_pkg_restore_path_internal) + if(NOT _extra_paths STREQUAL "") + # Restore the environment variable + set(ENV{PKG_CONFIG_PATH} "${_pkgconfig_path_old}") + endif() + + unset(_extra_paths) + unset(_pkgconfig_path_old) +endmacro() + +# pkg-config returns frameworks in --libs-only-other +# they need to be in ${_prefix}_LIBRARIES so "-framework a -framework b" does +# not incorrectly be combined to "-framework a b" +function(_pkgconfig_extract_frameworks _prefix) + set(ldflags "${${_prefix}_LDFLAGS_OTHER}") + list(FIND ldflags "-framework" FR_POS) + list(LENGTH ldflags LD_LENGTH) + + # reduce length by 1 as we need "-framework" and the next entry + math(EXPR LD_LENGTH "${LD_LENGTH} - 1") + while (FR_POS GREATER -1 AND LD_LENGTH GREATER FR_POS) + list(REMOVE_AT ldflags ${FR_POS}) + list(GET ldflags ${FR_POS} HEAD) + list(REMOVE_AT ldflags ${FR_POS}) + math(EXPR LD_LENGTH "${LD_LENGTH} - 2") + + list(APPEND LIBS "-framework ${HEAD}") + + list(FIND ldflags "-framework" FR_POS) + endwhile () + set(${_prefix}_LIBRARIES ${${_prefix}_LIBRARIES} ${LIBS} PARENT_SCOPE) + set(${_prefix}_LDFLAGS_OTHER "${ldflags}" PARENT_SCOPE) +endfunction() + +# pkg-config returns -isystem include directories in --cflags-only-other, +# depending on the version and if there is a space between -isystem and +# the actual path +function(_pkgconfig_extract_isystem _prefix) + set(cflags "${${_prefix}_CFLAGS_OTHER}") + set(outflags "") + set(incdirs "${${_prefix}_INCLUDE_DIRS}") + + set(next_is_isystem FALSE) + foreach (THING IN LISTS cflags) + # This may filter "-isystem -isystem". That would not work anyway, + # so let it happen. + if (THING STREQUAL "-isystem") + set(next_is_isystem TRUE) + continue() + endif () + if (next_is_isystem) + set(next_is_isystem FALSE) + list(APPEND incdirs "${THING}") + elseif (THING MATCHES "^-isystem") + string(SUBSTRING "${THING}" 8 -1 THING) + list(APPEND incdirs "${THING}") + else () + list(APPEND outflags "${THING}") + endif () + endforeach () + set(${_prefix}_CFLAGS_OTHER "${outflags}" PARENT_SCOPE) + set(${_prefix}_INCLUDE_DIRS "${incdirs}" PARENT_SCOPE) +endfunction() + ### macro(_pkg_check_modules_internal _is_required _is_silent _no_cmake_path _no_cmake_environment_path _imp_target _imp_target_global _prefix) _pkgconfig_unset(${_prefix}_FOUND) @@ -280,6 +448,7 @@ macro(_pkg_check_modules_internal _is_required _is_silent _no_cmake_path _no_cma _pkgconfig_unset(${_prefix}_PREFIX) _pkgconfig_unset(${_prefix}_INCLUDEDIR) _pkgconfig_unset(${_prefix}_LIBDIR) + _pkgconfig_unset(${_prefix}_MODULE_NAME) _pkgconfig_unset(${_prefix}_LIBS) _pkgconfig_unset(${_prefix}_LIBS_L) _pkgconfig_unset(${_prefix}_LIBS_PATHS) @@ -313,88 +482,7 @@ macro(_pkg_check_modules_internal _is_required _is_silent _no_cmake_path _no_cma set(_pkg_check_modules_packages) set(_pkg_check_modules_failed) - set(_extra_paths) - - if(NOT _no_cmake_path) - _pkgconfig_add_extra_path(_extra_paths CMAKE_PREFIX_PATH) - _pkgconfig_add_extra_path(_extra_paths CMAKE_FRAMEWORK_PATH) - _pkgconfig_add_extra_path(_extra_paths CMAKE_APPBUNDLE_PATH) - endif() - - if(NOT _no_cmake_environment_path) - _pkgconfig_add_extra_path(_extra_paths ENV CMAKE_PREFIX_PATH) - _pkgconfig_add_extra_path(_extra_paths ENV CMAKE_FRAMEWORK_PATH) - _pkgconfig_add_extra_path(_extra_paths ENV CMAKE_APPBUNDLE_PATH) - endif() - - if(NOT "${_extra_paths}" STREQUAL "") - # Save the PKG_CONFIG_PATH environment variable, and add paths - # from the CMAKE_PREFIX_PATH variables - set(_pkgconfig_path_old "$ENV{PKG_CONFIG_PATH}") - set(_pkgconfig_path "${_pkgconfig_path_old}") - if(NOT "${_pkgconfig_path}" STREQUAL "") - file(TO_CMAKE_PATH "${_pkgconfig_path}" _pkgconfig_path) - endif() - - # Create a list of the possible pkgconfig subfolder (depending on - # the system - set(_lib_dirs) - if(NOT DEFINED CMAKE_SYSTEM_NAME - OR (CMAKE_SYSTEM_NAME MATCHES "^(Linux|kFreeBSD|GNU)$" - AND NOT CMAKE_CROSSCOMPILING)) - if(EXISTS "/etc/debian_version") # is this a debian system ? - if(CMAKE_LIBRARY_ARCHITECTURE) - list(APPEND _lib_dirs "lib/${CMAKE_LIBRARY_ARCHITECTURE}/pkgconfig") - endif() - else() - # not debian, check the FIND_LIBRARY_USE_LIB32_PATHS and FIND_LIBRARY_USE_LIB64_PATHS properties - get_property(uselib32 GLOBAL PROPERTY FIND_LIBRARY_USE_LIB32_PATHS) - if(uselib32 AND CMAKE_SIZEOF_VOID_P EQUAL 4) - list(APPEND _lib_dirs "lib32/pkgconfig") - endif() - get_property(uselib64 GLOBAL PROPERTY FIND_LIBRARY_USE_LIB64_PATHS) - if(uselib64 AND CMAKE_SIZEOF_VOID_P EQUAL 8) - list(APPEND _lib_dirs "lib64/pkgconfig") - endif() - get_property(uselibx32 GLOBAL PROPERTY FIND_LIBRARY_USE_LIBX32_PATHS) - if(uselibx32 AND CMAKE_INTERNAL_PLATFORM_ABI STREQUAL "ELF X32") - list(APPEND _lib_dirs "libx32/pkgconfig") - endif() - endif() - endif() - if(CMAKE_SYSTEM_NAME STREQUAL "FreeBSD" AND NOT CMAKE_CROSSCOMPILING) - list(APPEND _lib_dirs "libdata/pkgconfig") - endif() - list(APPEND _lib_dirs "lib/pkgconfig") - list(APPEND _lib_dirs "share/pkgconfig") - - # Check if directories exist and eventually append them to the - # pkgconfig path list - foreach(_prefix_dir ${_extra_paths}) - foreach(_lib_dir ${_lib_dirs}) - if(EXISTS "${_prefix_dir}/${_lib_dir}") - list(APPEND _pkgconfig_path "${_prefix_dir}/${_lib_dir}") - list(REMOVE_DUPLICATES _pkgconfig_path) - endif() - endforeach() - endforeach() - - # Prepare and set the environment variable - if(NOT "${_pkgconfig_path}" STREQUAL "") - # remove empty values from the list - list(REMOVE_ITEM _pkgconfig_path "") - file(TO_NATIVE_PATH "${_pkgconfig_path}" _pkgconfig_path) - if(UNIX) - string(REPLACE ";" ":" _pkgconfig_path "${_pkgconfig_path}") - string(REPLACE "\\ " " " _pkgconfig_path "${_pkgconfig_path}") - endif() - set(ENV{PKG_CONFIG_PATH} "${_pkgconfig_path}") - endif() - - # Unset variables - unset(_lib_dirs) - unset(_pkgconfig_path) - endif() + _pkg_set_path_internal() # iterate through module list and check whether they exist and match the required version foreach (_pkg_check_modules_pkg ${_pkg_check_modules_list}) @@ -478,6 +566,7 @@ macro(_pkg_check_modules_internal _is_required _is_silent _no_cmake_path _no_cma foreach (variable IN ITEMS PREFIX INCLUDEDIR LIBDIR) _pkgconfig_set("${_pkg_check_prefix}_${variable}" "${${_pkg_check_prefix}_${variable}}") endforeach () + _pkgconfig_set("${_pkg_check_prefix}_MODULE_NAME" "${_pkg_check_modules_pkg}") if (NOT ${_is_silent}) message(STATUS " Found ${_pkg_check_modules_pkg}, version ${_pkgconfig_VERSION}") @@ -485,25 +574,27 @@ macro(_pkg_check_modules_internal _is_required _is_silent _no_cmake_path _no_cma endforeach() # set variables which are combined for multiple modules - _pkgconfig_invoke_dyn("${_pkg_check_modules_packages}" "${_prefix}" LIBRARIES "(^| )-l" --libs-only-l ) - _pkgconfig_invoke_dyn("${_pkg_check_modules_packages}" "${_prefix}" LIBRARY_DIRS "(^| )-L" --libs-only-L ) - _pkgconfig_invoke_dyn("${_pkg_check_modules_packages}" "${_prefix}" LDFLAGS "" --libs ) - _pkgconfig_invoke_dyn("${_pkg_check_modules_packages}" "${_prefix}" LDFLAGS_OTHER "" --libs-only-other ) + _pkgconfig_invoke_dyn("${_pkg_check_modules_packages}" "${_prefix}" LIBRARIES "(^| )-l" --libs-only-l ) + _pkgconfig_invoke_dyn("${_pkg_check_modules_packages}" "${_prefix}" LIBRARY_DIRS "(^| )-L" --libs-only-L ) + _pkgconfig_invoke_dyn("${_pkg_check_modules_packages}" "${_prefix}" LDFLAGS "" --libs ) + _pkgconfig_invoke_dyn("${_pkg_check_modules_packages}" "${_prefix}" LDFLAGS_OTHER "" --libs-only-other ) - _pkgconfig_invoke_dyn("${_pkg_check_modules_packages}" "${_prefix}" INCLUDE_DIRS "(^| )-I" --cflags-only-I ) - _pkgconfig_invoke_dyn("${_pkg_check_modules_packages}" "${_prefix}" CFLAGS "" --cflags ) - _pkgconfig_invoke_dyn("${_pkg_check_modules_packages}" "${_prefix}" CFLAGS_OTHER "" --cflags-only-other ) + if (APPLE AND "-framework" IN_LIST ${_prefix}_LDFLAGS_OTHER) + _pkgconfig_extract_frameworks("${_prefix}") + endif() - _pkg_recalculate("${_prefix}" ${_no_cmake_path} ${_no_cmake_environment_path} ${_imp_target} ${_imp_target_global}) - endif() + _pkgconfig_invoke_dyn("${_pkg_check_modules_packages}" "${_prefix}" INCLUDE_DIRS "(^| )(-I|-isystem ?)" --cflags-only-I ) + _pkgconfig_invoke_dyn("${_pkg_check_modules_packages}" "${_prefix}" CFLAGS "" --cflags ) + _pkgconfig_invoke_dyn("${_pkg_check_modules_packages}" "${_prefix}" CFLAGS_OTHER "" --cflags-only-other ) + + if (${_prefix}_CFLAGS_OTHER MATCHES "-isystem") + _pkgconfig_extract_isystem("${_prefix}") + endif () - if(NOT "${_extra_paths}" STREQUAL "") - # Restore the environment variable - set(ENV{PKG_CONFIG_PATH} "${_pkgconfig_path_old}") + _pkg_recalculate("${_prefix}" ${_no_cmake_path} ${_no_cmake_environment_path} ${_imp_target} ${_imp_target_global}) endif() - unset(_extra_paths) - unset(_pkgconfig_path_old) + _pkg_restore_path_internal() else() if (${_is_required}) message(SEND_ERROR "pkg-config tool not found") @@ -516,7 +607,9 @@ endmacro() .. command:: pkg_check_modules Checks for all the given modules, setting a variety of result variables in - the calling scope. :: + the calling scope. + + .. code-block:: cmake pkg_check_modules( [REQUIRED] [QUIET] @@ -547,27 +640,36 @@ endmacro() Each ```` can be either a bare module name or it can be a module name with a version constraint (operators ``=``, ``<``, ``>``, ``<=`` and ``>=`` are supported). The following are examples for a module - named ``foo`` with various constraints:: - - foo # Any version matches - foo<2 # Only match versions before 2 - foo>=3.1 # Match any version from 3.1 or later - foo=1.2.3 # Foo must be exactly version 1.2.3 - - The following variables may be set upon return. Two sets of values exist, - one for the common case (`` = ``) and another for the - information ``pkg-config`` provides when it is called with the ``--static`` - option (`` = _STATIC``):: - - _FOUND ... set to 1 if module(s) exist - _LIBRARIES ... only the libraries (without the '-l') - _LINK_LIBRARIES ... the libraries and their absolute paths - _LIBRARY_DIRS ... the paths of the libraries (without the '-L') - _LDFLAGS ... all required linker flags - _LDFLAGS_OTHER ... all other linker flags - _INCLUDE_DIRS ... the '-I' preprocessor flags (without the '-I') - _CFLAGS ... all required cflags - _CFLAGS_OTHER ... the other compiler flags + named ``foo`` with various constraints: + + - ``foo`` matches any version. + - ``foo<2`` only matches versions before 2. + - ``foo>=3.1`` matches any version from 3.1 or later. + - ``foo=1.2.3`` requires that foo must be exactly version 1.2.3. + + The following variables may be set upon return. Two sets of values exist: + One for the common case (`` = ``) and another for the + information ``pkg-config`` provides when called with the ``--static`` + option (`` = _STATIC``). + + ``_FOUND`` + set to 1 if module(s) exist + ``_LIBRARIES`` + only the libraries (without the '-l') + ``_LINK_LIBRARIES`` + the libraries and their absolute paths + ``_LIBRARY_DIRS`` + the paths of the libraries (without the '-L') + ``_LDFLAGS`` + all required linker flags + ``_LDFLAGS_OTHER`` + all other linker flags + ``_INCLUDE_DIRS`` + the '-I' preprocessor flags (without the '-I') + ``_CFLAGS`` + all required cflags + ``_CFLAGS_OTHER`` + the other compiler flags All but ``_FOUND`` may be a :ref:`;-list ` if the associated variable returned from ``pkg-config`` has multiple values. @@ -575,14 +677,18 @@ endmacro() There are some special variables whose prefix depends on the number of ```` given. When there is only one ````, ```` will simply be ````, but if two or more ```` - items are given, ```` will be ``_``:: + items are given, ```` will be ``_``. - _VERSION ... version of the module - _PREFIX ... prefix directory of the module - _INCLUDEDIR ... include directory of the module - _LIBDIR ... lib directory of the module + ``_VERSION`` + version of the module + ``_PREFIX`` + prefix directory of the module + ``_INCLUDEDIR`` + include directory of the module + ``_LIBDIR`` + lib directory of the module - Examples + Examples: .. code-block:: cmake @@ -642,7 +748,9 @@ endmacro() The behavior of this command is the same as :command:`pkg_check_modules`, except that rather than checking for all the specified modules, it searches - for just the first successful match. :: + for just the first successful match. + + .. code-block:: cmake pkg_search_module( [REQUIRED] [QUIET] @@ -651,7 +759,11 @@ endmacro() [IMPORTED_TARGET [GLOBAL]] [...]) - Examples + If a module is found, the ``_MODULE_NAME`` variable will contain the + name of the matching module. This variable can be used if you need to run + :command:`pkg_get_variable`. + + Example: .. code-block:: cmake @@ -675,6 +787,7 @@ macro(pkg_search_module _prefix _module0) if (${_prefix}_FOUND) set(_pkg_modules_found 1) + break() endif() endforeach() @@ -690,6 +803,34 @@ macro(pkg_search_module _prefix _module0) endif() endmacro() +#[========================================[.rst: +.. command:: pkg_get_variable + + Retrieves the value of a pkg-config variable ``varName`` and stores it in the + result variable ``resultVar`` in the calling scope. + + .. code-block:: cmake + + pkg_get_variable( ) + + If ``pkg-config`` returns multiple values for the specified variable, + ``resultVar`` will contain a :ref:`;-list `. + + For example: + + .. code-block:: cmake + + pkg_get_variable(GI_GIRDIR gobject-introspection-1.0 girdir) +#]========================================] +function (pkg_get_variable result pkg variable) + _pkg_set_path_internal() + _pkgconfig_invoke("${pkg}" "prefix" "result" "" "--variable=${variable}") + set("${result}" + "${prefix_result}" + PARENT_SCOPE) + _pkg_restore_path_internal() +endfunction () + #[========================================[.rst: Variables Affecting Behavior @@ -718,3 +859,5 @@ Variables Affecting Behavior ### Local Variables: ### mode: cmake ### End: + +cmake_policy(POP) diff --git a/cmake/upstream/FindPython.cmake b/cmake/upstream/FindPython.cmake index 018956b6a9..01b82c4b21 100644 --- a/cmake/upstream/FindPython.cmake +++ b/cmake/upstream/FindPython.cmake @@ -13,13 +13,24 @@ The following components are supported: * ``Interpreter``: search for Python interpreter. * ``Compiler``: search for Python compiler. Only offered by IronPython. * ``Development``: search for development artifacts (include directories and - libraries). + libraries). This component includes two sub-components which can be specified + independently: + + * ``Development.Module``: search for artifacts for Python module + developments. + * ``Development.Embed``: search for artifacts for Python embedding + developments. + * ``NumPy``: search for NumPy include directories. If no ``COMPONENTS`` are specified, ``Interpreter`` is assumed. +If component ``Development`` is specified, it implies sub-components +``Development.Module`` and ``Development.Embed``. + To ensure consistent versions between components ``Interpreter``, ``Compiler``, -``Development`` and ``NumPy``, specify all components at the same time:: +``Development`` (or one of its sub-components) and ``NumPy``, specify all +components at the same time:: find_package (Python COMPONENTS Interpreter Development) @@ -30,10 +41,11 @@ To manage concurrent versions 3 and 2 of Python, use :module:`FindPython3` and .. note:: - If components ``Interpreter`` and ``Development`` are both specified, this - module search only for interpreter with same platform architecture as the one - defined by ``CMake`` configuration. This contraint does not apply if only - ``Interpreter`` component is specified. + If components ``Interpreter`` and ``Development`` (or one of its + sub-components) are both specified, this module search only for interpreter + with same platform architecture as the one defined by ``CMake`` + configuration. This contraint does not apply if only ``Interpreter`` + component is specified. Imported Targets ^^^^^^^^^^^^^^^^ @@ -45,12 +57,12 @@ This module defines the following :ref:`Imported Targets ` Python interpreter. Target defined if component ``Interpreter`` is found. ``Python::Compiler`` Python compiler. Target defined if component ``Compiler`` is found. +``Python::Module`` + Python library for Python module. Target defined if component + ``Development.Module`` is found. ``Python::Python`` Python library for Python embedding. Target defined if component - ``Development`` is found. -``Python::Module`` - Python library for Python module. Target defined if component ``Development`` - is found. + ``Development.Embed`` is found. ``Python::NumPy`` NumPy Python library. Target defined if component ``NumPy`` is found. @@ -73,33 +85,40 @@ This module will set the following variables in your project * Anaconda * Canopy * IronPython + * PyPy ``Python_STDLIB`` Standard platform independent installation directory. Information returned by - ``distutils.sysconfig.get_python_lib(plat_specific=False,standard_lib=True)``. + ``distutils.sysconfig.get_python_lib(plat_specific=False,standard_lib=True)`` + or else ``sysconfig.get_path('stdlib')``. ``Python_STDARCH`` Standard platform dependent installation directory. Information returned by - ``distutils.sysconfig.get_python_lib(plat_specific=True,standard_lib=True)``. + ``distutils.sysconfig.get_python_lib(plat_specific=True,standard_lib=True)`` + or else ``sysconfig.get_path('platstdlib')``. ``Python_SITELIB`` Third-party platform independent installation directory. Information returned by - ``distutils.sysconfig.get_python_lib(plat_specific=False,standard_lib=False)``. + ``distutils.sysconfig.get_python_lib(plat_specific=False,standard_lib=False)`` + or else ``sysconfig.get_path('purelib')``. ``Python_SITEARCH`` Third-party platform dependent installation directory. Information returned by - ``distutils.sysconfig.get_python_lib(plat_specific=True,standard_lib=False)``. + ``distutils.sysconfig.get_python_lib(plat_specific=True,standard_lib=False)`` + or else ``sysconfig.get_path('platlib')``. ``Python_SOABI`` Extension suffix for modules. Information returned by - ``distutils.sysconfig.get_config_flag('SOABI')`` or computed from - ``distutils.sysconfig.get_config_flag('EXT_SUFFIX')`` or - ``python-config --extension-suffix``. + ``distutils.sysconfig.get_config_var('SOABI')`` or computed from + ``distutils.sysconfig.get_config_var('EXT_SUFFIX')`` or + ``python-config --extension-suffix``. If package ``distutils.sysconfig`` is + not available, ``sysconfig.get_config_var('SOABI')`` or + ``sysconfig.get_config_var('EXT_SUFFIX')`` are used. ``Python_Compiler_FOUND`` System has the Python compiler. ``Python_COMPILER`` @@ -107,8 +126,14 @@ This module will set the following variables in your project ``Python_COMPILER_ID`` A short string unique to the compiler. Possible values include: * IronPython +``Python_DOTNET_LAUNCHER`` + The ``.Net`` interpreter. Only used by ``IronPython`` implementation. ``Python_Development_FOUND`` System has the Python development artifacts. +``Python_Development.Module_FOUND`` + System has the Python development artifacts for Python module. +``Python_Development.Embed_FOUND`` + System has the Python development artifacts for Python embedding. ``Python_INCLUDE_DIRS`` The Python include directories. ``Python_LIBRARIES`` @@ -125,6 +150,8 @@ This module will set the following variables in your project Python minor version. ``Python_VERSION_PATCH`` Python patch version. +``Python_PyPy_VERSION`` + Python PyPy version. ``Python_NumPy_FOUND`` System has the NumPy. ``Python_NumPy_INCLUDE_DIRS`` @@ -237,8 +264,9 @@ Hints * ``ONLY``: Only the virtual environment is used to look-up for the interpreter. * ``STANDARD``: The virtual environment is not used to look-up for the - interpreter. In this case, variable ``Python_FIND_REGISTRY`` (Windows) - or ``CMAKE_FIND_FRAMEWORK`` (macOS) can be set with value ``LAST`` or + interpreter but environment variable ``PATH`` is always considered. + In this case, variable ``Python_FIND_REGISTRY`` (Windows) or + ``CMAKE_FIND_FRAMEWORK`` (macOS) can be set with value ``LAST`` or ``NEVER`` to select preferably the interpreter from the virtual environment. @@ -248,6 +276,39 @@ Hints recommended to also include the component ``Interpreter`` to get expected result. +``Python_FIND_IMPLEMENTATIONS`` + This variable defines, in an ordered list, the different implementations + which will be searched. The ``Python_FIND_IMPLEMENTATIONS`` variable can + hold the following values: + + * ``CPython``: this is the standard implementation. Various products, like + ``Anaconda`` or ``ActivePython``, rely on this implementation. + * ``IronPython``: This implementation use the ``CSharp`` language for + ``.NET Framework`` on top of the `Dynamic Language Runtime` (``DLR``). + See `IronPython `_. + * ``PyPy``: This implementation use ``RPython`` language and + ``RPython translation toolchain`` to produce the python interpreter. + See `PyPy `_. + + The default value is: + + * Windows platform: ``CPython``, ``IronPython`` + * Other platforms: ``CPython`` + + .. note:: + + This hint has the lowest priority of all hints, so even if, for example, + you specify ``IronPython`` first and ``CPython`` in second, a python + product based on ``CPython`` can be selected because, for example with + ``Python_FIND_STRATEGY=LOCATION``, each location will be search first for + ``IronPython`` and second for ``CPython``. + + .. note:: + + When ``IronPython`` is specified, on platforms other than ``Windows``, the + ``.Net`` interpreter (i.e. ``mono`` command) is expected to be available + through the ``PATH`` variable. + Artifacts Specification ^^^^^^^^^^^^^^^^^^^^^^^ @@ -260,6 +321,9 @@ setting the following variables: ``Python_COMPILER`` The path to the compiler. +``Python_DOTNET_LAUNCHER`` + The ``.Net`` interpreter. Only used by ``IronPython`` implementation. + ``Python_LIBRARY`` The path to the library. It will be used to compute the variables ``Python_LIBRARIES``, ``Python_LIBRAY_DIRS`` and diff --git a/cmake/upstream/FindPython/Support.cmake b/cmake/upstream/FindPython/Support.cmake index d4b20fea7b..4453647b41 100644 --- a/cmake/upstream/FindPython/Support.cmake +++ b/cmake/upstream/FindPython/Support.cmake @@ -6,7 +6,7 @@ # if (POLICY CMP0094) - cmake_policy (GET CMP0094 _${_PYTHON_PREFIX}_LOOKUP_POLICY) + cmake_policy (GET CMP0094 _${_PYTHON_PREFIX}_LOOKUP_POLICY) endif() cmake_policy (VERSION 3.7) @@ -24,15 +24,19 @@ endif() if (NOT DEFINED _${_PYTHON_PREFIX}_REQUIRED_VERSION_MAJOR) message (FATAL_ERROR "FindPython: INTERNAL ERROR") endif() -if (_${_PYTHON_PREFIX}_REQUIRED_VERSION_MAJOR EQUAL 3) +if (_${_PYTHON_PREFIX}_REQUIRED_VERSION_MAJOR EQUAL "3") set(_${_PYTHON_PREFIX}_VERSIONS 3.9 3.8 3.7 3.6 3.5 3.4 3.3 3.2 3.1 3.0) -elseif (_${_PYTHON_PREFIX}_REQUIRED_VERSION_MAJOR EQUAL 2) +elseif (_${_PYTHON_PREFIX}_REQUIRED_VERSION_MAJOR EQUAL "2") set(_${_PYTHON_PREFIX}_VERSIONS 2.7 2.6 2.5 2.4 2.3 2.2 2.1 2.0) else() message (FATAL_ERROR "FindPython: INTERNAL ERROR") endif() -get_property(_${_PYTHON_PREFIX}_CMAKE_ROLE GLOBAL PROPERTY CMAKE_ROLE) +if (CMAKE_VERSION VERSION_GREATER_EQUAL 3.14) + get_property(_${_PYTHON_PREFIX}_CMAKE_ROLE GLOBAL PROPERTY CMAKE_ROLE) +elseif (DEFINED CMAKE_PROJECT_NAME) + set(_${_PYTHON_PREFIX}_CMAKE_ROLE "PROJECT") +endif() # @@ -95,51 +99,117 @@ endmacro() macro (_PYTHON_FIND_FRAMEWORKS) - set (${_PYTHON_PREFIX}_FRAMEWORKS) if (CMAKE_HOST_APPLE OR APPLE) file(TO_CMAKE_PATH "$ENV{CMAKE_FRAMEWORK_PATH}" _pff_CMAKE_FRAMEWORK_PATH) set (_pff_frameworks ${CMAKE_FRAMEWORK_PATH} - ${_pff_CMAKE_FRAMEWORK_PATH} - ~/Library/Frameworks - /usr/local/Frameworks - ${CMAKE_SYSTEM_FRAMEWORK_PATH}) + ${_pff_CMAKE_FRAMEWORK_PATH} + ~/Library/Frameworks + /usr/local/Frameworks + ${CMAKE_SYSTEM_FRAMEWORK_PATH}) list (REMOVE_DUPLICATES _pff_frameworks) - foreach (_pff_framework IN LISTS _pff_frameworks) - if (EXISTS ${_pff_framework}/Python${_${_PYTHON_PREFIX}_REQUIRED_VERSION_MAJOR}.framework) - list (APPEND ${_PYTHON_PREFIX}_FRAMEWORKS ${_pff_framework}/Python${_${_PYTHON_PREFIX}_REQUIRED_VERSION_MAJOR}.framework) - endif() - if (EXISTS ${_pff_framework}/Python.framework) - list (APPEND ${_PYTHON_PREFIX}_FRAMEWORKS ${_pff_framework}/Python.framework) + foreach (_pff_implementation IN LISTS _${_PYTHON_PREFIX}_FIND_IMPLEMENTATIONS) + unset (_${_PYTHON_PREFIX}_${_pff_implementation}_FRAMEWORKS) + if (_pff_implementation STREQUAL "CPython") + foreach (_pff_framework IN LISTS _pff_frameworks) + if (EXISTS ${_pff_framework}/Python${_${_PYTHON_PREFIX}_REQUIRED_VERSION_MAJOR}.framework) + list (APPEND _${_PYTHON_PREFIX}_${_pff_implementation}_FRAMEWORKS ${_pff_framework}/Python${_${_PYTHON_PREFIX}_REQUIRED_VERSION_MAJOR}.framework) + endif() + if (EXISTS ${_pff_framework}/Python.framework) + list (APPEND _${_PYTHON_PREFIX}_${_pff_implementation}_FRAMEWORKS ${_pff_framework}/Python.framework) + endif() + endforeach() + elseif (_pff_implementation STREQUAL "IronPython") + foreach (_pff_framework IN LISTS _pff_frameworks) + if (EXISTS ${_pff_framework}/IronPython.framework) + list (APPEND _${_PYTHON_PREFIX}_${_pff_implementation}_FRAMEWORKS ${_pff_framework}/IronPython.framework) + endif() + endforeach() endif() endforeach() + unset (_pff_implementation) unset (_pff_frameworks) unset (_pff_framework) endif() endmacro() -function (_PYTHON_GET_FRAMEWORKS _PYTHON_PGF_FRAMEWORK_PATHS _PYTHON_VERSION) - set (_PYTHON_FRAMEWORK_PATHS) - foreach (_PYTHON_FRAMEWORK IN LISTS ${_PYTHON_PREFIX}_FRAMEWORKS) - list (APPEND _PYTHON_FRAMEWORK_PATHS - "${_PYTHON_FRAMEWORK}/Versions/${_PYTHON_VERSION}") +function (_PYTHON_GET_FRAMEWORKS _PYTHON_PGF_FRAMEWORK_PATHS) + cmake_parse_arguments (PARSE_ARGV 1 _PGF "" "" "IMPLEMENTATIONS;VERSION") + + if (NOT _PGF_IMPLEMENTATIONS) + set (_PGF_IMPLEMENTATIONS ${_${_PYTHON_PREFIX}_FIND_IMPLEMENTATIONS}) + endif() + + set (framework_paths) + + foreach (implementation IN LISTS _PGF_IMPLEMENTATIONS) + if (implementation STREQUAL "CPython") + foreach (version IN LISTS _PGF_VERSION) + foreach (framework IN LISTS _${_PYTHON_PREFIX}_${implementation}_FRAMEWORKS) + if (EXISTS "${framework}/Versions/${version}") + list (APPEND framework_paths "${framework}/Versions/${version}") + endif() + endforeach() + endforeach() + elseif (implementation STREQUAL "IronPython") + foreach (version IN LISTS _PGF_VERSION) + foreach (framework IN LISTS _${_PYTHON_PREFIX}_${implementation}_FRAMEWORKS) + # pick-up all available versions + file (GLOB versions LIST_DIRECTORIES true RELATIVE "${framework}/Versions/" + "${framework}/Versions/${version}*") + list (SORT versions ORDER DESCENDING) + list (TRANSFORM versions PREPEND "${framework}/Versions/") + list (APPEND framework_paths ${versions}) + endforeach() + endforeach() + endif() endforeach() - set (${_PYTHON_PGF_FRAMEWORK_PATHS} ${_PYTHON_FRAMEWORK_PATHS} PARENT_SCOPE) + + set (${_PYTHON_PGF_FRAMEWORK_PATHS} ${framework_paths} PARENT_SCOPE) endfunction() -function (_PYTHON_GET_REGISTRIES _PYTHON_PGR_REGISTRY_PATHS _PYTHON_VERSION) - string (REPLACE "." "" _PYTHON_VERSION_NO_DOTS ${_PYTHON_VERSION}) - set (${_PYTHON_PGR_REGISTRY_PATHS} - [HKEY_CURRENT_USER\\SOFTWARE\\Python\\PythonCore\\${_PYTHON_VERSION}-${_${_PYTHON_PREFIX}_ARCH}\\InstallPath] - [HKEY_CURRENT_USER\\SOFTWARE\\Python\\PythonCore\\${_PYTHON_VERSION}-${_${_PYTHON_PREFIX}_ARCH2}\\InstallPath] - [HKEY_CURRENT_USER\\SOFTWARE\\Python\\PythonCore\\${_PYTHON_VERSION}\\InstallPath] - [HKEY_CURRENT_USER\\SOFTWARE\\Python\\ContinuumAnalytics\\Anaconda${_PYTHON_VERSION_NO_DOTS}-${_${_PYTHON_PREFIX}_ARCH}\\InstallPath] - [HKEY_CURRENT_USER\\SOFTWARE\\Python\\ContinuumAnalytics\\Anaconda${_PYTHON_VERSION_NO_DOTS}-${_${_PYTHON_PREFIX}_ARCH2}\\InstallPath] - [HKEY_LOCAL_MACHINE\\SOFTWARE\\Python\\PythonCore\\${_PYTHON_VERSION}-${_${_PYTHON_PREFIX}_ARCH}\\InstallPath] - [HKEY_LOCAL_MACHINE\\SOFTWARE\\Python\\PythonCore\\${_PYTHON_VERSION}-${_${_PYTHON_PREFIX}_ARCH2}\\InstallPath] - [HKEY_LOCAL_MACHINE\\SOFTWARE\\Python\\PythonCore\\${_PYTHON_VERSION}\\InstallPath] - [HKEY_LOCAL_MACHINE\\SOFTWARE\\Python\\ContinuumAnalytics\\Anaconda${_PYTHON_VERSION_NO_DOTS}-${_${_PYTHON_PREFIX}_ARCH}\\InstallPath] - [HKEY_LOCAL_MACHINE\\SOFTWARE\\Python\\ContinuumAnalytics\\Anaconda${_PYTHON_VERSION_NO_DOTS}-${_${_PYTHON_PREFIX}_ARCH2}\\InstallPath] - PARENT_SCOPE) +function (_PYTHON_GET_REGISTRIES _PYTHON_PGR_REGISTRY_PATHS) + cmake_parse_arguments (PARSE_ARGV 1 _PGR "" "" "IMPLEMENTATIONS;VERSION") + + if (NOT _PGR_IMPLEMENTATIONS) + set (_PGR_IMPLEMENTATIONS ${_${_PYTHON_PREFIX}_FIND_IMPLEMENTATIONS}) + endif() + + set (registries) + + foreach (implementation IN LISTS _PGR_IMPLEMENTATIONS) + if (implementation STREQUAL "CPython") + foreach (version IN LISTS _PGR_VERSION) + string (REPLACE "." "" version_no_dots ${version}) + list (APPEND registries + [HKEY_CURRENT_USER\\SOFTWARE\\Python\\PythonCore\\${version}-${_${_PYTHON_PREFIX}_ARCH}\\InstallPath] + [HKEY_CURRENT_USER\\SOFTWARE\\Python\\PythonCore\\${version}-${_${_PYTHON_PREFIX}_ARCH2}\\InstallPath]) + if (version VERSION_GREATER_EQUAL "3.5") + get_filename_component (arch "[HKEY_CURRENT_USER\\Software\\Python\\PythonCore\\${version};SysArchitecture]" NAME) + if (arch MATCHES "(${_${_PYTHON_PREFIX}_ARCH}|${_${_PYTHON_PREFIX}_ARCH2})bit") + list (APPEND registries + [HKEY_CURRENT_USER\\SOFTWARE\\Python\\PythonCore\\${version}\\InstallPath]) + endif() + else() + list (APPEND registries + [HKEY_CURRENT_USER\\SOFTWARE\\Python\\PythonCore\\${version}\\InstallPath]) + endif() + list (APPEND registries + [HKEY_CURRENT_USER\\SOFTWARE\\Python\\ContinuumAnalytics\\Anaconda${version_no_dots}-${_${_PYTHON_PREFIX}_ARCH}\\InstallPath] + [HKEY_CURRENT_USER\\SOFTWARE\\Python\\ContinuumAnalytics\\Anaconda${version_no_dots}-${_${_PYTHON_PREFIX}_ARCH2}\\InstallPath] + [HKEY_LOCAL_MACHINE\\SOFTWARE\\Python\\PythonCore\\${version}-${_${_PYTHON_PREFIX}_ARCH}\\InstallPath] + [HKEY_LOCAL_MACHINE\\SOFTWARE\\Python\\PythonCore\\${version}-${_${_PYTHON_PREFIX}_ARCH2}\\InstallPath] + [HKEY_LOCAL_MACHINE\\SOFTWARE\\Python\\PythonCore\\${version}\\InstallPath] + [HKEY_LOCAL_MACHINE\\SOFTWARE\\Python\\ContinuumAnalytics\\Anaconda${version_no_dots}-${_${_PYTHON_PREFIX}_ARCH}\\InstallPath] + [HKEY_LOCAL_MACHINE\\SOFTWARE\\Python\\ContinuumAnalytics\\Anaconda${version_no_dots}-${_${_PYTHON_PREFIX}_ARCH2}\\InstallPath]) + endforeach() + elseif (implementation STREQUAL "IronPython") + foreach (version IN LISTS _PGR_VERSION) + list (APPEND registries [HKEY_LOCAL_MACHINE\\SOFTWARE\\IronPython\\${version}\\InstallPath]) + endforeach() + endif() + endforeach() + + set (${_PYTHON_PGR_REGISTRY_PATHS} "${registries}" PARENT_SCOPE) endfunction() @@ -187,7 +257,11 @@ function (_PYTHON_GET_ABIFLAGS _PGABIFLAGS) endfunction() function (_PYTHON_GET_PATH_SUFFIXES _PYTHON_PGPS_PATH_SUFFIXES) - cmake_parse_arguments (PARSE_ARGV 1 _PGPS "LIBRARY;INCLUDE" "VERSION" "") + cmake_parse_arguments (PARSE_ARGV 1 _PGPS "INTERPRETER;COMPILER;LIBRARY;INCLUDE" "" "IMPLEMENTATIONS;VERSION") + + if (NOT _PGPS_IMPLEMENTATIONS) + set (_PGPS_IMPLEMENTATIONS ${_${_PYTHON_PREFIX}_FIND_IMPLEMENTATIONS}) + endif() if (DEFINED _${_PYTHON_PREFIX}_ABIFLAGS) set (abi "${_${_PYTHON_PREFIX}_ABIFLAGS}") @@ -196,93 +270,163 @@ function (_PYTHON_GET_PATH_SUFFIXES _PYTHON_PGPS_PATH_SUFFIXES) endif() set (path_suffixes) - if (_PGPS_LIBRARY) - if (CMAKE_LIBRARY_ARCHITECTURE) - list (APPEND path_suffixes lib/${CMAKE_LIBRARY_ARCHITECTURE}) - endif() - list (APPEND path_suffixes lib libs) - if (CMAKE_LIBRARY_ARCHITECTURE) - set (suffixes "${abi}") - if (suffixes) - list (TRANSFORM suffixes PREPEND "lib/python${_PGPS_VERSION}/config-${_PGPS_VERSION}") - list (TRANSFORM suffixes APPEND "-${CMAKE_LIBRARY_ARCHITECTURE}") + foreach (implementation IN LISTS _PGPS_IMPLEMENTATIONS) + if (implementation STREQUAL "CPython") + if (_PGPS_INTERPRETER) + list (APPEND path_suffixes bin Scripts) else() - set (suffixes "lib/python${_PGPS_VERSION}/config-${_PGPS_VERSION}-${CMAKE_LIBRARY_ARCHITECTURE}") + foreach (version IN LISTS _PGPS_VERSION) + if (_PGPS_LIBRARY) + if (CMAKE_LIBRARY_ARCHITECTURE) + list (APPEND path_suffixes lib/${CMAKE_LIBRARY_ARCHITECTURE}) + endif() + list (APPEND path_suffixes lib libs) + + if (CMAKE_LIBRARY_ARCHITECTURE) + set (suffixes "${abi}") + if (suffixes) + list (TRANSFORM suffixes PREPEND "lib/python${_PGPS_VERSION}/config-${_PGPS_VERSION}") + list (TRANSFORM suffixes APPEND "-${CMAKE_LIBRARY_ARCHITECTURE}") + else() + set (suffixes "lib/python${_PGPS_VERSION}/config-${_PGPS_VERSION}-${CMAKE_LIBRARY_ARCHITECTURE}") + endif() + list (APPEND path_suffixes ${suffixes}) + endif() + set (suffixes "${abi}") + if (suffixes) + list (TRANSFORM suffixes PREPEND "lib/python${_PGPS_VERSION}/config-${_PGPS_VERSION}") + else() + set (suffixes "lib/python${_PGPS_VERSION}/config-${_PGPS_VERSION}") + endif() + list (APPEND path_suffixes ${suffixes}) + elseif (_PGPS_INCLUDE) + set (suffixes "${abi}") + if (suffixes) + list (TRANSFORM suffixes PREPEND "include/python${_PGPS_VERSION}") + else() + set (suffixes "include/python${_PGPS_VERSION}") + endif() + list (APPEND path_suffixes ${suffixes} include) + endif() + endforeach() + endif() + elseif (implementation STREQUAL "IronPython") + if (_PGPS_INTERPRETER OR _PGPS_COMPILER) + foreach (version IN LISTS _PGPS_VERSION) + list (APPEND path_suffixes "share/ironpython${version}") + endforeach() + list (APPEND path_suffixes ${_${_PYTHON_PREFIX}_IRON_PYTHON_PATH_SUFFIXES}) + endif() + elseif (implementation STREQUAL "PyPy") + if (_PGPS_INTERPRETER) + list (APPEND path_suffixes ${_${_PYTHON_PREFIX}_PYPY_EXECUTABLE_PATH_SUFFIXES}) + elseif (_PGPS_LIBRARY) + list (APPEND path_suffixes ${_${_PYTHON_PREFIX}_PYPY_LIBRARY_PATH_SUFFIXES}) + elseif (_PGPS_INCLUDE) + list (APPEND path_suffixes ${_${_PYTHON_PREFIX}_PYPY_INCLUDE_PATH_SUFFIXES}) endif() - list (APPEND path_suffixes ${suffixes}) - endif() - set (suffixes "${abi}") - if (suffixes) - list (TRANSFORM suffixes PREPEND "lib/python${_PGPS_VERSION}/config-${_PGPS_VERSION}") - else() - set (suffixes "lib/python${_PGPS_VERSION}/config-${_PGPS_VERSION}") - endif() - list (APPEND path_suffixes ${suffixes}) - elseif (_PGPS_INCLUDE) - set (suffixes "${abi}") - if (suffixes) - list (TRANSFORM suffixes PREPEND "include/python${_PGPS_VERSION}") - else() - set (suffixes "include/python${_PGPS_VERSION}") endif() - list (APPEND path_suffixes ${suffixes} include) - endif() + endforeach() + list (REMOVE_DUPLICATES path_suffixes) set (${_PYTHON_PGPS_PATH_SUFFIXES} ${path_suffixes} PARENT_SCOPE) endfunction() function (_PYTHON_GET_NAMES _PYTHON_PGN_NAMES) - cmake_parse_arguments (PARSE_ARGV 1 _PGN "POSIX;EXECUTABLE;CONFIG;LIBRARY;WIN32;DEBUG" "VERSION" "") + cmake_parse_arguments (PARSE_ARGV 1 _PGN "POSIX;INTERPRETER;COMPILER;CONFIG;LIBRARY;WIN32;DEBUG" "" "IMPLEMENTATIONS;VERSION") + + if (NOT _PGN_IMPLEMENTATIONS) + set (_PGN_IMPLEMENTATIONS ${_${_PYTHON_PREFIX}_FIND_IMPLEMENTATIONS}) + endif() set (names) - if (_PGN_WIN32) - string (REPLACE "." "" _PYTHON_VERSION_NO_DOTS ${_PGN_VERSION}) + foreach (implementation IN LISTS _PGN_IMPLEMENTATIONS) + if (implementation STREQUAL "CPython") + foreach (version IN LISTS _PGN_VERSION) + if (_PGN_WIN32) + string (REPLACE "." "" version_no_dots ${version}) - set (name python${_PYTHON_VERSION_NO_DOTS}) - if (_PGN_DEBUG) - string (APPEND name "_d") - endif() + set (name python${version_no_dots}) + if (_PGN_DEBUG) + string (APPEND name "_d") + endif() - list (APPEND names "${name}") - endif() + list (APPEND names "${name}") + endif() - if (_PGN_POSIX) - if (DEFINED _${_PYTHON_PREFIX}_ABIFLAGS) - set (abi "${_${_PYTHON_PREFIX}_ABIFLAGS}") - else() - if (_PGN_EXECUTABLE OR _PGN_CONFIG) - set (abi "") - else() - set (abi "mu" "m" "u" "") - endif() - endif() + if (_PGN_POSIX) + if (DEFINED _${_PYTHON_PREFIX}_ABIFLAGS) + set (abi "${_${_PYTHON_PREFIX}_ABIFLAGS}") + else() + if (_PGN_INTERPRETER OR _PGN_CONFIG) + set (abi "") + else() + set (abi "mu" "m" "u" "") + endif() + endif() - if (abi) - if (_PGN_CONFIG AND DEFINED CMAKE_LIBRARY_ARCHITECTURE) - set (abinames "${abi}") - list (TRANSFORM abinames PREPEND "${CMAKE_LIBRARY_ARCHITECTURE}-python${_PGN_VERSION}") - list (TRANSFORM abinames APPEND "-config") - list (APPEND names ${abinames}) - endif() - set (abinames "${abi}") - list (TRANSFORM abinames PREPEND "python${_PGN_VERSION}") - if (_PGN_CONFIG) - list (TRANSFORM abinames APPEND "-config") + if (abi) + if (_PGN_CONFIG AND DEFINED CMAKE_LIBRARY_ARCHITECTURE) + set (abinames "${abi}") + list (TRANSFORM abinames PREPEND "${CMAKE_LIBRARY_ARCHITECTURE}-python${version}") + list (TRANSFORM abinames APPEND "-config") + list (APPEND names ${abinames}) + endif() + set (abinames "${abi}") + list (TRANSFORM abinames PREPEND "python${version}") + if (_PGN_CONFIG) + list (TRANSFORM abinames APPEND "-config") + endif() + list (APPEND names ${abinames}) + else() + unset (abinames) + if (_PGN_CONFIG AND DEFINED CMAKE_LIBRARY_ARCHITECTURE) + set (abinames "${CMAKE_LIBRARY_ARCHITECTURE}-python${version}") + endif() + list (APPEND abinames "python${version}") + if (_PGN_CONFIG) + list (TRANSFORM abinames APPEND "-config") + endif() + list (APPEND names ${abinames}) + endif() + endif() + endforeach() + if (_PGN_INTERPRETER) + list (APPEND names python${_${_PYTHON_PREFIX}_REQUIRED_VERSION_MAJOR} python) endif() - list (APPEND names ${abinames}) - else() - if (_PGN_CONFIG AND DEFINED CMAKE_LIBRARY_ARCHITECTURE) - set (abinames "${CMAKE_LIBRARY_ARCHITECTURE}-python${_PGN_VERSION}") + elseif (implementation STREQUAL "IronPython") + if (_PGN_INTERPRETER) + if (NOT CMAKE_SYSTEM_NAME STREQUAL "Linux") + # Do not use wrapper script on Linux because it is buggy: -c interpreter option cannot be used + foreach (version IN LISTS _PGN_VERSION) + list (APPEND names "ipy${version}") + endforeach() + endif() + list (APPEND names ${_${_PYTHON_PREFIX}_IRON_PYTHON_INTERPRETER_NAMES}) + elseif (_PGN_COMPILER) + list (APPEND names ${_${_PYTHON_PREFIX}_IRON_PYTHON_COMPILER_NAMES}) endif() - list (APPEND abinames "python${_PGN_VERSION}") - if (_PGN_CONFIG) - list (TRANSFORM abinames APPEND "-config") + elseif (implementation STREQUAL "PyPy") + if (_PGN_INTERPRETER) + list (APPEND names ${_${_PYTHON_PREFIX}_PYPY_NAMES}) + elseif (_PGN_LIBRARY) + if (_PGN_WIN32) + foreach (version IN LISTS _PGN_VERSION) + string (REPLACE "." "" version_no_dots ${version}) + + set (name "python${version_no_dots}") + if (_PGN_DEBUG) + string (APPEND name "_d") + endif() + list (APPEND names "${name}") + endforeach() + endif() + list (APPEND names ${_${_PYTHON_PREFIX}_PYPY_LIB_NAMES}) endif() - list (APPEND names ${abinames}) endif() - endif() + endforeach() set (${_PYTHON_PGN_NAMES} ${names} PARENT_SCOPE) endfunction() @@ -323,7 +467,7 @@ function (_PYTHON_GET_CONFIG_VAR _PYTHON_PGCV_VALUE NAME) if (_${_PYTHON_PREFIX}_EXECUTABLE AND NOT CMAKE_CROSSCOMPILING) if (NAME STREQUAL "PREFIX") - execute_process (COMMAND "${_${_PYTHON_PREFIX}_EXECUTABLE}" -c "import sys\ntry:\n from distutils import sysconfig\n sys.stdout.write(';'.join([sysconfig.PREFIX,sysconfig.EXEC_PREFIX,sysconfig.BASE_EXEC_PREFIX]))\nexcept Exception:\n import sysconfig\n sys.stdout.write(';'.join([sysconfig.get_config_var('base') or '', sysconfig.get_config_var('installed_base') or '']))" + execute_process (COMMAND ${_${_PYTHON_PREFIX}_INTERPRETER_LAUNCHER} "${_${_PYTHON_PREFIX}_EXECUTABLE}" -c "import sys\ntry:\n from distutils import sysconfig\n sys.stdout.write(';'.join([sysconfig.PREFIX,sysconfig.EXEC_PREFIX,sysconfig.BASE_EXEC_PREFIX]))\nexcept Exception:\n import sysconfig\n sys.stdout.write(';'.join([sysconfig.get_config_var('base') or '', sysconfig.get_config_var('installed_base') or '']))" RESULT_VARIABLE _result OUTPUT_VARIABLE _values ERROR_QUIET @@ -339,7 +483,8 @@ function (_PYTHON_GET_CONFIG_VAR _PYTHON_PGCV_VALUE NAME) else() set (_scheme "posix_prefix") endif() - execute_process (COMMAND "${_${_PYTHON_PREFIX}_EXECUTABLE}" -c "import sys\ntry:\n from distutils import sysconfig\n sys.stdout.write(';'.join([sysconfig.get_python_inc(plat_specific=True),sysconfig.get_python_inc(plat_specific=False)]))\nexcept Exception:\n import sysconfig\n sys.stdout.write(';'.join([sysconfig.get_path('platinclude'),sysconfig.get_path('platinclude','${_scheme}'),sysconfig.get_path('include'),sysconfig.get_path('include','${_scheme}')]))" + execute_process (COMMAND ${_${_PYTHON_PREFIX}_INTERPRETER_LAUNCHER} "${_${_PYTHON_PREFIX}_EXECUTABLE}" -c + "import sys\ntry:\n from distutils import sysconfig\n sys.stdout.write(';'.join([sysconfig.get_python_inc(plat_specific=True),sysconfig.get_python_inc(plat_specific=False)]))\nexcept Exception:\n import sysconfig\n sys.stdout.write(';'.join([sysconfig.get_path('platinclude'),sysconfig.get_path('platinclude','${_scheme}'),sysconfig.get_path('include'),sysconfig.get_path('include','${_scheme}')]))" RESULT_VARIABLE _result OUTPUT_VARIABLE _values ERROR_QUIET @@ -350,7 +495,8 @@ function (_PYTHON_GET_CONFIG_VAR _PYTHON_PGCV_VALUE NAME) list (REMOVE_DUPLICATES _values) endif() elseif (NAME STREQUAL "SOABI") - execute_process (COMMAND "${_${_PYTHON_PREFIX}_EXECUTABLE}" -c "import sys\ntry:\n from distutils import sysconfig\n sys.stdout.write(';'.join([sysconfig.get_config_var('SOABI') or '',sysconfig.get_config_var('EXT_SUFFIX') or '']))\nexcept Exception:\n import sysconfig;sys.stdout.write(';'.join([sysconfig.get_config_var('SOABI') or '',sysconfig.get_config_var('EXT_SUFFIX') or '']))" + execute_process (COMMAND ${_${_PYTHON_PREFIX}_INTERPRETER_LAUNCHER} "${_${_PYTHON_PREFIX}_EXECUTABLE}" -c + "import sys\ntry:\n from distutils import sysconfig\n sys.stdout.write(';'.join([sysconfig.get_config_var('SOABI') or '',sysconfig.get_config_var('EXT_SUFFIX') or '']))\nexcept Exception:\n import sysconfig;sys.stdout.write(';'.join([sysconfig.get_config_var('SOABI') or '',sysconfig.get_config_var('EXT_SUFFIX') or '']))" RESULT_VARIABLE _result OUTPUT_VARIABLE _soabi ERROR_QUIET @@ -374,7 +520,8 @@ function (_PYTHON_GET_CONFIG_VAR _PYTHON_PGCV_VALUE NAME) if (NAME STREQUAL "CONFIGDIR") set (config_flag "LIBPL") endif() - execute_process (COMMAND "${_${_PYTHON_PREFIX}_EXECUTABLE}" -c "import sys\ntry:\n from distutils import sysconfig\n sys.stdout.write(sysconfig.get_config_var('${config_flag}'))\nexcept Exception:\n import sysconfig\n sys.stdout.write(sysconfig.get_config_var('${config_flag}'))" + execute_process (COMMAND ${_${_PYTHON_PREFIX}_INTERPRETER_LAUNCHER} "${_${_PYTHON_PREFIX}_EXECUTABLE}" -c + "import sys\ntry:\n from distutils import sysconfig\n sys.stdout.write(sysconfig.get_config_var('${config_flag}'))\nexcept Exception:\n import sysconfig\n sys.stdout.write(sysconfig.get_config_var('${config_flag}'))" RESULT_VARIABLE _result OUTPUT_VARIABLE _values ERROR_QUIET @@ -432,6 +579,16 @@ function (_PYTHON_GET_VERSION) set (${_PGV_PREFIX}VERSION_MINOR "${CMAKE_MATCH_2}" PARENT_SCOPE) set (${_PGV_PREFIX}VERSION "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}" PARENT_SCOPE) set (${_PGV_PREFIX}ABI "${CMAKE_MATCH_3}" PARENT_SCOPE) + elseif (_${_PYTHON_PREFIX}_LIBRARY_RELEASE MATCHES "pypy(3)?") + set (version "${CMAKE_MATCH_1}") + if (version EQUAL "3") + set (${_PGV_PREFIX}VERSION_MAJOR "3" PARENT_SCOPE) + set (${_PGV_PREFIX}VERSION "3" PARENT_SCOPE) + else() + set (${_PGV_PREFIX}VERSION_MAJOR "2" PARENT_SCOPE) + set (${_PGV_PREFIX}VERSION "2" PARENT_SCOPE) + endif() + set (${_PGV_PREFIX}ABI "" PARENT_SCOPE) endif() endif() else() @@ -446,13 +603,13 @@ function (_PYTHON_GET_VERSION) list (GET versions 1 version_minor) list (GET versions 2 version_patch) - set (${_PGV_PREFIX}VERSION "${version_major}.${version_minor}" PARENT_SCOPE) + set (${_PGV_PREFIX}VERSION "${version_major}.${version_minor}.${version_patch}" PARENT_SCOPE) set (${_PGV_PREFIX}VERSION_MAJOR ${version_major} PARENT_SCOPE) set (${_PGV_PREFIX}VERSION_MINOR ${version_minor} PARENT_SCOPE) set (${_PGV_PREFIX}VERSION_PATCH ${version_patch} PARENT_SCOPE) # compute ABI flags - if (version_major VERSION_GREATER 2) + if (version_major VERSION_GREATER "2") file (STRINGS "${_${_PYTHON_PREFIX}_INCLUDE_DIR}/pyconfig.h" config REGEX "(Py_DEBUG|WITH_PYMALLOC|Py_UNICODE_SIZE|MS_WIN32)") set (abi) if (config MATCHES "#[ ]*define[ ]+MS_WIN32") @@ -478,6 +635,36 @@ function (_PYTHON_GET_VERSION) endif() endfunction() +function (_PYTHON_GET_LAUNCHER _PYTHON_PGL_NAME) + cmake_parse_arguments (PARSE_ARGV 1 _PGL "INTERPRETER;COMPILER" "" "") + + unset ({_PYTHON_PGL_NAME} PARENT_SCOPE) + + if ((_PGL_INTERPRETER AND NOT _${_PYTHON_PREFIX}_EXECUTABLE) + OR (_PGL_COMPILER AND NOT _${_PYTHON_PREFIX}_COMPILER)) + return() + endif() + + if ("IronPython" IN_LIST _${_PYTHON_PREFIX}_FIND_IMPLEMENTATIONS + AND NOT SYSTEM_NAME MATCHES "Windows|Linux") + if (_PGL_INTERPRETER) + get_filename_component (name "${_${_PYTHON_PREFIX}_EXECUTABLE}" NAME) + get_filename_component (ext "${_${_PYTHON_PREFIX}_EXECUTABLE}" LAST_EXT) + if (name IN_LIST _${_PYTHON_PREFIX}_IRON_PYTHON_INTERPRETER_NAMES + AND ext STREQUAL ".exe") + set (${_PYTHON_PGL_NAME} "${${_PYTHON_PREFIX}_DOTNET_LAUNCHER}" PARENT_SCOPE) + endif() + else() + get_filename_component (name "${_${_PYTHON_PREFIX}_COMPILER}" NAME) + get_filename_component (ext "${_${_PYTHON_PREFIX}_COMPILER}" LAST_EXT) + if (name IN_LIST _${_PYTHON_PREFIX}_IRON_PYTHON_COMPILER_NAMES + AND ext STREQUAL ".exe") + set (${_PYTHON_PGL_NAME} "${${_PYTHON_PREFIX}_DOTNET_LAUNCHER}" PARENT_SCOPE) + endif() + endif() + endif() +endfunction() + function (_PYTHON_VALIDATE_INTERPRETER) if (NOT _${_PYTHON_PREFIX}_EXECUTABLE) @@ -486,21 +673,23 @@ function (_PYTHON_VALIDATE_INTERPRETER) cmake_parse_arguments (PARSE_ARGV 0 _PVI "EXACT;CHECK_EXISTS" "" "") if (_PVI_UNPARSED_ARGUMENTS) - set (expected_version ${_PVI_UNPARSED_ARGUMENTS}) + set (expected_version "${_PVI_UNPARSED_ARGUMENTS}") else() unset (expected_version) endif() if (_PVI_CHECK_EXISTS AND NOT EXISTS "${_${_PYTHON_PREFIX}_EXECUTABLE}") # interpreter does not exist anymore - set (_${_PYTHON_PREFIX}_Interpreter_REASON_FAILURE "Cannot find the interpreter \"${_${_PYTHON_PREFIX}_EXECUTABLE}\"") + set (_${_PYTHON_PREFIX}_Interpreter_REASON_FAILURE "Cannot find the interpreter \"${_${_PYTHON_PREFIX}_EXECUTABLE}\"" PARENT_SCOPE) set_property (CACHE _${_PYTHON_PREFIX}_EXECUTABLE PROPERTY VALUE "${_PYTHON_PREFIX}_EXECUTABLE-NOTFOUND") return() endif() + _python_get_launcher (launcher INTERPRETER) + # validate ABI compatibility if (DEFINED _${_PYTHON_PREFIX}_FIND_ABI) - execute_process (COMMAND "${_${_PYTHON_PREFIX}_EXECUTABLE}" -c + execute_process (COMMAND ${launcher} "${_${_PYTHON_PREFIX}_EXECUTABLE}" -c "import sys; sys.stdout.write(sys.abiflags)" RESULT_VARIABLE result OUTPUT_VARIABLE abi @@ -512,7 +701,7 @@ function (_PYTHON_VALIDATE_INTERPRETER) endif() if (NOT abi IN_LIST _${_PYTHON_PREFIX}_ABIFLAGS) # incompatible ABI - set (_${_PYTHON_PREFIX}_Interpreter_REASON_FAILURE "Wrong ABI for the interpreter \"${_${_PYTHON_PREFIX}_EXECUTABLE}\"") + set (_${_PYTHON_PREFIX}_Interpreter_REASON_FAILURE "Wrong ABI for the interpreter \"${_${_PYTHON_PREFIX}_EXECUTABLE}\"" PARENT_SCOPE) set_property (CACHE _${_PYTHON_PREFIX}_EXECUTABLE PROPERTY VALUE "${_PYTHON_PREFIX}_EXECUTABLE-NOTFOUND") return() endif() @@ -520,42 +709,52 @@ function (_PYTHON_VALIDATE_INTERPRETER) get_filename_component (python_name "${_${_PYTHON_PREFIX}_EXECUTABLE}" NAME) - if (expected_version AND NOT python_name STREQUAL "python${expected_version}${abi}${CMAKE_EXECUTABLE_SUFFIX}") - # executable found must have a specific version - execute_process (COMMAND "${_${_PYTHON_PREFIX}_EXECUTABLE}" -c - "import sys; sys.stdout.write('.'.join([str(x) for x in sys.version_info[:2]]))" - RESULT_VARIABLE result - OUTPUT_VARIABLE version - ERROR_QUIET - OUTPUT_STRIP_TRAILING_WHITESPACE) - if (result) - # interpreter is not usable - set (_${_PYTHON_PREFIX}_Interpreter_REASON_FAILURE "Cannot use the interpreter \"${_${_PYTHON_PREFIX}_EXECUTABLE}\"") - set_property (CACHE _${_PYTHON_PREFIX}_EXECUTABLE PROPERTY VALUE "${_PYTHON_PREFIX}_EXECUTABLE-NOTFOUND") - else() - if (_PVI_EXACT AND NOT version VERSION_EQUAL expected_version) - # interpreter has wrong version - set (_${_PYTHON_PREFIX}_Interpreter_REASON_FAILURE "Wrong version for the interpreter \"${_${_PYTHON_PREFIX}_EXECUTABLE}\"") + if (expected_version) + if (NOT python_name STREQUAL "python${expected_version}${abi}${CMAKE_EXECUTABLE_SUFFIX}") + # compute number of components for version + string (REGEX REPLACE "[^.]" "" dots "${expected_version}") + # add one dot because there is one dot less than there are components + string (LENGTH "${dots}." count) + if (count GREATER 3) + set (count 3) + endif() + + # executable found must have a specific version + execute_process (COMMAND ${launcher} "${_${_PYTHON_PREFIX}_EXECUTABLE}" -c + "import sys; sys.stdout.write('.'.join([str(x) for x in sys.version_info[:${count}]]))" + RESULT_VARIABLE result + OUTPUT_VARIABLE version + ERROR_QUIET + OUTPUT_STRIP_TRAILING_WHITESPACE) + if (result) + # interpreter is not usable + set (_${_PYTHON_PREFIX}_Interpreter_REASON_FAILURE "Cannot use the interpreter \"${_${_PYTHON_PREFIX}_EXECUTABLE}\"" PARENT_SCOPE) set_property (CACHE _${_PYTHON_PREFIX}_EXECUTABLE PROPERTY VALUE "${_PYTHON_PREFIX}_EXECUTABLE-NOTFOUND") else() - # check that version is OK - string(REGEX REPLACE "^([0-9]+)\\..*$" "\\1" major_version "${version}") - string(REGEX REPLACE "^([0-9]+)\\.?.*$" "\\1" expected_major_version "${expected_version}") - if (NOT major_version VERSION_EQUAL expected_major_version - OR NOT version VERSION_GREATER_EQUAL expected_version) - set (_${_PYTHON_PREFIX}_Interpreter_REASON_FAILURE "Wrong version for the interpreter \"${_${_PYTHON_PREFIX}_EXECUTABLE}\"") + if (_PVI_EXACT AND NOT version VERSION_EQUAL expected_version) + # interpreter has wrong version + set (_${_PYTHON_PREFIX}_Interpreter_REASON_FAILURE "Wrong version for the interpreter \"${_${_PYTHON_PREFIX}_EXECUTABLE}\"" PARENT_SCOPE) set_property (CACHE _${_PYTHON_PREFIX}_EXECUTABLE PROPERTY VALUE "${_PYTHON_PREFIX}_EXECUTABLE-NOTFOUND") + else() + # check that version is OK + string(REGEX REPLACE "^([0-9]+)\\.?.*$" "\\1" major_version "${version}") + string(REGEX REPLACE "^([0-9]+)\\.?.*$" "\\1" expected_major_version "${expected_version}") + if (NOT major_version VERSION_EQUAL expected_major_version + OR NOT version VERSION_GREATER_EQUAL expected_version) + set (_${_PYTHON_PREFIX}_Interpreter_REASON_FAILURE "Wrong version for the interpreter \"${_${_PYTHON_PREFIX}_EXECUTABLE}\"" PARENT_SCOPE) + set_property (CACHE _${_PYTHON_PREFIX}_EXECUTABLE PROPERTY VALUE "${_PYTHON_PREFIX}_EXECUTABLE-NOTFOUND") + endif() endif() endif() - endif() - if (NOT _${_PYTHON_PREFIX}_EXECUTABLE) - return() + if (NOT _${_PYTHON_PREFIX}_EXECUTABLE) + return() + endif() endif() else() if (NOT python_name STREQUAL "python${_${_PYTHON_PREFIX}_REQUIRED_VERSION_MAJOR}${CMAKE_EXECUTABLE_SUFFIX}") # executable found do not have version in name # ensure major version is OK - execute_process (COMMAND "${_${_PYTHON_PREFIX}_EXECUTABLE}" -c + execute_process (COMMAND ${launcher} "${_${_PYTHON_PREFIX}_EXECUTABLE}" -c "import sys; sys.stdout.write(str(sys.version_info[0]))" RESULT_VARIABLE result OUTPUT_VARIABLE version @@ -564,9 +763,9 @@ function (_PYTHON_VALIDATE_INTERPRETER) if (result OR NOT version EQUAL _${_PYTHON_PREFIX}_REQUIRED_VERSION_MAJOR) # interpreter not usable or has wrong major version if (result) - set (_${_PYTHON_PREFIX}_Interpreter_REASON_FAILURE "Cannot use the interpreter \"${_${_PYTHON_PREFIX}_EXECUTABLE}\"") + set (_${_PYTHON_PREFIX}_Interpreter_REASON_FAILURE "Cannot use the interpreter \"${_${_PYTHON_PREFIX}_EXECUTABLE}\"" PARENT_SCOPE) else() - set (_${_PYTHON_PREFIX}_Interpreter_REASON_FAILURE "Wrong major version for the interpreter \"${_${_PYTHON_PREFIX}_EXECUTABLE}\"") + set (_${_PYTHON_PREFIX}_Interpreter_REASON_FAILURE "Wrong major version for the interpreter \"${_${_PYTHON_PREFIX}_EXECUTABLE}\"" PARENT_SCOPE) endif() set_property (CACHE _${_PYTHON_PREFIX}_EXECUTABLE PROPERTY VALUE "${_PYTHON_PREFIX}_EXECUTABLE-NOTFOUND") return() @@ -574,10 +773,11 @@ function (_PYTHON_VALIDATE_INTERPRETER) endif() endif() - if (CMAKE_SIZEOF_VOID_P AND "Development" IN_LIST ${_PYTHON_PREFIX}_FIND_COMPONENTS + if (CMAKE_SIZEOF_VOID_P AND ("Development.Module" IN_LIST ${_PYTHON_PREFIX}_FIND_COMPONENTS + OR "Development.Embed" IN_LIST ${_PYTHON_PREFIX}_FIND_COMPONENTS) AND NOT CMAKE_CROSSCOMPILING) # In this case, interpreter must have same architecture as environment - execute_process (COMMAND "${_${_PYTHON_PREFIX}_EXECUTABLE}" -c + execute_process (COMMAND ${launcher} "${_${_PYTHON_PREFIX}_EXECUTABLE}" -c "import sys, struct; sys.stdout.write(str(struct.calcsize(\"P\")))" RESULT_VARIABLE result OUTPUT_VARIABLE size @@ -586,9 +786,9 @@ function (_PYTHON_VALIDATE_INTERPRETER) if (result OR NOT size EQUAL CMAKE_SIZEOF_VOID_P) # interpreter not usable or has wrong architecture if (result) - set (_${_PYTHON_PREFIX}_Interpreter_REASON_FAILURE "Cannot use the interpreter \"${_${_PYTHON_PREFIX}_EXECUTABLE}\"") + set (_${_PYTHON_PREFIX}_Interpreter_REASON_FAILURE "Cannot use the interpreter \"${_${_PYTHON_PREFIX}_EXECUTABLE}\"" PARENT_SCOPE) else() - set (_${_PYTHON_PREFIX}_Interpreter_REASON_FAILURE "Wrong architecture for the interpreter \"${_${_PYTHON_PREFIX}_EXECUTABLE}\"") + set (_${_PYTHON_PREFIX}_Interpreter_REASON_FAILURE "Wrong architecture for the interpreter \"${_${_PYTHON_PREFIX}_EXECUTABLE}\"" PARENT_SCOPE) endif() set_property (CACHE _${_PYTHON_PREFIX}_EXECUTABLE PROPERTY VALUE "${_PYTHON_PREFIX}_EXECUTABLE-NOTFOUND") return() @@ -597,55 +797,68 @@ function (_PYTHON_VALIDATE_INTERPRETER) endfunction() -function (_PYTHON_VALIDATE_COMPILER expected_version) +function (_PYTHON_VALIDATE_COMPILER) if (NOT _${_PYTHON_PREFIX}_COMPILER) return() endif() - cmake_parse_arguments (_PVC "EXACT;CHECK_EXISTS" "" "" ${ARGN}) + cmake_parse_arguments (PARSE_ARGV 0 _PVC "EXACT;CHECK_EXISTS" "" "") if (_PVC_UNPARSED_ARGUMENTS) set (major_version FALSE) - set (expected_version ${_PVC_UNPARSED_ARGUMENTS}) + set (expected_version "${_PVC_UNPARSED_ARGUMENTS}") else() set (major_version TRUE) - set (expected_version ${_${_PYTHON_PREFIX}_REQUIRED_VERSION_MAJOR}) + set (expected_version "${_${_PYTHON_PREFIX}_REQUIRED_VERSION_MAJOR}") set (_PVC_EXACT TRUE) endif() if (_PVC_CHECK_EXISTS AND NOT EXISTS "${_${_PYTHON_PREFIX}_COMPILER}") # Compiler does not exist anymore - set (_${_PYTHON_PREFIX}_Compiler_REASON_FAILURE "Cannot find the compiler \"${_${_PYTHON_PREFIX}_COMPILER}\"") + set (_${_PYTHON_PREFIX}_Compiler_REASON_FAILURE "Cannot find the compiler \"${_${_PYTHON_PREFIX}_COMPILER}\"" PARENT_SCOPE) set_property (CACHE _${_PYTHON_PREFIX}_COMPILER PROPERTY VALUE "${_PYTHON_PREFIX}_COMPILER-NOTFOUND") return() endif() + _python_get_launcher (launcher COMPILER) + # retrieve python environment version from compiler set (working_dir "${CMAKE_CURRENT_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/PythonCompilerVersion.dir") if (major_version) # check only major version file (WRITE "${working_dir}/version.py" "import sys; sys.stdout.write(str(sys.version_info[0]))") else() - file (WRITE "${working_dir}/version.py" "import sys; sys.stdout.write('.'.join([str(x) for x in sys.version_info[:2]]))\n") + # compute number of components for version + string (REGEX REPLACE "[^.]" "" dots "${expected_version}") + # add one dot because there is one dot less than there are components + string (LENGTH "${dots}." count) + if (count GREATER 3) + set (count 3) + endif() + file (WRITE "${working_dir}/version.py" "import sys; sys.stdout.write('.'.join([str(x) for x in sys.version_info[:${count}]]))\n") endif() - execute_process (COMMAND "${_${_PYTHON_PREFIX}_COMPILER}" /target:exe /embed "${working_dir}/version.py" + execute_process (COMMAND ${launcher} "${_${_PYTHON_PREFIX}_COMPILER}" + ${_${_PYTHON_PREFIX}_IRON_PYTHON_COMPILER_ARCH_FLAGS} + /target:exe /embed "${working_dir}/version.py" WORKING_DIRECTORY "${working_dir}" OUTPUT_QUIET ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE) - execute_process (COMMAND "${working_dir}/version" + get_filename_component (ir_dir "${_${_PYTHON_PREFIX}_COMPILER}" DIRECTORY) + execute_process (COMMAND "${CMAKE_COMMAND}" -E env "MONO_PATH=${ir_dir}" + ${${_PYTHON_PREFIX}_DOTNET_LAUNCHER} "${working_dir}/version.exe" WORKING_DIRECTORY "${working_dir}" RESULT_VARIABLE result OUTPUT_VARIABLE version ERROR_QUIET) - file (REMOVE_RECURSE "${_${_PYTHON_PREFIX}_VERSION_DIR}") - - if (result OR (_PVC_EXACT AND NOT version VERSION_EQUAL expected_version) OR (version VERSION_LESS expected_version)) - # Compiler not usable or has wrong version - if (result) - set (_${_PYTHON_PREFIX}_Compiler_REASON_FAILURE "Cannot use the compiler \"${_${_PYTHON_PREFIX}_COMPILER}\"") - else() - set (_${_PYTHON_PREFIX}_Compiler_REASON_FAILURE "Wrong version for the compiler \"${_${_PYTHON_PREFIX}_COMPILER}\"") - endif() + file (REMOVE_RECURSE "${working_dir}") + if (result) + # compiler is not usable + set (_${_PYTHON_PREFIX}_Compiler_REASON_FAILURE "Cannot use the compiler \"${_${_PYTHON_PREFIX}_COMPILER}\"" PARENT_SCOPE) + set_property (CACHE _${_PYTHON_PREFIX}_COMPILER PROPERTY VALUE "${_PYTHON_PREFIX}_COMPILER-NOTFOUND") + elseif ((_PVC_EXACT AND NOT version VERSION_EQUAL expected_version) + OR NOT version VERSION_GREATER_EQUAL expected_version) + # Compiler has wrong version + set (_${_PYTHON_PREFIX}_Compiler_REASON_FAILURE "Wrong version for the compiler \"${_${_PYTHON_PREFIX}_COMPILER}\"" PARENT_SCOPE) set_property (CACHE _${_PYTHON_PREFIX}_COMPILER PROPERTY VALUE "${_PYTHON_PREFIX}_COMPILER-NOTFOUND") endif() endfunction() @@ -653,6 +866,7 @@ endfunction() function (_PYTHON_VALIDATE_LIBRARY) if (NOT _${_PYTHON_PREFIX}_LIBRARY_RELEASE) + unset (_${_PYTHON_PREFIX}_LIBRARY_DEBUG) return() endif() @@ -665,7 +879,7 @@ function (_PYTHON_VALIDATE_LIBRARY) if (_PVL_CHECK_EXISTS AND NOT EXISTS "${_${_PYTHON_PREFIX}_LIBRARY_RELEASE}") # library does not exist anymore - set (_${_PYTHON_PREFIX}_Development_REASON_FAILURE "Cannot find the library \"${_${_PYTHON_PREFIX}_LIBRARY_RELEASE}\"") + set (_${_PYTHON_PREFIX}_Development_REASON_FAILURE "Cannot find the library \"${_${_PYTHON_PREFIX}_LIBRARY_RELEASE}\"" PARENT_SCOPE) set_property (CACHE _${_PYTHON_PREFIX}_LIBRARY_RELEASE PROPERTY VALUE "${_PYTHON_PREFIX}_LIBRARY_RELEASE-NOTFOUND") if (WIN32) set_property (CACHE _${_PYTHON_PREFIX}_LIBRARY_DEBUG PROPERTY VALUE "${_PYTHON_PREFIX}_LIBRARY_DEBUG-NOTFOUND") @@ -679,19 +893,21 @@ function (_PYTHON_VALIDATE_LIBRARY) if (DEFINED _${_PYTHON_PREFIX}_FIND_ABI AND NOT lib_ABI IN_LIST _${_PYTHON_PREFIX}_ABIFLAGS) # incompatible ABI - set (_${_PYTHON_PREFIX}_Development_REASON_FAILURE "Wrong ABI for the library \"${_${_PYTHON_PREFIX}_LIBRARY_RELEASE}\"") + set (_${_PYTHON_PREFIX}_Development_REASON_FAILURE "Wrong ABI for the library \"${_${_PYTHON_PREFIX}_LIBRARY_RELEASE}\"" PARENT_SCOPE) set_property (CACHE _${_PYTHON_PREFIX}_LIBRARY_RELEASE PROPERTY VALUE "${_PYTHON_PREFIX}_LIBRARY_RELEASE-NOTFOUND") else() if (expected_version) - if ((_PVL_EXACT AND NOT lib_VERSION VERSION_EQUAL expected_version) OR (lib_VERSION VERSION_LESS expected_version)) + # library have only major.minor information + string (REGEX MATCH "[0-9](\\.[0-9]+)?" version "${expected_version}") + if ((_PVL_EXACT AND NOT lib_VERSION VERSION_EQUAL version) OR (lib_VERSION VERSION_LESS version)) # library has wrong version - set (_${_PYTHON_PREFIX}_Development_REASON_FAILURE "Wrong version for the library \"${_${_PYTHON_PREFIX}_LIBRARY_RELEASE}\"") + set (_${_PYTHON_PREFIX}_Development_REASON_FAILURE "Wrong version for the library \"${_${_PYTHON_PREFIX}_LIBRARY_RELEASE}\"" PARENT_SCOPE) set_property (CACHE _${_PYTHON_PREFIX}_LIBRARY_RELEASE PROPERTY VALUE "${_PYTHON_PREFIX}_LIBRARY_RELEASE-NOTFOUND") endif() else() if (NOT lib_VERSION_MAJOR VERSION_EQUAL _${_PYTHON_PREFIX}_REQUIRED_VERSION_MAJOR) # library has wrong major version - set (_${_PYTHON_PREFIX}_Development_REASON_FAILURE "Wrong major version for the library \"${_${_PYTHON_PREFIX}_LIBRARY_RELEASE}\"") + set (_${_PYTHON_PREFIX}_Development_REASON_FAILURE "Wrong major version for the library \"${_${_PYTHON_PREFIX}_LIBRARY_RELEASE}\"" PARENT_SCOPE) set_property (CACHE _${_PYTHON_PREFIX}_LIBRARY_RELEASE PROPERTY VALUE "${_PYTHON_PREFIX}_LIBRARY_RELEASE-NOTFOUND") endif() endif() @@ -722,7 +938,7 @@ function (_PYTHON_VALIDATE_INCLUDE_DIR) if (_PVID_CHECK_EXISTS AND NOT EXISTS "${_${_PYTHON_PREFIX}_INCLUDE_DIR}") # include file does not exist anymore - set (_${_PYTHON_PREFIX}_Development_REASON_FAILURE "Cannot find the directory \"${_${_PYTHON_PREFIX}_INCLUDE_DIR}\"") + set (_${_PYTHON_PREFIX}_Development_REASON_FAILURE "Cannot find the directory \"${_${_PYTHON_PREFIX}_INCLUDE_DIR}\"" PARENT_SCOPE) set_property (CACHE _${_PYTHON_PREFIX}_INCLUDE_DIR PROPERTY VALUE "${_PYTHON_PREFIX}_INCLUDE_DIR-NOTFOUND") return() endif() @@ -732,19 +948,19 @@ function (_PYTHON_VALIDATE_INCLUDE_DIR) if (DEFINED _${_PYTHON_PREFIX}_FIND_ABI AND NOT inc_ABI IN_LIST _${_PYTHON_PREFIX}_ABIFLAGS) # incompatible ABI - set (_${_PYTHON_PREFIX}_Development_REASON_FAILURE "Wrong ABI for the directory \"${_${_PYTHON_PREFIX}_INCLUDE_DIR}\"") + set (_${_PYTHON_PREFIX}_Development_REASON_FAILURE "Wrong ABI for the directory \"${_${_PYTHON_PREFIX}_INCLUDE_DIR}\"" PARENT_SCOPE) set_property (CACHE _${_PYTHON_PREFIX}_INCLUDE_DIR PROPERTY VALUE "${_PYTHON_PREFIX}_INCLUDE_DIR-NOTFOUND") else() if (expected_version) if ((_PVID_EXACT AND NOT inc_VERSION VERSION_EQUAL expected_version) OR (inc_VERSION VERSION_LESS expected_version)) # include dir has wrong version - set (_${_PYTHON_PREFIX}_Development_REASON_FAILURE "Wrong version for the directory \"${_${_PYTHON_PREFIX}_INCLUDE_DIR}\"") + set (_${_PYTHON_PREFIX}_Development_REASON_FAILURE "Wrong version for the directory \"${_${_PYTHON_PREFIX}_INCLUDE_DIR}\"" PARENT_SCOPE) set_property (CACHE _${_PYTHON_PREFIX}_INCLUDE_DIR PROPERTY VALUE "${_PYTHON_PREFIX}_INCLUDE_DIR-NOTFOUND") endif() else() if (NOT inc_VERSION_MAJOR VERSION_EQUAL _${_PYTHON_PREFIX}_REQUIRED_VERSION_MAJOR) # include dir has wrong major version - set (_${_PYTHON_PREFIX}_Development_REASON_FAILURE "Wrong major version for the directory \"${_${_PYTHON_PREFIX}_INCLUDE_DIR}\"") + set (_${_PYTHON_PREFIX}_Development_REASON_FAILURE "Wrong major version for the directory \"${_${_PYTHON_PREFIX}_INCLUDE_DIR}\"" PARENT_SCOPE) set_property (CACHE _${_PYTHON_PREFIX}_INCLUDE_DIR PROPERTY VALUE "${_PYTHON_PREFIX}_INCLUDE_DIR-NOTFOUND") endif() endif() @@ -777,13 +993,30 @@ function (_PYTHON_SET_LIBRARY_DIRS _PYTHON_SLD_RESULT) list (APPEND _PYTHON_DIRS "${_PYTHON_DIR}") endif() endforeach() - if (_PYTHON_DIRS) - list (REMOVE_DUPLICATES _PYTHON_DIRS) - endif() + list (REMOVE_DUPLICATES _PYTHON_DIRS) set (${_PYTHON_SLD_RESULT} ${_PYTHON_DIRS} PARENT_SCOPE) endfunction() +function (_PYTHON_SET_DEVELOPMENT_MODULE_FOUND module) + if ("Development.${module}" IN_LIST ${_PYTHON_PREFIX}_FIND_COMPONENTS) + string(TOUPPER "${module}" id) + set (module_found TRUE) + + if ("LIBRARY" IN_LIST _${_PYTHON_PREFIX}_FIND_DEVELOPMENT_${id}_ARTIFACTS + AND NOT _${_PYTHON_PREFIX}_LIBRARY_RELEASE) + set (module_found FALSE) + endif() + if ("INCLUDE_DIR" IN_LIST _${_PYTHON_PREFIX}_FIND_DEVELOPMENT_${id}_ARTIFACTS + AND NOT _${_PYTHON_PREFIX}_INCLUDE_DIR) + set (module_found FALSE) + endif() + + set (${_PYTHON_PREFIX}_Development.${module}_FOUND ${module_found} PARENT_SCOPE) + endif() +endfunction() + + # If major version is specified, it must be the same as internal major version if (DEFINED ${_PYTHON_PREFIX}_FIND_VERSION_MAJOR AND NOT ${_PYTHON_PREFIX}_FIND_VERSION_MAJOR VERSION_EQUAL _${_PYTHON_PREFIX}_REQUIRED_VERSION_MAJOR) @@ -797,26 +1030,49 @@ if (NOT ${_PYTHON_PREFIX}_FIND_COMPONENTS) set (${_PYTHON_PREFIX}_FIND_REQUIRED_Interpreter TRUE) endif() if ("NumPy" IN_LIST ${_PYTHON_PREFIX}_FIND_COMPONENTS) - list (APPEND ${_PYTHON_PREFIX}_FIND_COMPONENTS "Interpreter" "Development") - list (REMOVE_DUPLICATES ${_PYTHON_PREFIX}_FIND_COMPONENTS) + list (APPEND ${_PYTHON_PREFIX}_FIND_COMPONENTS "Interpreter" "Development.Module") endif() -foreach (_${_PYTHON_PREFIX}_COMPONENT IN ITEMS Interpreter Compiler Development NumPy) +if ("Development" IN_LIST ${_PYTHON_PREFIX}_FIND_COMPONENTS) + list (APPEND ${_PYTHON_PREFIX}_FIND_COMPONENTS "Development.Module" "Development.Embed") +endif() +list (REMOVE_DUPLICATES ${_PYTHON_PREFIX}_FIND_COMPONENTS) +foreach (_${_PYTHON_PREFIX}_COMPONENT IN ITEMS Interpreter Compiler Development Development.Module Development.Embed NumPy) set (${_PYTHON_PREFIX}_${_${_PYTHON_PREFIX}_COMPONENT}_FOUND FALSE) endforeach() -unset (_${_PYTHON_PREFIX}_FIND_VERSIONS) +if (${_PYTHON_PREFIX}_FIND_REQUIRED_Development) + set (${_PYTHON_PREFIX}_FIND_REQUIRED_Development.Module TRUE) + set (${_PYTHON_PREFIX}_FIND_REQUIRED_Development.Embed TRUE) +endif() + +unset (_${_PYTHON_PREFIX}_FIND_DEVELOPMENT_ARTIFACTS) +unset (_${_PYTHON_PREFIX}_FIND_DEVELOPMENT_MODULE_ARTIFACTS) +unset (_${_PYTHON_PREFIX}_FIND_DEVELOPMENT_EMBED_ARTIFACTS) +if ("Development.Module" IN_LIST ${_PYTHON_PREFIX}_FIND_COMPONENTS) + if (CMAKE_SYSTEM_NAME MATCHES "^(Windows.*|CYGWIN|MSYS)$") + list (APPEND _${_PYTHON_PREFIX}_FIND_DEVELOPMENT_MODULE_ARTIFACTS "LIBRARY") + endif() + list (APPEND _${_PYTHON_PREFIX}_FIND_DEVELOPMENT_MODULE_ARTIFACTS "INCLUDE_DIR") +endif() +if ("Development.Embed" IN_LIST ${_PYTHON_PREFIX}_FIND_COMPONENTS) + list (APPEND _${_PYTHON_PREFIX}_FIND_DEVELOPMENT_EMBED_ARTIFACTS "LIBRARY" "INCLUDE_DIR") +endif() +set (_${_PYTHON_PREFIX}_FIND_DEVELOPMENT_ARTIFACTS ${_${_PYTHON_PREFIX}_FIND_DEVELOPMENT_MODULE_ARTIFACTS} ${_${_PYTHON_PREFIX}_FIND_DEVELOPMENT_EMBED_ARTIFACTS}) +list (REMOVE_DUPLICATES _${_PYTHON_PREFIX}_FIND_DEVELOPMENT_ARTIFACTS) # Set versions to search ## default: search any version set (_${_PYTHON_PREFIX}_FIND_VERSIONS ${_${_PYTHON_PREFIX}_VERSIONS}) +unset (_${_PYTHON_PREFIX}_FIND_VERSION_EXACT) -if (${_PYTHON_PREFIX}_FIND_VERSION_COUNT GREATER 1) +if (${_PYTHON_PREFIX}_FIND_VERSION_COUNT) if (${_PYTHON_PREFIX}_FIND_VERSION_EXACT) + set (_${_PYTHON_PREFIX}_FIND_VERSION_EXACT "EXACT") set (_${_PYTHON_PREFIX}_FIND_VERSIONS ${${_PYTHON_PREFIX}_FIND_VERSION_MAJOR}.${${_PYTHON_PREFIX}_FIND_VERSION_MINOR}) else() unset (_${_PYTHON_PREFIX}_FIND_VERSIONS) # add all compatible versions foreach (_${_PYTHON_PREFIX}_VERSION IN LISTS _${_PYTHON_PREFIX}_VERSIONS) - if (_${_PYTHON_PREFIX}_VERSION VERSION_GREATER_EQUAL ${_PYTHON_PREFIX}_FIND_VERSION) + if (_${_PYTHON_PREFIX}_VERSION VERSION_GREATER_EQUAL "${${_PYTHON_PREFIX}_FIND_VERSION_MAJOR}.${${_PYTHON_PREFIX}_FIND_VERSION_MINOR}") list (APPEND _${_PYTHON_PREFIX}_FIND_VERSIONS ${_${_PYTHON_PREFIX}_VERSION}) endif() endforeach() @@ -825,7 +1081,7 @@ endif() # Set ABIs to search ## default: search any ABI -if (_${_PYTHON_PREFIX}_REQUIRED_VERSION_MAJOR VERSION_LESS 3) +if (_${_PYTHON_PREFIX}_REQUIRED_VERSION_MAJOR VERSION_LESS "3") # ABI not supported unset (_${_PYTHON_PREFIX}_FIND_ABI) set (_${_PYTHON_PREFIX}_ABIFLAGS "") @@ -873,15 +1129,75 @@ else() endif() # IronPython support +unset (_${_PYTHON_PREFIX}_IRON_PYTHON_INTERPRETER_NAMES) +unset (_${_PYTHON_PREFIX}_IRON_PYTHON_COMPILER_NAMES) +unset (_${_PYTHON_PREFIX}_IRON_PYTHON_COMPILER_ARCH_FLAGS) if (CMAKE_SIZEOF_VOID_P) - # In this case, search only for 64bit or 32bit - math (EXPR _${_PYTHON_PREFIX}_ARCH "${CMAKE_SIZEOF_VOID_P} * 8") - set (_${_PYTHON_PREFIX}_IRON_PYTHON_NAMES ipy${_${_PYTHON_PREFIX}_ARCH} ipy) + if (_${_PYTHON_PREFIX}_ARCH EQUAL "32") + set (_${_PYTHON_PREFIX}_IRON_PYTHON_COMPILER_ARCH_FLAGS "/platform:x86") + else() + set (_${_PYTHON_PREFIX}_IRON_PYTHON_COMPILER_ARCH_FLAGS "/platform:x64") + endif() +endif() +if (NOT CMAKE_SYSTEM_NAME STREQUAL "Linux") + # Do not use wrapper script on Linux because it is buggy: -c interpreter option cannot be used + list (APPEND _${_PYTHON_PREFIX}_IRON_PYTHON_INTERPRETER_NAMES "ipy${_${_PYTHON_PREFIX}_REQUIRED_VERSION_MAJOR}" "ipy64" "ipy32" "ipy") + list (APPEND _${_PYTHON_PREFIX}_IRON_PYTHON_COMPILER_NAMES "ipyc") +endif() +list (APPEND _${_PYTHON_PREFIX}_IRON_PYTHON_INTERPRETER_NAMES "ipy.exe") +list (APPEND _${_PYTHON_PREFIX}_IRON_PYTHON_COMPILER_NAMES "ipyc.exe") +set (_${_PYTHON_PREFIX}_IRON_PYTHON_PATH_SUFFIXES net45 net40 bin) + +# PyPy support +if (_${_PYTHON_PREFIX}_REQUIRED_VERSION_MAJOR EQUAL "3") + set (_${_PYTHON_PREFIX}_PYPY_NAMES pypy3) + set (_${_PYTHON_PREFIX}_PYPY_LIB_NAMES pypy3-c) + if (WIN32) + # special name for runtime part + list (APPEND _${_PYTHON_PREFIX}_PYPY_LIB_NAMES libpypy3-c) + endif() + set (_${_PYTHON_PREFIX}_PYPY_INCLUDE_PATH_SUFFIXES lib/pypy3) +else() + set (_${_PYTHON_PREFIX}_PYPY_NAMES pypy) + set (_${_PYTHON_PREFIX}_PYPY_LIB_NAMES pypy-c) + if (WIN32) + # special name for runtime part + list (APPEND _${_PYTHON_PREFIX}_PYPY_LIB_NAMES libpypy-c) + endif() + set (_${_PYTHON_PREFIX}_PYPY_INCLUDE_PATH_SUFFIXES lib/pypy) +endif() +set (_${_PYTHON_PREFIX}_PYPY_EXECUTABLE_PATH_SUFFIXES bin) +set (_${_PYTHON_PREFIX}_PYPY_LIBRARY_PATH_SUFFIXES lib libs bin) +list (APPEND _${_PYTHON_PREFIX}_PYPY_INCLUDE_PATH_SUFFIXES include) + +# Python Implementations handling +unset (_${_PYTHON_PREFIX}_FIND_IMPLEMENTATIONS) +if (DEFINED ${_PYTHON_PREFIX}_FIND_IMPLEMENTATIONS) + foreach (_${_PYTHON_PREFIX}_IMPLEMENTATION IN LISTS ${_PYTHON_PREFIX}_FIND_IMPLEMENTATIONS) + if (NOT _${_PYTHON_PREFIX}_IMPLEMENTATION MATCHES "^(CPython|IronPython|PyPy)$") + message (AUTHOR_WARNING "Find${_PYTHON_PREFIX}: ${_${_PYTHON_PREFIX}_IMPLEMENTATION}: invalid value for '${_PYTHON_PREFIX}_FIND_IMPLEMENTATIONS'. 'CPython', 'IronPython' or 'PyPy' expected. Value will be ignored.") + else() + list (APPEND _${_PYTHON_PREFIX}_FIND_IMPLEMENTATIONS ${_${_PYTHON_PREFIX}_IMPLEMENTATION}) + endif() + endforeach() else() - # architecture unknown, search for natural interpreter - set (_${_PYTHON_PREFIX}_IRON_PYTHON_NAMES ipy) + if (WIN32) + set (_${_PYTHON_PREFIX}_FIND_IMPLEMENTATIONS CPython IronPython) + else() + set (_${_PYTHON_PREFIX}_FIND_IMPLEMENTATIONS CPython) + endif() endif() -set (_${_PYTHON_PREFIX}_IRON_PYTHON_PATH_SUFFIXES net45 net40) + +# compute list of names for header file +unset (_${_PYTHON_PREFIX}_INCLUDE_NAMES) +foreach (_${_PYTHON_PREFIX}_IMPLEMENTATION IN LISTS _${_PYTHON_PREFIX}_FIND_IMPLEMENTATIONS) + if (_${_PYTHON_PREFIX}_IMPLEMENTATION STREQUAL "CPython") + list (APPEND _${_PYTHON_PREFIX}_INCLUDE_NAMES "Python.h") + elseif (_${_PYTHON_PREFIX}_IMPLEMENTATION STREQUAL "PyPy") + list (APPEND _${_PYTHON_PREFIX}_INCLUDE_NAMES "PyPy.h") + endif() +endforeach() + # Apple frameworks handling _python_find_frameworks () @@ -953,7 +1269,7 @@ endif() # Compute search signature # This signature will be used to check validity of cached variables on new search -set (_${_PYTHON_PREFIX}_SIGNATURE "${${_PYTHON_PREFIX}_ROOT_DIR}:${_${_PYTHON_PREFIX}_FIND_STRATEGY}:${${_PYTHON_PREFIX}_FIND_VIRTUALENV}") +set (_${_PYTHON_PREFIX}_SIGNATURE "${${_PYTHON_PREFIX}_ROOT_DIR}:${_${_PYTHON_PREFIX}_FIND_IMPLEMENTATIONS}:${_${_PYTHON_PREFIX}_FIND_STRATEGY}:${${_PYTHON_PREFIX}_FIND_VIRTUALENV}") if (NOT WIN32) string (APPEND _${_PYTHON_PREFIX}_SIGNATURE ":${${_PYTHON_PREFIX}_USE_STATIC_LIBS}:") endif() @@ -964,6 +1280,64 @@ if (CMAKE_HOST_WIN32) string (APPEND _${_PYTHON_PREFIX}_SIGNATURE ":${_${_PYTHON_PREFIX}_FIND_REGISTRY}") endif() +function (_PYTHON_CHECK_DEVELOPMENT_SIGNATURE module) + if ("Development.${module}" IN_LIST ${_PYTHON_PREFIX}_FIND_COMPONENTS) + string (TOUPPER "${module}" id) + set (signature "${_${_PYTHON_PREFIX}_SIGNATURE}:") + if ("LIBRARY" IN_LIST _${_PYTHON_PREFIX}_FIND_DEVELOPMENT_${id}_ARTIFACTS) + list (APPEND signature "${_${_PYTHON_PREFIX}_LIBRARY_RELEASE}:") + endif() + if ("INCLUDE_DIR" IN_LIST _${_PYTHON_PREFIX}_FIND_DEVELOPMENT_${id}_ARTIFACTS) + list (APPEND signature "${_${_PYTHON_PREFIX}_INCLUDE_DIR}:") + endif() + string (MD5 signature "${signature}") + if (signature STREQUAL _${_PYTHON_PREFIX}_DEVELOPMENT_${id}_SIGNATURE) + if ("LIBRARY" IN_LIST _${_PYTHON_PREFIX}_FIND_DEVELOPMENT_${id}_ARTIFACTS) + _python_validate_library (${${_PYTHON_PREFIX}_FIND_VERSION} + ${_${_PYTHON_PREFIX}_FIND_VERSION_EXACT} + CHECK_EXISTS) + endif() + if ("INCLUDE_DIR" IN_LIST _${_PYTHON_PREFIX}_FIND_DEVELOPMENT_${id}_ARTIFACTS) + _python_validate_include_dir (${${_PYTHON_PREFIX}_FIND_VERSION} + ${_${_PYTHON_PREFIX}_FIND_VERSION_EXACT} + CHECK_EXISTS) + endif() + else() + if ("LIBRARY" IN_LIST _${_PYTHON_PREFIX}_FIND_DEVELOPMENT_${id}_ARTIFACTS) + unset (_${_PYTHON_PREFIX}_LIBRARY_RELEASE CACHE) + unset (_${_PYTHON_PREFIX}_LIBRARY_DEBUG CACHE) + endif() + if ("INCLUDE_DIR" IN_LIST _${_PYTHON_PREFIX}_FIND_DEVELOPMENT_${id}_ARTIFACTS) + unset (_${_PYTHON_PREFIX}_INCLUDE_DIR CACHE) + endif() + endif() + if (("LIBRARY" IN_LIST _${_PYTHON_PREFIX}_FIND_DEVELOPMENT_${id}_ARTIFACTS + AND NOT _${_PYTHON_PREFIX}_LIBRARY_RELEASE) + OR ("INCLUDE_DIR" IN_LIST _${_PYTHON_PREFIX}_FIND_DEVELOPMENT_${id}_ARTIFACTS + AND NOT _${_PYTHON_PREFIX}_INCLUDE_DIR)) + unset (_${_PYTHON_PREFIX}_CONFIG CACHE) + unset (_${_PYTHON_PREFIX}_DEVELOPMENT_${id}_SIGNATURE CACHE) + endif() + endif() +endfunction() + +function (_PYTHON_COMPUTE_DEVELOPMENT_SIGNATURE module) + string (TOUPPER "${module}" id) + if (${_PYTHON_PREFIX}_Development.${module}_FOUND) + set (signature "${_${_PYTHON_PREFIX}_SIGNATURE}:") + if ("LIBRARY" IN_LIST _${_PYTHON_PREFIX}_FIND_DEVELOPMENT_${id}_ARTIFACTS) + list (APPEND signature "${_${_PYTHON_PREFIX}_LIBRARY_RELEASE}:") + endif() + if ("INCLUDE_DIR" IN_LIST _${_PYTHON_PREFIX}_FIND_DEVELOPMENT_${id}_ARTIFACTS) + list (APPEND signature "${_${_PYTHON_PREFIX}_INCLUDE_DIR}:") + endif() + string (MD5 signature "${signature}") + set (_${_PYTHON_PREFIX}_DEVELOPMENT_${id}_SIGNATURE "${signature}" CACHE INTERNAL "") + else() + unset (_${_PYTHON_PREFIX}_DEVELOPMENT_${id}_SIGNATURE CACHE) + endif() +endfunction() + unset (_${_PYTHON_PREFIX}_REQUIRED_VARS) unset (_${_PYTHON_PREFIX}_CACHED_VARS) @@ -973,9 +1347,19 @@ unset (_${_PYTHON_PREFIX}_Development_REASON_FAILURE) unset (_${_PYTHON_PREFIX}_NumPy_REASON_FAILURE) +# preamble +## For IronPython on platforms other than Windows, search for the .Net interpreter +if ("IronPython" IN_LIST _${_PYTHON_PREFIX}_FIND_IMPLEMENTATIONS + AND NOT WIN32) + find_program (${_PYTHON_PREFIX}_DOTNET_LAUNCHER + NAMES "mono") +endif() + + # first step, search for the interpreter if ("Interpreter" IN_LIST ${_PYTHON_PREFIX}_FIND_COMPONENTS) - list (APPEND _${_PYTHON_PREFIX}_CACHED_VARS _${_PYTHON_PREFIX}_EXECUTABLE) + list (APPEND _${_PYTHON_PREFIX}_CACHED_VARS _${_PYTHON_PREFIX}_EXECUTABLE + _${_PYTHON_PREFIX}_INTERPRETER_PROPERTIES) if (${_PYTHON_PREFIX}_FIND_REQUIRED_Interpreter) list (APPEND _${_PYTHON_PREFIX}_REQUIRED_VARS ${_PYTHON_PREFIX}_EXECUTABLE) endif() @@ -1010,25 +1394,14 @@ if ("Interpreter" IN_LIST ${_PYTHON_PREFIX}_FIND_COMPONENTS) set (_${_PYTHON_PREFIX}_HINTS "${${_PYTHON_PREFIX}_ROOT_DIR}" ENV ${_PYTHON_PREFIX}_ROOT_DIR) if (_${_PYTHON_PREFIX}_FIND_STRATEGY STREQUAL "LOCATION") - unset (_${_PYTHON_PREFIX}_NAMES) - unset (_${_PYTHON_PREFIX}_FRAMEWORK_PATHS) - unset (_${_PYTHON_PREFIX}_REGISTRY_PATHS) + # build all executable names + _python_get_names (_${_PYTHON_PREFIX}_NAMES VERSION ${_${_PYTHON_PREFIX}_FIND_VERSIONS} POSIX INTERPRETER) + _python_get_path_suffixes (_${_PYTHON_PREFIX}_PATH_SUFFIXES VERSION ${_${_PYTHON_PREFIX}_FIND_VERSIONS} INTERPRETER) - foreach (_${_PYTHON_PREFIX}_VERSION IN LISTS _${_PYTHON_PREFIX}_FIND_VERSIONS) - # build all executable names - _python_get_names (_${_PYTHON_PREFIX}_VERSION_NAMES VERSION ${_${_PYTHON_PREFIX}_VERSION} POSIX EXECUTABLE) - list (APPEND _${_PYTHON_PREFIX}_NAMES ${_${_PYTHON_PREFIX}_VERSION_NAMES}) - - # Framework Paths - _python_get_frameworks (_${_PYTHON_PREFIX}_VERSION_PATHS ${_${_PYTHON_PREFIX}_VERSION}) - list (APPEND _${_PYTHON_PREFIX}_FRAMEWORK_PATHS ${_${_PYTHON_PREFIX}_VERSION_PATHS}) - - # Registry Paths - _python_get_registries (_${_PYTHON_PREFIX}_VERSION_PATHS ${_${_PYTHON_PREFIX}_VERSION}) - list (APPEND _${_PYTHON_PREFIX}_REGISTRY_PATHS ${_${_PYTHON_PREFIX}_VERSION_PATHS} - [HKEY_LOCAL_MACHINE\\SOFTWARE\\IronPython\\${_${_PYTHON_PREFIX}_VERSION}\\InstallPath]) - endforeach() - list (APPEND _${_PYTHON_PREFIX}_NAMES python${_${_PYTHON_PREFIX}_REQUIRED_VERSION_MAJOR} python) + # Framework Paths + _python_get_frameworks (_${_PYTHON_PREFIX}_FRAMEWORK_PATHS VERSION ${_${_PYTHON_PREFIX}_FIND_VERSIONS}) + # Registry Paths + _python_get_registries (_${_PYTHON_PREFIX}_REGISTRY_PATHS VERSION ${_${_PYTHON_PREFIX}_FIND_VERSIONS}) while (TRUE) # Virtual environments handling @@ -1038,17 +1411,17 @@ if ("Interpreter" IN_LIST ${_PYTHON_PREFIX}_FIND_COMPONENTS) NAMES_PER_DIR HINTS ${_${_PYTHON_PREFIX}_HINTS} PATHS ENV VIRTUAL_ENV ENV CONDA_PREFIX - PATH_SUFFIXES bin Scripts + PATH_SUFFIXES ${_${_PYTHON_PREFIX}_PATH_SUFFIXES} NO_CMAKE_PATH NO_CMAKE_ENVIRONMENT_PATH NO_SYSTEM_ENVIRONMENT_PATH NO_CMAKE_SYSTEM_PATH) - _python_validate_interpreter (${${_PYTHON_PREFIX}_FIND_VERSION}) + _python_validate_interpreter (${${_PYTHON_PREFIX}_FIND_VERSION} ${_${_PYTHON_PREFIX}_FIND_VERSION_EXACT}) if (_${_PYTHON_PREFIX}_EXECUTABLE) break() endif() - if (NOT _${_PYTHON_PREFIX}_FIND_VIRTUALENV STREQUAL "ONLY") + if (_${_PYTHON_PREFIX}_FIND_VIRTUALENV STREQUAL "ONLY") break() endif() endif() @@ -1060,12 +1433,12 @@ if ("Interpreter" IN_LIST ${_PYTHON_PREFIX}_FIND_COMPONENTS) NAMES_PER_DIR HINTS ${_${_PYTHON_PREFIX}_HINTS} PATHS ${_${_PYTHON_PREFIX}_FRAMEWORK_PATHS} - PATH_SUFFIXES bin + PATH_SUFFIXES ${_${_PYTHON_PREFIX}_PATH_SUFFIXES} NO_CMAKE_PATH NO_CMAKE_ENVIRONMENT_PATH NO_SYSTEM_ENVIRONMENT_PATH NO_CMAKE_SYSTEM_PATH) - _python_validate_interpreter (${${_PYTHON_PREFIX}_FIND_VERSION}) + _python_validate_interpreter (${${_PYTHON_PREFIX}_FIND_VERSION} ${_${_PYTHON_PREFIX}_FIND_VERSION_EXACT}) if (_${_PYTHON_PREFIX}_EXECUTABLE) break() endif() @@ -1074,14 +1447,13 @@ if ("Interpreter" IN_LIST ${_PYTHON_PREFIX}_FIND_COMPONENTS) if (CMAKE_HOST_WIN32 AND _${_PYTHON_PREFIX}_FIND_REGISTRY STREQUAL "FIRST") find_program (_${_PYTHON_PREFIX}_EXECUTABLE NAMES ${_${_PYTHON_PREFIX}_NAMES} - ${_${_PYTHON_PREFIX}_IRON_PYTHON_NAMES} NAMES_PER_DIR HINTS ${_${_PYTHON_PREFIX}_HINTS} PATHS ${_${_PYTHON_PREFIX}_REGISTRY_PATHS} - PATH_SUFFIXES bin ${_${_PYTHON_PREFIX}_IRON_PYTHON_PATH_SUFFIXES} + PATH_SUFFIXES ${_${_PYTHON_PREFIX}_PATH_SUFFIXES} NO_SYSTEM_ENVIRONMENT_PATH NO_CMAKE_SYSTEM_PATH) - _python_validate_interpreter (${${_PYTHON_PREFIX}_FIND_VERSION}) + _python_validate_interpreter (${${_PYTHON_PREFIX}_FIND_VERSION} ${_${_PYTHON_PREFIX}_FIND_VERSION_EXACT}) if (_${_PYTHON_PREFIX}_EXECUTABLE) break() endif() @@ -1090,23 +1462,21 @@ if ("Interpreter" IN_LIST ${_PYTHON_PREFIX}_FIND_COMPONENTS) # try using HINTS find_program (_${_PYTHON_PREFIX}_EXECUTABLE NAMES ${_${_PYTHON_PREFIX}_NAMES} - ${_${_PYTHON_PREFIX}_IRON_PYTHON_NAMES} NAMES_PER_DIR HINTS ${_${_PYTHON_PREFIX}_HINTS} - PATH_SUFFIXES bin ${_${_PYTHON_PREFIX}_IRON_PYTHON_PATH_SUFFIXES} + PATH_SUFFIXES ${_${_PYTHON_PREFIX}_PATH_SUFFIXES} NO_SYSTEM_ENVIRONMENT_PATH NO_CMAKE_SYSTEM_PATH) - _python_validate_interpreter (${${_PYTHON_PREFIX}_FIND_VERSION}) + _python_validate_interpreter (${${_PYTHON_PREFIX}_FIND_VERSION} ${_${_PYTHON_PREFIX}_FIND_VERSION_EXACT}) if (_${_PYTHON_PREFIX}_EXECUTABLE) break() endif() # try using standard paths find_program (_${_PYTHON_PREFIX}_EXECUTABLE NAMES ${_${_PYTHON_PREFIX}_NAMES} - ${_${_PYTHON_PREFIX}_IRON_PYTHON_NAMES} NAMES_PER_DIR - PATH_SUFFIXES bin ${_${_PYTHON_PREFIX}_IRON_PYTHON_PATH_SUFFIXES}) - _python_validate_interpreter (${${_PYTHON_PREFIX}_FIND_VERSION}) + PATH_SUFFIXES ${_${_PYTHON_PREFIX}_PATH_SUFFIXES}) + _python_validate_interpreter (${${_PYTHON_PREFIX}_FIND_VERSION} ${_${_PYTHON_PREFIX}_FIND_VERSION_EXACT}) if (_${_PYTHON_PREFIX}_EXECUTABLE) break() endif() @@ -1117,9 +1487,9 @@ if ("Interpreter" IN_LIST ${_PYTHON_PREFIX}_FIND_COMPONENTS) NAMES ${_${_PYTHON_PREFIX}_NAMES} NAMES_PER_DIR PATHS ${_${_PYTHON_PREFIX}_FRAMEWORK_PATHS} - PATH_SUFFIXES bin + PATH_SUFFIXES ${_${_PYTHON_PREFIX}_PATH_SUFFIXES} NO_DEFAULT_PATH) - _python_validate_interpreter (${${_PYTHON_PREFIX}_FIND_VERSION}) + _python_validate_interpreter (${${_PYTHON_PREFIX}_FIND_VERSION} ${_${_PYTHON_PREFIX}_FIND_VERSION_EXACT}) if (_${_PYTHON_PREFIX}_EXECUTABLE) break() endif() @@ -1128,12 +1498,11 @@ if ("Interpreter" IN_LIST ${_PYTHON_PREFIX}_FIND_COMPONENTS) if (CMAKE_HOST_WIN32 AND _${_PYTHON_PREFIX}_FIND_REGISTRY STREQUAL "LAST") find_program (_${_PYTHON_PREFIX}_EXECUTABLE NAMES ${_${_PYTHON_PREFIX}_NAMES} - ${_${_PYTHON_PREFIX}_IRON_PYTHON_NAMES} NAMES_PER_DIR PATHS ${_${_PYTHON_PREFIX}_REGISTRY_PATHS} - PATH_SUFFIXES bin ${_${_PYTHON_PREFIX}_IRON_PYTHON_PATH_SUFFIXES} + PATH_SUFFIXES ${_${_PYTHON_PREFIX}_PATH_SUFFIXES} NO_DEFAULT_PATH) - _python_validate_interpreter (${${_PYTHON_PREFIX}_FIND_VERSION}) + _python_validate_interpreter (${${_PYTHON_PREFIX}_FIND_VERSION} ${_${_PYTHON_PREFIX}_FIND_VERSION_EXACT}) if (_${_PYTHON_PREFIX}_EXECUTABLE) break() endif() @@ -1144,12 +1513,11 @@ if ("Interpreter" IN_LIST ${_PYTHON_PREFIX}_FIND_COMPONENTS) else() # look-up for various versions and locations foreach (_${_PYTHON_PREFIX}_VERSION IN LISTS _${_PYTHON_PREFIX}_FIND_VERSIONS) - _python_get_names (_${_PYTHON_PREFIX}_NAMES VERSION ${_${_PYTHON_PREFIX}_VERSION} POSIX EXECUTABLE) - list (APPEND _${_PYTHON_PREFIX}_NAMES python${_${_PYTHON_PREFIX}_REQUIRED_VERSION_MAJOR} - python) + _python_get_names (_${_PYTHON_PREFIX}_NAMES VERSION ${_${_PYTHON_PREFIX}_VERSION} POSIX INTERPRETER) + _python_get_path_suffixes (_${_PYTHON_PREFIX}_PATH_SUFFIXES VERSION ${_${_PYTHON_PREFIX}_VERSION} INTERPRETER) - _python_get_frameworks (_${_PYTHON_PREFIX}_FRAMEWORK_PATHS ${_${_PYTHON_PREFIX}_VERSION}) - _python_get_registries (_${_PYTHON_PREFIX}_REGISTRY_PATHS ${_${_PYTHON_PREFIX}_VERSION}) + _python_get_frameworks (_${_PYTHON_PREFIX}_FRAMEWORK_PATHS VERSION ${_${_PYTHON_PREFIX}_VERSION}) + _python_get_registries (_${_PYTHON_PREFIX}_REGISTRY_PATHS VERSION ${_${_PYTHON_PREFIX}_VERSION}) # Virtual environments handling if (_${_PYTHON_PREFIX}_FIND_VIRTUALENV MATCHES "^(FIRST|ONLY)$") @@ -1158,7 +1526,7 @@ if ("Interpreter" IN_LIST ${_PYTHON_PREFIX}_FIND_COMPONENTS) NAMES_PER_DIR HINTS ${_${_PYTHON_PREFIX}_HINTS} PATHS ENV VIRTUAL_ENV ENV CONDA_PREFIX - PATH_SUFFIXES bin Scripts + PATH_SUFFIXES ${_${_PYTHON_PREFIX}_PATH_SUFFIXES} NO_CMAKE_PATH NO_CMAKE_ENVIRONMENT_PATH NO_SYSTEM_ENVIRONMENT_PATH @@ -1179,7 +1547,7 @@ if ("Interpreter" IN_LIST ${_PYTHON_PREFIX}_FIND_COMPONENTS) NAMES_PER_DIR HINTS ${_${_PYTHON_PREFIX}_HINTS} PATHS ${_${_PYTHON_PREFIX}_FRAMEWORK_PATHS} - PATH_SUFFIXES bin + PATH_SUFFIXES ${_${_PYTHON_PREFIX}_PATH_SUFFIXES} NO_CMAKE_PATH NO_CMAKE_ENVIRONMENT_PATH NO_SYSTEM_ENVIRONMENT_PATH @@ -1190,12 +1558,10 @@ if ("Interpreter" IN_LIST ${_PYTHON_PREFIX}_FIND_COMPONENTS) if (CMAKE_HOST_WIN32 AND _${_PYTHON_PREFIX}_FIND_REGISTRY STREQUAL "FIRST") find_program (_${_PYTHON_PREFIX}_EXECUTABLE NAMES ${_${_PYTHON_PREFIX}_NAMES} - ${_${_PYTHON_PREFIX}_IRON_PYTHON_NAMES} NAMES_PER_DIR HINTS ${_${_PYTHON_PREFIX}_HINTS} PATHS ${_${_PYTHON_PREFIX}_REGISTRY_PATHS} - [HKEY_LOCAL_MACHINE\\SOFTWARE\\IronPython\\${_${_PYTHON_PREFIX}_VERSION}\\InstallPath] - PATH_SUFFIXES bin ${_${_PYTHON_PREFIX}_IRON_PYTHON_PATH_SUFFIXES} + PATH_SUFFIXES ${_${_PYTHON_PREFIX}_PATH_SUFFIXES} NO_SYSTEM_ENVIRONMENT_PATH NO_CMAKE_SYSTEM_PATH) endif() @@ -1208,10 +1574,9 @@ if ("Interpreter" IN_LIST ${_PYTHON_PREFIX}_FIND_COMPONENTS) # try using HINTS find_program (_${_PYTHON_PREFIX}_EXECUTABLE NAMES ${_${_PYTHON_PREFIX}_NAMES} - ${_${_PYTHON_PREFIX}_IRON_PYTHON_NAMES} NAMES_PER_DIR HINTS ${_${_PYTHON_PREFIX}_HINTS} - PATH_SUFFIXES bin ${_${_PYTHON_PREFIX}_IRON_PYTHON_PATH_SUFFIXES} + PATH_SUFFIXES ${_${_PYTHON_PREFIX}_PATH_SUFFIXES} NO_SYSTEM_ENVIRONMENT_PATH NO_CMAKE_SYSTEM_PATH) _python_validate_interpreter (${_${_PYTHON_PREFIX}_VERSION} EXACT) @@ -1224,15 +1589,9 @@ if ("Interpreter" IN_LIST ${_PYTHON_PREFIX}_FIND_COMPONENTS) # For example, typical systems have 'python' for version 2.* and 'python3' # for version 3.*. So looking for names per dir will find, potentially, # systematically 'python' (i.e. version 2) even if version 3 is searched. - if (CMAKE_HOST_WIN32) - find_program (_${_PYTHON_PREFIX}_EXECUTABLE - NAMES ${_${_PYTHON_PREFIX}_NAMES} - python - ${_${_PYTHON_PREFIX}_IRON_PYTHON_NAMES}) - else() - find_program (_${_PYTHON_PREFIX}_EXECUTABLE - NAMES ${_${_PYTHON_PREFIX}_NAMES}) - endif() + find_program (_${_PYTHON_PREFIX}_EXECUTABLE + NAMES ${_${_PYTHON_PREFIX}_NAMES} + PATH_SUFFIXES ${_${_PYTHON_PREFIX}_PATH_SUFFIXES}) _python_validate_interpreter (${_${_PYTHON_PREFIX}_VERSION} EXACT) if (_${_PYTHON_PREFIX}_EXECUTABLE) break() @@ -1244,7 +1603,7 @@ if ("Interpreter" IN_LIST ${_PYTHON_PREFIX}_FIND_COMPONENTS) NAMES ${_${_PYTHON_PREFIX}_NAMES} NAMES_PER_DIR PATHS ${_${_PYTHON_PREFIX}_FRAMEWORK_PATHS} - PATH_SUFFIXES bin + PATH_SUFFIXES ${_${_PYTHON_PREFIX}_PATH_SUFFIXES} NO_DEFAULT_PATH) endif() @@ -1252,11 +1611,9 @@ if ("Interpreter" IN_LIST ${_PYTHON_PREFIX}_FIND_COMPONENTS) if (CMAKE_HOST_WIN32 AND _${_PYTHON_PREFIX}_FIND_REGISTRY STREQUAL "LAST") find_program (_${_PYTHON_PREFIX}_EXECUTABLE NAMES ${_${_PYTHON_PREFIX}_NAMES} - ${_${_PYTHON_PREFIX}_IRON_PYTHON_NAMES} NAMES_PER_DIR PATHS ${_${_PYTHON_PREFIX}_REGISTRY_PATHS} - [HKEY_LOCAL_MACHINE\\SOFTWARE\\IronPython\\${_${_PYTHON_PREFIX}_VERSION}\\InstallPath] - PATH_SUFFIXES bin ${_${_PYTHON_PREFIX}_IRON_PYTHON_PATH_SUFFIXES} + PATH_SUFFIXES ${_${_PYTHON_PREFIX}_PATH_SUFFIXES} NO_DEFAULT_PATH) endif() @@ -1274,20 +1631,20 @@ if ("Interpreter" IN_LIST ${_PYTHON_PREFIX}_FIND_COMPONENTS) # For example, typical systems have 'python' for version 2.* and 'python3' # for version 3.*. So looking for names per dir will find, potentially, # systematically 'python' (i.e. version 2) even if version 3 is searched. + _python_get_names (_${_PYTHON_PREFIX}_NAMES POSIX INTERPRETER) find_program (_${_PYTHON_PREFIX}_EXECUTABLE - NAMES python${_${_PYTHON_PREFIX}_REQUIRED_VERSION_MAJOR} - python - ${_${_PYTHON_PREFIX}_IRON_PYTHON_NAMES}) + NAMES ${_${_PYTHON_PREFIX}_NAMES}) _python_validate_interpreter () endif() endif() endif() set (${_PYTHON_PREFIX}_EXECUTABLE "${_${_PYTHON_PREFIX}_EXECUTABLE}") + _python_get_launcher (_${_PYTHON_PREFIX}_INTERPRETER_LAUNCHER INTERPRETER) # retrieve exact version of executable found if (_${_PYTHON_PREFIX}_EXECUTABLE) - execute_process (COMMAND "${_${_PYTHON_PREFIX}_EXECUTABLE}" -c + execute_process (COMMAND ${_${_PYTHON_PREFIX}_INTERPRETER_LAUNCHER} "${_${_PYTHON_PREFIX}_EXECUTABLE}" -c "import sys; sys.stdout.write('.'.join([str(x) for x in sys.version_info[:3]]))" RESULT_VARIABLE _${_PYTHON_PREFIX}_RESULT OUTPUT_VARIABLE ${_PYTHON_PREFIX}_VERSION @@ -1334,7 +1691,8 @@ if ("Interpreter" IN_LIST ${_PYTHON_PREFIX}_FIND_COMPONENTS) # Use interpreter version and ABI for future searches to ensure consistency set (_${_PYTHON_PREFIX}_FIND_VERSIONS ${${_PYTHON_PREFIX}_VERSION_MAJOR}.${${_PYTHON_PREFIX}_VERSION_MINOR}) - execute_process (COMMAND "${_${_PYTHON_PREFIX}_EXECUTABLE}" -c "import sys; sys.stdout.write(sys.abiflags)" + execute_process (COMMAND ${_${_PYTHON_PREFIX}_INTERPRETR_LAUNCHER} "${_${_PYTHON_PREFIX}_EXECUTABLE}" -c + "import sys; sys.stdout.write(sys.abiflags)" RESULT_VARIABLE _${_PYTHON_PREFIX}_RESULT OUTPUT_VARIABLE _${_PYTHON_PREFIX}_ABIFLAGS ERROR_QUIET @@ -1346,13 +1704,16 @@ if ("Interpreter" IN_LIST ${_PYTHON_PREFIX}_FIND_COMPONENTS) endif() if (${_PYTHON_PREFIX}_Interpreter_FOUND) + unset (_${_PYTHON_PREFIX}_Interpreter_REASON_FAILURE) + # compute and save interpreter signature string (MD5 __${_PYTHON_PREFIX}_INTERPRETER_SIGNATURE "${_${_PYTHON_PREFIX}_SIGNATURE}:${_${_PYTHON_PREFIX}_EXECUTABLE}") set (_${_PYTHON_PREFIX}_INTERPRETER_SIGNATURE "${__${_PYTHON_PREFIX}_INTERPRETER_SIGNATURE}" CACHE INTERNAL "") if (NOT CMAKE_SIZEOF_VOID_P) # determine interpreter architecture - execute_process (COMMAND "${_${_PYTHON_PREFIX}_EXECUTABLE}" -c "import sys; print(sys.maxsize > 2**32)" + execute_process (COMMAND ${_${_PYTHON_PREFIX}_INTERPRETER_LAUNCHER} "${_${_PYTHON_PREFIX}_EXECUTABLE}" -c + "import sys; sys.stdout.write(str(sys.maxsize > 2**32))" RESULT_VARIABLE _${_PYTHON_PREFIX}_RESULT OUTPUT_VARIABLE ${_PYTHON_PREFIX}_IS64BIT ERROR_VARIABLE ${_PYTHON_PREFIX}_IS64BIT) @@ -1368,7 +1729,7 @@ if ("Interpreter" IN_LIST ${_PYTHON_PREFIX}_FIND_COMPONENTS) endif() # retrieve interpreter identity - execute_process (COMMAND "${_${_PYTHON_PREFIX}_EXECUTABLE}" -V + execute_process (COMMAND ${_${_PYTHON_PREFIX}_INTERPRETER_LAUNCHER} "${_${_PYTHON_PREFIX}_EXECUTABLE}" -V RESULT_VARIABLE _${_PYTHON_PREFIX}_RESULT OUTPUT_VARIABLE ${_PYTHON_PREFIX}_INTERPRETER_ID ERROR_VARIABLE ${_PYTHON_PREFIX}_INTERPRETER_ID) @@ -1377,11 +1738,15 @@ if ("Interpreter" IN_LIST ${_PYTHON_PREFIX}_FIND_COMPONENTS) set (${_PYTHON_PREFIX}_INTERPRETER_ID "Anaconda") elseif (${_PYTHON_PREFIX}_INTERPRETER_ID MATCHES "Enthought") set (${_PYTHON_PREFIX}_INTERPRETER_ID "Canopy") + elseif (${_PYTHON_PREFIX}_INTERPRETER_ID MATCHES "PyPy ([0-9.]+)") + set (${_PYTHON_PREFIX}_INTERPRETER_ID "PyPy") + set (${_PYTHON_PREFIX}_PyPy_VERSION "${CMAKE_MATCH_1}") else() string (REGEX REPLACE "^([^ ]+).*" "\\1" ${_PYTHON_PREFIX}_INTERPRETER_ID "${${_PYTHON_PREFIX}_INTERPRETER_ID}") if (${_PYTHON_PREFIX}_INTERPRETER_ID STREQUAL "Python") # try to get a more precise ID - execute_process (COMMAND "${_${_PYTHON_PREFIX}_EXECUTABLE}" -c "import sys; print(sys.copyright)" + execute_process (COMMAND ${_${_PYTHON_PREFIX}_INTERPRETER_LAUNCHER} "${_${_PYTHON_PREFIX}_EXECUTABLE}" -c + "import sys; sys.stdout.write(sys.copyright)" RESULT_VARIABLE _${_PYTHON_PREFIX}_RESULT OUTPUT_VARIABLE ${_PYTHON_PREFIX}_COPYRIGHT ERROR_QUIET) @@ -1395,10 +1760,11 @@ if ("Interpreter" IN_LIST ${_PYTHON_PREFIX}_FIND_COMPONENTS) endif() # retrieve various package installation directories - execute_process (COMMAND "${_${_PYTHON_PREFIX}_EXECUTABLE}" -c "import sys\ntry:\n from distutils import sysconfig\n sys.stdout.write(';'.join([sysconfig.get_python_lib(plat_specific=False,standard_lib=True),sysconfig.get_python_lib(plat_specific=True,standard_lib=True),sysconfig.get_python_lib(plat_specific=False,standard_lib=False),sysconfig.get_python_lib(plat_specific=True,standard_lib=False)]))\nexcept Exception:\n import sysconfig\n sys.stdout.write(';'.join([sysconfig.get_path('stdlib'),sysconfig.get_path('platstdlib'),sysconfig.get_path('purelib'),sysconfig.get_path('platlib')]))" - RESULT_VARIABLE _${_PYTHON_PREFIX}_RESULT - OUTPUT_VARIABLE _${_PYTHON_PREFIX}_LIBPATHS - ERROR_QUIET) + execute_process (COMMAND ${_${_PYTHON_PREFIX}_INTERPRETER_LAUNCHER} "${_${_PYTHON_PREFIX}_EXECUTABLE}" -c + "import sys\ntry:\n from distutils import sysconfig\n sys.stdout.write(';'.join([sysconfig.get_python_lib(plat_specific=False,standard_lib=True),sysconfig.get_python_lib(plat_specific=True,standard_lib=True),sysconfig.get_python_lib(plat_specific=False,standard_lib=False),sysconfig.get_python_lib(plat_specific=True,standard_lib=False)]))\nexcept Exception:\n import sysconfig\n sys.stdout.write(';'.join([sysconfig.get_path('stdlib'),sysconfig.get_path('platstdlib'),sysconfig.get_path('purelib'),sysconfig.get_path('platlib')]))" + RESULT_VARIABLE _${_PYTHON_PREFIX}_RESULT + OUTPUT_VARIABLE _${_PYTHON_PREFIX}_LIBPATHS + ERROR_QUIET) if (NOT _${_PYTHON_PREFIX}_RESULT) list (GET _${_PYTHON_PREFIX}_LIBPATHS 0 ${_PYTHON_PREFIX}_STDLIB) list (GET _${_PYTHON_PREFIX}_LIBPATHS 1 ${_PYTHON_PREFIX}_STDARCH) @@ -1411,7 +1777,7 @@ if ("Interpreter" IN_LIST ${_PYTHON_PREFIX}_FIND_COMPONENTS) unset (${_PYTHON_PREFIX}_SITEARCH) endif() - if (_${_PYTHON_PREFIX}_REQUIRED_VERSION_MAJOR VERSION_GREATER_EQUAL 3) + if (_${_PYTHON_PREFIX}_REQUIRED_VERSION_MAJOR VERSION_GREATER_EQUAL "3") _python_get_config_var (${_PYTHON_PREFIX}_SOABI SOABI) endif() @@ -1442,7 +1808,10 @@ if ("Compiler" IN_LIST ${_PYTHON_PREFIX}_FIND_COMPONENTS) list (APPEND _${_PYTHON_PREFIX}_REQUIRED_VARS ${_PYTHON_PREFIX}_COMPILER) endif() - if (DEFINED ${_PYTHON_PREFIX}_COMPILER + if (NOT "IronPython" IN_LIST _${_PYTHON_PREFIX}_FIND_IMPLEMENTATIONS) + unset (_${_PYTHON_PREFIX}_COMPILER CACHE) + unset (_${_PYTHON_PREFIX}_COMPILER_SIGNATURE CACHE) + elseif (DEFINED ${_PYTHON_PREFIX}_COMPILER AND IS_ABSOLUTE "${${_PYTHON_PREFIX}_COMPILER}") set (_${_PYTHON_PREFIX}_COMPILER "${${_PYTHON_PREFIX}_COMPILER}" CACHE INTERNAL "") elseif (DEFINED _${_PYTHON_PREFIX}_COMPILER) @@ -1461,7 +1830,8 @@ if ("Compiler" IN_LIST ${_PYTHON_PREFIX}_FIND_COMPONENTS) endif() endif() - if (NOT _${_PYTHON_PREFIX}_COMPILER) + if ("IronPython" IN_LIST _${_PYTHON_PREFIX}_FIND_IMPLEMENTATIONS + AND NOT _${_PYTHON_PREFIX}_COMPILER) # IronPython specific artifacts # If IronPython interpreter is found, use its path unset (_${_PYTHON_PREFIX}_IRON_ROOT) @@ -1470,96 +1840,215 @@ if ("Compiler" IN_LIST ${_PYTHON_PREFIX}_FIND_COMPONENTS) endif() if (_${_PYTHON_PREFIX}_FIND_STRATEGY STREQUAL "LOCATION") - set (_${_PYTHON_PREFIX}_REGISTRY_PATHS) - - foreach (_${_PYTHON_PREFIX}_VERSION IN LISTS _${_PYTHON_PREFIX}_FIND_VERSIONS) - # Registry Paths - list (APPEND _${_PYTHON_PREFIX}_REGISTRY_PATHS - [HKEY_LOCAL_MACHINE\\SOFTWARE\\IronPython\\${_${_PYTHON_PREFIX}_VERSION}\\InstallPath]) - endforeach() + _python_get_names (_${_PYTHON_PREFIX}_COMPILER_NAMES + IMPLEMENTATIONS IronPython + VERSION ${_${_PYTHON_PREFIX}_FIND_VERSIONS} + COMPILER) + + _python_get_path_suffixes (_${_PYTHON_PREFIX}_PATH_SUFFIXES + IMPLEMENTATIONS IronPython + VERSION ${_${_PYTHON_PREFIX}_FIND_VERSIONS} + COMPILER) + + _python_get_frameworks (_${_PYTHON_PREFIX}_FRAMEWORK_PATHS + IMPLEMENTATIONS IronPython + VERSION ${_${_PYTHON_PREFIX}_FIND_VERSIONS}) + _python_get_registries (_${_PYTHON_PREFIX}_REGISTRY_PATHS + IMPLEMENTATIONS IronPython + VERSION ${_${_PYTHON_PREFIX}_FIND_VERSIONS}) while (TRUE) - if (_${_PYTHON_PREFIX}_FIND_REGISTRY STREQUAL "FIRST") + # Apple frameworks handling + if (CMAKE_HOST_APPLE AND _${_PYTHON_PREFIX}_FIND_FRAMEWORK STREQUAL "FIRST") find_program (_${_PYTHON_PREFIX}_COMPILER - NAMES ipyc + NAMES ${_${_PYTHON_PREFIX}_COMPILER_NAMES} + NAMES_PER_DIR HINTS ${_${_PYTHON_PREFIX}_IRON_ROOT} ${_${_PYTHON_PREFIX}_HINTS} - PATHS ${_${_PYTHON_PREFIX}_REGISTRY_PATHS} - PATH_SUFFIXES ${_${_PYTHON_PREFIX}_IRON_PYTHON_PATH_SUFFIXES} + PATHS ${_${_PYTHON_PREFIX}_FRAMEWORK_PATHS} + PATH_SUFFIXES ${_${_PYTHON_PREFIX}_PATH_SUFFIXES} + NO_CMAKE_PATH + NO_CMAKE_ENVIRONMENT_PATH NO_SYSTEM_ENVIRONMENT_PATH NO_CMAKE_SYSTEM_PATH) - _python_validate_compiler (${${_PYTHON_PREFIX}_FIND_VERSION}) + _python_validate_compiler (${${_PYTHON_PREFIX}_FIND_VERSION} ${_${_PYTHON_PREFIX}_FIND_VERSION_EXACT}) if (_${_PYTHON_PREFIX}_COMPILER) break() endif() endif() - - find_program (_${_PYTHON_PREFIX}_COMPILER - NAMES ipyc - HINTS ${_${_PYTHON_PREFIX}_IRON_ROOT} ${_${_PYTHON_PREFIX}_HINTS} - PATH_SUFFIXES ${_${_PYTHON_PREFIX}_IRON_PYTHON_PATH_SUFFIXES} - NO_SYSTEM_ENVIRONMENT_PATH - NO_CMAKE_SYSTEM_PATH) - _python_validate_compiler (${${_PYTHON_PREFIX}_FIND_VERSION}) - if (_${_PYTHON_PREFIX}_COMPILER) - break() - endif() - - if (_${_PYTHON_PREFIX}_FIND_REGISTRY STREQUAL "LAST") - find_program (_${_PYTHON_PREFIX}_COMPILER - NAMES ipyc - PATHS ${_${_PYTHON_PREFIX}_REGISTRY_PATHS} - PATH_SUFFIXES ${_${_PYTHON_PREFIX}_IRON_PYTHON_PATH_SUFFIXES} - NO_DEFAULT_PATH) - endif() - - break() - endwhile() - else() - # try using root dir and registry - foreach (_${_PYTHON_PREFIX}_VERSION IN LISTS _${_PYTHON_PREFIX}_FIND_VERSIONS) - if (_${_PYTHON_PREFIX}_FIND_REGISTRY STREQUAL "FIRST") + # Windows registry + if (CMAKE_HOST_WIN32 AND _${_PYTHON_PREFIX}_FIND_REGISTRY STREQUAL "FIRST") find_program (_${_PYTHON_PREFIX}_COMPILER - NAMES ipyc + NAMES ${_${_PYTHON_PREFIX}_COMPILER_NAMES} + NAMES_PER_DIR HINTS ${_${_PYTHON_PREFIX}_IRON_ROOT} ${_${_PYTHON_PREFIX}_HINTS} - PATHS [HKEY_LOCAL_MACHINE\\SOFTWARE\\IronPython\\${_${_PYTHON_PREFIX}_VERSION}\\InstallPath] - PATH_SUFFIXES ${_${_PYTHON_PREFIX}_IRON_PYTHON_PATH_SUFFIXES} + PATHS ${_${_PYTHON_PREFIX}_REGISTRY_PATHS} + PATH_SUFFIXES ${_${_PYTHON_PREFIX}_PATH_SUFFIXES} NO_SYSTEM_ENVIRONMENT_PATH NO_CMAKE_SYSTEM_PATH) - _python_validate_compiler (${_${_PYTHON_PREFIX}_VERSION} EXACT) + _python_validate_compiler (${${_PYTHON_PREFIX}_FIND_VERSION} ${_${_PYTHON_PREFIX}_FIND_VERSION_EXACT}) if (_${_PYTHON_PREFIX}_COMPILER) break() endif() endif() + # try using HINTS find_program (_${_PYTHON_PREFIX}_COMPILER - NAMES ipyc + NAMES ${_${_PYTHON_PREFIX}_COMPILER_NAMES} + NAMES_PER_DIR HINTS ${_${_PYTHON_PREFIX}_IRON_ROOT} ${_${_PYTHON_PREFIX}_HINTS} - PATH_SUFFIXES ${_${_PYTHON_PREFIX}_IRON_PYTHON_PATH_SUFFIXES} + PATH_SUFFIXES ${_${_PYTHON_PREFIX}_PATH_SUFFIXES} NO_SYSTEM_ENVIRONMENT_PATH NO_CMAKE_SYSTEM_PATH) - _python_validate_compiler (${_${_PYTHON_PREFIX}_VERSION} EXACT) + _python_validate_compiler (${${_PYTHON_PREFIX}_FIND_VERSION} ${_${_PYTHON_PREFIX}_FIND_VERSION_EXACT}) if (_${_PYTHON_PREFIX}_COMPILER) break() endif() - if (_${_PYTHON_PREFIX}_FIND_REGISTRY STREQUAL "LAST") - find_program (_${_PYTHON_PREFIX}_COMPILER - NAMES ipyc - PATHS [HKEY_LOCAL_MACHINE\\SOFTWARE\\IronPython\\${_${_PYTHON_PREFIX}_VERSION}\\InstallPath] - PATH_SUFFIXES ${_${_PYTHON_PREFIX}_IRON_PYTHON_PATH_SUFFIXES} - NO_DEFAULT_PATH) - _python_validate_compiler (${_${_PYTHON_PREFIX}_VERSION} EXACT) - if (_${_PYTHON_PREFIX}_COMPILER) - break() - endif() + # try using standard paths + find_program (_${_PYTHON_PREFIX}_COMPILER + NAMES ${_${_PYTHON_PREFIX}_COMPILER_NAMES} + NAMES_PER_DIR + PATH_SUFFIXES ${_${_PYTHON_PREFIX}_PATH_SUFFIXES}) + _python_validate_compiler (${${_PYTHON_PREFIX}_FIND_VERSION} ${_${_PYTHON_PREFIX}_FIND_VERSION_EXACT}) + if (_${_PYTHON_PREFIX}_COMPILER) + break() + endif() + + # Apple frameworks handling + if (CMAKE_HOST_APPLE AND _${_PYTHON_PREFIX}_FIND_FRAMEWORK STREQUAL "LAST") + find_program (_${_PYTHON_PREFIX}_COMPILER + NAMES ${_${_PYTHON_PREFIX}_COMPILER_NAMES} + NAMES_PER_DIR + PATHS ${_${_PYTHON_PREFIX}_FRAMEWORK_PATHS} + PATH_SUFFIXES ${_${_PYTHON_PREFIX}_PATH_SUFFIXES} + NO_DEFAULT_PATH) + _python_validate_compiler (${${_PYTHON_PREFIX}_FIND_VERSION} ${_${_PYTHON_PREFIX}_FIND_VERSION_EXACT}) + if (_${_PYTHON_PREFIX}_COMPILER) + break() + endif() + endif() + # Windows registry + if (CMAKE_HOST_WIN32 AND _${_PYTHON_PREFIX}_FIND_REGISTRY STREQUAL "LAST") + find_program (_${_PYTHON_PREFIX}_COMPILER + NAMES ${_${_PYTHON_PREFIX}_COMPILER_NAMES} + NAMES_PER_DIR + PATHS ${_${_PYTHON_PREFIX}_REGISTRY_PATHS} + PATH_SUFFIXES ${_${_PYTHON_PREFIX}_PATH_SUFFIXES} + NO_DEFAULT_PATH) + if (_${_PYTHON_PREFIX}_COMPILER) + break() + endif() + endif() + + break() + endwhile() + else() + # try using root dir and registry + foreach (_${_PYTHON_PREFIX}_VERSION IN LISTS _${_PYTHON_PREFIX}_FIND_VERSIONS) + _python_get_names (_${_PYTHON_PREFIX}_COMPILER_NAMES + IMPLEMENTATIONS IronPython + VERSION ${_${_PYTHON_PREFIX}_FIND_VERSIONS} + COMPILER) + + _python_get_path_suffixes (_${_PYTHON_PREFIX}_PATH_SUFFIXES + IMPLEMENTATIONS IronPython + VERSION ${_${_PYTHON_PREFIX}_FIND_VERSION} + COMPILER) + + _python_get_frameworks (_${_PYTHON_PREFIX}_FRAMEWORK_PATHS + IMPLEMENTATIONS IronPython + VERSION ${_${_PYTHON_PREFIX}_VERSION}) + _python_get_registries (_${_PYTHON_PREFIX}_REGISTRY_PATHS + IMPLEMENTATIONS IronPython + VERSION ${_${_PYTHON_PREFIX}_VERSION}) + + # Apple frameworks handling + if (CMAKE_HOST_APPLE AND _${_PYTHON_PREFIX}_FIND_FRAMEWORK STREQUAL "FIRST") + find_program (_${_PYTHON_PREFIX}_COMPILER + NAMES ${_${_PYTHON_PREFIX}_COMPILER_NAMES} + NAMES_PER_DIR + HINTS ${_${_PYTHON_PREFIX}_IRON_ROOT} ${_${_PYTHON_PREFIX}_HINTS} + PATHS ${_${_PYTHON_PREFIX}_FRAMEWORK_PATHS} + PATH_SUFFIXES ${_${_PYTHON_PREFIX}_PATH_SUFFIXES} + NO_CMAKE_PATH + NO_CMAKE_ENVIRONMENT_PATH + NO_SYSTEM_ENVIRONMENT_PATH + NO_CMAKE_SYSTEM_PATH) + _python_validate_compiler (${_${_PYTHON_PREFIX}_VERSION} EXACT) + if (_${_PYTHON_PREFIX}_COMPILER) + break() + endif() + endif() + # Windows registry + if (CMAKE_HOST_WIN32 AND _${_PYTHON_PREFIX}_FIND_REGISTRY STREQUAL "FIRST") + find_program (_${_PYTHON_PREFIX}_COMPILER + NAMES ${_${_PYTHON_PREFIX}_COMPILER_NAMES} + NAMES_PER_DIR + HINTS ${_${_PYTHON_PREFIX}_IRON_ROOT} ${_${_PYTHON_PREFIX}_HINTS} + PATHS ${_${_PYTHON_PREFIX}_REGISTRY_PATHS} + PATH_SUFFIXES ${_${_PYTHON_PREFIX}_PATH_SUFFIXES} + NO_SYSTEM_ENVIRONMENT_PATH + NO_CMAKE_SYSTEM_PATH) + _python_validate_compiler (${_${_PYTHON_PREFIX}_VERSION} EXACT) + if (_${_PYTHON_PREFIX}_COMPILER) + break() + endif() + endif() + + # try using HINTS + find_program (_${_PYTHON_PREFIX}_COMPILER + NAMES ${_${_PYTHON_PREFIX}_COMPILER_NAMES} + NAMES_PER_DIR + HINTS ${_${_PYTHON_PREFIX}_IRON_ROOT} ${_${_PYTHON_PREFIX}_HINTS} + PATH_SUFFIXES ${_${_PYTHON_PREFIX}_PATH_SUFFIXES} + NO_SYSTEM_ENVIRONMENT_PATH + NO_CMAKE_SYSTEM_PATH) + _python_validate_compiler (${_${_PYTHON_PREFIX}_VERSION} EXACT) + if (_${_PYTHON_PREFIX}_COMPILER) + break() + endif() + + # Apple frameworks handling + if (CMAKE_HOST_APPLE AND _${_PYTHON_PREFIX}_FIND_FRAMEWORK STREQUAL "LAST") + find_program (_${_PYTHON_PREFIX}_COMPILER + NAMES ${_${_PYTHON_PREFIX}_COMPILER_NAMES} + NAMES_PER_DIR + PATHS ${_${_PYTHON_PREFIX}_FRAMEWORK_PATHS} + PATH_SUFFIXES ${_${_PYTHON_PREFIX}_PATH_SUFFIXES} + NO_DEFAULT_PATH) + _python_validate_compiler (${_${_PYTHON_PREFIX}_VERSION} EXACT) + if (_${_PYTHON_PREFIX}_COMPILER) + break() + endif() + endif() + # Windows registry + if (CMAKE_HOST_WIN32 AND _${_PYTHON_PREFIX}_FIND_REGISTRY STREQUAL "LAST") + find_program (_${_PYTHON_PREFIX}_COMPILER + NAMES ${_${_PYTHON_PREFIX}_COMPILER_NAMES} + NAMES_PER_DIR + PATHS ${_${_PYTHON_PREFIX}_REGISTRY_PATHS} + PATH_SUFFIXES ${_${_PYTHON_PREFIX}_PATH_SUFFIXES} + NO_DEFAULT_PATH) + _python_validate_compiler (${_${_PYTHON_PREFIX}_VERSION} EXACT) + if (_${_PYTHON_PREFIX}_COMPILER) + break() + endif() endif() endforeach() # no specific version found, re-try in standard paths + _python_get_names (_${_PYTHON_PREFIX}_COMPILER_NAMES + IMPLEMENTATIONS IronPython + VERSION ${_${_PYTHON_PREFIX}_FIND_VERSIONS} + COMPILER) + _python_get_path_suffixes (_${_PYTHON_PREFIX}_PATH_SUFFIXES + IMPLEMENTATIONS IronPython + VERSION ${_${_PYTHON_PREFIX}_FIND_VERSIONS} + COMPILER) find_program (_${_PYTHON_PREFIX}_COMPILER - NAMES ipyc + NAMES ${_${_PYTHON_PREFIX}_COMPILER_NAMES} HINTS ${_${_PYTHON_PREFIX}_IRON_ROOT} ${_${_PYTHON_PREFIX}_HINTS} - PATH_SUFFIXES ${_${_PYTHON_PREFIX}_IRON_PYTHON_PATH_SUFFIXES}) + PATH_SUFFIXES ${_${_PYTHON_PREFIX}_PATH_SUFFIXES}) endif() endif() @@ -1567,13 +2056,18 @@ if ("Compiler" IN_LIST ${_PYTHON_PREFIX}_FIND_COMPONENTS) if (_${_PYTHON_PREFIX}_COMPILER) # retrieve python environment version from compiler + _python_get_launcher (_${_PYTHON_PREFIX}_COMPILER_LAUNCHER COMPILER) set (_${_PYTHON_PREFIX}_VERSION_DIR "${CMAKE_CURRENT_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/PythonCompilerVersion.dir") file (WRITE "${_${_PYTHON_PREFIX}_VERSION_DIR}/version.py" "import sys; sys.stdout.write('.'.join([str(x) for x in sys.version_info[:3]]))\n") - execute_process (COMMAND "${_${_PYTHON_PREFIX}_COMPILER}" /target:exe /embed "${_${_PYTHON_PREFIX}_VERSION_DIR}/version.py" + execute_process (COMMAND ${_${_PYTHON_PREFIX}_COMPILER_LAUNCHER} "${_${_PYTHON_PREFIX}_COMPILER}" + ${_${_PYTHON_PREFIX}_IRON_PYTHON_COMPILER_ARCH_FLAGS} + /target:exe /embed "${_${_PYTHON_PREFIX}_VERSION_DIR}/version.py" WORKING_DIRECTORY "${_${_PYTHON_PREFIX}_VERSION_DIR}" OUTPUT_QUIET ERROR_QUIET) - execute_process (COMMAND "${_${_PYTHON_PREFIX}_VERSION_DIR}/version" + get_filename_component (_${_PYTHON_PREFIX}_IR_DIR "${_${_PYTHON_PREFIX}_COMPILER}" DIRECTORY) + execute_process (COMMAND "${CMAKE_COMMAND}" -E env "MONO_PATH=${_${_PYTHON_PREFIX}_IR_DIR}" + ${${_PYTHON_PREFIX}_DOTNET_LAUNCHER} "${_${_PYTHON_PREFIX}_VERSION_DIR}/version.exe" WORKING_DIRECTORY "${_${_PYTHON_PREFIX}_VERSION_DIR}" RESULT_VARIABLE _${_PYTHON_PREFIX}_RESULT OUTPUT_VARIABLE _${_PYTHON_PREFIX}_VERSION @@ -1603,17 +2097,19 @@ if ("Compiler" IN_LIST ${_PYTHON_PREFIX}_FIND_COMPONENTS) if (_${_PYTHON_PREFIX}_COMPILER AND _${_PYTHON_PREFIX}_COMPILER_USABLE) if (${_PYTHON_PREFIX}_Interpreter_FOUND) # Compiler must be compatible with interpreter - if (${_${_PYTHON_PREFIX}_VERSION_MAJOR}.${_${_PYTHON_PREFIX}_VERSION_MINOR} VERSION_EQUAL ${${_PYTHON_PREFIX}_VERSION_MAJOR}.${${_PYTHON_PREFIX}_VERSION_MINOR}) + if ("${_${_PYTHON_PREFIX}_VERSION_MAJOR}.${_${_PYTHON_PREFIX}_VERSION_MINOR}" VERSION_EQUAL "${${_PYTHON_PREFIX}_VERSION_MAJOR}.${${_PYTHON_PREFIX}_VERSION_MINOR}") set (${_PYTHON_PREFIX}_Compiler_FOUND TRUE) endif() elseif (${_PYTHON_PREFIX}_VERSION_MAJOR VERSION_EQUAL _${_PYTHON_PREFIX}_REQUIRED_VERSION_MAJOR) set (${_PYTHON_PREFIX}_Compiler_FOUND TRUE) - # Use compiler version for future searches to ensure consistency - set (_${_PYTHON_PREFIX}_FIND_VERSIONS ${${_PYTHON_PREFIX}_VERSION_MAJOR}.${${_PYTHON_PREFIX}_VERSION_MINOR}) + # Use compiler version for future searches to ensure consistency + set (_${_PYTHON_PREFIX}_FIND_VERSIONS ${${_PYTHON_PREFIX}_VERSION_MAJOR}.${${_PYTHON_PREFIX}_VERSION_MINOR}) endif() endif() if (${_PYTHON_PREFIX}_Compiler_FOUND) + unset (_${_PYTHON_PREFIX}_Compiler_REASON_FAILURE) + # compute and save compiler signature string (MD5 __${_PYTHON_PREFIX}_COMPILER_SIGNATURE "${_${_PYTHON_PREFIX}_SIGNATURE}:${_${_PYTHON_PREFIX}_COMPILER}") set (_${_PYTHON_PREFIX}_COMPILER_SIGNATURE "${__${_PYTHON_PREFIX}_COMPILER_SIGNATURE}" CACHE INTERNAL "") @@ -1632,50 +2128,53 @@ if ("Compiler" IN_LIST ${_PYTHON_PREFIX}_FIND_COMPONENTS) _${_PYTHON_PREFIX}_COMPILER_SIGNATURE) endif() - # third step, search for the development artifacts +if (${_PYTHON_PREFIX}_FIND_REQUIRED_Development.Module) + if ("LIBRARY" IN_LIST _${_PYTHON_PREFIX}_FIND_DEVELOPMENT_MODULE_ARTIFACTS) + list (APPEND _${_PYTHON_PREFIX}_REQUIRED_VARS ${_PYTHON_PREFIX}_LIBRARIES) + endif() + if ("INCLUDE_DIR" IN_LIST _${_PYTHON_PREFIX}_FIND_DEVELOPMENT_MODULE_ARTIFACTS) + list (APPEND _${_PYTHON_PREFIX}_REQUIRED_VARS ${_PYTHON_PREFIX}_INCLUDE_DIRS) + endif() +endif() +if (${_PYTHON_PREFIX}_FIND_REQUIRED_Development.Embed) + if ("LIBRARY" IN_LIST _${_PYTHON_PREFIX}_FIND_DEVELOPMENT_EMBED_ARTIFACTS) + list (APPEND _${_PYTHON_PREFIX}_REQUIRED_VARS ${_PYTHON_PREFIX}_LIBRARIES) + endif() + if ("INCLUDE_DIR" IN_LIST _${_PYTHON_PREFIX}_FIND_DEVELOPMENT_EMBED_ARTIFACTS) + list (APPEND _${_PYTHON_PREFIX}_REQUIRED_VARS ${_PYTHON_PREFIX}_INCLUDE_DIRS) + endif() +endif() +list (REMOVE_DUPLICATES _${_PYTHON_PREFIX}_REQUIRED_VARS) ## Development environment is not compatible with IronPython interpreter -if ("Development" IN_LIST ${_PYTHON_PREFIX}_FIND_COMPONENTS - AND NOT ${_PYTHON_PREFIX}_INTERPRETER_ID STREQUAL "IronPython") +if (("Development.Module" IN_LIST ${_PYTHON_PREFIX}_FIND_COMPONENTS + OR "Development.Embed" IN_LIST ${_PYTHON_PREFIX}_FIND_COMPONENTS) + AND ((${_PYTHON_PREFIX}_Interpreter_FOUND + AND NOT ${_PYTHON_PREFIX}_INTERPRETER_ID STREQUAL "IronPython") + OR NOT ${_PYTHON_PREFIX}_Interpreter_FOUND)) + if (${_PYTHON_PREFIX}_Interpreter_FOUND) + # reduce possible implementations to the interpreter one + if (${_PYTHON_PREFIX}_INTERPRETER_ID STREQUAL "PyPy") + set (_${_PYTHON_PREFIX}_FIND_IMPLEMENTATIONS "PyPy") + else() + set (_${_PYTHON_PREFIX}_FIND_IMPLEMENTATIONS "CPython") + endif() + else() + list (REMOVE_ITEM _${_PYTHON_PREFIX}_FIND_IMPLEMENTATIONS "IronPython") + endif() + if ("LIBRARY" IN_LIST _${_PYTHON_PREFIX}_FIND_DEVELOPMENT_ARTIFACTS) list (APPEND _${_PYTHON_PREFIX}_CACHED_VARS _${_PYTHON_PREFIX}_LIBRARY_RELEASE _${_PYTHON_PREFIX}_RUNTIME_LIBRARY_RELEASE _${_PYTHON_PREFIX}_LIBRARY_DEBUG - _${_PYTHON_PREFIX}_RUNTIME_LIBRARY_DEBUG - _${_PYTHON_PREFIX}_INCLUDE_DIR) - if (${_PYTHON_PREFIX}_FIND_REQUIRED_Development) - list (APPEND _${_PYTHON_PREFIX}_REQUIRED_VARS ${_PYTHON_PREFIX}_LIBRARIES - ${_PYTHON_PREFIX}_INCLUDE_DIRS) - endif() - - if (DEFINED _${_PYTHON_PREFIX}_LIBRARY_RELEASE OR DEFINED _${_PYTHON_PREFIX}_INCLUDE_DIR) - # compute development signature and check validity of definition - string (MD5 __${_PYTHON_PREFIX}_DEVELOPMENT_SIGNATURE "${_${_PYTHON_PREFIX}_SIGNATURE}:${_${_PYTHON_PREFIX}_LIBRARY_RELEASE}:${_${_PYTHON_PREFIX}_INCLUDE_DIR}") - if (WIN32 AND NOT DEFINED _${_PYTHON_PREFIX}_LIBRARY_DEBUG) - set (_${_PYTHON_PREFIX}_LIBRARY_DEBUG "${_PYTHON_PREFIX}_LIBRARY_DEBUG-NOTFOUND" CACHE INTERNAL "") - endif() - if (NOT DEFINED _${_PYTHON_PREFIX}_INCLUDE_DIR) - set (_${_PYTHON_PREFIX}_INCLUDE_DIR "${_PYTHON_PREFIX}_INCLUDE_DIR-NOTFOUND" CACHE INTERNAL "") - endif() - if (__${_PYTHON_PREFIX}_DEVELOPMENT_SIGNATURE STREQUAL _${_PYTHON_PREFIX}_DEVELOPMENT_SIGNATURE) - # check version validity - if (${_PYTHON_PREFIX}_FIND_VERSION_EXACT) - _python_validate_library (${${_PYTHON_PREFIX}_FIND_VERSION} EXACT CHECK_EXISTS) - _python_validate_include_dir (${${_PYTHON_PREFIX}_FIND_VERSION} EXACT CHECK_EXISTS) - else() - _python_validate_library (${${_PYTHON_PREFIX}_FIND_VERSION} CHECK_EXISTS) - _python_validate_include_dir (${${_PYTHON_PREFIX}_FIND_VERSION} CHECK_EXISTS) - endif() - else() - unset (_${_PYTHON_PREFIX}_LIBRARY_RELEASE CACHE) - unset (_${_PYTHON_PREFIX}_LIBRARY_DEBUG CACHE) - unset (_${_PYTHON_PREFIX}_INCLUDE_DIR CACHE) - endif() + _${_PYTHON_PREFIX}_RUNTIME_LIBRARY_DEBUG) endif() - if (NOT _${_PYTHON_PREFIX}_LIBRARY_RELEASE OR NOT _${_PYTHON_PREFIX}_INCLUDE_DIR) - unset (_${_PYTHON_PREFIX}_CONFIG CACHE) - unset (_${_PYTHON_PREFIX}_DEVELOPMENT_SIGNATURE CACHE) + if ("INCLUDE_DIR" IN_LIST _${_PYTHON_PREFIX}_FIND_DEVELOPMENT_ARTIFACTS) + list (APPEND _${_PYTHON_PREFIX}_CACHED_VARS _${_PYTHON_PREFIX}_INCLUDE_DIR) endif() + _python_check_development_signature (Module) + _python_check_development_signature (Embed) + if (DEFINED ${_PYTHON_PREFIX}_LIBRARY AND IS_ABSOLUTE "${${_PYTHON_PREFIX}_LIBRARY}") set (_${_PYTHON_PREFIX}_LIBRARY_RELEASE "${${_PYTHON_PREFIX}_LIBRARY}" CACHE INTERNAL "") @@ -1702,25 +2201,18 @@ if ("Development" IN_LIST ${_PYTHON_PREFIX}_FIND_COMPONENTS # if python interpreter is found, use it to look-up for artifacts # to ensure consistency between interpreter and development environments. # If not, try to locate a compatible config tool - if (NOT ${_PYTHON_PREFIX}_Interpreter_FOUND OR CMAKE_CROSSCOMPILING) + if ((NOT ${_PYTHON_PREFIX}_Interpreter_FOUND OR CMAKE_CROSSCOMPILING) + AND "CPython" IN_LIST _${_PYTHON_PREFIX}_FIND_IMPLEMENTATIONS) set (_${_PYTHON_PREFIX}_HINTS "${${_PYTHON_PREFIX}_ROOT_DIR}" ENV ${_PYTHON_PREFIX}_ROOT_DIR) unset (_${_PYTHON_PREFIX}_VIRTUALENV_PATHS) if (_${_PYTHON_PREFIX}_FIND_VIRTUALENV MATCHES "^(FIRST|ONLY)$") set (_${_PYTHON_PREFIX}_VIRTUALENV_PATHS ENV VIRTUAL_ENV ENV CONDA_PREFIX) endif() - unset (_${_PYTHON_PREFIX}_FRAMEWORK_PATHS) if (_${_PYTHON_PREFIX}_FIND_STRATEGY STREQUAL "LOCATION") - set (_${_PYTHON_PREFIX}_CONFIG_NAMES) - - foreach (_${_PYTHON_PREFIX}_VERSION IN LISTS _${_PYTHON_PREFIX}_FIND_VERSIONS) - _python_get_names (_${_PYTHON_PREFIX}_VERSION_NAMES VERSION ${_${_PYTHON_PREFIX}_VERSION} POSIX CONFIG) - list (APPEND _${_PYTHON_PREFIX}_CONFIG_NAMES ${_${_PYTHON_PREFIX}_VERSION_NAMES}) - + _python_get_names (_${_PYTHON_PREFIX}_CONFIG_NAMES VERSION ${_${_PYTHON_PREFIX}_FIND_VERSIONS} POSIX CONFIG) # Framework Paths - _python_get_frameworks (_${_PYTHON_PREFIX}_VERSION_PATHS ${_${_PYTHON_PREFIX}_VERSION}) - list (APPEND _${_PYTHON_PREFIX}_FRAMEWORK_PATHS ${_${_PYTHON_PREFIX}_VERSION_PATHS}) - endforeach() + _python_get_frameworks (_${_PYTHON_PREFIX}_FRAMEWORK_PATHS VERSION ${_${_PYTHON_PREFIX}_FIND_VERSIONS}) # Apple frameworks handling if (CMAKE_HOST_APPLE AND _${_PYTHON_PREFIX}_FIND_FRAMEWORK STREQUAL "FIRST") @@ -1804,7 +2296,7 @@ if ("Development" IN_LIST ${_PYTHON_PREFIX}_FIND_COMPONENTS _python_get_names (_${_PYTHON_PREFIX}_CONFIG_NAMES VERSION ${_${_PYTHON_PREFIX}_VERSION} POSIX CONFIG) # Framework Paths - _python_get_frameworks (_${_PYTHON_PREFIX}_FRAMEWORK_PATHS ${_${_PYTHON_PREFIX}_VERSION}) + _python_get_frameworks (_${_PYTHON_PREFIX}_FRAMEWORK_PATHS VERSION ${_${_PYTHON_PREFIX}_VERSION}) # Apple frameworks handling if (CMAKE_HOST_APPLE AND _${_PYTHON_PREFIX}_FIND_FRAMEWORK STREQUAL "FIRST") @@ -1897,140 +2389,61 @@ if ("Development" IN_LIST ${_PYTHON_PREFIX}_FIND_COMPONENTS endif() endif() - if (NOT _${_PYTHON_PREFIX}_LIBRARY_RELEASE) - if ((${_PYTHON_PREFIX}_Interpreter_FOUND AND NOT CMAKE_CROSSCOMPILING) OR _${_PYTHON_PREFIX}_CONFIG) - # retrieve root install directory - _python_get_config_var (_${_PYTHON_PREFIX}_PREFIX PREFIX) - - # enforce current ABI - _python_get_config_var (_${_PYTHON_PREFIX}_ABIFLAGS ABIFLAGS) - - set (_${_PYTHON_PREFIX}_HINTS "${_${_PYTHON_PREFIX}_PREFIX}") - - # retrieve library - ## compute some paths and artifact names - if (_${_PYTHON_PREFIX}_CONFIG) - string (REGEX REPLACE "^.+python([0-9.]+)[a-z]*-config" "\\1" _${_PYTHON_PREFIX}_VERSION "${_${_PYTHON_PREFIX}_CONFIG}") - else() - set (_${_PYTHON_PREFIX}_VERSION "${${_PYTHON_PREFIX}_VERSION_MAJOR}.${${_PYTHON_PREFIX}_VERSION_MINOR}") - endif() - _python_get_path_suffixes (_${_PYTHON_PREFIX}_PATH_SUFFIXES VERSION ${_${_PYTHON_PREFIX}_VERSION} LIBRARY) - _python_get_names (_${_PYTHON_PREFIX}_LIB_NAMES VERSION ${_${_PYTHON_PREFIX}_VERSION} WIN32 POSIX LIBRARY) - - _python_get_config_var (_${_PYTHON_PREFIX}_CONFIGDIR CONFIGDIR) - list (APPEND _${_PYTHON_PREFIX}_HINTS "${_${_PYTHON_PREFIX}_CONFIGDIR}") - - list (APPEND _${_PYTHON_PREFIX}_HINTS "${${_PYTHON_PREFIX}_ROOT_DIR}" ENV ${_PYTHON_PREFIX}_ROOT_DIR) - - find_library (_${_PYTHON_PREFIX}_LIBRARY_RELEASE - NAMES ${_${_PYTHON_PREFIX}_LIB_NAMES} - NAMES_PER_DIR - HINTS ${_${_PYTHON_PREFIX}_HINTS} - PATH_SUFFIXES ${_${_PYTHON_PREFIX}_PATH_SUFFIXES} - NO_SYSTEM_ENVIRONMENT_PATH - NO_CMAKE_SYSTEM_PATH) - endif() - - # Rely on HINTS and standard paths if interpreter or config tool failed to locate artifacts + if ("LIBRARY" IN_LIST _${_PYTHON_PREFIX}_FIND_DEVELOPMENT_ARTIFACTS) if (NOT _${_PYTHON_PREFIX}_LIBRARY_RELEASE) - set (_${_PYTHON_PREFIX}_HINTS "${${_PYTHON_PREFIX}_ROOT_DIR}" ENV ${_PYTHON_PREFIX}_ROOT_DIR) - - unset (_${_PYTHON_PREFIX}_VIRTUALENV_PATHS) - if (_${_PYTHON_PREFIX}_FIND_VIRTUALENV MATCHES "^(FIRST|ONLY)$") - set (_${_PYTHON_PREFIX}_VIRTUALENV_PATHS ENV VIRTUAL_ENV ENV CONDA_PREFIX) - endif() - - if (_${_PYTHON_PREFIX}_FIND_STRATEGY STREQUAL "LOCATION") - unset (_${_PYTHON_PREFIX}_LIB_NAMES) - unset (_${_PYTHON_PREFIX}_LIB_NAMES_DEBUG) - unset (_${_PYTHON_PREFIX}_FRAMEWORK_PATHS) - unset (_${_PYTHON_PREFIX}_REGISTRY_PATHS) - unset (_${_PYTHON_PREFIX}_PATH_SUFFIXES) - - foreach (_${_PYTHON_PREFIX}_LIB_VERSION IN LISTS _${_PYTHON_PREFIX}_FIND_VERSIONS) - # library names - _python_get_names (_${_PYTHON_PREFIX}_VERSION_NAMES VERSION ${_${_PYTHON_PREFIX}_LIB_VERSION} WIN32 POSIX LIBRARY) - list (APPEND _${_PYTHON_PREFIX}_LIB_NAMES ${_${_PYTHON_PREFIX}_VERSION_NAMES}) - _python_get_names (_${_PYTHON_PREFIX}_VERSION_NAMES VERSION ${_${_PYTHON_PREFIX}_LIB_VERSION} WIN32 DEBUG) - list (APPEND _${_PYTHON_PREFIX}_LIB_NAMES_DEBUG ${_${_PYTHON_PREFIX}_VERSION_NAMES}) - - # Framework Paths - _python_get_frameworks (_${_PYTHON_PREFIX}_VERSION_PATHS ${_${_PYTHON_PREFIX}_LIB_VERSION}) - list (APPEND _${_PYTHON_PREFIX}_FRAMEWORK_PATHS ${_${_PYTHON_PREFIX}_VERSION_PATHS}) + if ((${_PYTHON_PREFIX}_Interpreter_FOUND AND NOT CMAKE_CROSSCOMPILING) OR _${_PYTHON_PREFIX}_CONFIG) + # retrieve root install directory + _python_get_config_var (_${_PYTHON_PREFIX}_PREFIX PREFIX) - # Registry Paths - _python_get_registries (_${_PYTHON_PREFIX}_VERSION_PATHS ${_${_PYTHON_PREFIX}_LIB_VERSION}) - list (APPEND _${_PYTHON_PREFIX}_REGISTRY_PATHS ${_${_PYTHON_PREFIX}_VERSION_PATHS}) + # enforce current ABI + _python_get_config_var (_${_PYTHON_PREFIX}_ABIFLAGS ABIFLAGS) - # Paths suffixes - _python_get_path_suffixes (_${_PYTHON_PREFIX}_VERSION_PATHS VERSION ${_${_PYTHON_PREFIX}_LIB_VERSION} LIBRARY) - list (APPEND _${_PYTHON_PREFIX}_PATH_SUFFIXES ${_${_PYTHON_PREFIX}_VERSION_PATHS}) - endforeach() + set (_${_PYTHON_PREFIX}_HINTS "${_${_PYTHON_PREFIX}_PREFIX}") - if (APPLE AND _${_PYTHON_PREFIX}_FIND_FRAMEWORK STREQUAL "FIRST") - find_library (_${_PYTHON_PREFIX}_LIBRARY_RELEASE - NAMES ${_${_PYTHON_PREFIX}_LIB_NAMES} - NAMES_PER_DIR - HINTS ${_${_PYTHON_PREFIX}_HINTS} - PATHS ${_${_PYTHON_PREFIX}_VIRTUALENV_PATHS} - ${_${_PYTHON_PREFIX}_FRAMEWORK_PATHS} - PATH_SUFFIXES ${_${_PYTHON_PREFIX}_PATH_SUFFIXES} - NO_CMAKE_PATH - NO_CMAKE_ENVIRONMENT_PATH - NO_SYSTEM_ENVIRONMENT_PATH - NO_CMAKE_SYSTEM_PATH) + # retrieve library + ## compute some paths and artifact names + if (_${_PYTHON_PREFIX}_CONFIG) + string (REGEX REPLACE "^.+python([0-9.]+)[a-z]*-config" "\\1" _${_PYTHON_PREFIX}_VERSION "${_${_PYTHON_PREFIX}_CONFIG}") + else() + set (_${_PYTHON_PREFIX}_VERSION "${${_PYTHON_PREFIX}_VERSION_MAJOR}.${${_PYTHON_PREFIX}_VERSION_MINOR}") endif() + _python_get_path_suffixes (_${_PYTHON_PREFIX}_PATH_SUFFIXES VERSION ${_${_PYTHON_PREFIX}_VERSION} LIBRARY) + _python_get_names (_${_PYTHON_PREFIX}_LIB_NAMES VERSION ${_${_PYTHON_PREFIX}_VERSION} WIN32 POSIX LIBRARY) - if (WIN32 AND _${_PYTHON_PREFIX}_FIND_REGISTRY STREQUAL "FIRST") - find_library (_${_PYTHON_PREFIX}_LIBRARY_RELEASE - NAMES ${_${_PYTHON_PREFIX}_LIB_NAMES} - NAMES_PER_DIR - HINTS ${_${_PYTHON_PREFIX}_HINTS} - PATHS ${_${_PYTHON_PREFIX}_VIRTUALENV_PATHS} - ${_${_PYTHON_PREFIX}_REGISTRY_PATHS} - PATH_SUFFIXES ${_${_PYTHON_PREFIX}_PATH_SUFFIXES} - NO_SYSTEM_ENVIRONMENT_PATH - NO_CMAKE_SYSTEM_PATH) - endif() + _python_get_config_var (_${_PYTHON_PREFIX}_CONFIGDIR CONFIGDIR) + list (APPEND _${_PYTHON_PREFIX}_HINTS "${_${_PYTHON_PREFIX}_CONFIGDIR}") + + list (APPEND _${_PYTHON_PREFIX}_HINTS "${${_PYTHON_PREFIX}_ROOT_DIR}" ENV ${_PYTHON_PREFIX}_ROOT_DIR) - # search in HINTS locations find_library (_${_PYTHON_PREFIX}_LIBRARY_RELEASE NAMES ${_${_PYTHON_PREFIX}_LIB_NAMES} NAMES_PER_DIR HINTS ${_${_PYTHON_PREFIX}_HINTS} - PATHS ${_${_PYTHON_PREFIX}_VIRTUALENV_PATHS} PATH_SUFFIXES ${_${_PYTHON_PREFIX}_PATH_SUFFIXES} NO_SYSTEM_ENVIRONMENT_PATH NO_CMAKE_SYSTEM_PATH) + endif() - if (APPLE AND _${_PYTHON_PREFIX}_FIND_FRAMEWORK STREQUAL "LAST") - set (__${_PYTHON_PREFIX}_FRAMEWORK_PATHS ${_${_PYTHON_PREFIX}_FRAMEWORK_PATHS}) - else() - unset (__${_PYTHON_PREFIX}_FRAMEWORK_PATHS) - endif() + # Rely on HINTS and standard paths if interpreter or config tool failed to locate artifacts + if (NOT _${_PYTHON_PREFIX}_LIBRARY_RELEASE) + set (_${_PYTHON_PREFIX}_HINTS "${${_PYTHON_PREFIX}_ROOT_DIR}" ENV ${_PYTHON_PREFIX}_ROOT_DIR) - if (WIN32 AND _${_PYTHON_PREFIX}_FIND_REGISTRY STREQUAL "LAST") - set (__${_PYTHON_PREFIX}_REGISTRY_PATHS ${_${_PYTHON_PREFIX}_REGISTRY_PATHS}) - else() - unset (__${_PYTHON_PREFIX}_REGISTRY_PATHS) + unset (_${_PYTHON_PREFIX}_VIRTUALENV_PATHS) + if (_${_PYTHON_PREFIX}_FIND_VIRTUALENV MATCHES "^(FIRST|ONLY)$") + set (_${_PYTHON_PREFIX}_VIRTUALENV_PATHS ENV VIRTUAL_ENV ENV CONDA_PREFIX) endif() - # search in all default paths - find_library (_${_PYTHON_PREFIX}_LIBRARY_RELEASE - NAMES ${_${_PYTHON_PREFIX}_LIB_NAMES} - NAMES_PER_DIR - PATHS ${__${_PYTHON_PREFIX}_FRAMEWORK_PATHS} - ${__${_PYTHON_PREFIX}_REGISTRY_PATHS} - PATH_SUFFIXES ${_${_PYTHON_PREFIX}_PATH_SUFFIXES}) - else() - foreach (_${_PYTHON_PREFIX}_LIB_VERSION IN LISTS _${_PYTHON_PREFIX}_FIND_VERSIONS) - _python_get_names (_${_PYTHON_PREFIX}_LIB_NAMES VERSION ${_${_PYTHON_PREFIX}_LIB_VERSION} WIN32 POSIX LIBRARY) - _python_get_names (_${_PYTHON_PREFIX}_LIB_NAMES_DEBUG VERSION ${_${_PYTHON_PREFIX}_LIB_VERSION} WIN32 DEBUG) - - _python_get_frameworks (_${_PYTHON_PREFIX}_FRAMEWORK_PATHS ${_${_PYTHON_PREFIX}_LIB_VERSION}) - _python_get_registries (_${_PYTHON_PREFIX}_REGISTRY_PATHS ${_${_PYTHON_PREFIX}_LIB_VERSION}) + if (_${_PYTHON_PREFIX}_FIND_STRATEGY STREQUAL "LOCATION") + # library names + _python_get_names (_${_PYTHON_PREFIX}_LIB_NAMES VERSION ${_${_PYTHON_PREFIX}_FIND_VERSIONS} WIN32 POSIX LIBRARY) + _python_get_names (_${_PYTHON_PREFIX}_LIB_NAMES_DEBUG VERSION ${_${_PYTHON_PREFIX}_FIND_VERSIONS} WIN32 DEBUG) + # Paths suffixes + _python_get_path_suffixes (_${_PYTHON_PREFIX}_PATH_SUFFIXES VERSION ${_${_PYTHON_PREFIX}_FIND_VERSIONS} LIBRARY) - _python_get_path_suffixes (_${_PYTHON_PREFIX}_PATH_SUFFIXES VERSION ${_${_PYTHON_PREFIX}_LIB_VERSION} LIBRARY) + # Framework Paths + _python_get_frameworks (_${_PYTHON_PREFIX}_FRAMEWORK_PATHS VERSION ${_${_PYTHON_PREFIX}_LIB_FIND_VERSIONS}) + # Registry Paths + _python_get_registries (_${_PYTHON_PREFIX}_REGISTRY_PATHS VERSION ${_${_PYTHON_PREFIX}_FIND_VERSIONS} ) if (APPLE AND _${_PYTHON_PREFIX}_FIND_FRAMEWORK STREQUAL "FIRST") find_library (_${_PYTHON_PREFIX}_LIBRARY_RELEASE @@ -2068,184 +2481,288 @@ if ("Development" IN_LIST ${_PYTHON_PREFIX}_FIND_COMPONENTS NO_SYSTEM_ENVIRONMENT_PATH NO_CMAKE_SYSTEM_PATH) - if (APPLE AND _${_PYTHON_PREFIX}_FIND_FRAMEWORK STREQUAL "LAST") - set (__${_PYTHON_PREFIX}_FRAMEWORK_PATHS ${_${_PYTHON_PREFIX}_FRAMEWORK_PATHS}) - else() - unset (__${_PYTHON_PREFIX}_FRAMEWORK_PATHS) - endif() + if (APPLE AND _${_PYTHON_PREFIX}_FIND_FRAMEWORK STREQUAL "LAST") + set (__${_PYTHON_PREFIX}_FRAMEWORK_PATHS ${_${_PYTHON_PREFIX}_FRAMEWORK_PATHS}) + else() + unset (__${_PYTHON_PREFIX}_FRAMEWORK_PATHS) + endif() - if (WIN32 AND _${_PYTHON_PREFIX}_FIND_REGISTRY STREQUAL "LAST") - set (__${_PYTHON_PREFIX}_REGISTRY_PATHS ${_${_PYTHON_PREFIX}_REGISTRY_PATHS}) - else() - unset (__${_PYTHON_PREFIX}_REGISTRY_PATHS) - endif() + if (WIN32 AND _${_PYTHON_PREFIX}_FIND_REGISTRY STREQUAL "LAST") + set (__${_PYTHON_PREFIX}_REGISTRY_PATHS ${_${_PYTHON_PREFIX}_REGISTRY_PATHS}) + else() + unset (__${_PYTHON_PREFIX}_REGISTRY_PATHS) + endif() - # search in all default paths - find_library (_${_PYTHON_PREFIX}_LIBRARY_RELEASE + # search in all default paths + find_library (_${_PYTHON_PREFIX}_LIBRARY_RELEASE NAMES ${_${_PYTHON_PREFIX}_LIB_NAMES} NAMES_PER_DIR PATHS ${__${_PYTHON_PREFIX}_FRAMEWORK_PATHS} ${__${_PYTHON_PREFIX}_REGISTRY_PATHS} PATH_SUFFIXES ${_${_PYTHON_PREFIX}_PATH_SUFFIXES}) + else() + foreach (_${_PYTHON_PREFIX}_LIB_VERSION IN LISTS _${_PYTHON_PREFIX}_FIND_VERSIONS) + _python_get_names (_${_PYTHON_PREFIX}_LIB_NAMES VERSION ${_${_PYTHON_PREFIX}_LIB_VERSION} WIN32 POSIX LIBRARY) + _python_get_names (_${_PYTHON_PREFIX}_LIB_NAMES_DEBUG VERSION ${_${_PYTHON_PREFIX}_LIB_VERSION} WIN32 DEBUG) + + _python_get_frameworks (_${_PYTHON_PREFIX}_FRAMEWORK_PATHS VERSION ${_${_PYTHON_PREFIX}_LIB_VERSION}) + _python_get_registries (_${_PYTHON_PREFIX}_REGISTRY_PATHS VERSION ${_${_PYTHON_PREFIX}_LIB_VERSION}) + + _python_get_path_suffixes (_${_PYTHON_PREFIX}_PATH_SUFFIXES VERSION ${_${_PYTHON_PREFIX}_LIB_VERSION} LIBRARY) + + if (APPLE AND _${_PYTHON_PREFIX}_FIND_FRAMEWORK STREQUAL "FIRST") + find_library (_${_PYTHON_PREFIX}_LIBRARY_RELEASE + NAMES ${_${_PYTHON_PREFIX}_LIB_NAMES} + NAMES_PER_DIR + HINTS ${_${_PYTHON_PREFIX}_HINTS} + PATHS ${_${_PYTHON_PREFIX}_VIRTUALENV_PATHS} + ${_${_PYTHON_PREFIX}_FRAMEWORK_PATHS} + PATH_SUFFIXES ${_${_PYTHON_PREFIX}_PATH_SUFFIXES} + NO_CMAKE_PATH + NO_CMAKE_ENVIRONMENT_PATH + NO_SYSTEM_ENVIRONMENT_PATH + NO_CMAKE_SYSTEM_PATH) + endif() - if (_${_PYTHON_PREFIX}_LIBRARY_RELEASE) - break() - endif() - endforeach() + if (WIN32 AND _${_PYTHON_PREFIX}_FIND_REGISTRY STREQUAL "FIRST") + find_library (_${_PYTHON_PREFIX}_LIBRARY_RELEASE + NAMES ${_${_PYTHON_PREFIX}_LIB_NAMES} + NAMES_PER_DIR + HINTS ${_${_PYTHON_PREFIX}_HINTS} + PATHS ${_${_PYTHON_PREFIX}_VIRTUALENV_PATHS} + ${_${_PYTHON_PREFIX}_REGISTRY_PATHS} + PATH_SUFFIXES ${_${_PYTHON_PREFIX}_PATH_SUFFIXES} + NO_SYSTEM_ENVIRONMENT_PATH + NO_CMAKE_SYSTEM_PATH) + endif() + + # search in HINTS locations + find_library (_${_PYTHON_PREFIX}_LIBRARY_RELEASE + NAMES ${_${_PYTHON_PREFIX}_LIB_NAMES} + NAMES_PER_DIR + HINTS ${_${_PYTHON_PREFIX}_HINTS} + PATHS ${_${_PYTHON_PREFIX}_VIRTUALENV_PATHS} + PATH_SUFFIXES ${_${_PYTHON_PREFIX}_PATH_SUFFIXES} + NO_SYSTEM_ENVIRONMENT_PATH + NO_CMAKE_SYSTEM_PATH) + + if (APPLE AND _${_PYTHON_PREFIX}_FIND_FRAMEWORK STREQUAL "LAST") + set (__${_PYTHON_PREFIX}_FRAMEWORK_PATHS ${_${_PYTHON_PREFIX}_FRAMEWORK_PATHS}) + else() + unset (__${_PYTHON_PREFIX}_FRAMEWORK_PATHS) + endif() + + if (WIN32 AND _${_PYTHON_PREFIX}_FIND_REGISTRY STREQUAL "LAST") + set (__${_PYTHON_PREFIX}_REGISTRY_PATHS ${_${_PYTHON_PREFIX}_REGISTRY_PATHS}) + else() + unset (__${_PYTHON_PREFIX}_REGISTRY_PATHS) + endif() + + # search in all default paths + find_library (_${_PYTHON_PREFIX}_LIBRARY_RELEASE + NAMES ${_${_PYTHON_PREFIX}_LIB_NAMES} + NAMES_PER_DIR + PATHS ${__${_PYTHON_PREFIX}_FRAMEWORK_PATHS} + ${__${_PYTHON_PREFIX}_REGISTRY_PATHS} + PATH_SUFFIXES ${_${_PYTHON_PREFIX}_PATH_SUFFIXES}) + + if (_${_PYTHON_PREFIX}_LIBRARY_RELEASE) + break() + endif() + endforeach() + endif() endif() endif() - endif() - # finalize library version information - _python_get_version (LIBRARY PREFIX _${_PYTHON_PREFIX}_) + # finalize library version information + _python_get_version (LIBRARY PREFIX _${_PYTHON_PREFIX}_) + if (_${_PYTHON_PREFIX}_VERSION EQUAL "${_${_PYTHON_PREFIX}_REQUIRED_VERSION_MAJOR}") + # not able to extract full version from library name + if (${_PYTHON_PREFIX}_Interpreter_FOUND) + # update from interpreter + set (_${_PYTHON_PREFIX}_VERSION ${${_PYTHON_PREFIX}_VERSION}) + set (_${_PYTHON_PREFIX}_VERSION_MAJOR ${${_PYTHON_PREFIX}_VERSION_MAJOR}) + set (_${_PYTHON_PREFIX}_VERSION_MINOR ${${_PYTHON_PREFIX}_VERSION_MINOR}) + set (_${_PYTHON_PREFIX}_VERSION_PATCH ${${_PYTHON_PREFIX}_VERSION_PATCH}) + endif() + endif() - set (${_PYTHON_PREFIX}_LIBRARY_RELEASE "${_${_PYTHON_PREFIX}_LIBRARY_RELEASE}") + set (${_PYTHON_PREFIX}_LIBRARY_RELEASE "${_${_PYTHON_PREFIX}_LIBRARY_RELEASE}") - if (_${_PYTHON_PREFIX}_LIBRARY_RELEASE AND NOT EXISTS "${_${_PYTHON_PREFIX}_LIBRARY_RELEASE}") - set (_${_PYTHON_PREFIX}_Development_REASON_FAILURE "Cannot find the library \"${_${_PYTHON_PREFIX}_LIBRARY_RELEASE}\"") - set_property (CACHE _${_PYTHON_PREFIX}_LIBRARY_RELEASE PROPERTY VALUE "${_PYTHON_PREFIX}_LIBRARY_RELEASE-NOTFOUND") - endif() + if (_${_PYTHON_PREFIX}_LIBRARY_RELEASE AND NOT EXISTS "${_${_PYTHON_PREFIX}_LIBRARY_RELEASE}") + set (_${_PYTHON_PREFIX}_Development_REASON_FAILURE "Cannot find the library \"${_${_PYTHON_PREFIX}_LIBRARY_RELEASE}\"") + set_property (CACHE _${_PYTHON_PREFIX}_LIBRARY_RELEASE PROPERTY VALUE "${_PYTHON_PREFIX}_LIBRARY_RELEASE-NOTFOUND") + endif() - set (_${_PYTHON_PREFIX}_HINTS "${${_PYTHON_PREFIX}_ROOT_DIR}" ENV ${_PYTHON_PREFIX}_ROOT_DIR) - - if (WIN32 AND _${_PYTHON_PREFIX}_LIBRARY_RELEASE) - # search for debug library - # use release library location as a hint - _python_get_names (_${_PYTHON_PREFIX}_LIB_NAMES_DEBUG VERSION ${_${_PYTHON_PREFIX}_VERSION} WIN32 DEBUG) - get_filename_component (_${_PYTHON_PREFIX}_PATH "${_${_PYTHON_PREFIX}_LIBRARY_RELEASE}" DIRECTORY) - find_library (_${_PYTHON_PREFIX}_LIBRARY_DEBUG - NAMES ${_${_PYTHON_PREFIX}_LIB_NAMES_DEBUG} - NAMES_PER_DIR - HINTS "${_${_PYTHON_PREFIX}_PATH}" ${_${_PYTHON_PREFIX}_HINTS} - NO_DEFAULT_PATH) - endif() - - # retrieve runtime libraries - if (_${_PYTHON_PREFIX}_LIBRARY_RELEASE) - _python_get_names (_${_PYTHON_PREFIX}_LIB_NAMES VERSION ${_${_PYTHON_PREFIX}_VERSION} WIN32 POSIX LIBRARY) - get_filename_component (_${_PYTHON_PREFIX}_PATH "${_${_PYTHON_PREFIX}_LIBRARY_RELEASE}" DIRECTORY) - get_filename_component (_${_PYTHON_PREFIX}_PATH2 "${_${_PYTHON_PREFIX}_PATH}" DIRECTORY) - _python_find_runtime_library (_${_PYTHON_PREFIX}_RUNTIME_LIBRARY_RELEASE - NAMES ${_${_PYTHON_PREFIX}_LIB_NAMES} - NAMES_PER_DIR - HINTS "${_${_PYTHON_PREFIX}_PATH}" "${_${_PYTHON_PREFIX}_PATH2}" ${_${_PYTHON_PREFIX}_HINTS} - PATH_SUFFIXES bin) - endif() - if (_${_PYTHON_PREFIX}_LIBRARY_DEBUG) - _python_get_names (_${_PYTHON_PREFIX}_LIB_NAMES_DEBUG VERSION ${_${_PYTHON_PREFIX}_VERSION} WIN32 DEBUG) - get_filename_component (_${_PYTHON_PREFIX}_PATH "${_${_PYTHON_PREFIX}_LIBRARY_DEBUG}" DIRECTORY) - get_filename_component (_${_PYTHON_PREFIX}_PATH2 "${_${_PYTHON_PREFIX}_PATH}" DIRECTORY) - _python_find_runtime_library (_${_PYTHON_PREFIX}_RUNTIME_LIBRARY_DEBUG - NAMES ${_${_PYTHON_PREFIX}_LIB_NAMES_DEBUG} - NAMES_PER_DIR - HINTS "${_${_PYTHON_PREFIX}_PATH}" "${_${_PYTHON_PREFIX}_PATH2}" ${_${_PYTHON_PREFIX}_HINTS} - PATH_SUFFIXES bin) - endif() - - # Don't search for include dir if no library was founded - if (_${_PYTHON_PREFIX}_LIBRARY_RELEASE AND NOT _${_PYTHON_PREFIX}_INCLUDE_DIR) - if ((${_PYTHON_PREFIX}_Interpreter_FOUND AND NOT CMAKE_CROSSCOMPILING) OR _${_PYTHON_PREFIX}_CONFIG) - _python_get_config_var (_${_PYTHON_PREFIX}_INCLUDE_DIRS INCLUDES) + set (_${_PYTHON_PREFIX}_HINTS "${${_PYTHON_PREFIX}_ROOT_DIR}" ENV ${_PYTHON_PREFIX}_ROOT_DIR) - find_path (_${_PYTHON_PREFIX}_INCLUDE_DIR - NAMES Python.h - HINTS ${_${_PYTHON_PREFIX}_INCLUDE_DIRS} - NO_SYSTEM_ENVIRONMENT_PATH - NO_CMAKE_SYSTEM_PATH) + if (WIN32 AND _${_PYTHON_PREFIX}_LIBRARY_RELEASE) + # search for debug library + # use release library location as a hint + _python_get_names (_${_PYTHON_PREFIX}_LIB_NAMES_DEBUG VERSION ${_${_PYTHON_PREFIX}_VERSION} WIN32 DEBUG) + get_filename_component (_${_PYTHON_PREFIX}_PATH "${_${_PYTHON_PREFIX}_LIBRARY_RELEASE}" DIRECTORY) + find_library (_${_PYTHON_PREFIX}_LIBRARY_DEBUG + NAMES ${_${_PYTHON_PREFIX}_LIB_NAMES_DEBUG} + NAMES_PER_DIR + HINTS "${_${_PYTHON_PREFIX}_PATH}" ${_${_PYTHON_PREFIX}_HINTS} + NO_DEFAULT_PATH) + # second try including CMAKE variables to catch-up non conventional layouts + find_library (_${_PYTHON_PREFIX}_LIBRARY_DEBUG + NAMES ${_${_PYTHON_PREFIX}_LIB_NAMES_DEBUG} + NAMES_PER_DIR + NO_SYSTEM_ENVIRONMENT_PATH + NO_CMAKE_SYSTEM_PATH) endif() - # Rely on HINTS and standard paths if interpreter or config tool failed to locate artifacts - if (NOT _${_PYTHON_PREFIX}_INCLUDE_DIR) - unset (_${_PYTHON_PREFIX}_VIRTUALENV_PATHS) - if (_${_PYTHON_PREFIX}_FIND_VIRTUALENV MATCHES "^(FIRST|ONLY)$") - set (_${_PYTHON_PREFIX}_VIRTUALENV_PATHS ENV VIRTUAL_ENV ENV CONDA_PREFIX) - endif() - unset (_${_PYTHON_PREFIX}_INCLUDE_HINTS) - - # Use the library's install prefix as a hint - if (${_${_PYTHON_PREFIX}_LIBRARY_RELEASE} MATCHES "^(.+/Frameworks/Python.framework/Versions/[0-9.]+)") - list (APPEND _${_PYTHON_PREFIX}_INCLUDE_HINTS "${CMAKE_MATCH_1}") - elseif (${_${_PYTHON_PREFIX}_LIBRARY_RELEASE} MATCHES "^(.+)/lib(64|32)?/python[0-9.]+/config") - list (APPEND _${_PYTHON_PREFIX}_INCLUDE_HINTS "${CMAKE_MATCH_1}") - elseif (DEFINED CMAKE_LIBRARY_ARCHITECTURE AND ${_${_PYTHON_PREFIX}_LIBRARY_RELEASE} MATCHES "^(.+)/lib/${CMAKE_LIBRARY_ARCHITECTURE}") - list (APPEND _${_PYTHON_PREFIX}_INCLUDE_HINTS "${CMAKE_MATCH_1}") - else() - # assume library is in a directory under root - get_filename_component (_${_PYTHON_PREFIX}_PREFIX "${_${_PYTHON_PREFIX}_LIBRARY_RELEASE}" DIRECTORY) - get_filename_component (_${_PYTHON_PREFIX}_PREFIX "${_${_PYTHON_PREFIX}_PREFIX}" DIRECTORY) - list (APPEND _${_PYTHON_PREFIX}_INCLUDE_HINTS "${_${_PYTHON_PREFIX}_PREFIX}") + # retrieve runtime libraries + if (_${_PYTHON_PREFIX}_LIBRARY_RELEASE) + _python_get_names (_${_PYTHON_PREFIX}_LIB_NAMES VERSION ${_${_PYTHON_PREFIX}_VERSION} WIN32 POSIX LIBRARY) + get_filename_component (_${_PYTHON_PREFIX}_PATH "${_${_PYTHON_PREFIX}_LIBRARY_RELEASE}" DIRECTORY) + get_filename_component (_${_PYTHON_PREFIX}_PATH2 "${_${_PYTHON_PREFIX}_PATH}" DIRECTORY) + _python_find_runtime_library (_${_PYTHON_PREFIX}_RUNTIME_LIBRARY_RELEASE + NAMES ${_${_PYTHON_PREFIX}_LIB_NAMES} + NAMES_PER_DIR + HINTS "${_${_PYTHON_PREFIX}_PATH}" + "${_${_PYTHON_PREFIX}_PATH2}" ${_${_PYTHON_PREFIX}_HINTS} + PATH_SUFFIXES bin) + endif() + if (_${_PYTHON_PREFIX}_LIBRARY_DEBUG) + _python_get_names (_${_PYTHON_PREFIX}_LIB_NAMES_DEBUG VERSION ${_${_PYTHON_PREFIX}_VERSION} WIN32 DEBUG) + get_filename_component (_${_PYTHON_PREFIX}_PATH "${_${_PYTHON_PREFIX}_LIBRARY_DEBUG}" DIRECTORY) + get_filename_component (_${_PYTHON_PREFIX}_PATH2 "${_${_PYTHON_PREFIX}_PATH}" DIRECTORY) + _python_find_runtime_library (_${_PYTHON_PREFIX}_RUNTIME_LIBRARY_DEBUG + NAMES ${_${_PYTHON_PREFIX}_LIB_NAMES_DEBUG} + NAMES_PER_DIR + HINTS "${_${_PYTHON_PREFIX}_PATH}" + "${_${_PYTHON_PREFIX}_PATH2}" ${_${_PYTHON_PREFIX}_HINTS} + PATH_SUFFIXES bin) + endif() + endif() + + if ("INCLUDE_DIR" IN_LIST _${_PYTHON_PREFIX}_FIND_DEVELOPMENT_ARTIFACTS) + while (NOT _${_PYTHON_PREFIX}_INCLUDE_DIR) + if ("LIBRARY" IN_LIST _${_PYTHON_PREFIX}_FIND_DEVELOPMENT_ARTIFACTS + AND NOT _${_PYTHON_PREFIX}_LIBRARY_RELEASE) + # Don't search for include dir if no library was founded + break() endif() - _python_get_frameworks (_${_PYTHON_PREFIX}_FRAMEWORK_PATHS ${_${_PYTHON_PREFIX}_VERSION}) - _python_get_registries (_${_PYTHON_PREFIX}_REGISTRY_PATHS ${_${_PYTHON_PREFIX}_VERSION}) - _python_get_path_suffixes (_${_PYTHON_PREFIX}_PATH_SUFFIXES VERSION ${_${_PYTHON_PREFIX}_VERSION} INCLUDE) + if ((${_PYTHON_PREFIX}_Interpreter_FOUND AND NOT CMAKE_CROSSCOMPILING) OR _${_PYTHON_PREFIX}_CONFIG) + _python_get_config_var (_${_PYTHON_PREFIX}_INCLUDE_DIRS INCLUDES) - if (APPLE AND _${_PYTHON_PREFIX}_FIND_FRAMEWORK STREQUAL "FIRST") find_path (_${_PYTHON_PREFIX}_INCLUDE_DIR - NAMES Python.h - HINTS ${_${_PYTHON_PREFIX}_INCLUDE_HINTS} ${_${_PYTHON_PREFIX}_HINTS} - PATHS ${_${_PYTHON_PREFIX}_VIRTUALENV_PATHS} - ${_${_PYTHON_PREFIX}_FRAMEWORK_PATHS} - PATH_SUFFIXES ${_${_PYTHON_PREFIX}_PATH_SUFFIXES} - NO_CMAKE_PATH - NO_CMAKE_ENVIRONMENT_PATH + NAMES ${_${_PYTHON_PREFIX}_INCLUDE_NAMES} + HINTS ${_${_PYTHON_PREFIX}_INCLUDE_DIRS} NO_SYSTEM_ENVIRONMENT_PATH NO_CMAKE_SYSTEM_PATH) endif() - if (WIN32 AND _${_PYTHON_PREFIX}_FIND_REGISTRY STREQUAL "FIRST") + # Rely on HINTS and standard paths if interpreter or config tool failed to locate artifacts + if (NOT _${_PYTHON_PREFIX}_INCLUDE_DIR) + unset (_${_PYTHON_PREFIX}_VIRTUALENV_PATHS) + if (_${_PYTHON_PREFIX}_FIND_VIRTUALENV MATCHES "^(FIRST|ONLY)$") + set (_${_PYTHON_PREFIX}_VIRTUALENV_PATHS ENV VIRTUAL_ENV ENV CONDA_PREFIX) + endif() + unset (_${_PYTHON_PREFIX}_INCLUDE_HINTS) + + if (_${_PYTHON_PREFIX}_LIBRARY_RELEASE) + # Use the library's install prefix as a hint + if (_${_PYTHON_PREFIX}_LIBRARY_RELEASE MATCHES "^(.+/Frameworks/Python.framework/Versions/[0-9.]+)") + list (APPEND _${_PYTHON_PREFIX}_INCLUDE_HINTS "${CMAKE_MATCH_1}") + elseif (_${_PYTHON_PREFIX}_LIBRARY_RELEASE MATCHES "^(.+)/lib(64|32)?/python[0-9.]+/config") + list (APPEND _${_PYTHON_PREFIX}_INCLUDE_HINTS "${CMAKE_MATCH_1}") + elseif (DEFINED CMAKE_LIBRARY_ARCHITECTURE AND ${_${_PYTHON_PREFIX}_LIBRARY_RELEASE} MATCHES "^(.+)/lib/${CMAKE_LIBRARY_ARCHITECTURE}") + list (APPEND _${_PYTHON_PREFIX}_INCLUDE_HINTS "${CMAKE_MATCH_1}") + else() + # assume library is in a directory under root + get_filename_component (_${_PYTHON_PREFIX}_PREFIX "${_${_PYTHON_PREFIX}_LIBRARY_RELEASE}" DIRECTORY) + get_filename_component (_${_PYTHON_PREFIX}_PREFIX "${_${_PYTHON_PREFIX}_PREFIX}" DIRECTORY) + list (APPEND _${_PYTHON_PREFIX}_INCLUDE_HINTS "${_${_PYTHON_PREFIX}_PREFIX}") + endif() + endif() + + _python_get_frameworks (_${_PYTHON_PREFIX}_FRAMEWORK_PATHS VERSION ${_${_PYTHON_PREFIX}_VERSION}) + _python_get_registries (_${_PYTHON_PREFIX}_REGISTRY_PATHS VERSION ${_${_PYTHON_PREFIX}_VERSION}) + _python_get_path_suffixes (_${_PYTHON_PREFIX}_PATH_SUFFIXES VERSION ${_${_PYTHON_PREFIX}_VERSION} INCLUDE) + + if (APPLE AND _${_PYTHON_PREFIX}_FIND_FRAMEWORK STREQUAL "FIRST") + find_path (_${_PYTHON_PREFIX}_INCLUDE_DIR + NAMES ${_${_PYTHON_PREFIX}_INCLUDE_NAMES} + HINTS ${_${_PYTHON_PREFIX}_INCLUDE_HINTS} ${_${_PYTHON_PREFIX}_HINTS} + PATHS ${_${_PYTHON_PREFIX}_VIRTUALENV_PATHS} + ${_${_PYTHON_PREFIX}_FRAMEWORK_PATHS} + PATH_SUFFIXES ${_${_PYTHON_PREFIX}_PATH_SUFFIXES} + NO_CMAKE_PATH + NO_CMAKE_ENVIRONMENT_PATH + NO_SYSTEM_ENVIRONMENT_PATH + NO_CMAKE_SYSTEM_PATH) + endif() + + if (WIN32 AND _${_PYTHON_PREFIX}_FIND_REGISTRY STREQUAL "FIRST") + find_path (_${_PYTHON_PREFIX}_INCLUDE_DIR + NAMES ${_${_PYTHON_PREFIX}_INCLUDE_NAMES} + HINTS ${_${_PYTHON_PREFIX}_INCLUDE_HINTS} ${_${_PYTHON_PREFIX}_HINTS} + PATHS ${_${_PYTHON_PREFIX}_VIRTUALENV_PATHS} + ${_${_PYTHON_PREFIX}_REGISTRY_PATHS} + PATH_SUFFIXES ${_${_PYTHON_PREFIX}_PATH_SUFFIXES} + NO_SYSTEM_ENVIRONMENT_PATH + NO_CMAKE_SYSTEM_PATH) + endif() + + if (APPLE AND _${_PYTHON_PREFIX}_FIND_FRAMEWORK STREQUAL "LAST") + set (__${_PYTHON_PREFIX}_FRAMEWORK_PATHS ${_${_PYTHON_PREFIX}_FRAMEWORK_PATHS}) + else() + unset (__${_PYTHON_PREFIX}_FRAMEWORK_PATHS) + endif() + + if (WIN32 AND _${_PYTHON_PREFIX}_FIND_REGISTRY STREQUAL "LAST") + set (__${_PYTHON_PREFIX}_REGISTRY_PATHS ${_${_PYTHON_PREFIX}_REGISTRY_PATHS}) + else() + unset (__${_PYTHON_PREFIX}_REGISTRY_PATHS) + endif() + find_path (_${_PYTHON_PREFIX}_INCLUDE_DIR - NAMES Python.h + NAMES ${_${_PYTHON_PREFIX}_INCLUDE_NAMES} HINTS ${_${_PYTHON_PREFIX}_INCLUDE_HINTS} ${_${_PYTHON_PREFIX}_HINTS} PATHS ${_${_PYTHON_PREFIX}_VIRTUALENV_PATHS} - ${_${_PYTHON_PREFIX}_REGISTRY_PATHS} + ${__${_PYTHON_PREFIX}_FRAMEWORK_PATHS} + ${__${_PYTHON_PREFIX}_REGISTRY_PATHS} PATH_SUFFIXES ${_${_PYTHON_PREFIX}_PATH_SUFFIXES} NO_SYSTEM_ENVIRONMENT_PATH NO_CMAKE_SYSTEM_PATH) endif() - if (APPLE AND _${_PYTHON_PREFIX}_FIND_FRAMEWORK STREQUAL "LAST") - set (__${_PYTHON_PREFIX}_FRAMEWORK_PATHS ${_${_PYTHON_PREFIX}_FRAMEWORK_PATHS}) - else() - unset (__${_PYTHON_PREFIX}_FRAMEWORK_PATHS) - endif() - - if (WIN32 AND _${_PYTHON_PREFIX}_FIND_REGISTRY STREQUAL "LAST") - set (__${_PYTHON_PREFIX}_REGISTRY_PATHS ${_${_PYTHON_PREFIX}_REGISTRY_PATHS}) - else() - unset (__${_PYTHON_PREFIX}_REGISTRY_PATHS) - endif() - + # search header file in standard locations find_path (_${_PYTHON_PREFIX}_INCLUDE_DIR - NAMES Python.h - HINTS ${_${_PYTHON_PREFIX}_INCLUDE_HINTS} ${_${_PYTHON_PREFIX}_HINTS} - PATHS ${_${_PYTHON_PREFIX}_VIRTUALENV_PATHS} - ${__${_PYTHON_PREFIX}_FRAMEWORK_PATHS} - ${__${_PYTHON_PREFIX}_REGISTRY_PATHS} - PATH_SUFFIXES ${_${_PYTHON_PREFIX}_PATH_SUFFIXES} - NO_SYSTEM_ENVIRONMENT_PATH - NO_CMAKE_SYSTEM_PATH) - endif() - - # search header file in standard locations - find_path (_${_PYTHON_PREFIX}_INCLUDE_DIR - NAMES Python.h) - endif() + NAMES ${_${_PYTHON_PREFIX}_INCLUDE_NAMES}) - set (${_PYTHON_PREFIX}_INCLUDE_DIRS "${_${_PYTHON_PREFIX}_INCLUDE_DIR}") + break() + endwhile() - if (_${_PYTHON_PREFIX}_INCLUDE_DIR AND NOT EXISTS "${_${_PYTHON_PREFIX}_INCLUDE_DIR}") - set (_${_PYTHON_PREFIX}_Development_REASON_FAILURE "Cannot find the directory \"${_${_PYTHON_PREFIX}_INCLUDE_DIR}\"") - set_property (CACHE _${_PYTHON_PREFIX}_INCLUDE_DIR PROPERTY VALUE "${_PYTHON_PREFIX}_INCLUDE_DIR-NOTFOUND") - endif() + set (${_PYTHON_PREFIX}_INCLUDE_DIRS "${_${_PYTHON_PREFIX}_INCLUDE_DIR}") - if (_${_PYTHON_PREFIX}_INCLUDE_DIR) - # retrieve version from header file - _python_get_version (INCLUDE PREFIX _${_PYTHON_PREFIX}_INC_) + if (_${_PYTHON_PREFIX}_INCLUDE_DIR AND NOT EXISTS "${_${_PYTHON_PREFIX}_INCLUDE_DIR}") + set (_${_PYTHON_PREFIX}_Development_REASON_FAILURE "Cannot find the directory \"${_${_PYTHON_PREFIX}_INCLUDE_DIR}\"") + set_property (CACHE _${_PYTHON_PREFIX}_INCLUDE_DIR PROPERTY VALUE "${_PYTHON_PREFIX}_INCLUDE_DIR-NOTFOUND") + endif() - # update versioning - if (_${_PYTHON_PREFIX}_INC_VERSION VERSION_EQUAL ${_${_PYTHON_PREFIX}_VERSION}) - set (_${_PYTHON_PREFIX}_VERSION_PATCH ${_${_PYTHON_PREFIX}_INC_VERSION_PATCH}) + if (_${_PYTHON_PREFIX}_INCLUDE_DIR) + # retrieve version from header file + _python_get_version (INCLUDE PREFIX _${_PYTHON_PREFIX}_INC_) + + if (_${_PYTHON_PREFIX}_LIBRARY_RELEASE) + if ("${_${_PYTHON_PREFIX}_INC_VERSION_MAJOR}.${_${_PYTHON_PREFIX}_INC_VERSION_MINOR}" + VERSION_EQUAL _${_PYTHON_PREFIX}_VERSION) + # update versioning + set (_${_PYTHON_PREFIX}_VERSION ${_${_PYTHON_PREFIX}_INC_VERSION}) + set (_${_PYTHON_PREFIX}_VERSION_PATCH ${_${_PYTHON_PREFIX}_INC_VERSION_PATCH}) + endif() + else() + set (_${_PYTHON_PREFIX}_VERSION ${_${_PYTHON_PREFIX}_INC_VERSION}) + set (_${_PYTHON_PREFIX}_VERSION_MAJOR ${_${_PYTHON_PREFIX}_INC_VERSION_MAJOR}) + set (_${_PYTHON_PREFIX}_VERSION_MINOR ${_${_PYTHON_PREFIX}_INC_VERSION_MINOR}) + set (_${_PYTHON_PREFIX}_VERSION_PATCH ${_${_PYTHON_PREFIX}_INC_VERSION_PATCH}) + endif() endif() endif() @@ -2258,61 +2775,88 @@ if ("Development" IN_LIST ${_PYTHON_PREFIX}_FIND_COMPONENTS endif() # define public variables - set (${_PYTHON_PREFIX}_LIBRARY_DEBUG "${_${_PYTHON_PREFIX}_LIBRARY_DEBUG}") - _python_select_library_configurations (${_PYTHON_PREFIX}) + if ("LIBRARY" IN_LIST _${_PYTHON_PREFIX}_FIND_DEVELOPMENT_ARTIFACTS) + set (${_PYTHON_PREFIX}_LIBRARY_DEBUG "${_${_PYTHON_PREFIX}_LIBRARY_DEBUG}") + _python_select_library_configurations (${_PYTHON_PREFIX}) - set (${_PYTHON_PREFIX}_RUNTIME_LIBRARY_RELEASE "${_${_PYTHON_PREFIX}_RUNTIME_LIBRARY_RELEASE}") - set (${_PYTHON_PREFIX}_RUNTIME_LIBRARY_DEBUG "${_${_PYTHON_PREFIX}_RUNTIME_LIBRARY_DEBUG}") + set (${_PYTHON_PREFIX}_RUNTIME_LIBRARY_RELEASE "${_${_PYTHON_PREFIX}_RUNTIME_LIBRARY_RELEASE}") + set (${_PYTHON_PREFIX}_RUNTIME_LIBRARY_DEBUG "${_${_PYTHON_PREFIX}_RUNTIME_LIBRARY_DEBUG}") - if (_${_PYTHON_PREFIX}_RUNTIME_LIBRARY_RELEASE) - set (${_PYTHON_PREFIX}_RUNTIME_LIBRARY "${_${_PYTHON_PREFIX}_RUNTIME_LIBRARY_RELEASE}") - elseif (_${_PYTHON_PREFIX}_RUNTIME_LIBRARY_DEBUG) - set (${_PYTHON_PREFIX}_RUNTIME_LIBRARY "${_${_PYTHON_PREFIX}_RUNTIME_LIBRARY_DEBUG}") - else() - set (${_PYTHON_PREFIX}_RUNTIME_LIBRARY "${_PYTHON_PREFIX}_RUNTIME_LIBRARY-NOTFOUND") - endif() - - _python_set_library_dirs (${_PYTHON_PREFIX}_LIBRARY_DIRS - _${_PYTHON_PREFIX}_LIBRARY_RELEASE _${_PYTHON_PREFIX}_LIBRARY_DEBUG) - if (UNIX) - if (_${_PYTHON_PREFIX}_LIBRARY_RELEASE MATCHES "${CMAKE_SHARED_LIBRARY_SUFFIX}$") - set (${_PYTHON_PREFIX}_RUNTIME_LIBRARY_DIRS ${${_PYTHON_PREFIX}_LIBRARY_DIRS}) + if (_${_PYTHON_PREFIX}_RUNTIME_LIBRARY_RELEASE) + set (${_PYTHON_PREFIX}_RUNTIME_LIBRARY "${_${_PYTHON_PREFIX}_RUNTIME_LIBRARY_RELEASE}") + elseif (_${_PYTHON_PREFIX}_RUNTIME_LIBRARY_DEBUG) + set (${_PYTHON_PREFIX}_RUNTIME_LIBRARY "${_${_PYTHON_PREFIX}_RUNTIME_LIBRARY_DEBUG}") + else() + set (${_PYTHON_PREFIX}_RUNTIME_LIBRARY "${_PYTHON_PREFIX}_RUNTIME_LIBRARY-NOTFOUND") endif() - else() + + _python_set_library_dirs (${_PYTHON_PREFIX}_LIBRARY_DIRS + _${_PYTHON_PREFIX}_LIBRARY_RELEASE + _${_PYTHON_PREFIX}_LIBRARY_DEBUG) + if (UNIX) + if (_${_PYTHON_PREFIX}_LIBRARY_RELEASE MATCHES "${CMAKE_SHARED_LIBRARY_SUFFIX}$") + set (${_PYTHON_PREFIX}_RUNTIME_LIBRARY_DIRS ${${_PYTHON_PREFIX}_LIBRARY_DIRS}) + endif() + else() _python_set_library_dirs (${_PYTHON_PREFIX}_RUNTIME_LIBRARY_DIRS - _${_PYTHON_PREFIX}_RUNTIME_LIBRARY_RELEASE _${_PYTHON_PREFIX}_RUNTIME_LIBRARY_DEBUG) + _${_PYTHON_PREFIX}_RUNTIME_LIBRARY_RELEASE + _${_PYTHON_PREFIX}_RUNTIME_LIBRARY_DEBUG) + endif() endif() - if (_${_PYTHON_PREFIX}_LIBRARY_RELEASE AND _${_PYTHON_PREFIX}_INCLUDE_DIR) + if (_${_PYTHON_PREFIX}_LIBRARY_RELEASE OR _${_PYTHON_PREFIX}_INCLUDE_DIR) if (${_PYTHON_PREFIX}_Interpreter_FOUND OR ${_PYTHON_PREFIX}_Compiler_FOUND) # development environment must be compatible with interpreter/compiler - if (${_${_PYTHON_PREFIX}_VERSION_MAJOR}.${_${_PYTHON_PREFIX}_VERSION_MINOR} VERSION_EQUAL ${${_PYTHON_PREFIX}_VERSION_MAJOR}.${${_PYTHON_PREFIX}_VERSION_MINOR} - AND ${_${_PYTHON_PREFIX}_INC_VERSION_MAJOR}.${_${_PYTHON_PREFIX}_INC_VERSION_MINOR} VERSION_EQUAL ${_${_PYTHON_PREFIX}_VERSION_MAJOR}.${_${_PYTHON_PREFIX}_VERSION_MINOR}) - set (${_PYTHON_PREFIX}_Development_FOUND TRUE) + if ("${_${_PYTHON_PREFIX}_VERSION_MAJOR}.${_${_PYTHON_PREFIX}_VERSION_MINOR}" VERSION_EQUAL "${${_PYTHON_PREFIX}_VERSION_MAJOR}.${${_PYTHON_PREFIX}_VERSION_MINOR}" + AND "${_${_PYTHON_PREFIX}_INC_VERSION_MAJOR}.${_${_PYTHON_PREFIX}_INC_VERSION_MINOR}" VERSION_EQUAL "${_${_PYTHON_PREFIX}_VERSION_MAJOR}.${_${_PYTHON_PREFIX}_VERSION_MINOR}") + _python_set_development_module_found (Module) + _python_set_development_module_found (Embed) endif() elseif (${_PYTHON_PREFIX}_VERSION_MAJOR VERSION_EQUAL _${_PYTHON_PREFIX}_REQUIRED_VERSION_MAJOR - AND ${_${_PYTHON_PREFIX}_INC_VERSION_MAJOR}.${_${_PYTHON_PREFIX}_INC_VERSION_MINOR} VERSION_EQUAL ${_${_PYTHON_PREFIX}_VERSION_MAJOR}.${_${_PYTHON_PREFIX}_VERSION_MINOR}) - set (${_PYTHON_PREFIX}_Development_FOUND TRUE) + AND "${_${_PYTHON_PREFIX}_INC_VERSION_MAJOR}.${_${_PYTHON_PREFIX}_INC_VERSION_MINOR}" VERSION_EQUAL "${_${_PYTHON_PREFIX}_VERSION_MAJOR}.${_${_PYTHON_PREFIX}_VERSION_MINOR}") + _python_set_development_module_found (Module) + _python_set_development_module_found (Embed) endif() if (DEFINED _${_PYTHON_PREFIX}_FIND_ABI AND (NOT _${_PYTHON_PREFIX}_ABI IN_LIST _${_PYTHON_PREFIX}_ABIFLAGS OR NOT _${_PYTHON_PREFIX}_INC_ABI IN_LIST _${_PYTHON_PREFIX}_ABIFLAGS)) - set (${_PYTHON_PREFIX}_Development_FOUND FALSE) + set (${_PYTHON_PREFIX}_Development.Module_FOUND FALSE) + set (${_PYTHON_PREFIX}_Development.Embed_FOUND FALSE) endif() endif() - if (_${_PYTHON_PREFIX}_REQUIRED_VERSION_MAJOR VERSION_GREATER_EQUAL 3 + if (( ${_PYTHON_PREFIX}_Development.Module_FOUND + AND ${_PYTHON_PREFIX}_Development.Embed_FOUND) + OR (NOT "Development.Module" IN_LIST ${_PYTHON_PREFIX}_FIND_COMPONENTS + AND ${_PYTHON_PREFIX}_Development.Embed_FOUND) + OR (NOT "Development.Embed" IN_LIST ${_PYTHON_PREFIX}_FIND_COMPONENTS + AND ${_PYTHON_PREFIX}_Development.Module_FOUND)) + unset (_${_PYTHON_PREFIX}_Development_REASON_FAILURE) + endif() + + if ("Development" IN_LIST ${_PYTHON_PREFIX}_FIND_COMPONENTS + AND ${_PYTHON_PREFIX}_Development.Module_FOUND + AND ${_PYTHON_PREFIX}_Development.Embed_FOUND) + set (${_PYTHON_PREFIX}_Development_FOUND TRUE) + endif() + + if ((${_PYTHON_PREFIX}_Development.Module_FOUND + OR ${_PYTHON_PREFIX}_Development.Embed_FOUND) + AND EXISTS "${_${_PYTHON_PREFIX}_INCLUDE_DIR}/PyPy.h") + # retrieve PyPy version + file (STRINGS "${_${_PYTHON_PREFIX}_INCLUDE_DIR}/patchlevel.h" ${_PYTHON_PREFIX}_PyPy_VERSION + REGEX "^#define[ \t]+PYPY_VERSION[ \t]+\"[^\"]+\"") + string (REGEX REPLACE "^#define[ \t]+PYPY_VERSION[ \t]+\"([^\"]+)\".*" "\\1" + ${_PYTHON_PREFIX}_PyPy_VERSION "${${_PYTHON_PREFIX}_PyPy_VERSION}") + endif() + + if (_${_PYTHON_PREFIX}_REQUIRED_VERSION_MAJOR VERSION_GREATER_EQUAL "3" AND NOT DEFINED ${_PYTHON_PREFIX}_SOABI) _python_get_config_var (${_PYTHON_PREFIX}_SOABI SOABI) endif() - if (${_PYTHON_PREFIX}_Development_FOUND) - # compute and save development signature - string (MD5 __${_PYTHON_PREFIX}_DEVELOPMENT_SIGNATURE "${_${_PYTHON_PREFIX}_SIGNATURE}:${_${_PYTHON_PREFIX}_LIBRARY_RELEASE}:${_${_PYTHON_PREFIX}_INCLUDE_DIR}") - set (_${_PYTHON_PREFIX}_DEVELOPMENT_SIGNATURE "${__${_PYTHON_PREFIX}_DEVELOPMENT_SIGNATURE}" CACHE INTERNAL "") - else() - unset (_${_PYTHON_PREFIX}_DEVELOPMENT_SIGNATURE CACHE) - endif() + _python_compute_development_signature (Module) + _python_compute_development_signature (Embed) # Restore the original find library ordering if (DEFINED _${_PYTHON_PREFIX}_CMAKE_FIND_LIBRARY_SUFFIXES) @@ -2320,8 +2864,12 @@ if ("Development" IN_LIST ${_PYTHON_PREFIX}_FIND_COMPONENTS endif() if (${_PYTHON_PREFIX}_ARTIFACTS_INTERACTIVE) - set (${_PYTHON_PREFIX}_LIBRARY "${_${_PYTHON_PREFIX}_LIBRARY_RELEASE}" CACHE FILEPATH "${_PYTHON_PREFIX} Library") - set (${_PYTHON_PREFIX}_INCLUDE_DIR "${_${_PYTHON_PREFIX}_INCLUDE_DIR}" CACHE FILEPATH "${_PYTHON_PREFIX} Include Directory") + if ("LIBRARY" IN_LIST _${_PYTHON_PREFIX}_FIND_DEVELOPMENT_ARTIFACTS) + set (${_PYTHON_PREFIX}_LIBRARY "${_${_PYTHON_PREFIX}_LIBRARY_RELEASE}" CACHE FILEPATH "${_PYTHON_PREFIX} Library") + endif() + if ("INCLUDE_DIR" IN_LIST _${_PYTHON_PREFIX}_FIND_DEVELOPMENT_ARTIFACTS) + set (${_PYTHON_PREFIX}_INCLUDE_DIR "${_${_PYTHON_PREFIX}_INCLUDE_DIR}" CACHE FILEPATH "${_PYTHON_PREFIX} Include Directory") + endif() endif() _python_mark_as_internal (_${_PYTHON_PREFIX}_LIBRARY_RELEASE @@ -2330,21 +2878,22 @@ if ("Development" IN_LIST ${_PYTHON_PREFIX}_FIND_COMPONENTS _${_PYTHON_PREFIX}_RUNTIME_LIBRARY_DEBUG _${_PYTHON_PREFIX}_INCLUDE_DIR _${_PYTHON_PREFIX}_CONFIG - _${_PYTHON_PREFIX}_DEVELOPMENT_SIGNATURE) + _${_PYTHON_PREFIX}_DEVELOPMENT_MODULE_SIGNATURE + _${_PYTHON_PREFIX}_DEVELOPMENT_EMBED_SIGNATURE) endif() +if (${_PYTHON_PREFIX}_FIND_REQUIRED_NumPy) + list (APPEND _${_PYTHON_PREFIX}_REQUIRED_VARS ${_PYTHON_PREFIX}_NumPy_INCLUDE_DIRS) +endif() if ("NumPy" IN_LIST ${_PYTHON_PREFIX}_FIND_COMPONENTS AND ${_PYTHON_PREFIX}_Interpreter_FOUND) list (APPEND _${_PYTHON_PREFIX}_CACHED_VARS _${_PYTHON_PREFIX}_NumPy_INCLUDE_DIR) - if (${_PYTHON_PREFIX}_FIND_REQUIRED_NumPy) - list (APPEND _${_PYTHON_PREFIX}_REQUIRED_VARS ${_PYTHON_PREFIX}_NumPy_INCLUDE_DIRS) - endif() if (DEFINED ${_PYTHON_PREFIX}_NumPy_INCLUDE_DIR AND IS_ABSOLUTE "${${_PYTHON_PREFIX}_NumPy_INCLUDE_DIR}") set (_${_PYTHON_PREFIX}_NumPy_INCLUDE_DIR "${${_PYTHON_PREFIX}_NumPy_INCLUDE_DIR}" CACHE INTERNAL "") elseif (DEFINED _${_PYTHON_PREFIX}_NumPy_INCLUDE_DIR) # compute numpy signature. Depends on interpreter and development signatures - string (MD5 __${_PYTHON_PREFIX}_NUMPY_SIGNATURE "${_${_PYTHON_PREFIX}_INTERPRETER_SIGNATURE}:${_${_PYTHON_PREFIX}_DEVELOPMENT_SIGNATURE}:${_${_PYTHON_PREFIX}_NumPy_INCLUDE_DIR}") + string (MD5 __${_PYTHON_PREFIX}_NUMPY_SIGNATURE "${_${_PYTHON_PREFIX}_INTERPRETER_SIGNATURE}:${_${_PYTHON_PREFIX}_DEVELOPMENT_MODULE_SIGNATURE}:${_${_PYTHON_PREFIX}_NumPy_INCLUDE_DIR}") if (NOT __${_PYTHON_PREFIX}_NUMPY_SIGNATURE STREQUAL _${_PYTHON_PREFIX}_NUMPY_SIGNATURE OR NOT EXISTS "${_${_PYTHON_PREFIX}_NumPy_INCLUDE_DIR}") unset (_${_PYTHON_PREFIX}_NumPy_INCLUDE_DIR CACHE) @@ -2353,13 +2902,12 @@ if ("NumPy" IN_LIST ${_PYTHON_PREFIX}_FIND_COMPONENTS AND ${_PYTHON_PREFIX}_Inte endif() if (NOT _${_PYTHON_PREFIX}_NumPy_INCLUDE_DIR) - execute_process( - COMMAND "${${_PYTHON_PREFIX}_EXECUTABLE}" -c - "from __future__ import print_function\ntry: import numpy; print(numpy.get_include(), end='')\nexcept:pass\n" - RESULT_VARIABLE _${_PYTHON_PREFIX}_RESULT - OUTPUT_VARIABLE _${_PYTHON_PREFIX}_NumPy_PATH - ERROR_QUIET - OUTPUT_STRIP_TRAILING_WHITESPACE) + execute_process(COMMAND ${${_PYTHON_PREFIX}_INTERPRETER_LAUNCHER} "${_${_PYTHON_PREFIX}_EXECUTABLE}" -c + "import sys\ntry: import numpy; sys.stdout.write(numpy.get_include())\nexcept:pass\n" + RESULT_VARIABLE _${_PYTHON_PREFIX}_RESULT + OUTPUT_VARIABLE _${_PYTHON_PREFIX}_NumPy_PATH + ERROR_QUIET + OUTPUT_STRIP_TRAILING_WHITESPACE) if (NOT _${_PYTHON_PREFIX}_RESULT) find_path (_${_PYTHON_PREFIX}_NumPy_INCLUDE_DIR @@ -2377,26 +2925,27 @@ if ("NumPy" IN_LIST ${_PYTHON_PREFIX}_FIND_COMPONENTS AND ${_PYTHON_PREFIX}_Inte endif() if (_${_PYTHON_PREFIX}_NumPy_INCLUDE_DIR) - execute_process ( - COMMAND "${${_PYTHON_PREFIX}_EXECUTABLE}" -c - "from __future__ import print_function\ntry: import numpy; print(numpy.__version__, end='')\nexcept:pass\n" - RESULT_VARIABLE _${_PYTHON_PREFIX}_RESULT - OUTPUT_VARIABLE _${_PYTHON_PREFIX}_NumPy_VERSION) + execute_process (COMMAND ${${_PYTHON_PREFIX}_INTERPRETER_LAUNCHER} "${_${_PYTHON_PREFIX}_EXECUTABLE}" -c + "import sys\ntry: import numpy; sys.stdout.write(numpy.__version__)\nexcept:pass\n" + RESULT_VARIABLE _${_PYTHON_PREFIX}_RESULT + OUTPUT_VARIABLE _${_PYTHON_PREFIX}_NumPy_VERSION) if (NOT _${_PYTHON_PREFIX}_RESULT) set (${_PYTHON_PREFIX}_NumPy_VERSION "${_${_PYTHON_PREFIX}_NumPy_VERSION}") else() unset (${_PYTHON_PREFIX}_NumPy_VERSION) endif() - # final step: set NumPy founded only if Development component is founded as well - set(${_PYTHON_PREFIX}_NumPy_FOUND ${${_PYTHON_PREFIX}_Development_FOUND}) + # final step: set NumPy founded only if Development.Module component is founded as well + set(${_PYTHON_PREFIX}_NumPy_FOUND ${${_PYTHON_PREFIX}_Development.Module_FOUND}) else() set (${_PYTHON_PREFIX}_NumPy_FOUND FALSE) endif() if (${_PYTHON_PREFIX}_NumPy_FOUND) + unset (_${_PYTHON_PREFIX}_NumPy_REASON_FAILURE) + # compute and save numpy signature - string (MD5 __${_PYTHON_PREFIX}_NUMPY_SIGNATURE "${_${_PYTHON_PREFIX}_INTERPRETER_SIGNATURE}:${_${_PYTHON_PREFIX}_DEVELOPMENT_SIGNATURE}:${${_PYTHON_PREFIX}_NumPyINCLUDE_DIR}") + string (MD5 __${_PYTHON_PREFIX}_NUMPY_SIGNATURE "${_${_PYTHON_PREFIX}_INTERPRETER_SIGNATURE}:${_${_PYTHON_PREFIX}_DEVELOPMENT_MODULE_SIGNATURE}:${${_PYTHON_PREFIX}_NumPyINCLUDE_DIR}") set (_${_PYTHON_PREFIX}_NUMPY_SIGNATURE "${__${_PYTHON_PREFIX}_NUMPY_SIGNATURE}" CACHE INTERNAL "") else() unset (_${_PYTHON_PREFIX}_NUMPY_SIGNATURE CACHE) @@ -2420,6 +2969,7 @@ unset (_${_PYTHON_PREFIX}_REASON_FAILURE) foreach (_${_PYTHON_PREFIX}_COMPONENT IN ITEMS Interpreter Compiler Development NumPy) if (_${_PYTHON_PREFIX}_${_${_PYTHON_PREFIX}_COMPONENT}_REASON_FAILURE) string (APPEND _${_PYTHON_PREFIX}_REASON_FAILURE "\n ${_${_PYTHON_PREFIX}_COMPONENT}: ${_${_PYTHON_PREFIX}_${_${_PYTHON_PREFIX}_COMPONENT}_REASON_FAILURE}") + unset (_${_PYTHON_PREFIX}_${_${_PYTHON_PREFIX}_COMPONENT}_REASON_FAILURE) endif() endforeach() @@ -2447,8 +2997,10 @@ if(_${_PYTHON_PREFIX}_CMAKE_ROLE STREQUAL "PROJECT") PROPERTY IMPORTED_LOCATION "${${_PYTHON_PREFIX}_COMPILER}") endif() - if ("Development" IN_LIST ${_PYTHON_PREFIX}_FIND_COMPONENTS - AND ${_PYTHON_PREFIX}_Development_FOUND) + if (("Development.Module" IN_LIST ${_PYTHON_PREFIX}_FIND_COMPONENTS + AND ${_PYTHON_PREFIX}_Development.Module_FOUND) + OR ("Development.Embed" IN_LIST ${_PYTHON_PREFIX}_FIND_COMPONENTS + AND ${_PYTHON_PREFIX}_Development.Embed_FOUND)) macro (__PYTHON_IMPORT_LIBRARY __name) if (${_PYTHON_PREFIX}_LIBRARY_RELEASE MATCHES "${CMAKE_SHARED_LIBRARY_SUFFIX}$" @@ -2509,31 +3061,35 @@ if(_${_PYTHON_PREFIX}_CMAKE_ROLE STREQUAL "PROJECT") endif() endmacro() - __python_import_library (${_PYTHON_PREFIX}::Python) - - if (CMAKE_SYSTEM_NAME MATCHES "^(Windows.*|CYGWIN|MSYS)$") - # On Windows/CYGWIN/MSYS, Python::Module is the same as Python::Python - # but ALIAS cannot be used because the imported library is not GLOBAL. - __python_import_library (${_PYTHON_PREFIX}::Module) - else() - if (NOT TARGET ${_PYTHON_PREFIX}::Module ) - add_library (${_PYTHON_PREFIX}::Module INTERFACE IMPORTED) - endif() - set_property (TARGET ${_PYTHON_PREFIX}::Module - PROPERTY INTERFACE_INCLUDE_DIRECTORIES "${${_PYTHON_PREFIX}_INCLUDE_DIRS}") + if (${_PYTHON_PREFIX}_Development.Embed_FOUND) + __python_import_library (${_PYTHON_PREFIX}::Python) + endif() - # When available, enforce shared library generation with undefined symbols - if (APPLE) - set_property (TARGET ${_PYTHON_PREFIX}::Module - PROPERTY INTERFACE_LINK_OPTIONS "LINKER:-undefined,dynamic_lookup") - endif() - if (CMAKE_SYSTEM_NAME STREQUAL "SunOS") - set_property (TARGET ${_PYTHON_PREFIX}::Module - PROPERTY INTERFACE_LINK_OPTIONS "LINKER:-z,nodefs") - endif() - if (CMAKE_SYSTEM_NAME STREQUAL "AIX") + if (${_PYTHON_PREFIX}_Development.Module_FOUND) + if ("LIBRARY" IN_LIST _${_PYTHON_PREFIX}_FIND_DEVELOPMENT_MODULE_ARTIFACTS) + # On Windows/CYGWIN/MSYS, Python::Module is the same as Python::Python + # but ALIAS cannot be used because the imported library is not GLOBAL. + __python_import_library (${_PYTHON_PREFIX}::Module) + else() + if (NOT TARGET ${_PYTHON_PREFIX}::Module) + add_library (${_PYTHON_PREFIX}::Module INTERFACE IMPORTED) + endif() set_property (TARGET ${_PYTHON_PREFIX}::Module - PROPERTY INTERFACE_LINK_OPTIONS "LINKER:-b,erok") + PROPERTY INTERFACE_INCLUDE_DIRECTORIES "${${_PYTHON_PREFIX}_INCLUDE_DIRS}") + + # When available, enforce shared library generation with undefined symbols + if (APPLE) + set_property (TARGET ${_PYTHON_PREFIX}::Module + PROPERTY INTERFACE_LINK_OPTIONS "LINKER:-undefined,dynamic_lookup") + endif() + if (CMAKE_SYSTEM_NAME STREQUAL "SunOS") + set_property (TARGET ${_PYTHON_PREFIX}::Module + PROPERTY INTERFACE_LINK_OPTIONS "LINKER:-z,nodefs") + endif() + if (CMAKE_SYSTEM_NAME STREQUAL "AIX") + set_property (TARGET ${_PYTHON_PREFIX}::Module + PROPERTY INTERFACE_LINK_OPTIONS "LINKER:-b,erok") + endif() endif() endif() @@ -2542,8 +3098,7 @@ if(_${_PYTHON_PREFIX}_CMAKE_ROLE STREQUAL "PROJECT") # It is used to build modules for python. # function (__${_PYTHON_PREFIX}_ADD_LIBRARY prefix name) - cmake_parse_arguments (PARSE_ARGV 2 PYTHON_ADD_LIBRARY - "STATIC;SHARED;MODULE;WITH_SOABI" "" "") + cmake_parse_arguments (PARSE_ARGV 2 PYTHON_ADD_LIBRARY "STATIC;SHARED;MODULE;WITH_SOABI" "" "") if (prefix STREQUAL "Python2" AND PYTHON_ADD_LIBRARY_WITH_SOABI) message (AUTHOR_WARNING "FindPython2: Option `WITH_SOABI` is not supported for Python2 and will be ignored.") @@ -2557,6 +3112,16 @@ if(_${_PYTHON_PREFIX}_CMAKE_ROLE STREQUAL "PROJECT") else() set (type MODULE) endif() + + if (type STREQUAL "MODULE" AND NOT TARGET ${prefix}::Module) + message (SEND_ERROR "${prefix}_ADD_LIBRARY: dependent target '${prefix}::Module' is not defined.\n Did you miss to request COMPONENT 'Development.Module'?") + return() + endif() + if (NOT type STREQUAL "MODULE" AND NOT TARGET ${prefix}::Python) + message (SEND_ERROR "${prefix}_ADD_LIBRARY: dependent target '${prefix}::Python' is not defined.\n Did you miss to request COMPONENT 'Development.Embed'?") + return() + endif() + add_library (${name} ${type} ${PYTHON_ADD_LIBRARY_UNPARSED_ARGUMENTS}) get_property (type TARGET ${name} PROPERTY TYPE) diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000000..81e4ea5019 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,16 @@ +# Documentation for The Adaptable Input Output System + +# Generate User Guide in html with Sphinx + +This user guide is hosted in readthedocs: https://adios2.readthedocs.io/en/latest/ + +To generate the User Guide under docs/user_guide/build/html format from the Sphinx source files: + +```bash +$ cd ADIOS2/docs +docs$ python3 -m venv . +docs$ . bin/activate +docs$ pip3 install -r requirements.txt +user_guide$ cd user_guide +user_guide$ make html +``` diff --git a/docs/ReadMe.md b/docs/ReadMe.md deleted file mode 100644 index 867be6c908..0000000000 --- a/docs/ReadMe.md +++ /dev/null @@ -1,21 +0,0 @@ -# Documentation for The Adaptable Input Output System - -# Generate User Guide in html with Sphinx - -This user guide is hosted in readthedocs: https://adios2.readthedocs.io/en/latest/ - -Python 2.7 or 3 is required to build the Sphinx based user guide. Sphinx can be downloaded using pip `pip install sphinx`. In addition: - -1. Required Python packages (all can be installed with `pip install package` or `pip3 install package`): - * blockdiag https://pypi.python.org/pypi/blockdiag/ - * sphinx v1.4 or above http://www.sphinx-doc.org/en/stable/ - * sphinxcontrib-blockdiag https://pypi.python.org/pypi/sphinxcontrib-blockdiag - * sphinx_rtd_theme https://sphinx-rtd-theme.readthedocs.io/en/latest/installing.html - * breathe https://breathe.readthedocs.io/en/latest/ - -2. To generate the User Guide under docs/user_guide/build/html format (default) from the existing Sphinx source files: - - ``` - $ cd ADIOS2/docs/user_guide - $ make html - ``` diff --git a/docs/requirements.txt b/docs/requirements.txt index 7dbf9e3f93..307c5edb0c 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,5 +1,31 @@ -sphinx -blockdiag>=1.5.4 -sphinxcontrib-blockdiag>=1.5.5 -sphinx_rtd_theme>=0.4.1 +alabaster==0.7.12 +Babel==2.8.0 +blockdiag==2.0.1 breathe==4.18.1 +certifi==2019.11.28 +chardet==3.0.4 +docutils==0.16 +funcparserlib==0.3.6 +idna==2.9 +imagesize==1.2.0 +Jinja2==2.11.1 +MarkupSafe==1.1.1 +packaging==20.1 +Pillow==7.0.0 +Pygments==2.5.2 +pyparsing==2.4.6 +pytz==2019.3 +requests==2.23.0 +six==1.14.0 +snowballstemmer==2.0.0 +Sphinx==3.1.2 +sphinx-rtd-theme==0.4.3 +sphinxcontrib-applehelp==1.0.2 +sphinxcontrib-blockdiag==2.0.0 +sphinxcontrib-devhelp==1.0.2 +sphinxcontrib-htmlhelp==1.0.3 +sphinxcontrib-jsmath==1.0.1 +sphinxcontrib-qthelp==1.0.3 +sphinxcontrib-serializinghtml==1.1.4 +urllib3==1.25.8 +webcolors==1.11.1 diff --git a/docs/user_guide/source/api_full/api_full.rst b/docs/user_guide/source/api_full/api_full.rst index fa91c4b571..b721b8b695 100644 --- a/docs/user_guide/source/api_full/api_full.rst +++ b/docs/user_guide/source/api_full/api_full.rst @@ -4,7 +4,8 @@ Full APIs .. note:: - Application developers who desire fine-grained control of IO tasks should use the Full APIs. In simple cases (e.g. reading a file for analysis, interactive Python, or saving some data for a small project or tests) please refer to the :ref:High-Level APIs. + Application developers who desire fine-grained control of IO tasks should use the Full APIs. + In simple cases (e.g. reading a file for analysis, interactive Python, or saving some data for a small project or tests) please refer to the :ref:`High-Level APIs`. Currently ADIOS2 support bindings for the following languages and their minimum standards: @@ -27,12 +28,13 @@ Currently ADIOS2 support bindings for the following languages and their minimum .. tip:: - Prefer the C++11 bindings if your application C++ compiler supports the 2011 (or later) standard. For code using previous C++ standards (98 or 03) use the C bindings for ABI compatibility. + Prefer the C++11 bindings if your application C++ compiler supports the 2011 (or later) standard. + For code using previous C++ standards (98 or 03) use the C bindings for ABI compatibility. .. caution:: - Statically linked libraries (``*.a``) might result in conflicting ABIs between an older C++ project, the C bindings, and the adios native C++11 library. Test to make sure it works for your platform. + Statically linked libraries (``*.a``) might result in conflicting ABIs between an older C++ project, the C bindings, and the adios native C++11 library. Test to make sure it works for your platform. The current interaction flow for each language binding API with the ADIOS2 library is specified as follows @@ -70,4 +72,3 @@ The following sections provide a summary of the API calls on each language and l .. include:: fortran.rst .. include:: c.rst .. include:: python.rst - diff --git a/docs/user_guide/source/api_full/cxx11.rst b/docs/user_guide/source/api_full/cxx11.rst index 2dbe9af466..d86db960e0 100644 --- a/docs/user_guide/source/api_full/cxx11.rst +++ b/docs/user_guide/source/api_full/cxx11.rst @@ -88,10 +88,11 @@ The following section provides a summary of the available functionality for each :members: -Debugging ---------- +**Debugging** -For debugging, ADIOS2 C++11 class instances and enums can be passed directly to ostreams, as well as converted to human-readable strings via the ubiquitous ``ToString(object)`` member variable. You can also directly pass objects to ``ostream``s. +For debugging, ADIOS2 C++11 class instances and enums can be passed directly to ostreams, +as well as converted to human-readable strings via the ubiquitous ``ToString(object)`` member variable. +You can also directly pass objects to an ``ostream``. Example: diff --git a/docs/user_guide/source/api_high/cxx11.rst b/docs/user_guide/source/api_high/cxx11.rst index cb84f866c1..f322bd07e8 100644 --- a/docs/user_guide/source/api_high/cxx11.rst +++ b/docs/user_guide/source/api_high/cxx11.rst @@ -34,7 +34,7 @@ C++11 Write example { if(rank == 0 && step == 0) // global variable { - oStream.write("size", size); + oStream.write("size", size); } // physicalTime double, is optional diff --git a/docs/user_guide/source/components/adios.rst b/docs/user_guide/source/components/adios.rst index a3e0cd30e1..32c046a80d 100644 --- a/docs/user_guide/source/components/adios.rst +++ b/docs/user_guide/source/components/adios.rst @@ -1,42 +1,24 @@ -*****ADIOS ***** +********* +ADIOS +********* - The `adios2::ADIOS` component is the initial contact point between an - application and the ADIOS2 library.Applications - can be classified as MPI and non - - MPI based.We start by focusing on MPI applications as their non - - MPI equivalent just removes the MPI - - specific communicator - . +The ``adios2::ADIOS`` component is the initial contact point between an application and the ADIOS2 library. +Applications can be classified as MPI and non-MPI based. +We start by focusing on MPI applications as their non-MPI equivalent just removes the MPI communicator. - ..code - - block::c++ +.. code-block:: c++ /** ADIOS class factory of IO class objects */ adios2::ADIOS adios("config.xml", MPI_COMM_WORLD); This component is created by passing : - 1. * *Runtime config file **(optional) -: ADIOS2 xml runtime config file, - see : ref :`Runtime Configuration Files` 2. * *MPI communicator * - * : which determines the scope of the ADIOS library components in an - application.3. * - *debug mode flag * * : turns on / - off exceptions triggered by user inputs - . - - ..note:: - - Debug mode is deprecated - and will not affect ADIOS2 behavior.Unexpected system failures and - runtime errors are always checked by throwing ``std::runtime_error`` - .Keep in mind that Segmentation Faults are NOT runtime exceptions.We - try to keep user interactions as friendly as possible, please report any bugs on GitHub: https://github.com/ornladios/ADIOS2/issues + 1. **Runtime config file** (optional): ADIOS2 xml runtime config file, see :ref:`Runtime Configuration Files`. + 2. **MPI communicator** : which determines the scope of the ADIOS library components in an application. ``adios2::ADIOS`` objects can be created in MPI and non-MPI (serial) mode. Optionally, a runtime configuration file can be passed to the constructor indicating the full file path, name and extension. -Thus resulting in: **Constructors for MPI applications** @@ -45,35 +27,27 @@ Thus resulting in: /** Constructors */ // version that accepts an optional runtime adios2 config file - adios2::ADIOS (const std::string configFile, - MPI_COMM mpiComm = MPI_COMM_SELF, - const bool debugMode = adios2::DebugON ); + adios2::ADIOS(const std::string configFile, + MPI_COMM mpiComm = MPI_COMM_SELF); -adios2::ADIOS(MPI_COMM mpiComm = MPI_COMM_SELF, - const bool debugMode = adios2::DebugON); + adios2::ADIOS(MPI_COMM mpiComm = MPI_COMM_SELF); -/** Examples */ -adios2::ADIOS adios(MPI_COMM_WORLD); -adios2::ADIOS adios("config.xml", MPI_COMM_WORLD, adios2::DebugOFF); + /** Examples */ + adios2::ADIOS adios(MPI_COMM_WORLD); + adios2::ADIOS adios("config.xml", MPI_COMM_WORLD); **Constructors for non-MPI (serial) applications** .. code-block:: c++ /** Constructors */ - adios2::ADIOS (const std::string configFile, - const bool debugMode = adios2::DebugON ); - -adios2::ADIOS(const bool debugMode = adios2::DebugON); + adios2::ADIOS(const std::string configFile); -/** Examples */ -adios2::ADIOS adios("config.xml"); -adios2::ADIOS adios; // Do not use () for empty constructor. + adios2::ADIOS(); -.. tip:: - - ``adios2::DebugON`` and ``adios::DebugOFF`` are aliases to true and false, respectively. - Use them for code clarity. + /** Examples */ + adios2::ADIOS adios("config.xml"); + adios2::ADIOS adios; // Do not use () for empty constructor. **Factory of IO components**: Multiple IO components (IO tasks) can be created from within the scope of an ADIOS object by calling the ``DeclareIO`` function: @@ -83,35 +57,32 @@ adios2::ADIOS adios; // Do not use () for empty constructor. /** Signature */ adios2::IO ADIOS::DeclareIO(const std::string ioName); -/** Examples */ -adios2::IO bpWriter = adios.DeclareIO("BPWriter"); -adios2::IO bpReader = adios.DeclareIO("BPReader"); + /** Examples */ + adios2::IO bpWriter = adios.DeclareIO("BPWriter"); + adios2::IO bpReader = adios.DeclareIO("BPReader"); This function returns a reference to an existing IO class object that lives inside the ADIOS object that created it. -The ``ioName`` identifier input must be unique for each IO. -Trying to declare an IO object with the same name twice will throw an exception. -IO names are used to identify IO components in the runtime configuration file, :ref:`Runtime Configuration Files` +The ``ioName`` string must be unique; declaring two IO objects with the same name will throw an exception. +IO names are used to identify IO components in the runtime configuration file, :ref:`Runtime Configuration Files`. -As shown in the diagram below, each resulting IO object is self-managed and independent, thus providing an adaptable way to perform different kinds of I/O operations. Users must be careful not to create conflicts between system level unique I/O identifiers: file names, IP address and port, MPI Send/Receive message rank and tag, etc. +As shown in the diagram below, each resulting IO object is self-managed and independent, thus providing an adaptable way to perform different kinds of I/O operations. +Users must be careful not to create conflicts between system level unique I/O identifiers: file names, IP address and port, MPI Send/Receive message rank and tag, etc. .. blockdiag:: - diagram -{ - default_fontsize = 18; - default_shape = roundedbox; - default_linecolor = blue; - span_width = 150; - - ADIOS->IO_1, B, IO_N[label = "DeclareIO", fontsize = 13]; - B[shape = "dots"]; - ADIOS->B[style = "none"]; -} - -..tip:: - - The ADIOS component is the only one whose memory is owned by the - application.Thus applications must decide on its scope.Any - other component of the ADIOS2 API refers to a component that - lives inside the ADIOS component(e.g.IO, Operator) or - indirectly in the IO component(Variable, Engine) + blockdiag { + default_fontsize = 18; + default_shape = roundedbox; + default_linecolor = blue; + span_width = 150; + + ADIOS -> IO_1, B, IO_N[label = "DeclareIO", fontsize = 13]; + B[shape = "dots"]; + ADIOS -> B[style = "none"]; + } + +.. tip:: + + The ADIOS component is the only one whose memory is owned by the application. + Thus applications must decide on its scope. + Any other component of the ADIOS2 API refers to a component that lives inside the ADIOS component(e.g. IO, Operator) or indirectly in the IO component(Variable, Engine) diff --git a/docs/user_guide/source/components/attribute.rst b/docs/user_guide/source/components/attribute.rst index 7324799ee1..0d3d150f08 100644 --- a/docs/user_guide/source/components/attribute.rst +++ b/docs/user_guide/source/components/attribute.rst @@ -2,13 +2,15 @@ Attribute ********* -Attributes are extra information associated with a particular IO component. They can be thought of a very simplified version of a Variable, but with the goal of adding extra metadata. The most common use is the addition of human-readable information available when producing data (*e.g.* ``"experiment name"``, ``"date and time"``, ``"04,27,2017"``, or a schema). +Attributes are extra information associated with a particular IO component. +They can be thought of as a very simplified ``Variable``, but with the goal of adding extra metadata. +The most common use is the addition of human-readable metadata (*e.g.* ``"experiment name"``, ``"date and time"``, ``"04,27,2017"``, or a schema). -Currently, ADIOS2 supports single values and arrays of primitive types (excluding ``complex``) for the template type in the ``IO::DefineAttribute`` and ``IO::InquireAttribute`` function (in C++). +Currently, ADIOS2 supports single values and arrays of primitive types (excluding ``complex``) for the template type in the ``IO::DefineAttribute`` and ``IO::InquireAttribute`` function (in C++). -.. code-block:: c++ +The data types supported for ADIOS2 ``Attributes`` are - Data types Attributes supported by ADIOS2: +.. code-block:: c++ std::string char @@ -30,4 +32,5 @@ The returned object (``DefineAttribute`` or ``InquireAttribute``) only serves th .. note: - Attributes are not forcibly associated to a particular variable in ADIOS2. Developers are free to create associations through their own naming conventions. + Attributes are not forcibly associated to a particular variable in ADIOS2. + Developers are free to create associations through their own naming conventions. diff --git a/docs/user_guide/source/components/engine.rst b/docs/user_guide/source/components/engine.rst index eb8b58a3c9..a9d18061d6 100644 --- a/docs/user_guide/source/components/engine.rst +++ b/docs/user_guide/source/components/engine.rst @@ -2,34 +2,41 @@ Engine ****** -The Engine abstraction component serves as the base interface to the actual IO Systems executing the heavy-load tasks performed when Producing and Consuming data. +The Engine abstraction component serves as the base interface to the actual IO systems executing the heavy-load tasks performed when producing and consuming data. -Engine functionality works around two concepts from the application's point-of-view: +Engine functionality works around two concepts: -1. Self-describing variables are published and consumed in "steps" in either "File" random-access (all steps are available) or "Streaming" (steps are available as they are produced in a step-by-step fashion). -2. Self-describing variables are published (Put) and consumed (Get) using a "sync" or "deferred" (lazy evaluation) policy. +1. Variables are published (``Put``) and consumed (``Get``) in "steps" in either "File" random-access (all steps are available) or "Streaming" (steps are available as they are produced in a step-by-step fashion). +2. Variables are published (``Put``) and consumed (``Get``) using a "sync" or "deferred" (lazy evaluation) policy. .. caution:: - The ADIOS 2 "step" is a logical abstraction that means different things depending on the application context. Examples: "time step", "iteration step", "inner loop step", or "interpolation step", "variable section", etc. It only indicates how the variables were passed into ADIOS 2 (e.g. I/O steps) without the user having to index this information on their own. + The ADIOS2 "step" is a logical abstraction that means different things depending on the application context. + Examples: "time step", "iteration step", "inner loop step", or "interpolation step", "variable section", etc. + It only indicates how the variables were passed into ADIOS2 (e.g. I/O steps) without the user having to index this information on their own. .. tip:: - Publishing/Consuming data can be seen as a round-trip in ADIOS 2. Put and Get APIs for write/append and read modes aim to be "symmetric". Hence, reusing similar functions, objects, semantics as much as possible. + Publishing and consuming data is a round-trip in ADIOS2. + ``Put`` and ``Get`` APIs for write/append and read modes aim to be "symmetric", reusing functions, objects, and semantics as much as possible. -The rest of the section explains the important concepts +The rest of the section explains the important concepts. BeginStep --------- - - Begin logical step and return the status (via an enum) of the stream to be read/written. In streaming engines BeginStep in where the receiver tries to acquire a new step in the reading process. The full signature allows for a mode and timeout parameters. See :ref:`Supported Engines` for more information on what engine allows. A simplified signature allows each engine to pick reasonable defaults. - + + Begins a logical step and return the status (via an enum) of the stream to be read/written. + In streaming engines ``BeginStep`` is where the receiver tries to acquire a new step in the reading process. + The full signature allows for a mode and timeout parameters. + See :ref:`Supported Engines` for more information on what engine allows. + A simplified signature allows each engine to pick reasonable defaults. + .. code-block:: c++ // Full signature StepStatus BeginStep(const StepMode mode, const float timeoutSeconds = -1.f); - + // Simplified signature StepStatus BeginStep(); @@ -41,29 +48,29 @@ EndStep .. tip:: - To write portable code for a step-by-step access across adios2 engines (file and streaming engines) use BeginStep and EndStep. + To write portable code for a step-by-step access across ADIOS2 engines (file and streaming engines) use ``BeginStep`` and ``EndStep``. .. danger:: - Accessing random steps in read mode (e.g. Variable::SetStepSelection in file engines) will create a conflict with BeginStep and EndStep and will throw an exception. In file engines, data is either consumed in a random-access or step-by-step mode, but not both. + Accessing random steps in read mode (e.g. ``Variable::SetStepSelection`` in file engines) will create a conflict with ``BeginStep`` and ``EndStep`` and will throw an exception. + In file engines, data is either consumed in a random-access or step-by-step mode, but not both. Close ----- - Close current engine and underlying transports. Engine object can't be used after this call. - -.. tip:: - - C++11: despite the fact that we use RAII, always use Close for explicit resource management and guarantee that the Engine data transport operations are concluded. + Close current engine and underlying transports. + An ``Engine`` object can't be used after this call. Put: modes and memory contracts ------------------------------- -``Put`` is the generalized abstract function for publishing data in adios2 when an Engine is created using Write, or Append, mode at ``IO::Open``. +``Put`` publishes data in ADIOS2. +It is unavailable unless the ``Engine`` is created in ``Write`` or ``Append`` mode. -The most common signature is the one that passes a ``Variable`` object for the metadata, a ``const`` piece of contiguous memory for the data, and a mode for either Deferred (data is collected until EndStep/PerformPuts/Close) or Sync (data is reusable immediately). This is the most common use case in applications. +The most common signature is the one that passes a ``Variable`` object for the metadata, a ``const`` piece of contiguous memory for the data, and a mode for either ``Deferred`` (data is collected until EndStep/PerformPuts/Close) or ``Sync`` (data is reusable immediately). +This is the most common use case in applications. 1. Deferred (default) or Sync mode, data is contiguous memory @@ -71,53 +78,55 @@ The most common signature is the one that passes a ``Variable`` object for th void Put(Variable variable, const T* data, const adios2::Mode = adios2::Mode::Deferred); -Optionally, adios2 Engines can provide direct access to its buffer memory using an overload that returns a piece of memory to a variable block, basically a zero-copy. Variable::Span is based on a subset of the upcoming `C++20 std::span `_, which is non-owning and typed contiguous memory piece (it helps to review what std::span is, formerly known as array_view). Spans act as a 1D memory container meant to be filled out by the application. It is safely used as any other STL sequence container, with iterators ``begin()`` and ``end()``, ``operator[]`` and ``at()``, while also providing ``data()`` and ``size()`` functions to manipulate the internal pointer. +ADIOS2 Engines also provide direct access to their buffer memory. +``Variable::Span`` is based on a subset of the upcoming `C++20 std::span `_, which is a non-owning reference to a block of contiguous memory. +Spans act as a 1D container meant to be filled out by the application. +They provide the standard API of an STL container, providing ``begin()`` and ``end()`` iterators, ``operator[]`` and ``at()``, as well as ``data()`` and ``size()``. -Variable::Span is helpful in situations in which temporaries are needed to create contiguous pieces of memory from non-contiguous pieces (``e.g.`` tables, arrays without ghost-cells), or just to save memory as the returned Variable::Span can be used for computation, thus avoiding an extra copy from user memory into the adios buffer. -Variable::Span combines a hybrid Sync and Deferred mode, in which the initial value and memory allocations are Sync, while data population and metadata collection are done at EndStep/PerformPuts/Close. Memory contracts are explained later in this chapter followed by examples. +``Variable::Span`` is helpful in situations in which temporaries are needed to create contiguous pieces of memory from non-contiguous pieces (*e.g.* tables, arrays without ghost-cells), or just to save memory as the returned ``Variable::Span`` can be used for computation, thus avoiding an extra copy from user memory into the ADIOS2 buffer. +``Variable::Span`` combines a hybrid ``Sync`` and ``Deferred`` mode, in which the initial value and memory allocations are ``Sync``, while data population and metadata collection are done at EndStep/PerformPuts/Close. +Memory contracts are explained later in this chapter followed by examples. -The following Variable::Span signatures are available: +The following ``Variable::Span`` signatures are available: -2. Return a span setting a default T() value into a default buffer +2. Return a span setting a default ``T()`` value into a default buffer .. code-block:: c++ Variable::Span Put(Variable variable); -3. Return a span setting an initial fill value into a certain buffer. If span is not returned then the fillValue is fixed for that block. +3. Return a span setting an initial fill value into a certain buffer. +If span is not returned then the ``fillValue`` is fixed for that block. .. code-block:: c++ Variable::Span Put(Variable variable, const size_t bufferID, const T fillValue); -.. warning:: - - As of version 2.4.0 only the default BP3 engine using the C++11 bindings supports ``Variable::Span`` Put signatures. We plan to support this feature and add this to streaming Engines. - - In summary, the following are the current Put signatures for publishing data in ADIOS 2: -1. Deferred (default) or Sync mode, data is contiguous memory put in an adios2 buffer +1. ``Deferred`` (default) or ``Sync`` mode, data is contiguous memory put in an ADIOS2 buffer. .. code-block:: c++ void Put(Variable variable, const T* data, const adios2::Mode = adios2::Mode::Deferred); -2. Return a span setting a default T() value into a default adios2 buffer. If span is not returned then the default T() is fixed for that block (e.g. zeros). +2. Return a span setting a default ``T()`` value into a default ADIOS2 buffer. +If span is not returned then the default ``T()`` is fixed for that block (e.g. zeros). .. code-block:: c++ Variable::Span Put(Variable variable); -3. Return a span setting an initial fill value into a certain buffer. If span is not returned then the fillValue is fixed for that block. +3. Return a span setting an initial fill value into a certain buffer. +If span is not returned then the ``fillValue`` is fixed for that block. .. code-block:: c++ Variable::Span Put(Variable variable, const size_t bufferID, const T fillValue); -The following table summarizes the memory contracts required by adios2 engines between Put signatures and the data memory coming from an application: +The following table summarizes the memory contracts required by ADIOS2 engines between ``Put`` signatures and the data memory coming from an application: +----------+-------------+----------------------------------------------------+ | Put | Data Memory | Contract | @@ -141,9 +150,9 @@ The following table summarizes the memory contracts required by adios2 engines b In Fortran (array) and Python (numpy array) avoid operations that modify the internal structure of an array (size) to preserve the address. -Each Engine will give a concrete meaning to each functions signatures, but all of them must follow the same memory contracts to the "data pointer": the memory address itself, and the "data contents": memory bits (values). +Each ``Engine`` will give a concrete meaning to each functions signatures, but all of them must follow the same memory contracts to the "data pointer": the memory address itself, and the "data contents": memory bits (values). -1. **Put in Deferred or lazy evaluation mode (default)**: this is the preferred mode as it allows Put calls to be "grouped" before potential data transport at the first encounter of ``PerformPuts``, ``EndStep`` or ``Close``. +1. **Put in Deferred or lazy evaluation mode (default)**: this is the preferred mode as it allows ``Put`` calls to be "grouped" before potential data transport at the first encounter of ``PerformPuts``, ``EndStep`` or ``Close``. .. code-block:: c++ @@ -185,10 +194,11 @@ Each Engine will give a concrete meaning to each functions signatures, but all .. tip:: - It's recommended practice to set all data contents before Put in deferred mode to minimize the risk of modifying the data pointer (not just the contents) before PerformPuts/EndStep/Close. + It's recommended practice to set all data contents before ``Put`` in deferred mode to minimize the risk of modifying the data pointer (not just the contents) before PerformPuts/EndStep/Close. -2. **Put in Sync mode**: this is the special case, data pointer becomes reusable right after Put. Only use it if absolutely necessary (*e.g.* memory bound application or out of scope data, temporary). +2. **Put in Sync mode**: this is the special case, data pointer becomes reusable right after ``Put``. +Only use it if absolutely necessary (*e.g.* memory bound application or out of scope data, temporary). .. code-block:: c++ @@ -233,7 +243,7 @@ Each Engine will give a concrete meaning to each functions signatures, but all Span memory contracts: - - "data pointer" provided by the engine and returned by span.data(), might change with the generation of a new span. It follows iterator invalidation rules from std::vector. Use `span.data()` or iterators, `span.begin()`, `span.end()` to keep an updated data pointer. + - "data pointer" provided by the engine and returned by ``span.data()``, might change with the generation of a new span. It follows iterator invalidation rules from std::vector. Use `span.data()` or iterators, `span.begin()`, `span.end()` to keep an updated data pointer. - span "data contents" are published at the first call to ``PerformPuts``, ``EndStep`` or ``Close`` @@ -282,35 +292,34 @@ Each Engine will give a concrete meaning to each functions signatures, but all PerformsPuts ------------ - - Executes all pending Put calls in deferred mode ad collect spans data + + Executes all pending ``Put`` calls in deferred mode ad collect spans data Get: modes and memory contracts ------------------------------- -``Get`` is the generalized abstract function for consuming data in adios2 when an Engine is created using Read mode at ``IO::Open``. ADIOS 2 Put and Get APIs semantics are as symmetric as possible considering that they are opposite operations (*e.g.* Put passes ``const T*``, while Get populates a non-const ``T*``). - -.. - Optionally, adios2 Engines can provide direct access to its buffer memory using an overload that returns a span to a variable block (base on a subset of the upcoming `C++20 std::span `_, non-owning contiguous memory piece). Spans act as a 1D memory container meant to be filled out by the application. See :ref:`Supported Engines` for engines that support the span feature (e.g. BP3). +``Get`` is the function for consuming data in ADIOS2. +It is available when an Engine is created using ``Read`` mode at ``IO::Open``. +ADIOS2 ``Put`` and ``Get`` semantics are as symmetric as possible considering that they are opposite operations (*e.g.* ``Put`` passes ``const T*``, while ``Get`` populates a non-const ``T*``). -The following are the current Get signatures: +The ``Get`` signatures are described below. -1. Deferred (default) or Sync mode, data is contiguous pre-allocated memory +1. ``Deferred`` (default) or ``Sync`` mode, data is contiguous pre-allocated memory: .. code-block:: c++ Get(Variable variable, const T* data, const adios2::Mode = adios2::Mode::Deferred); - - -2. C++11 only, dataV is automatically resized by adios2 based on Variable selection + + +2. In this signature, ``dataV`` is automatically resized by ADIOS2 based on the ``Variable`` selection: .. code-block:: c++ - + Get(Variable variable, std::vector& dataV, const adios2::Mode = adios2::Mode::Deferred); - - -The following table summarizes the memory contracts required by adios2 engines between Get signatures and the pre-allocated (except when using C++11 ``std::vector``) data memory coming from an application: + + +The following table summarizes the memory contracts required by ADIOS2 engines between ``Get`` signatures and the pre-allocated (except when using C++11 ``std::vector``) data memory coming from an application: +----------+-------------+-----------------------------------------------+ | Get | Data Memory | Contract | @@ -325,7 +334,7 @@ The following table summarizes the memory contracts required by adios2 engines b +----------+-------------+-----------------------------------------------+ -1. **Get in Deferred or lazy evaluation mode (default)**: this is the preferred mode as it allows Get calls to be "grouped" before potential data transport at the first encounter of ``PerformPuts``, ``EndStep`` or ``Close``. +1. **Get in Deferred or lazy evaluation mode (default)**: this is the preferred mode as it allows ``Get`` calls to be "grouped" before potential data transport at the first encounter of ``PerformPuts``, ``EndStep`` or ``Close``. .. code-block:: c++ @@ -342,36 +351,34 @@ The following table summarizes the memory contracts required by adios2 engines b Usage: .. code-block:: c++ - + std::vector data; - + // resize memory to expected size data.resize(varBlockSize); // valid if all memory is populated // data.reserve(varBlockSize); - + // Gets data pointer to adios2 engine // associated with current variable metadata engine.Get(variable, data.data() ); - + // optionally pass data std::vector // leave resize to adios2 //engine.Get(variable, data); - + // "data contents" must be ready // "data pointer" must be the same as in Get engine.EndStep(); //engine.PerformPuts(); //engine.Close(); - + // now data pointer can be reused or modified - - .. caution:: - Use uninitialized memory at your own risk (e.g. vector reserve, new, malloc). Accessing unitiliazed memory is undefined behavior. -2. **Put in Sync mode**: this is the special case, data pointer becomes reusable right after Put. Only use it if absolutely necessary (*e.g.* memory bound application or out of scope data, temporary). +2. **Put in Sync mode**: this is the special case, data pointer becomes reusable right after Put. +Only use it if absolutely necessary (*e.g.* memory bound application or out of scope data, temporary). .. code-block:: c++ @@ -404,15 +411,15 @@ The following table summarizes the memory contracts required by adios2 engines b // "data pointer" can be reused by the application .. note:: - - As of v2.4 Get doesn't support returning spans. This is future work required in streaming engines if the application wants a non-owning view into the data buffer for a particular variable block. + + ``Get`` doesn't support returning spans. PerformsGets ------------ - - Executes all pending Get calls in deferred mode - + + Executes all pending ``Get`` calls in deferred mode. + Engine usage example -------------------- @@ -503,22 +510,24 @@ The following example illustrates the basic API usage in write mode for data gen .. tip:: - Prefer default Deferred (lazy evaluation) functions as they have the potential to group several variables with the trade-off of not being able to reuse the pointers memory space until ``EndStep``, ``PerformPuts``, ``PerformGets``, or ``Close``. - Only use Sync if you really have to (*e.g.* reuse memory space from pointer). + Prefer default ``Deferred`` (lazy evaluation) functions as they have the potential to group several variables with the trade-off of not being able to reuse the pointers memory space until ``EndStep``, ``PerformPuts``, ``PerformGets``, or ``Close``. + Only use ``Sync`` if you really have to (*e.g.* reuse memory space from pointer). ADIOS2 prefers a step-based IO in which everything is known ahead of time when writing an entire step. .. danger:: - The default behavior of adios2 ``Put`` and ``Get`` calls IS NOT synchronized, but rather deferred. + The default behavior of ADIOS2 ``Put`` and ``Get`` calls IS NOT synchronized, but rather deferred. It's actually the opposite of ``MPI_Put`` and more like ``MPI_rPut``. Do not assume the data pointer is usable after a ``Put`` and ``Get``, before ``EndStep``, ``Close`` or the corresponding ``PerformPuts``/``PerformGets``. - Avoid using TEMPORARIES, r-values, and out-of-scope variables in ``Deferred`` mode, use adios2::Mode::Sync if required. + Avoid using temporaries, r-values, and out-of-scope variables in ``Deferred`` mode. + Use ``adios2::Mode::Sync`` in these cases. Available Engines ----------------- -A particular engine is set within the IO object that creates it with the ``IO::SetEngine`` function in a case insensitive manner. If the SetEngine function is not invoked the default engine is the ``BPFile`` for writing and reading self-describing bp (binary-pack) files. +A particular engine is set within the ``IO`` object that creates it with the ``IO::SetEngine`` function in a case insensitive manner. +If the ``SetEngine`` function is not invoked the default engine is the ``BPFile``. +-------------------------+---------+---------------------------------------------+ | Application | Engine | Description | @@ -533,12 +542,13 @@ A particular engine is set within the IO object that creates it with the ``IO::S +-------------------------+---------+---------------------------------------------+ -Engine Polymorphism has a two-fold goal: - -1. Each Engine implements an orthogonal IO scenario targeting a use case (e.g. Files, WAN, InSitu MPI, etc) using a simple, unified API. - -2. Allow developers to build their own custom system solution based on their particular requirements in the own playground space. Reusable toolkit objects are available inside ADIOS2 for common tasks: bp buffering, transport management, transports, etc. +``Engine`` polymorphism has two goals: -A class that extends Engine must be thought of as a solution to a range of IO applications. Each engine must provide a list of supported parameters, set in the IO object creating this engine using ``IO::SetParameters, IO::SetParameter``, and supported transports (and their parameters) in ``IO::AddTransport``. Each Engine's particular options are documented in :ref:`Supported Engines`. +1. Each ``Engine`` implements an orthogonal IO scenario targeting a use case (e.g. Files, WAN, InSitu MPI, etc) using a simple, unified API. +2. Allow developers to build their own custom system solution based on their particular requirements in the own playground space. +Reusable toolkit objects are available inside ADIOS2 for common tasks: bp buffering, transport management, transports, etc. +A class that extends ``Engine`` must be thought of as a solution to a range of IO applications. +Each engine must provide a list of supported parameters, set in the IO object creating this engine using ``IO::SetParameters``, and supported transports (and their parameters) in ``IO::AddTransport``. +Each Engine's particular options are documented in :ref:`Supported Engines`. diff --git a/docs/user_guide/source/components/io.rst b/docs/user_guide/source/components/io.rst index b7a6602673..ae2355baa6 100644 --- a/docs/user_guide/source/components/io.rst +++ b/docs/user_guide/source/components/io.rst @@ -2,14 +2,18 @@ IO ** -The IO component is the connection between how applications set up their input/output options by selecting an Engine and its specific parameters, subscribing variables to self-describe the data, and setting supported transport modes to a particular Engine. Think of IO as a control panel for all the user-defined parameters that specific applications would like to be able to fine tune. None of the IO operations are heavyweight until the Open function that generates an Engine is called. Its API allows for: +The ``IO`` component is the connection between how applications set up their input/output options by selecting an ``Engine`` and its specific parameters, subscribing variables to data, and setting supported transport modes to a particular ``Engine``. +Think of ``IO`` as a control panel for all the user-defined parameters that applications would like to fine tune. +None of the ``IO`` operations are heavyweight until the ``Open`` function that generates an ``Engine`` is called. +Its API allows -* Being a factory of Variable and Attribute components containing self-describing information about the overall data in the input output process -* Setting Engine-specific parameters and adding supported modes of transport -* Being a factory of (same type) Engine to execute the actual IO tasks +* generation of ``Variable`` and ``Attribute`` components containing information about the data in the input output process +* setting ``Engine``-specific parameters and adding supported modes of transport +* generation of ``Engine`` objects to execute the actual IO tasks. .. note:: - If two different engine types are needed (*e.g.* BPFile, SST) define two IOs. Also, at reading always define separate IOs to avoid name-clashing of Variables. + If two different engine types are needed (*e.g.* ``BPFile``, ``SST``), you must define two ``IO`` objects. + Also, at reading always define separate IOs to avoid ``Variable`` name clashes. .. blockdiag:: @@ -41,7 +45,9 @@ The IO component is the connection between how applications set up their input/o Setting a Particular Engine and its Parameters ---------------------------------------------- -Engines are the actual components executing the heavy operations in ADIOS2. Each IO must select a type of Engine depending on the application needs through the SetEngine function. The default is the BPFile engine, for writing and reading bp files, if SetEngine is not called. +Engines execute the heavy operations in ADIOS2. +Each ``IO`` may select a type of ``Engine`` through the ``SetEngine`` function. +If ``SetEngine`` is not called, then the ``BPFile`` engine is used. .. code-block:: c++ @@ -49,9 +55,11 @@ Engines are the actual components executing the heavy operations in ADIOS2. Each void adios2::IO::SetEngine( const std::string engineType ); /** Example */ - bpIO.SetEngine("BPFileWriter"); + bpIO.SetEngine("BPFile"); -Each Engine allows the user to fine tune execution of buffering and output tasks through passing parameters to the IO object that will then propagate to the Engine. For a list of parameters allowed by each engine see :ref:`Available Engines`. +Each ``Engine`` allows the user to fine tune execution of buffering and output tasks via parameters passed to the ``IO`` object. +These parameters are then propagated to the ``Engine``. +For a list of parameters allowed by each engine see :ref:`Available Engines`. .. note:: @@ -72,13 +80,15 @@ Each Engine allows the user to fine tune execution of buffering and output tasks {"BufferGrowthFactor", "1.5" } {"FlushStepsCount", "5" } } ); - io.SetParameter( "Threads", "4 ); + io.SetParameter( "Threads", "4" ); Adding Supported Transports with Parameters ------------------------------------------- -The AddTransport function returns an unsigned int handler for each transport that can be used with the Engine Close function at different times. AddTransport must provide library specific settings that the low-level system library interface allows. These options are expected to become more complex as new modes of transport are allowed beyond files (*e.g.* RDMA). +The ``AddTransport`` function allows the user to specify how data is moved through the system, *e.g.* RDMA, wide-area networks, or files. +It returns an ``unsigned int`` handler for each transport that can be used with the ``Engine::Close`` function at different times. +``AddTransport`` must provide library specific settings that the low-level system library interface allows. .. code-block:: c++ @@ -107,10 +117,10 @@ The AddTransport function returns an unsigned int handler for each transport tha Defining, Inquiring and Removing Variables and Attributes --------------------------------------------------------- -The template functions ``DefineVariable`` allows subscribing self-describing data into ADIOS2 by returning a reference to a Variable class object whose scope is the same as the IO object that created it. -The user must provide a unique name (among Variables), the dimensions: MPI global: shape, MPI local: start and offset, optionally a flag indicating that dimensions are known to be constant, and a data pointer if defined in the application. -Note: actual data is not passed at this stage. -This is done by the Engine functions ``Put``/``Get`` for Variables. +The template functions ``DefineVariable`` allows subscribing to data into ADIOS2 by returning a reference to a ``Variable`` class object whose scope is the same as the ``IO`` object that created it. +The user must provide a unique name, the dimensions: MPI global: shape, MPI local: start and offset, optionally a flag indicating that dimensions are known to be constant, and a data pointer if defined in the application. +Note: data is not passed at this stage. +This is done by the ``Engine`` functions ``Put`` and ``Get`` for Variables. See the :ref:`Variable` section for supported types and shapes. .. tip:: @@ -135,7 +145,9 @@ See the :ref:`Variable` section for supported types and shapes. {Nx}, adios2::ConstantDims); -Attributes are extra-information associated with the current IO object. The function ``DefineAttribute`` allows for defining single value and array attributes. Keep in mind that Attributes apply to all Engines created by the IO object and, unlike Variables which are passed to each Engine explicitly, their definition contains their actual data. +Attributes are extra-information associated with the current ``IO`` object. +The function ``DefineAttribute`` allows for defining single value and array attributes. +Keep in mind that Attributes apply to all Engines created by the ``IO`` object and, unlike Variables which are passed to each ``Engine`` explicitly, their definition contains their actual data. .. code-block:: c++ @@ -151,7 +163,8 @@ Attributes are extra-information associated with the current IO object. The func const size_t elements); In situations in which a variable and attribute has been previously defined: -1) a variable/attribute reference goes out of scope, or 2) when reading from an incoming stream, IO can inquire the current variables and attributes and return a pointer acting as reference. If the inquired variable/attribute is not found, then ``nullptr`` is returned. +1) a variable/attribute reference goes out of scope, or 2) when reading from an incoming stream, the ``IO`` can inquire about the status of variables and attributes. +If the inquired variable/attribute is not found, then the overloaded ``bool()`` operator of returns ``false``. .. code-block:: c++ @@ -160,7 +173,7 @@ In situations in which a variable and attribute has been previously defined: adios2::Attribute InquireAttribute(const std::string &name) noexcept; /** Example */ - adios2::Variable varPressure = io.InquireVariable("pressure"); + adios2::Variable varPressure = io.InquireVariable("pressure"); if( varPressure ) // it exists { ... @@ -168,43 +181,22 @@ In situations in which a variable and attribute has been previously defined: .. note:: - The reason for returning a pointer when inquiring, unlike references when defining, is because ``nullptr`` is a valid state (e.g. variables hasn't arrived in a stream, wasn't previously defined or wasn't written in a file). - - Always check for ``nullptr`` in the pointer returned by ``InquireVariable`` or ``InquireAttribute`` + ``adios2::Variable`` overloads ``operator bool()`` so that we can check for invalid states (e.g. variables haven't arrived in a stream, weren't previously defined, or weren't written in a file). .. caution:: - Since Inquire are template functions, name and type must both match the variable/attribute you are looking for. - - -Removing Variables and Attributes can be done one at a time or by removing all existing variables or attributes in IO. - -.. code-block:: c++ - - /** Signature */ - bool IO::RemoveVariable(const std::string &name) noexcept; - void IO::RemoveAllVariables( ) noexcept; - - bool IO::RemoveAttribute(const std::string &name) noexcept; - void IO::RemoveAllAttributes( ) noexcept; - -.. caution:: - - Remove functions must be used with caution as they generate dangling Variable/Attributes pointers or references if they didn't go out of scope. - -.. tip:: - - It is good practice to check the bool flag returned by ``RemoveVariable`` or ``RemoveAttribute``. + Since ``InquireVariable`` and ``InquireAttribute`` are template functions, both the name and type must match the data you are looking for. Opening an Engine ----------------- -The ``IO::Open`` function creates a new derived object of the abstract Engine class and returns a reference handler to the user. A particular Engine type is set to the current IO component with the ``IO::SetEngine`` function. Engine polymorphism is handled internally by the IO class, which allows subscribing future derived Engine types without changing the basic API. - -.. note:: +The ``IO::Open`` function creates a new derived object of the abstract ``Engine`` class and returns a reference handler to the user. +A particular ``Engine`` type is set to the current ``IO`` component with the ``IO::SetEngine`` function. +Engine polymorphism is handled internally by the ``IO`` class, which allows subclassing future derived ``Engine`` types without changing the basic API. - Currently only ``adios2::Mode:Write`` and ``adios2::Mode::Read`` are supported, ``adios2::Mode::Append`` is under development +``Engine`` objects are created in various modes. +The available modes are ``adios2::Mode::Read``, ``adios2::Mode::Write``, ``adios2::Mode::Append``, ``adios2::Mode::Sync``, ``adios2::Mode::Deferred``, and ``adios2::Mode::Undefined``. .. code-block:: c++ @@ -223,33 +215,17 @@ The ``IO::Open`` function creates a new derived object of the abstract Engine cl /** Examples */ /** Engine derived class, spawned to start Write operations */ - adios2::Engine bpWriter = io.Open("myVector.bp", adios2::Mode::Write); - - if(bpWriter) // good practice - { - ... - } - + adios2::Engine& bpWriter = io.Open("myVector.bp", adios2::Mode::Write); /** Engine derived class, spawned to start Read operations on rank 0 */ if( rank == 0 ) { - adios2::Engine bpReader = io.Open("myVector.bp", + adios2::Engine& bpReader = io.Open("myVector.bp", adios2::Mode::Read, MPI_COMM_SELF); - if(bpReader) // good practice - { - ... - } } -.. tip:: - - It is good practice to always check the validity of each ADIOS object before operating on it using the explicit bool operator. - ``if( engine ){ }`` - .. caution:: - Always pass ``MPI_COMM_SELF`` if an ``Engine`` lives in only one MPI process. ``Open`` and ``Close`` are collective operations. - - + Always pass ``MPI_COMM_SELF`` if an ``Engine`` lives in only one MPI process. + ``Open`` and ``Close`` are collective operations. diff --git a/docs/user_guide/source/components/operator.rst b/docs/user_guide/source/components/operator.rst index f9be356022..1c3bcb8dbf 100644 --- a/docs/user_guide/source/components/operator.rst +++ b/docs/user_guide/source/components/operator.rst @@ -2,7 +2,8 @@ Operator ******** -The Operator abstraction allows ADIOS2 to act upon the user application data, either from a ``adios2::Variable`` or a set of Variables in an ``adios2::IO`` object. Current supported operations are: +The Operator abstraction allows ADIOS2 to act upon the user application data, either from a ``adios2::Variable`` or a set of Variables in an ``adios2::IO`` object. +Current supported operations are: 1. Data compression/decompression, lossy and lossless. 2. Callback functions (C++11 bindings only) supported by specific engines @@ -11,4 +12,5 @@ ADIOS2 enables the use of third-party libraries to execute these tasks. .. warning:: - Make sure your ADIOS2 library installation used for writing and reading was linked with a compatible version of a third-party dependency when working with operators. ADIOS2 will issue an exception if an operator library dependency is missing. + Make sure your ADIOS2 library installation used for writing and reading was linked with a compatible version of a third-party dependency when working with operators. + ADIOS2 will issue an exception if an operator library dependency is missing. diff --git a/docs/user_guide/source/components/overview.rst b/docs/user_guide/source/components/overview.rst index 89d422ed0f..c6a28b46b2 100644 --- a/docs/user_guide/source/components/overview.rst +++ b/docs/user_guide/source/components/overview.rst @@ -4,7 +4,8 @@ Components Overview .. note:: - If you are doing simple tasks where performance is a non-critical aspect please go to the :ref:`High-Level APIs` section for a quick start. If you are an HPC application developer or you want to use ADIOS2 functionality in full please read this chapter. + If you are doing simple tasks where performance is a non-critical aspect please go to the :ref:`High-Level APIs` section for a quick start. + If you are an HPC application developer or you want to use ADIOS2 functionality in full please read this chapter. The simple way to understand the big picture for the ADIOS2 unified user interface components is to map each class to the actual definition of the ADIOS acronym. @@ -46,7 +47,7 @@ The following section provides a common overview to all languages based on the C The following figure depicts the components hierarchy from the application's point of view. -.. image:: https://i.imgur.com/y7bkQQt.png : alt: my-picture2 +.. image:: https://i.imgur.com/y7bkQQt.png * **ADIOS**: the ADIOS component is the starting point between an application and the ADIOS2 library. Applications provide: 1. the scope of the ADIOS object through the MPI communicator, @@ -65,4 +66,4 @@ The following figure depicts the components hierarchy from the application's poi * **Engine**: Engines define the actual system executing the heavy IO tasks at Open, BeginStep, Put, Get, EndStep and Close. Due to polymorphism, new IO system solutions can be developed quickly reusing internal components and reusing the same API. If IO.SetEngine is not called, the default engine is the binary-pack bp file reader and writer: **BPFile**. -* **Operator**: (under development) this component defines possible operations to be applied on adios2 self-describing data. This higher level abstraction is needed to provide support for: Callback functions, Transforms, Analytics functions, Data Models functionality, etc. Any required task will be executed within the Engine. One or many operators can be associated with any of the adios2 objects or a group of them. +* **Operator**: These define possible operations to be applied on adios2-managed data, for example, compression. This higher level abstraction is needed to provide support for callbacks, transforms, analytics, data models, etc. Any required task will be executed within the Engine. One or many operators can be associated with any of the adios2 objects or a group of them. diff --git a/docs/user_guide/source/components/runtime.rst b/docs/user_guide/source/components/runtime.rst index 2e66899a0d..d79398f976 100644 --- a/docs/user_guide/source/components/runtime.rst +++ b/docs/user_guide/source/components/runtime.rst @@ -6,7 +6,8 @@ ADIOS2 supports passing an optional runtime configuration file to the :ref:`ADIO This file contains key-value pairs equivalent to the compile time ``IO::SetParameters`` (``adios2_set_parameter`` in C, Fortran), and ``IO::AddTransport`` (``adios2_set_transport_parameter`` in C, Fortran). -Each Engine and Operator must provide a set of available parameters as described in the :ref:`Supported Engines` section. Up to version v2.6.0 only XML is supported, v2.6.0 and beyond support XML as well as YAML. +Each ``Engine`` and ``Operator`` must provide a set of available parameters as described in the :ref:`Supported Engines` section. +Up to version v2.6.0 only XML is supported, v2.6.0 and beyond support XML as well as YAML. .. warning:: @@ -50,7 +51,10 @@ XML YAML ---- -Starting with v2.6.0, ADIOS supports YAML configuration files. The syntax follows strict use of the YAML node keywords mapping to the ADIOS2 components hierarchy. If a keyword is unknown ADIOS2 simply ignores it. For an example file refer to `adios2 config file example in our repo. `_ +Starting with v2.6.0, ADIOS supports YAML configuration files. +The syntax follows strict use of the YAML node keywords mapping to the ADIOS2 components hierarchy. +If a keyword is unknown ADIOS2 simply ignores it. +For an example file refer to `adios2 config file example in our repo. `_ .. code-block:: yaml @@ -100,4 +104,3 @@ Starting with v2.6.0, ADIOS supports YAML configuration files. The syntax follow .. tip:: Run a YAML validator or use a YAML editor to make sure the provided file is YAML compatible. - diff --git a/docs/user_guide/source/components/variable.rst b/docs/user_guide/source/components/variable.rst index 23d35d1dfb..4e8eb6112a 100644 --- a/docs/user_guide/source/components/variable.rst +++ b/docs/user_guide/source/components/variable.rst @@ -2,15 +2,19 @@ Variable ******** -Self-describing Variables are the atomic unit of data representation in the ADIOS2 library when interacting with applications. Thus, the Variable component is the link between a piece of data coming from an application and its self-describing information or metadata. This component handles all application variables classified by data type and shape type. +An ``adios2::Variable`` is the link between a piece of data coming from an application and its metadata. +This component handles all application variables classified by data type and shape. -Each IO holds its own set of Variables, each Variable is identified with a unique name. They are created using the reference from ``IO::DefineVariable`` or retrieved using the pointer from ``IO::InquireVariable`` functions in :ref:`IO`. +Each ``IO`` holds a set of Variables, and each ``Variable`` is identified with a unique name. +They are created using the reference from ``IO::DefineVariable`` or retrieved using the pointer from ``IO::InquireVariable`` functions in :ref:`IO`. -Variables Data Types +Data Types -------------------- -Currently, only primitive types are supported in ADIOS 2. -Fixed-width types from ` and `_ should be preferred when writing portable code. ADIOS 2 maps primitive "natural" types to its equivalent fixed-width type (e.g. ``int`` -> ``int32_t``). Acceptable values for the type ``T`` in ``Variable`` (this is C++ only, see below for other bindings) along with their preferred fix-width equivalent in 64-bit platforms: +Only primitive types are supported in ADIOS2. +Fixed-width types from ` and `_ should be preferred when writing portable code. +ADIOS2 maps primitive types to equivalent fixed-width types (e.g. ``int`` -> ``int32_t``). +In C++, acceptable types ``T`` in ``Variable`` along with their preferred fix-width equivalent in 64-bit platforms are given below: .. code-block:: c++ @@ -36,34 +40,38 @@ Fixed-width types from ` and ``. + It's recommended to be consistent when using types for portability. + If data is defined as a fixed-width integer, define variables in ADIOS2 using a fixed-width type, *e.g.* for ``int32_t`` data types use ``DefineVariable``. .. note:: - C, Fortran APIs: the enum and parameter adios2_type_XXX only provides fixed-width types + C, Fortran APIs: the enum and parameter adios2_type_XXX only provides fixed-width types. .. note:: - Python APIs: use the equivalent fixed-width types from numpy. If dtype is not specified, ADIOS 2 would handle numpy defaults just fine as long as primitive types are passed. + Python APIs: use the equivalent fixed-width types from numpy. + If ``dtype`` is not specified, ADIOS2 handles numpy defaults just fine as long as primitive types are passed. -Variables Shape Types +Shapes --------------------- -.. note:: - As of beta release version 2.2.0 local variable reads are not supported, yet. This is work in progress. Please use global arrays and values as a workaround. - -ADIOS2 is designed *out-of-the-box* for MPI applications. Thus different application data shape types must be covered depending on their scope within a particular MPI communicator. The shape type is defined at creation from the IO object by providing the dimensions: shape, start, count in the ``IO::DeclareVariable`` template function. The supported Variables by shape types can be classified as: +ADIOS2 is designed for MPI applications. +Thus different application data shapes must be supported depending on their scope within a particular MPI communicator. +The shape is defined at creation from the ``IO`` object by providing the dimensions: shape, start, count in the ``IO::DefineVariable``. +The supported shapes are described below. -1. **Global Single Value**: only name is required in their definition. This variables are helpful for storing global information, preferably managed by only one MPI process, that may or may not change over steps: *e.g.* total number of particles, collective norm, number of nodes/cells, etc. +1. **Global Single Value**: +Only a name is required for their definition. +These variables are helpful for storing global information, preferably managed by only one MPI process, that may or may not change over steps: *e.g.* total number of particles, collective norm, number of nodes/cells, etc. .. code-block:: c++ if( rank == 0 ) { - adios2::Variable varNodes = adios2::DefineVariable("Nodes"); - adios2::Variable varFlag = adios2::DefineVariable("Nodes flag"); + adios2::Variable varNodes = io.DefineVariable("Nodes"); + adios2::Variable varFlag = io.DefineVariable("Nodes flag"); // ... engine.Put( varNodes, nodes ); engine.Put( varFlag, "increased" ); @@ -72,52 +80,46 @@ ADIOS2 is designed *out-of-the-box* for MPI applications. Thus different applica .. note:: - Variables of type string are defined just like global single values. In the current adios2 version multidimensional strings are supported for fixed size strings through variables of type ``char``. + Variables of type ``string`` are defined just like global single values. + Multidimensional strings are supported for fixed size strings through variables of type ``char``. -2. **Global Array**: the most common shape used for storing self-describing data used for analysis that lives in several MPI processes. The image below illustrates the definitions of the dimension components in a global array: shape, start, and count. +2. **Global Array**: +This is the most common shape used for storing data that lives in several MPI processes. +The image below illustrates the definitions of the dimension components in a global array: shape, start, and count. - .. image:: https://i.imgur.com/MKwNe5e.png : alt: my-picture2 + .. image:: https://i.imgur.com/MKwNe5e.png .. warning:: - Be aware of data ordering in your language of choice (Row-Major or Column-Major) as depicted in the above figure. Data decomposition is done by the application based on their requirements, not by adios2. + Be aware of data ordering in your language of choice (row-major or column-major) as depicted in the figure above. + Data decomposition is done by the application, not by ADIOS2. Start and Count local dimensions can be later modified with the ``Variable::SetSelection`` function if it is not a constant dimensions variable. -3. **Local Value**: single value-per-rank variables that are local to the MPI process. They are defined by passing the ``adios2::LocalValueDim`` enum as follows: +3. **Local Value**: +Values that are local to the MPI process. +They are defined by passing the ``adios2::LocalValueDim`` enum as follows: .. code-block:: c++ - adios2::Variable varProcessID = - io.DefineVariable("ProcessID", {adios2::LocalValueDim}) + adios2::Variable varProcessID = + io.DefineVariable("ProcessID", {adios2::LocalValueDim}) //... - engine.Put(varProcessID, rank); + engine.Put(varProcessID, rank); -4. **Local Array**: single array variables that are local to the MPI process. These are more commonly used to write Checkpoint data, that is later read for Restart. Reading, however, needs to be handled differently: each process' array has to be read separately, using SetSelection per rank. The size of each process selection should be discovered by the reading application by inquiring per-block size information of the variable, and allocate memory accordingly. +4. **Local Array**: +Arrays that are local to the MPI process. +These are commonly used to write checkpoint-restart data. +Reading, however, needs to be handled differently: each process' array has to be read separately, using ``SetSelection`` per rank. +The size of each process selection should be discovered by the reading application by inquiring per-block size information of the variable, and allocate memory accordingly. - .. image:: https://i.imgur.com/XLh2TUG.png : alt: my-picture3 - - -5. **Joined Array (NOT YET SUPPORTED)**: in certain circumstances every process has an array that is different only in one dimension. ADIOS2 allows user to present them as a global array by joining the arrays together. For example, if every process has a table with a different number of rows, and one does not want to do a global communication to calculate the offsets in the global table, one can just write the local arrays and let ADIOS2 calculate the offsets at read time (when all sizes are known by any process). - - .. code-block:: c++ - - adios2::Variable varTable = io.DefineVariable( - "table", {adios2::JoinedDim, Ncolumns}, {}, {Nrows, Ncolumns}); - - .. note:: - - Only one dimension can be joinable, every other dimension must be the same on each process. - - .. note: - - The local dimension size in the joinable dimension is allowed to change over time within each processor. However, if the sum of all local sizes changes over time, the result will look like a local array. Since global arrays with changing global dimension over time can only be handled as local arrays in ADIOS2. + .. image:: https://i.imgur.com/XLh2TUG.png .. note:: - Constants are not handled separately from step-varying values in ADIOS2. Simply write them only once from one rank. - + Constants are not handled separately from step-varying values in ADIOS2. + Simply write them only once from one rank. diff --git a/docs/user_guide/source/conf.py b/docs/user_guide/source/conf.py index 23b99413a4..950cdfc325 100644 --- a/docs/user_guide/source/conf.py +++ b/docs/user_guide/source/conf.py @@ -66,7 +66,7 @@ # General information about the project. project = u'ADIOS2' -copyright = u'2018, Oak Ridge National Laboratory' +copyright = u'2020, Oak Ridge National Laboratory' author = u'Oak Ridge National Laboratory' # The version info for the project you're documenting, acts as replacement for diff --git a/docs/user_guide/source/ecosystem/visualization.rst b/docs/user_guide/source/ecosystem/visualization.rst index 6b2200f1c5..a9c0b30327 100644 --- a/docs/user_guide/source/ecosystem/visualization.rst +++ b/docs/user_guide/source/ecosystem/visualization.rst @@ -2,11 +2,7 @@ Visualizing Data ################ -.. note:: +Certain ADIOS2 bp files can be recognized by third party visualization tools. +This section describes how to create an ADIOS2 bp file to accomodate the visualization product requirements. - As of version 2.6.0 this work is experimental. As it builds on top of the library in third party ecosystems, and doesn't ship with ADIOS 2 source code. - -Certain ADIOS 2 bp files, and in-memory streams in the future), can be recognized by third party products to enable data visualization. The expectation is to keep adding support to the list of products enabling ADIOS 2 for visualization purposes. The goal of this section is to describe the currently cover cases and how to create an ADIOS 2 BP file to accomodate the visualization product requirements. - - .. include:: visualization/vtk.rst diff --git a/docs/user_guide/source/ecosystem/visualization/vtk.rst b/docs/user_guide/source/ecosystem/visualization/vtk.rst index e7507a9905..9f4ebeac41 100644 --- a/docs/user_guide/source/ecosystem/visualization/vtk.rst +++ b/docs/user_guide/source/ecosystem/visualization/vtk.rst @@ -2,25 +2,27 @@ Using VTK and Paraview ********************** -ADIOS BP files can now be seamlessly integrated into the `Visualization Toolkit `_ (VTK) and `Paraview `_. Datasets can be described with an additional attribute that conforms to the `VTK XML data model formats `_ as high-level descriptors that will allow interpretation of ADIOS 2 variables into a hierarchy understood by the VTK infrastructure. This XML data format is saved in ADIOS as either an attribute, and optionally, as an additional vtk.xml file inside file.bp.dir/vtk.xml from the default engine. For more details, see :ref:`BP3 (Default)`. +ADIOS BP files can now be seamlessly integrated into the `Visualization Toolkit `_ and `Paraview `_. +Datasets can be described with an additional attribute that conforms to the `VTK XML data model formats `_ as high-level descriptors that will allow interpretation of ADIOS2 variables into a hierarchy understood by the VTK infrastructure. +This XML data format is saved in ADIOS2 as either an attribute or as an additional ``vtk.xml`` file inside the ``file.bp.dir`` directory. +There are currently a number of limitations: -Current limitations: - -* Only works with MPI builds of VTK and Paraview -* Support only one Block per ADIOS dataset +* It only works with MPI builds of VTK and Paraview +* Support only one block per ADIOS dataset * Only supports BP Files, streams are not supported * Currently working up to 3D (and linearized 1D) variables for scalars and vectors. * Image Data, vti, is supported with ADIOS2 Global Array Variables only * Unstructured Grid, vtu, is supported with ADIOS2 Local Arrays Variables only -Two "VTK File" types are supported: +Two VTK file types are supported: 1. Image data (.vti) 2. Unstructured Grid (.vtu) -The main idea is to populate the above XML format contents describing the extent and the data arrays with ADIOS variables in the BP data set. The result is a more-than-compact typical VTK data file since extra information about the variable, such as dimensions and type, as they already exist in the BP data set. +The main idea is to populate the above XML format contents describing the extent and the data arrays with ADIOS variables in the BP data set. +The result is a more-than-compact typical VTK data file since extra information about the variable, such as dimensions and type, as they already exist in the BP data set. A typical VTK image data XML descriptor (.vti): @@ -64,13 +66,14 @@ A typical VTK unstructured grid XML descriptor (.vtu): -In addition, VTK can interpret physical-time or output-step varying data stored with ADIOS by resusing the special "TIME" tag. This is better illustrated in the following section. +In addition, VTK can interpret physical-time or output-step varying data stored with ADIOS by reusing the special "TIME" tag. +This is better illustrated in the following section. Saving the vtk.xml data model ----------------------------- -For the full source code of the following illustration example see the `gray-scott adios2 tutorial `_ +For the full source code of the following illustration example see the `gray-scott adios2 tutorial `_. To incorporate the data model in a BP data file, the application has two options: @@ -99,7 +102,7 @@ To incorporate the data model in a BP data file, the application has two options .. tip:: - C++11 users should take advantage C++11 string literals (R"( xml_here )") to simplify escaping characters in the XML. + C++11 users should take advantage C++11 string literals (``R"( xml_here )"``) to simplify escaping characters in the XML. The resulting bpls output should contain the "vtk.xml" attribute and the variables in the model: @@ -158,14 +161,14 @@ The resulting bpls output should contain the "vtk.xml" attribute and the variabl -The result is that the generated BP file should be recognize by a branch of Paraview/VTK that must be built from source: +This BP file should be recognize by Paraview: .. image:: https://i.imgur.com/ap3l9Z5.png : alt: my-picture2 Similarly, unstructured grid (.vtu) support can be added with the limitations of using specific labels for the variable names setting the "connectivity", "vertices", and cell "types". -The following example is taken from example 2 of the `MFEM product examples website `_ using ADIOS 2: +The following example is taken from example 2 of the `MFEM product examples website `_ using ADIOS2: The resulting `bpls` output for unstructured grid data types: @@ -207,20 +210,3 @@ The resulting `bpls` output for unstructured grid data types: and resulting visualization in Paraview for different "cell" types: .. image:: https://i.imgur.com/ke1xiNh.png : alt: my-picture3 - -Build VTK and Paraview with ADIOS 2 Support -------------------------------------------- - -.. note:: - - Currently the implementation for ADIOS 2 readers exist in VTK and Paraview nightly releases and master branches. We expect this to be part of the VTK and Paraview release cycle with their upcoming releases. Users must build from source and point to these branches until formal merge into their master branches is done. - - -Paraview and its VTK dependency must be built with the following CMake options: - -1. `-DVTK_MODULE_ENABLE_VTK_IOADIOS2=YES` -2. `-DVTK_USE_MPI=ON` -3. `-DPARAVIEW_USE_MPI=ON` - -For comprehensive build instructions see the documentation for `VTK `_ (VTK) and `Paraview `_. - diff --git a/docs/user_guide/source/engines/bp3.rst b/docs/user_guide/source/engines/bp3.rst index 7534f1f26a..4f46c7b44e 100644 --- a/docs/user_guide/source/engines/bp3.rst +++ b/docs/user_guide/source/engines/bp3.rst @@ -48,9 +48,11 @@ This engine allows the user to fine tune the buffering operations through the fo 8. **FlushStepsCount**: users can select how often to produce the more expensive collective metadata file in terms of steps: default is 1. Increase to reduce adios2 collective operations footprint, with the trade-off of reducing checkpoint frequency. Buffer size will increase until first steps count if ``MaxBufferSize`` is not set. -9. **SubStreams**: (MPI-only) users can select how many sub-streams (``M`` sub-files) are produced during a run, ranges between 1 and the number of mpi processes from ``MPI_Size`` (``N``), adios2 will internally aggregate data buffers (``N-to-M``) to output the required number of sub-files. If Substream is out of bounds it will pick either 1 (``SubStreams`` < ``1 -> N-to-1``) or ``N`` ((``SubStreams`` > ``N -> N-to-N``) and ADIOS2 will issue a WARNING message. Use for performance tuning. +9. **NumAggregators** (or **SubStreams**): Users can select how many sub-files (``M``) are produced during a run, ranges between 1 and the number of mpi processes from ``MPI_Size`` (``N``), adios2 will internally aggregate data buffers (``N-to-M``) to output the required number of sub-files. Default is 0, which will let adios2 to group processes per shared-memory-access (i.e. one per compute node) and use one process per node as an aggregator. If NumAggregators is larger than the number of processes then it will be set to the number of processes. -10. **Node-Local**: For distributed file system. Every writer process must make sure the .bp/ directory is created on the local file system. Required for using local disk/SSD/NVMe in a cluster. +10. **AggregatorRatio**: An alternative option to NumAggregators to pick every Nth process as aggregator. An integer divider of the number of processes is required, otherwise a runtime exception is thrown. + +11. **Node-Local**: For distributed file system. Every writer process must make sure the .bp/ directory is created on the local file system. Required for using local disk/SSD/NVMe in a cluster. ==================== ===================== =========================================================== **Key** **Value Format** **Default** and Examples @@ -63,7 +65,8 @@ This engine allows the user to fine tune the buffering operations through the fo MaxBufferSize float+units >= 16Kb **at EndStep**, 10Mb, 0.5Gb BufferGrowthFactor float > 1 **1.05**, 1.01, 1.5, 2 FlushStepsCount integer > 1 **1**, 5, 1000, 50000 - SubStreams integer >= 1 **MPI_Size (N-to-N)**, ``MPI_Size``/2, ... , 2, (N-to-1) 1 + NumAggregators integer >= 1 **0 (one file per compute node)**, ``MPI_Size``/2, ... , 2, (N-to-1) 1 + AggregatorRatio integer >= 1 not used unless set, ``MPI_Size``/N must be an integer value Node-Local string On/Off **Off**, On ==================== ===================== =========================================================== diff --git a/docs/user_guide/source/engines/bp4.rst b/docs/user_guide/source/engines/bp4.rst index e43a51a714..9391476123 100644 --- a/docs/user_guide/source/engines/bp4.rst +++ b/docs/user_guide/source/engines/bp4.rst @@ -58,25 +58,27 @@ This engine allows the user to fine tune the buffering operations through the fo 7. **FlushStepsCount**: users can select how often to produce the more expensive collective metadata file in terms of steps: default is 1. Increase to reduce adios2 collective operations footprint, with the trade-off of reducing checkpoint frequency. Buffer size will increase until first steps count if ``MaxBufferSize`` is not set. -8. **SubStreams**: (MPI-only) users can select how many sub-streams (``M`` sub-files) are produced during a run, ranges between 1 and the number of mpi processes from ``MPI_Size`` (``N``), adios2 will internally aggregate data buffers (``N-to-M``) to output the required number of sub-files. If Substream is out of bounds it will pick either 1 (``SubStreams`` < ``1 -> N-to-1``) or ``N`` ((``SubStreams`` > ``N -> N-to-N``) and ADIOS2 will issue a WARNING message. Use for performance tuning. +8. **NumAggregators** (or **SubStreams**): Users can select how many sub-files (``M``) are produced during a run, ranges between 1 and the number of mpi processes from ``MPI_Size`` (``N``), adios2 will internally aggregate data buffers (``N-to-M``) to output the required number of sub-files. Default is 0, which will let adios2 to group processes per shared-memory-access (i.e. one per compute node) and use one process per node as an aggregator. If NumAggregators is larger than the number of processes then it will be set to the number of processes. -9. **OpenTimeoutSecs**: (Streaming mode) Reader may want to wait for the creation of the file in ``io.Open()``. By default the Open() function returns with an error if file is not found. +9. **AggregatorRatio**: An alternative option to NumAggregators to pick every Nth process as aggregator. An integer divider of the number of processes is required, otherwise a runtime exception is thrown. -10. **BeginStepPollingFrequencySecs**: (Streaming mode) Reader can set how frequently to check the file (and file system) for new steps. Default is 1 seconds which may be stressful for the file system and unnecessary for the application. +10. **OpenTimeoutSecs**: (Streaming mode) Reader may want to wait for the creation of the file in ``io.Open()``. By default the Open() function returns with an error if file is not found. -11. **StatsLevel**: Turn on/off calculating statistics for every variable (Min/Max). Default is On. It has some cost to generate this metadata so it can be turned off if there is no need for this information. +11. **BeginStepPollingFrequencySecs**: (Streaming mode) Reader can set how frequently to check the file (and file system) for new steps. Default is 1 seconds which may be stressful for the file system and unnecessary for the application. -12. **StatsBlockSize**: Calculate Min/Max for a given size of each process output. Default is one Min/Max per writer. More fine-grained min/max can be useful for querying the data. +12. **StatsLevel**: Turn on/off calculating statistics for every variable (Min/Max). Default is On. It has some cost to generate this metadata so it can be turned off if there is no need for this information. -13. **NodeLocal** or **Node-Local**: For distributed file system. Every writer process must make sure the .bp/ directory is created on the local file system. Required when writing to local disk/SSD/NVMe in a cluster. Note: the BurstBuffer* parameters are newer and should be used for using the local storage as temporary instead of this parameter. +13. **StatsBlockSize**: Calculate Min/Max for a given size of each process output. Default is one Min/Max per writer. More fine-grained min/max can be useful for querying the data. -14. **BurstBufferPath**: Redirect output file to another location and drain it to the original target location in an asynchronous thread. It requires to be able to launch one thread per aggregator (see SubStreams) on the system. This feature can be used on machines that have local NVMe/SSDs on each node to accelerate the output writing speed. On Summit at OLCF, use "/mnt/bb/" for the path where is your user account name. Temporary files on the accelerated storage will be automatically deleted after the application closes the output and ADIOS drains all data to the file system, unless draining is turned off (see the next parameter). Note: at this time, this feature cannot be used to append data to an existing dataset on the target system. +14. **NodeLocal** or **Node-Local**: For distributed file system. Every writer process must make sure the .bp/ directory is created on the local file system. Required when writing to local disk/SSD/NVMe in a cluster. Note: the BurstBuffer* parameters are newer and should be used for using the local storage as temporary instead of this parameter. -15. **BurstBufferDrain**: To write only to the accelerated storage but to not drain it to the target file system, set this flag to false. Data will NOT be deleted from the accelerated storage on close. By default, setting the BurstBufferPath will turn on draining. +15. **BurstBufferPath**: Redirect output file to another location and drain it to the original target location in an asynchronous thread. It requires to be able to launch one thread per aggregator (see SubStreams) on the system. This feature can be used on machines that have local NVMe/SSDs on each node to accelerate the output writing speed. On Summit at OLCF, use "/mnt/bb/" for the path where is your user account name. Temporary files on the accelerated storage will be automatically deleted after the application closes the output and ADIOS drains all data to the file system, unless draining is turned off (see the next parameter). Note: at this time, this feature cannot be used to append data to an existing dataset on the target system. -16. **BurstBufferVerbose**: Verbose level 1 will cause each draining thread to print a one line report at the end (to standard output) about where it has spent its time and the number of bytes moved. Verbose level 2 will cause each thread to print a line for each draining operation (file creation, copy block, write block from memory, etc). +16. **BurstBufferDrain**: To write only to the accelerated storage but to not drain it to the target file system, set this flag to false. Data will NOT be deleted from the accelerated storage on close. By default, setting the BurstBufferPath will turn on draining. +17. **BurstBufferVerbose**: Verbose level 1 will cause each draining thread to print a one line report at the end (to standard output) about where it has spent its time and the number of bytes moved. Verbose level 2 will cause each thread to print a line for each draining operation (file creation, copy block, write block from memory, etc). +18. **StreamReader**: By default the BP4 engine parses all available metadata in Open(). An application may turn this flag on to parse a limited number of steps at once, and update metadata when those steps have been processed. If the flag is ON, reading only works in streaming mode (using BeginStep/EndStep); file reading mode will not work as there will be zero steps processed in Open(). ============================== ===================== =========================================================== **Key** **Value Format** **Default** and Examples @@ -88,7 +90,8 @@ This engine allows the user to fine tune the buffering operations through the fo MaxBufferSize float+units >= 16Kb **at EndStep**, 10Mb, 0.5Gb BufferGrowthFactor float > 1 **1.05**, 1.01, 1.5, 2 FlushStepsCount integer > 1 **1**, 5, 1000, 50000 - SubStreams integer >= 1 **MPI_Size (N-to-N)**, ``MPI_Size``/2, ... , 2, (N-to-1) 1 + NumAggregators integer >= 1 **0 (one file per compute node)**, ``MPI_Size``/2, ... , 2, (N-to-1) 1 + AggregatorRatio integer >= 1 not used unless set, ``MPI_Size``/N must be an integer value OpenTimeoutSecs float **0**, ``10.0``, ``5`` BeginStepPollingFrequencySecs float **1**, ``10.0`` StatsLevel integer, 0 or 1 **1**, ``0`` @@ -98,6 +101,7 @@ This engine allows the user to fine tune the buffering operations through the fo BurstBufferPath string **""**, /mnt/bb/norbert, /ssd BurstBufferDrain string On/Off **On**, Off BurstBufferVerbose integer, 0-2 **0**, ``1``, ``2`` + StreamReader string On/Off On, **Off** ============================== ===================== =========================================================== diff --git a/docs/user_guide/source/engines/dataman.rst b/docs/user_guide/source/engines/dataman.rst index 7bec5d7254..90c63fe97d 100644 --- a/docs/user_guide/source/engines/dataman.rst +++ b/docs/user_guide/source/engines/dataman.rst @@ -6,16 +6,8 @@ The DataMan engine is designed for data staging over the wide area network. It is supposed to be used in cases where a few writers send data to a few readers over long distance. -DataMan does NOT guarantee that readers receive EVERY data step -from writers. The idea behind this is that for experimental data, which is the target -use case of this engine, the data rate of writers should not be slowed down -by readers. If readers cannot keep up with the experiment, the experiment should still -continue, and the readers should read the latest data steps. The design also helps -improving performance because it saves the communication time for checking step completeness, -which usually means ~100 milliseconds every step for transoceanic connections. - -For wide area data staging applications that require readers to receive EVERY data step, -the SST engine is recommended. +DataMan supports compression operators such as ZFP lossy compression and BZip2 lossless compression. +Please refer to the operator section for usage. The DataMan engine takes the following parameters: diff --git a/docs/user_guide/source/engines/dataspaces.rst b/docs/user_guide/source/engines/dataspaces.rst index ff92e5a0ff..a9627a833b 100644 --- a/docs/user_guide/source/engines/dataspaces.rst +++ b/docs/user_guide/source/engines/dataspaces.rst @@ -19,17 +19,17 @@ how to create an DataSpaces reader: .. code-block:: c++ - adios2::IO dspacesIO = adios.DeclareIO("SomeName"); - dspacesIO.SetEngine("DATASPACES"); - adios2::Engine dspacesReader = dspacesIO.Open(filename, adios2::Mode::Read); + adios2::IO dspacesIO = adios.DeclareIO("SomeName"); + dspacesIO.SetEngine("DATASPACES"); + adios2::Engine dspacesReader = dspacesIO.Open(filename, adios2::Mode::Read); and a sample code for DataSpaces writer is: .. code-block:: c++ - adios2::IO dspacesIO = adios.DeclareIO("SomeName"); - dspacesIO.SetEngine("DATASPACES"); - adios2::Engine dspacesWriter = dspacesIO.Open(filename, adios2::Mode::Write); + adios2::IO dspacesIO = adios.DeclareIO("SomeName"); + dspacesIO.SetEngine("DATASPACES"); + adios2::Engine dspacesWriter = dspacesIO.Open(filename, adios2::Mode::Write); To make use of the DataSpaces engine, an application job needs to also run the dataspaces_server component together with the application. The server should be configured and started @@ -52,9 +52,10 @@ How many output timesteps of the same dataset (called versions) should be kept i and served to readers should be specified in the file. If this file does not exist in the current directory, the server will assume default values (only 1 timestep stored). .. code-block:: - ## Config file for DataSpaces - max_versions = 5 - lock_type = 3 + + ## Config file for DataSpaces + max_versions = 5 + lock_type = 3 The dataspaces_server module is a stand-alone service that runs independently of a simulation on a set of dedicated nodes in the staging area. It transfers data from the application through RDMA, diff --git a/docs/user_guide/source/engines/inline.rst b/docs/user_guide/source/engines/inline.rst index 11bf59aa6c..e116e6691c 100644 --- a/docs/user_guide/source/engines/inline.rst +++ b/docs/user_guide/source/engines/inline.rst @@ -2,9 +2,10 @@ Inline for zero-copy ******************** -The Inline engine provides in-process communication between the writer and reader, and seeks to avoid copying data buffers. +The ``Inline`` engine provides in-process communication between the writer and reader, and seeks to avoid copying data buffers. -This engine is experimental, and is focused on the N -> N case: N writers share a process with N readers, and the analysis happens 'inline' without writing the data to a file or copying to another buffer. It has similar considerations to the streaming SST engine, since analysis must happen per step. +This engine is focused on the N -> N case: N writers share a process with N readers, and the analysis happens 'inline' without writing the data to a file or copying to another buffer. +It has similar considerations to the streaming SST engine, since analysis must happen per step. To use this engine, you can either specify it in your XML config file, with tag ```` or set it in your application code: @@ -12,26 +13,33 @@ To use this engine, you can either specify it in your XML config file, with tag adios2::IO inlineIO = adios.DeclareIO("ioName"); inlineIO.SetEngine("Inline"); - inlineIO.SetParameters({{"writerID", "inline_write"}, {"readerID", "inline_read"}}); adios2::Engine inlineWriter = inlineIO.Open("inline_write", adios2::Mode::Write); adios2::Engine inlineReader = inlineIO.Open("inline_read", adios2::Mode::Read); -Notice that unlike other engines, the reader and writer share an IO instance. Also, the ``writerID`` parameter allows the reader to connect to the writer, and ``readerID`` allows writer to connect to the reader. Both the writer and reader must be opened before either tries to call BeginStep/PerformPuts/PeformGets. +Notice that unlike other engines, the reader and writer share an IO instance. +Both the writer and reader must be opened before either tries to call ``BeginStep()``/``PerformPuts()``/``PerformGets()``. +There must be exactly one writer, and exactly one reader. -For successful operation, the writer will perform a step, then the reader will perform a step in the same process. Data is decomposed between processes, and the writer can write its portion of the data like other ADIOS engines. When the reader starts its step, the only data it has available is that written by the writer in its process. To select this data in ADIOS, use a block selection. The reader then can retrieve whatever data was written by the writer. The reader does require the use of a new ``Get()`` call that was added to the API: +For successful operation, the writer will perform a step, then the reader will perform a step in the same process. +Data is decomposed between processes, and the writer can write its portion of the data like other ADIOS2 engines. +When the reader starts its step, the only data it has available is that written by the writer in its process. +To select this data in ADIOS2, use a block selection. +The reader then can retrieve whatever data was written by the writer. +The reader requires the use of a new ``Get()`` call that was added to the API: .. code-block:: c++ void Engine::Get( \ Variable, typename Variable::Info & info, const Mode); -This version of ``Get`` is only used for the inline engine and requires passing a ``Variable::Info`` object, which can be obtained from calling the reader's ``BlocksInfo()``. See the example below for details. +This version of ``Get`` is only used for the inline engine and requires passing a ``Variable::Info`` object, which can be obtained from calling the reader's ``BlocksInfo()``. +See the example below for details. .. note:: This ``Get()`` method is preliminary and may be removed in the future when the span interface on the read side becomes available. .. note:: - The inline engine does not support Sync mode for writing. In addition, since the inline engine does not do any data copy, the writer should avoid changing the data contents before the reader has read the data. + The inline engine does not support ``Sync`` mode for writing. In addition, since the inline engine does not do any data copy, the writer should avoid changing the data contents before the reader has read the data. Typical access pattern: @@ -59,16 +67,3 @@ Typical access pattern: // use info.Data() to get the pointer for each element in blocksInfo // After any desired analysis is finished, writer can now reuse data pointer - - -Parameters: - -1. **writerID**: Match the string passed to the ``IO::Open()`` call when creating the writer. The reader uses this parameter to fetch the correct writer. -2. **readerID**: Match the string passed to the ``IO::Open()`` call when creating the reader. The writer uses this parameter to fetch the correct reader. - -=========== ===================== =============================== - **Key** **Value Type** **Default** and Examples -=========== ===================== =============================== - writerID string none, match the writer name - readerID string none, match the reader name -=========== ===================== =============================== diff --git a/docs/user_guide/source/engines/ssc.rst b/docs/user_guide/source/engines/ssc.rst index 6181a5e086..d34a2c8250 100644 --- a/docs/user_guide/source/engines/ssc.rst +++ b/docs/user_guide/source/engines/ssc.rst @@ -6,23 +6,14 @@ The SSC engine is designed specifically for strong code coupling. Currently SSC The SSC engine takes the following parameters: -1. ``RendezvousAppCount``: Default **2**. The number of applications, including both writers and readers, that will work on this stream. The SSC engine's open function will block until all these applications reach the open call. If there are multiple applications in a workflow, this parameter needs to be set respectively for every application. For example, in a three-app coupling scenario: App 0 writes Stream A to App 1; App 1 writes Stream B to App 0; App 2 writes Stream C to App 1; App 1 writes Stream D to App 2, the parameter RendezvousAppCount for engine instances of every stream should be all set to 2, because for each of the streams, two applications will work on it. In another example, where App 0 writes Stream A to App 1 and App 2; App 1 writes Stream B to App 2, the parameter RendezvousAppCount for engine instances of Stream A and B should be set to 3 and 2 respectively, because three applications will work on Stream A, while two applications will work on Stream B. +1. ``OpenTimeoutSecs``: Default **10**. Timeout in seconds for opening a stream. The SSC engine's open function will block until the RendezvousAppCount is reached, or timeout, whichever comes first. If it reaches the timeout, SSC will throw an exception. -2. ``MaxStreamsPerApp``: Default **1**. The maximum number of streams that all applications sharing this MPI_COMM_WORLD can possibly open. It is required that this number is consistent across all ranks from all applications. This is used for pre-allocating the vectors holding MPI handshake informations and due to the fundamental communication mechanism of MPI, this information must be set statically through engine parameters, and the SSC engine cannot provide any mechanism to check if this parameter is set correctly. If this parameter is wrongly set, the SSC engine's open function will either exit early than expected without gathering all applications' handshake information, or it will block until timeout. It may cause other unpredictable errors too. - -3. ``OpenTimeoutSecs``: Default **10**. Timeout in seconds for opening a stream. The SSC engine's open function will block until the RendezvousAppCount is reached, or timeout, whichever comes first. If it reaches the timeout, SSC will throw an exception. - -4. ``MaxFilenameLength``: Default **128**. The maximum length of filenames across all ranks from all applications. It is used for allocating the handshake buffer. Due to the limitation of MPI communication, this number must be set statically. The default number should work for most use cases. SSC will throw an exception if any rank opens a stream with a filename longer than this number. - -5. ``MpiMode``: Default **TwoSided**. MPI communication modes to use. Besides the default TwoSided mode using two sided MPI communications, MPI_Isend and MPI_Irecv, for data transport, there are four one sided MPI modes: OneSidedFencePush, OneSidedPostPush, OneSidedFencePull, and OneSidedPostPull. Modes with **Push** are based on the push model and use MPI_Put for data transport, while modes with **Pull** are based on the pull model and use MPI_Get. Modes with **Fence** use MPI_Win_fence for synchronization, while modes with **Post** use MPI_Win_start, MPI_Win_complete, MPI_Win_post and MPI_Win_wait. +2. ``MpiMode``: Default **TwoSided**. MPI communication modes to use. Besides the default TwoSided mode using two sided MPI communications, MPI_Isend and MPI_Irecv, for data transport, there are four one sided MPI modes: OneSidedFencePush, OneSidedPostPush, OneSidedFencePull, and OneSidedPostPull. Modes with **Push** are based on the push model and use MPI_Put for data transport, while modes with **Pull** are based on the pull model and use MPI_Get. Modes with **Fence** use MPI_Win_fence for synchronization, while modes with **Post** use MPI_Win_start, MPI_Win_complete, MPI_Win_post and MPI_Win_wait. =============================== ================== ================================================ **Key** **Value Format** **Default** and Examples =============================== ================== ================================================ - RendezvousAppCount integer **2**, 3, 5, 10 - MaxStreamsPerApp integer **1**, 2, 4, 8 OpenTimeoutSecs integer **10**, 2, 20, 200 - MaxFilenameLength integer **128**, 32, 64, 512 MpiMode string **TwoSided**, OneSidedFencePush, OneSidedPostPush, OneSidedFencePull, OneSidedPostPull =============================== ================== ================================================ diff --git a/docs/user_guide/source/engines/sst.rst b/docs/user_guide/source/engines/sst.rst index bb85da1f36..39a1f4cbd4 100644 --- a/docs/user_guide/source/engines/sst.rst +++ b/docs/user_guide/source/engines/sst.rst @@ -168,7 +168,15 @@ applications running on different interconnects, the Wide Area Network (WAN) option should be chosen. This value is interpreted by both SST Writer and Reader engines. -7. ``ControlTransport``: Default **tcp**. This string value specifies +7. ``WANDataTransport``: Default **sockets**. If the SST +**DataTransport** parameter is **"WAN**, this string value specifies +the EVPath-level data transport to use for exchanging data. The value +must be a data transport known to EVPath, such as **"sockets"**, +**"enet"**, or **"ib"**. Generally both the reader and writer should +be using the same EVPath-level data transport. This value is +interpreted by both SST Writer and Reader engines. + +8. ``ControlTransport``: Default **tcp**. This string value specifies the underlying network communication mechanism to use for performing control operations in SST. SST can be configured to standard TCP sockets, which are very reliable and efficient, but which are limited @@ -180,7 +188,7 @@ equivalent to **scalable**. Generally both the reader and writer should be using the same control transport. This value is interpreted by both SST Writer and Reader engines. -8. ``NetworkInterface``: Default **NULL**. In situations in which +9. ``NetworkInterface``: Default **NULL**. In situations in which there are multiple possible network interfaces available to SST, this string value specifies which should be used to generate SST's contact information for writers. Generally this should *NOT* be specified @@ -193,14 +201,14 @@ will result in SST generating contact information that uses the network address associated with the loopback interface (127.0.0.1). This value is interpreted by only by the SST Writer engine. -9. ``ControlInterface``: Default **NULL**. This value is similar to the +10. ``ControlInterface``: Default **NULL**. This value is similar to the NetworkInterface parameter, but only applies to the SST layer which does messaging for control (open, close, flow and timestep management, but not actual data transfer). Generally the NetworkInterface parameter can be used to control this, but that also aplies to the Data Plane. Use ControlInterface in the event of conflicting specifications. -10. ``DataInterface``: Default **NULL**. This value is similar to the +11. ``DataInterface``: Default **NULL**. This value is similar to the NetworkInterface parameter, but only applies to the SST layer which does messaging for data transfer, not control (open, close, flow and timestep management). Generally the NetworkInterface parameter can be used to @@ -208,7 +216,7 @@ control this, but that also aplies to the Control Plane. Use DataInterface in the event of conflicting specifications. In the case of the RDMA data plane, this parameter controls the libfabric interface choice. -11. ``FirstTimestepPrecious``: Default **FALSE**. +12. ``FirstTimestepPrecious``: Default **FALSE**. FirstTimestepPrecious is a boolean parameter that affects the queueing of the first timestep presented to the SST Writer engine. If FirstTimestepPrecious is **TRUE***, then the first timestep is @@ -224,7 +232,7 @@ other reader-side operations (like requesting the LatestAvailable timestep in Engine parameters) might still cause the timestep to be skipped. This value is interpreted by only by the SST Writer engine. -12. ``AlwaysProvideLatestTimestep``: Default **FALSE**. +13. ``AlwaysProvideLatestTimestep``: Default **FALSE**. AlwaysProvideLatestTimestep is a boolean parameter that affects what of the available timesteps will be provided to the reader engine. If AlwaysProvideLatestTimestep is **TRUE***, then if there are multiple @@ -232,14 +240,14 @@ timesteps available to the reader, older timesteps will be skipped and the reader will see only the newest available upon BeginStep. This value is interpreted by only by the SST Reader engine. -13. ``OpenTimeoutSecs``: Default **60**. OpenTimeoutSecs is an integer +14. ``OpenTimeoutSecs``: Default **60**. OpenTimeoutSecs is an integer parameter that specifies the number of seconds SST is to wait for a peer connection on Open(). Currently this is only implemented on the Reader side of SST, and is a timeout for locating the contact information file created by Writer-side Open, not for completing the entire Open() handshake. Currently value is interpreted by only by the SST Reader engine. -14. ``SpeculativePreloadMode``: Default **AUTO**. In some +15. ``SpeculativePreloadMode``: Default **AUTO**. In some circumstances, SST eagerly sends all data from writers to every readers without first waiting for read requests. Generally this improves performance if every reader needs all the data, but can be @@ -252,7 +260,7 @@ is less than or equal to the value of the ``SpecAutoNodeThreshold`` engine parameter (Default value 1), eager sending is initiated. Currently value is interpreted by only by the SST Reader engine. -15. ``SpecAutoNodeThreshold``: Default **1**. If the size of the +16. ``SpecAutoNodeThreshold``: Default **1**. If the size of the reader cohort is less than or equal to this value *and* the ``SpeculativePreloadMode`` parameter is **AUTO**, SST will initiate eager data sending of all data from each writer to all readers. @@ -268,6 +276,7 @@ Currently value is interpreted by only by the SST Reader engine. QueueFullPolicy string **Block**, Discard ReserveQueueLimit integer **0** (no queue limits) DataTransport string **default varies by platform**, RDMA, WAN + WANDataTransport string **sockets**, enet, ib ControlTransport string **TCP**, Scalable NetworkInterface string **NULL** ControlInterface string **NULL** diff --git a/docs/user_guide/source/engines/virtual_engines.rst b/docs/user_guide/source/engines/virtual_engines.rst index e532c62a25..4aa2a11c30 100644 --- a/docs/user_guide/source/engines/virtual_engines.rst +++ b/docs/user_guide/source/engines/virtual_engines.rst @@ -55,6 +55,7 @@ These are the actual settings in ADIOS when a virtual engine is selected. The pa ============================== ===================== =========================================================== OpenTimeoutSecs float **3600** (wait for up to an hour) BeginStepPollingFrequencySecs float **1** (poll the file system with 1 second frequency + StreamReader bool **On** (process metadata in streaming mode) ============================== ===================== =========================================================== 3. ``InSituAnalysis``. The engine is ``SST``. The parameters are set to: diff --git a/docs/user_guide/source/introduction/nutshell.rst b/docs/user_guide/source/introduction/nutshell.rst index 0358fe19a2..9720cc7b56 100644 --- a/docs/user_guide/source/introduction/nutshell.rst +++ b/docs/user_guide/source/introduction/nutshell.rst @@ -32,7 +32,7 @@ The key aspects ADIOS2 are #. **Commitment:** ADIOS2 is committed to the HPC community, releasing a new version every 6 months. -*ADIOS2 is funded by the Department of Energy as part of the* `Exascale Computing Program `_. +*ADIOS2 is funded by the Department of Energy as part of the* `Exascale Computing Project `_. ************************ What ADIOS2 is and isn't diff --git a/docs/user_guide/source/setting_up/linking.rst b/docs/user_guide/source/setting_up/linking.rst index 55221f840a..3df8cd8e82 100644 --- a/docs/user_guide/source/setting_up/linking.rst +++ b/docs/user_guide/source/setting_up/linking.rst @@ -10,7 +10,7 @@ ADIOS exports a CMake package configuration file that allows its targets to be d .. code-block:: cmake - cmake_minimum_required(VERSION 3.6) + cmake_minimum_required(VERSION 3.12) project(MySimulation C CXX) find_package(MPI REQUIRED) diff --git a/docs/user_guide/source/setting_up/package.rst b/docs/user_guide/source/setting_up/package.rst index 878266da26..6b8f7a7586 100644 --- a/docs/user_guide/source/setting_up/package.rst +++ b/docs/user_guide/source/setting_up/package.rst @@ -16,7 +16,7 @@ Currently ADIOS 2 can be obtained from anaconda cloud: Spack ***** -ADIOS 2 is fully supported in the latest Spack `adios2 package `_ +ADIOS 2 is packaged in Spack `adios2 package `_ ****** diff --git a/docs/user_guide/source/setting_up/source.rst b/docs/user_guide/source/setting_up/source.rst index 69d323ed69..d3378967a0 100644 --- a/docs/user_guide/source/setting_up/source.rst +++ b/docs/user_guide/source/setting_up/source.rst @@ -2,7 +2,7 @@ Install from Source ################### -ADIOS2 uses `CMake `_ version 3.6 or above, for building, +ADIOS2 uses `CMake `_ for building, testing and installing the library and utilities. .. include:: source/cmake.rst @@ -11,4 +11,4 @@ testing and installing the library and utilities. .. include:: source/python.rst .. include:: source/fortran.rst .. include:: source/ctest.rst -.. include:: source/examples.rst \ No newline at end of file +.. include:: source/examples.rst diff --git a/docs/user_guide/source/setting_up/source/cmake.rst b/docs/user_guide/source/setting_up/source/cmake.rst index 03d3f27e2f..d988189653 100644 --- a/docs/user_guide/source/setting_up/source/cmake.rst +++ b/docs/user_guide/source/setting_up/source/cmake.rst @@ -2,11 +2,6 @@ Building, Testing, and Installing ADIOS 2 ***************************************** -.. caution:: - - Always do a fresh build from scratch if your source is updated with considerable changes *e.g.* `git pull` - - To build ADIOS v2.x, clone the repository and invoke the canonical CMake build sequence: .. code-block:: bash diff --git a/docs/user_guide/source/setting_up/source/examples.rst b/docs/user_guide/source/setting_up/source/examples.rst index 10bbb29f69..4434fc8ca7 100644 --- a/docs/user_guide/source/setting_up/source/examples.rst +++ b/docs/user_guide/source/setting_up/source/examples.rst @@ -10,6 +10,6 @@ A few very basic examples are described below: Directory Description ================================ ========================================================================================================================== ``ADIOS2/examples/hello`` very basic "hello world"-style examples for reading and writing `.bp` files. -``ADIOS2/examples/heatTransfer`` 2D Poisson solver for transients in Fourier's modeo of heat transfer. Outputs ``bp.dir`` or HDF5. +``ADIOS2/examples/heatTransfer`` 2D Poisson solver for transients in Fourier's model of heat transfer. Outputs ``bp.dir`` or HDF5. ``ADIOS2/examples/basics`` covers different ``Variable`` use cases classified by the dimension. ================================ ========================================================================================================================== diff --git a/docs/user_guide/source/setting_up/source/hpc_systems.rst b/docs/user_guide/source/setting_up/source/hpc_systems.rst index 1f0934d203..6caa4d3f98 100644 --- a/docs/user_guide/source/setting_up/source/hpc_systems.rst +++ b/docs/user_guide/source/setting_up/source/hpc_systems.rst @@ -15,8 +15,6 @@ Building on HPC Systems #. **Big Endian and 32-bit systems:** ADIOS2 hasn't been tested on big endian or 32-bit systems. Please be aware before attempting to run. -#. **CMake minimum version:** The ADIOS2 build system requires a minimum CMake version of 3.6.0. However, IBM XL, Cray, and PGI compilers require version 3.9.0 or newer. - #. **PGI compilers and C++11 support:** Version 15 of the PGI compiler is C++11 compliant. However it relies on the C++ standard library headers supplied by the system version of GCC, which may or may support all the C++11 features used in ADIOS2. On many systems (Titan at OLCF, for example) even though the PGI compiler supports C++11, the configured GCC and its headers do not (4.3.x on Cray Linux Environment, and v5 systems like Titan). diff --git a/docs/user_guide/source/setting_up/source/python.rst b/docs/user_guide/source/setting_up/source/python.rst index 19b69d0276..e34498a91b 100644 --- a/docs/user_guide/source/setting_up/source/python.rst +++ b/docs/user_guide/source/setting_up/source/python.rst @@ -18,18 +18,9 @@ To enable the Python bindings in ADIOS2, based on `PyBind11 `_ and `helloBPTimeWriter.py `_ - - * In Python 2.7: + * Run `helloBPWriter.py `_ and `helloBPTimeWriter.py `_ via .. code-block:: bash $ mpirun -n 4 python helloBPWriter.py $ python helloBPWriter.py - - * In Python 3: - - .. code-block:: bash - - $ mpirun -n 4 python3 helloBPWriter.py - $ python3 helloBPWriter.py diff --git a/examples/heatTransfer/write/HeatTransfer.cpp b/examples/heatTransfer/write/HeatTransfer.cpp index 1fd78206cd..45510fbac3 100644 --- a/examples/heatTransfer/write/HeatTransfer.cpp +++ b/examples/heatTransfer/write/HeatTransfer.cpp @@ -11,6 +11,9 @@ * */ +#include + +#include #include #include #include @@ -20,36 +23,28 @@ #include "HeatTransfer.h" -HeatTransfer::HeatTransfer(const Settings &settings) : m_s{settings} +HeatTransfer::HeatTransfer(const Settings &settings) +: m_s(settings), m_T1Buf(new double[(m_s.ndx + 2) * (m_s.ndy + 2)]), + m_T2Buf(new double[(m_s.ndx + 2) * (m_s.ndy + 2)]), + m_T1(new double *[m_s.ndx + 2]), m_T2(new double *[m_s.ndx + 2]) { - m_T1 = new double *[m_s.ndx + 2]; - m_T1[0] = new double[(m_s.ndx + 2) * (m_s.ndy + 2)]; - m_T2 = new double *[m_s.ndx + 2]; - m_T2[0] = new double[(m_s.ndx + 2) * (m_s.ndy + 2)]; - for (unsigned int i = 1; i < m_s.ndx + 2; i++) + m_T1[0] = m_T1Buf.get(); + m_T2[0] = m_T2Buf.get(); + for (size_t i = 1; i < m_s.ndx + 2; ++i) { - m_T1[i] = m_T1[i - 1] + m_s.ndy + 2; - m_T2[i] = m_T2[i - 1] + m_s.ndy + 2; + m_T1[i] = m_T1[0] + i * (m_s.ndy + 2); + m_T2[i] = m_T2[0] + i * (m_s.ndy + 2); } - m_TCurrent = m_T1; - m_TNext = m_T2; -} - -HeatTransfer::~HeatTransfer() -{ - delete[] m_T1[0]; - delete[] m_T1; - delete[] m_T2[0]; - delete[] m_T2; + m_TCurrent = m_T1.get(); + m_TNext = m_T2.get(); } void HeatTransfer::init(bool init_with_rank) { if (init_with_rank) { - for (unsigned int i = 0; i < m_s.ndx + 2; i++) - for (unsigned int j = 0; j < m_s.ndy + 2; j++) - m_T1[i][j] = m_s.rank; + std::fill_n(m_T1Buf.get(), (m_s.ndx + 2) * (m_s.ndy + 2), + static_cast(m_s.rank)); } else { @@ -69,8 +64,8 @@ void HeatTransfer::init(bool init_with_rank) } } } - m_TCurrent = m_T1; - m_TNext = m_T2; + m_TCurrent = m_T1.get(); + m_TNext = m_T2.get(); } void HeatTransfer::printT(std::string message, MPI_Comm comm) const @@ -104,13 +99,6 @@ void HeatTransfer::printT(std::string message, MPI_Comm comm) const } } -void HeatTransfer::switchCurrentNext() -{ - double **tmp = m_TCurrent; - m_TCurrent = m_TNext; - m_TNext = tmp; -} - void HeatTransfer::iterate() { for (unsigned int i = 1; i <= m_s.ndx; ++i) @@ -123,19 +111,17 @@ void HeatTransfer::iterate() (1.0 - omega) * m_TCurrent[i][j]; } } - switchCurrentNext(); + std::swap(m_TCurrent, m_TNext); } void HeatTransfer::heatEdges() { // Heat the whole global edges if (m_s.posx == 0) - for (unsigned int j = 0; j < m_s.ndy + 2; ++j) - m_TCurrent[0][j] = edgetemp; + std::fill_n(m_TCurrent[0], m_s.ndy + 2, edgetemp); if (m_s.posx == m_s.npx - 1) - for (unsigned int j = 0; j < m_s.ndy + 2; ++j) - m_TCurrent[m_s.ndx + 1][j] = edgetemp; + std::fill_n(m_TCurrent[m_s.ndx + 1], m_s.ndy + 2, edgetemp); if (m_s.posy == 0) for (unsigned int i = 0; i < m_s.ndx + 2; ++i) @@ -148,10 +134,12 @@ void HeatTransfer::heatEdges() void HeatTransfer::exchange(MPI_Comm comm) { - // Exchange ghost cells, in the order left-right-up-down + // Build a custom MPI type for the column vector to allow strided access + MPI_Datatype tColumnVector; + MPI_Type_vector(m_s.ndx + 2, 1, m_s.ndy + 2, MPI_REAL8, &tColumnVector); + MPI_Type_commit(&tColumnVector); - double *send_x = new double[m_s.ndx + 2]; - double *recv_x = new double[m_s.ndx + 2]; + // Exchange ghost cells, in the order left-right-up-down // send to left + receive from right int tag = 1; @@ -160,18 +148,14 @@ void HeatTransfer::exchange(MPI_Comm comm) { // std::cout << "Rank " << m_s.rank << " send left to rank " // << m_s.rank_left << std::endl; - for (unsigned int i = 0; i < m_s.ndx + 2; ++i) - send_x[i] = m_TCurrent[i][1]; - MPI_Send(send_x, m_s.ndx + 2, MPI_REAL8, m_s.rank_left, tag, comm); + MPI_Send(m_TCurrent[0] + 1, 1, tColumnVector, m_s.rank_left, tag, comm); } if (m_s.rank_right >= 0) { // std::cout << "Rank " << m_s.rank << " receive from right from rank " // << m_s.rank_right << std::endl; - MPI_Recv(recv_x, m_s.ndx + 2, MPI_REAL8, m_s.rank_right, tag, comm, - &status); - for (unsigned int i = 0; i < m_s.ndx + 2; ++i) - m_TCurrent[i][m_s.ndy + 1] = recv_x[i]; + MPI_Recv(m_TCurrent[0] + (m_s.ndy + 1), 1, tColumnVector, + m_s.rank_right, tag, comm, &status); } // send to right + receive from left @@ -180,20 +164,20 @@ void HeatTransfer::exchange(MPI_Comm comm) { // std::cout << "Rank " << m_s.rank << " send right to rank " // << m_s.rank_right << std::endl; - for (unsigned int i = 0; i < m_s.ndx + 2; ++i) - send_x[i] = m_TCurrent[i][m_s.ndy]; - MPI_Send(send_x, m_s.ndx + 2, MPI_REAL8, m_s.rank_right, tag, comm); + MPI_Send(m_TCurrent[0] + m_s.ndy, 1, tColumnVector, m_s.rank_right, tag, + comm); } if (m_s.rank_left >= 0) { // std::cout << "Rank " << m_s.rank << " receive from left from rank " // << m_s.rank_left << std::endl; - MPI_Recv(recv_x, m_s.ndx + 2, MPI_REAL8, m_s.rank_left, tag, comm, + MPI_Recv(m_TCurrent[0], 1, tColumnVector, m_s.rank_left, tag, comm, &status); - for (unsigned int i = 0; i < m_s.ndx + 2; ++i) - m_TCurrent[i][0] = recv_x[i]; } + // Cleanup the custom column vector type + MPI_Type_free(&tColumnVector); + // send down + receive from above tag = 3; if (m_s.rank_down >= 0) @@ -227,12 +211,8 @@ void HeatTransfer::exchange(MPI_Comm comm) MPI_Recv(m_TCurrent[m_s.ndx + 1], m_s.ndy + 2, MPI_REAL8, m_s.rank_down, tag, comm, &status); } - - delete[] send_x; - delete[] recv_x; } -#include /* Copies the internal ndx*ndy section of the ndx+2 * ndy+2 local array * into a separate contiguous vector and returns it. * @return A vector with ndx*ndy elements diff --git a/examples/heatTransfer/write/HeatTransfer.h b/examples/heatTransfer/write/HeatTransfer.h index a52acbb742..d504c8a966 100644 --- a/examples/heatTransfer/write/HeatTransfer.h +++ b/examples/heatTransfer/write/HeatTransfer.h @@ -13,6 +13,7 @@ #include +#include #include #include "Settings.h" @@ -22,7 +23,7 @@ class HeatTransfer public: HeatTransfer(const Settings &settings); // Create two 2D arrays with ghost // cells to compute - ~HeatTransfer(); + ~HeatTransfer() = default; void init(bool init_with_rank); // set up array values with either rank or // real demo values void iterate(); // one local calculation step @@ -41,15 +42,23 @@ class HeatTransfer MPI_Comm comm) const; // debug: print local TCurrent on stdout private: + const Settings &m_s; + const double edgetemp = 3.0; // temperature at the edges of the global plate const double omega = - 0.8; // weight for current temp is (1-omega) in iteration - double **m_T1; // 2D array (ndx+2) * (ndy+2) size, including ghost cells - double **m_T2; // another 2D array - double **m_TCurrent; // pointer to T1 or T2 - double **m_TNext; // pointer to T2 or T1 - const Settings &m_s; - void switchCurrentNext(); // switch the current array with the next array + 0.8; // weight for current temp is (1-omega) in iteration + + // 2D data arrays (ndx+2) * (ndy+2) size, including ghost cells + std::unique_ptr m_T1Buf; + std::unique_ptr m_T2Buf; + + // Double indexable view into the data arrays to allow for m_T1[i][j] + std::unique_ptr m_T1; + std::unique_ptr m_T2; + + // Track which data array is active + double **m_TCurrent; + double **m_TNext; }; #endif /* HEATTRANSFER_H_ */ diff --git a/examples/hello/bpReader/helloBPReader_nompi.cpp b/examples/hello/bpReader/helloBPReader_nompi.cpp index b40fe38b83..776543d8a5 100644 --- a/examples/hello/bpReader/helloBPReader_nompi.cpp +++ b/examples/hello/bpReader/helloBPReader_nompi.cpp @@ -32,6 +32,16 @@ int main(int argc, char *argv[]) /** Engine derived class, spawned to start IO operations */ adios2::Engine bpReader = bpIO.Open(filename, adios2::Mode::Read); + const std::map variables = + bpIO.AvailableVariables(true); + + std::cout << "List of variables:"; + for (const auto variablePair : variables) + { + std::cout << " " << variablePair.first; + } + std::cout << std::endl; + /** Write variable for buffering */ adios2::Variable bpFloats = bpIO.InquireVariable("bpFloats"); diff --git a/examples/hello/bpTimeWriter/helloBPTimeWriter.py b/examples/hello/bpTimeWriter/helloBPTimeWriter.py index ca9039bda5..7cadf886eb 100644 --- a/examples/hello/bpTimeWriter/helloBPTimeWriter.py +++ b/examples/hello/bpTimeWriter/helloBPTimeWriter.py @@ -38,7 +38,7 @@ for t in range(0, 10): bpFileWriter.BeginStep() - if(rank == 0): + if rank == 0: bpFileWriter.Put(bpTimeStep, np.array([t])) bpFileWriter.Put(bpArray, myArray) bpFileWriter.EndStep() diff --git a/examples/hello/bpWriter/helloBPPutDeferred.cpp b/examples/hello/bpWriter/helloBPPutDeferred.cpp index 284b2efdd1..6ac10f72dd 100644 --- a/examples/hello/bpWriter/helloBPPutDeferred.cpp +++ b/examples/hello/bpWriter/helloBPPutDeferred.cpp @@ -20,8 +20,8 @@ * Author: William F Godoy godoywf@ornl.gov */ -#include //std::ios_base::failure -#include //std::cout +#include //std::ios_base::failure +#include //std::cout #include //std::invalid_argument std::exception #include diff --git a/examples/hello/bpWriter/helloBPSZ.cpp b/examples/hello/bpWriter/helloBPSZ.cpp index 8d354e6d08..3b24095e1e 100644 --- a/examples/hello/bpWriter/helloBPSZ.cpp +++ b/examples/hello/bpWriter/helloBPSZ.cpp @@ -8,8 +8,8 @@ * Author: William F Godoy godoywf@ornl.gov */ -#include //std::ios_base::failure -#include //std::cout +#include //std::ios_base::failure +#include //std::cout #include //std::iota #include //std::invalid_argument std::exception #include diff --git a/examples/hello/bpWriter/helloBPSubStreams.cpp b/examples/hello/bpWriter/helloBPSubStreams.cpp index 3ad9534011..73e9412775 100644 --- a/examples/hello/bpWriter/helloBPSubStreams.cpp +++ b/examples/hello/bpWriter/helloBPSubStreams.cpp @@ -8,8 +8,8 @@ * Author: William F Godoy godoywf@ornl.gov */ -#include //std::ios_base::failure -#include //std::cout +#include //std::ios_base::failure +#include //std::cout #include //std::invalid_argument std::exception #include diff --git a/examples/hello/bpWriter/helloBPWriter.cpp b/examples/hello/bpWriter/helloBPWriter.cpp index 113fc5c202..e67c82ba92 100644 --- a/examples/hello/bpWriter/helloBPWriter.cpp +++ b/examples/hello/bpWriter/helloBPWriter.cpp @@ -9,8 +9,8 @@ * Author: William F Godoy godoywf@ornl.gov */ -#include //std::ios_base::failure -#include //std::cout +#include //std::ios_base::failure +#include //std::cout #include //std::invalid_argument std::exception #include diff --git a/examples/hello/hdf5Writer/helloHDF5Writer.cpp b/examples/hello/hdf5Writer/helloHDF5Writer.cpp index 7d45a091dd..90a0c549ca 100644 --- a/examples/hello/hdf5Writer/helloHDF5Writer.cpp +++ b/examples/hello/hdf5Writer/helloHDF5Writer.cpp @@ -39,6 +39,8 @@ int main(int argc, char *argv[]) * Parameters, Transports, and Execution: Engines */ adios2::IO hdf5IO = adios.DeclareIO("HDFFileIO"); hdf5IO.SetEngine("HDF5"); + hdf5IO.SetParameter("IdleH5Writer", + "true"); // set this if not all ranks are writting /** global array : name, { shape (total) }, { start (local) }, { count * (local) }, all are constant dimensions */ @@ -53,12 +55,25 @@ int main(int argc, char *argv[]) /** Engine derived class, spawned to start IO operations */ adios2::Engine hdf5Writer = hdf5IO.Open("myVector.h5", adios2::Mode::Write); - +#ifdef ALL_RANKS_WRITE + // all Ranks must call Put /** Write variable for buffering */ hdf5Writer.Put(h5Floats, myFloats.data()); hdf5Writer.Put(h5Ints, myInts.data()); hdf5Writer.Put(h5ScalarDouble, &myScalar); - +#else + // using collective Begin/EndStep() to run the + // collective HDF5 calls. Now Ranks can skip writting if no data + // presented + hdf5Writer.BeginStep(); + if (rank == 0) + { + hdf5Writer.Put(h5Floats, myFloats.data()); + hdf5Writer.Put(h5Ints, myInts.data()); + hdf5Writer.Put(h5ScalarDouble, &myScalar); + } + hdf5Writer.EndStep(); +#endif std::vector m_globalDims = {10, 20, 30, 40}; hdf5IO.DefineAttribute( "adios2_schema/version_major", @@ -79,13 +94,7 @@ int main(int argc, char *argv[]) hdf5IO.DefineAttribute("adios2_schema/mesh/dimension-num", m_globalDims.size()); -#ifdef NEVER - /** Create h5 file, engine becomes unreachable after this*/ hdf5Writer.Close(); -#else - hdf5Writer.Flush(); - hdf5Writer.Flush(); -#endif } catch (std::invalid_argument &e) { diff --git a/examples/hello/inlineReaderWriter/helloInlineReaderWriter.cpp b/examples/hello/inlineReaderWriter/helloInlineReaderWriter.cpp index 908f0ed65b..ae0a54cdf7 100644 --- a/examples/hello/inlineReaderWriter/helloInlineReaderWriter.cpp +++ b/examples/hello/inlineReaderWriter/helloInlineReaderWriter.cpp @@ -22,84 +22,61 @@ void DoAnalysis(adios2::IO &inlineIO, adios2::Engine &inlineReader, int rank, unsigned int step) { - try - { - inlineReader.BeginStep(); - /////////////////////READ - adios2::Variable inlineFloats000 = - inlineIO.InquireVariable("inlineFloats000"); + inlineReader.BeginStep(); + /////////////////////READ + adios2::Variable inlineFloats000 = + inlineIO.InquireVariable("inlineFloats000"); - adios2::Variable inlineString = - inlineIO.InquireVariable("inlineString"); + adios2::Variable inlineString = + inlineIO.InquireVariable("inlineString"); - if (inlineFloats000) - { - auto blocksInfo = inlineReader.BlocksInfo(inlineFloats000, step); - - std::cout << "Data StepsStart " << inlineFloats000.StepsStart() - << " from rank " << rank << ": "; - for (auto &info : blocksInfo) - { - // bp file reader would see all blocks, inline only sees local - // writer's block(s). - size_t myBlock = info.BlockID; - inlineFloats000.SetBlockSelection(myBlock); - - // info passed by reference - // engine must remember data pointer (or info) to fill it out at - // PerformGets() - inlineReader.Get(inlineFloats000, info, - adios2::Mode::Deferred); - } - inlineReader.PerformGets(); + if (inlineFloats000) + { + auto blocksInfo = inlineReader.BlocksInfo(inlineFloats000, step); - for (const auto &info : blocksInfo) - { - adios2::Dims count = info.Count; - const float *vectData = info.Data(); - for (size_t i = 0; i < count[0]; ++i) - { - float datum = vectData[i]; - std::cout << datum << " "; - } - std::cout << "\n"; - } - } - else + std::cout << "Data StepsStart " << inlineFloats000.StepsStart() + << " from rank " << rank << ": "; + for (auto &info : blocksInfo) { - std::cout << "Variable inlineFloats000 not found\n"; + // bp file reader would see all blocks, inline only sees local + // writer's block(s). + size_t myBlock = info.BlockID; + inlineFloats000.SetBlockSelection(myBlock); + + // info passed by reference + // engine must remember data pointer (or info) to fill it out at + // PerformGets() + inlineReader.Get(inlineFloats000, info, + adios2::Mode::Deferred); } + inlineReader.PerformGets(); - if (inlineString && rank == 0) + for (const auto &info : blocksInfo) { - // inlineString.SetStepSelection({step, 1}); - - std::string myString; - inlineReader.Get(inlineString, myString, adios2::Mode::Sync); - std::cout << "inlineString: " << myString << "\n"; + adios2::Dims count = info.Count; + const float *vectData = info.Data(); + for (size_t i = 0; i < count[0]; ++i) + { + float datum = vectData[i]; + std::cout << datum << " "; + } + std::cout << "\n"; } - inlineReader.EndStep(); - // all deferred block info are now valid - need data pointers to be - // valid, filled with data } - catch (std::invalid_argument &e) + else { - std::cout << "Invalid argument exception, STOPPING PROGRAM from rank " - << rank << "\n"; - std::cout << e.what() << "\n"; + std::cout << "Variable inlineFloats000 not found\n"; } - catch (std::ios_base::failure &e) - { - std::cout << "IO System base failure exception, STOPPING PROGRAM " - "from rank " - << rank << "\n"; - std::cout << e.what() << "\n"; - } - catch (std::exception &e) + + if (inlineString && rank == 0) { - std::cout << "Exception, STOPPING PROGRAM from rank " << rank << "\n"; - std::cout << e.what() << "\n"; + std::string myString; + inlineReader.Get(inlineString, myString, adios2::Mode::Sync); + std::cout << "inlineString: " << myString << "\n"; } + inlineReader.EndStep(); + // all deferred block info are now valid - need data pointers to be + // valid, filled with data } int main(int argc, char *argv[]) @@ -109,6 +86,9 @@ int main(int argc, char *argv[]) MPI_Init(&argc, &argv); MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &size); + adios2::ADIOS adios(MPI_COMM_WORLD); +#else + adios2::ADIOS adios; #endif // Application variable @@ -117,114 +97,88 @@ int main(int argc, char *argv[]) try { -#if ADIOS2_USE_MPI - /** ADIOS class factory of IO class objects */ - adios2::ADIOS adios(MPI_COMM_WORLD); -#else - adios2::ADIOS adios; -#endif - /*** IO class object: settings and factory of Settings: Variables, - * Parameters, Transports, and Execution: Engines - * Inline uses single IO for write/read */ + // Inline uses single IO for write/read adios2::IO inlineIO = adios.DeclareIO("InlineReadWrite"); /// WRITE - { - inlineIO.SetEngine("Inline"); - inlineIO.SetParameters({{"verbose", "4"}, - {"writerID", "myWriteID"}, - {"readerID", "myReadID"}}); + inlineIO.SetEngine("Inline"); + inlineIO.SetParameter("verbose", "4"); - /** global array: name, { shape (total dimensions) }, { start - * (local) }, - * { count (local) }, all are constant dimensions */ - const unsigned int variablesSize = 10; - std::vector> inlineFloats(variablesSize); + /** global array: name, { shape (total dimensions) }, { start + * (local) }, + * { count (local) }, all are constant dimensions */ + const unsigned int variablesSize = 10; + std::vector> inlineFloats(variablesSize); - adios2::Variable inlineString = - inlineIO.DefineVariable("inlineString"); + adios2::Variable inlineString = + inlineIO.DefineVariable("inlineString"); - for (unsigned int v = 0; v < variablesSize; ++v) + for (unsigned int v = 0; v < variablesSize; ++v) + { + std::string namev("inlineFloats"); + if (v < 10) + { + namev += "00"; + } + else if (v < 100) { - std::string namev("inlineFloats"); - if (v < 10) - { - namev += "00"; - } - else if (v < 100) - { - namev += "0"; - } - namev += std::to_string(v); - - inlineFloats[v] = inlineIO.DefineVariable( - namev, {size * Nx}, {rank * Nx}, {Nx}, - adios2::ConstantDims); + namev += "0"; } + namev += std::to_string(v); - /** global single value variable: name */ - adios2::Variable inlineTimeStep = - inlineIO.DefineVariable("timeStep"); + inlineFloats[v] = inlineIO.DefineVariable( + namev, {size * Nx}, {rank * Nx}, {Nx}, adios2::ConstantDims); + } + + /** global single value variable: name */ + adios2::Variable inlineTimeStep = + inlineIO.DefineVariable("timeStep"); - /** Engine derived class, spawned to start IO operations */ - adios2::Engine inlineWriter = - inlineIO.Open("myWriteID", adios2::Mode::Write); + adios2::Engine inlineWriter = + inlineIO.Open("myWriteID", adios2::Mode::Write); - adios2::Engine inlineReader = - inlineIO.Open("myReadID", adios2::Mode::Read); + adios2::Engine inlineReader = + inlineIO.Open("myReadID", adios2::Mode::Read); - for (unsigned int timeStep = 0; timeStep < 3; ++timeStep) + for (unsigned int timeStep = 0; timeStep < 3; ++timeStep) + { + inlineWriter.BeginStep(); + if (rank == 0) // global single value, only saved by rank 0 { - inlineWriter.BeginStep(); - if (rank == 0) // global single value, only saved by rank 0 - { - inlineWriter.Put(inlineTimeStep, timeStep); - } - - // template type is optional, but recommended - for (unsigned int v = 0; v < variablesSize; ++v) - { - // Note: Put is deferred, so all variables will see v == 9 - // and myFloats[0] == 9, 10, or 11 - myFloats[rank] = static_cast(v + timeStep + rank); - inlineWriter.Put(inlineFloats[v], myFloats.data()); - } - - const std::string myString( - "Hello from rank: " + std::to_string(rank) + - " and timestep: " + std::to_string(timeStep)); - - if (rank == 0) - { - inlineWriter.Put(inlineString, myString); - } - - inlineWriter.EndStep(); - - DoAnalysis(inlineIO, inlineReader, rank, timeStep); + inlineWriter.Put(inlineTimeStep, timeStep); } - inlineWriter.Close(); - inlineReader.Close(); + // template type is optional, but recommended + for (unsigned int v = 0; v < variablesSize; ++v) + { + // Note: Put is deferred, so all variables will see v == 9 + // and myFloats[0] == 9, 10, or 11 + myFloats[rank] = static_cast(v + timeStep + rank); + inlineWriter.Put(inlineFloats[v], myFloats.data()); + } + + const std::string myString( + "Hello from rank: " + std::to_string(rank) + + " and timestep: " + std::to_string(timeStep)); + + if (rank == 0) + { + inlineWriter.Put(inlineString, myString); + } + + inlineWriter.EndStep(); + + DoAnalysis(inlineIO, inlineReader, rank, timeStep); } - // MPI_Barrier(MPI_COMM_WORLD); - } - catch (std::invalid_argument &e) - { - std::cout << "Invalid argument exception, STOPPING PROGRAM from rank " - << rank << "\n"; - std::cout << e.what() << "\n"; } - catch (std::ios_base::failure &e) + catch (std::exception const &e) { - std::cout << "IO System base failure exception, STOPPING PROGRAM " - "from rank " - << rank << "\n"; - std::cout << e.what() << "\n"; - } - catch (std::exception &e) - { - std::cout << "Exception, STOPPING PROGRAM from rank " << rank << "\n"; + std::cout << "Caught exception from rank " << rank << "\n"; std::cout << e.what() << "\n"; +#if ADIOS2_USE_MPI + return MPI_Abort(MPI_COMM_WORLD, 1); +#else + return 1; +#endif } #if ADIOS2_USE_MPI diff --git a/examples/hello/insituMPI/CMakeLists.txt b/examples/hello/insituMPI/CMakeLists.txt index 36d6cbea6f..5168c8f048 100644 --- a/examples/hello/insituMPI/CMakeLists.txt +++ b/examples/hello/insituMPI/CMakeLists.txt @@ -5,6 +5,10 @@ if(ADIOS2_HAVE_Fortran) add_library(helloInsituArgs OBJECT helloInsituArgs.F90) + target_compile_definitions(helloInsituArgs PRIVATE + $<$:ADIOS2_HAVE_FORTRAN_F03_ARGS> + $<$:ADIOS2_HAVE_FORTRAN_GNU_ARGS> + ) add_executable(hello_insituMPIWriter_f helloInsituMPIWriter.f90 diff --git a/examples/hello/insituMPI/helloInsituArgs.F90 b/examples/hello/insituMPI/helloInsituArgs.F90 index cf6275f526..59a00c2667 100644 --- a/examples/hello/insituMPI/helloInsituArgs.F90 +++ b/examples/hello/insituMPI/helloInsituArgs.F90 @@ -35,11 +35,15 @@ end subroutine usage !!*************************** subroutine processArgs(rank, nproc, isWriter) -#if defined(_CRAYFTN) || !defined(__GFORTRAN__) && !defined(__GNUC__) - interface - integer function iargc() - end function iargc - end interface +#if defined(ADIOS2_HAVE_FORTRAN_F03_ARGS) +# define ADIOS2_ARGC() command_argument_count() +# define ADIOS2_ARGV(i, v) call get_command_argument(i, v) +#elif defined(ADIOS2_HAVE_FORTRAN_GNU_ARGS) +# define ADIOS2_ARGC() iargc() +# define ADIOS2_ARGV(i, v) call getarg(i, v) +#else +# define ADIOS2_ARGC() 1 +# define ADIOS2_ARGV(i, v) #endif integer, intent(in) :: rank @@ -57,20 +61,20 @@ end function iargc endif !! process arguments - numargs = iargc() + numargs = ADIOS2_ARGC() !print *,"Number of arguments:",numargs if ( numargs < expargs ) then call usage(isWriter) call exit(1) endif - call getarg(1, xmlfile) - call getarg(2, npx_str) - call getarg(3, npy_str) + ADIOS2_ARGV(1, xmlfile) + ADIOS2_ARGV(2, npx_str) + ADIOS2_ARGV(3, npy_str) if (isWriter) then - call getarg(4, ndx_str) - call getarg(5, ndy_str) - call getarg(6, steps_str) - call getarg(7, time_str) + ADIOS2_ARGV(4, ndx_str) + ADIOS2_ARGV(5, ndy_str) + ADIOS2_ARGV(6, steps_str) + ADIOS2_ARGV(7, time_str) endif read (npx_str,'(i5)') npx read (npy_str,'(i5)') npy diff --git a/examples/hello/sstWriter/helloSstWriter.cpp b/examples/hello/sstWriter/helloSstWriter.cpp index 5238476b5a..5cf47aceb5 100644 --- a/examples/hello/sstWriter/helloSstWriter.cpp +++ b/examples/hello/sstWriter/helloSstWriter.cpp @@ -30,7 +30,6 @@ int main(int argc, char *argv[]) #else rank = 0; size = 1; -#define MPI_COMM_WORLD 0 #endif std::vector myFloats = { @@ -42,7 +41,11 @@ int main(int argc, char *argv[]) try { +#if ADIOS2_USE_MPI adios2::ADIOS adios(MPI_COMM_WORLD); +#else + adios2::ADIOS adios; +#endif adios2::IO sstIO = adios.DeclareIO("myIO"); sstIO.SetEngine("Sst"); diff --git a/scripts/ci/azure/macos-setup.sh b/scripts/ci/azure/macos-setup.sh index b98f781a0d..7717ddca23 100755 --- a/scripts/ci/azure/macos-setup.sh +++ b/scripts/ci/azure/macos-setup.sh @@ -16,14 +16,12 @@ case "$SYSTEM_JOBNAME" in ;; esac -echo "Installing CMake Nightly" -curl -L https://cmake.org/files/dev/cmake-3.16.20191218-g8262562-Darwin-x86_64.tar.gz | sudo tar -C /Applications --strip-components=1 -xz - -echo "Removing all existing brew packages" +echo "Removing all existing brew package and update the formule" brew remove --force $(brew list) +brew update -echo "Installing Kitware Ninja" -brew install ninja +echo "Installing Kitware CMake and Ninja" +brew install cmake ninja echo "Installing GCC" brew install gcc @@ -31,6 +29,10 @@ brew install gcc echo "Installing blosc compression" brew install c-blosc +echo "Installing python and friends" +brew install python numpy +brew link --overwrite python + if [[ "$SYSTEM_JOBNAME" =~ .*openmpi.* ]] then echo "Installing OpenMPI" diff --git a/scripts/ci/azure/run.sh b/scripts/ci/azure/run.sh index c8a7ce6608..81eff1f72b 100755 --- a/scripts/ci/azure/run.sh +++ b/scripts/ci/azure/run.sh @@ -66,6 +66,9 @@ then export OMPI_MCA_hwloc_base_binding_policy=none fi +# Make sure staging tests use localhost +export ADIOS2_IP=127.0.0.1 + echo "**********Env Begin**********" env | sort echo "**********Env End************" diff --git a/scripts/ci/cmake/ci-debian-sid-openmpi.cmake b/scripts/ci/cmake/ci-debian-sid-openmpi.cmake index fca35b3231..889a7ed5a0 100644 --- a/scripts/ci/cmake/ci-debian-sid-openmpi.cmake +++ b/scripts/ci/cmake/ci-debian-sid-openmpi.cmake @@ -6,6 +6,9 @@ execute_process( OUTPUT_STRIP_TRAILING_WHITESPACE ) +set(ENV{CFLAGS} "-Wno-deprecated -Wno-deprecated-declarations") +set(ENV{CXXFLAGS} "-Wno-deprecated -Wno-deprecated-declarations") + set(dashboard_cache " ADIOS2_USE_EXTERNAL_DEPENDENCIES:BOOL=ON ADIOS2_USE_EXTERNAL_EVPATH:BOOL=OFF @@ -35,6 +38,9 @@ HDF5_C_COMPILER_EXECUTABLE:FILEPATH=/usr/bin/h5pcc.openmpi set(CTEST_TEST_ARGS PARALLEL_LEVEL 1 + + # Unclear why this test currently dies. Disabling until it can be addressed. + EXCLUDE "Engine.SSC.SscEngineTest.TestSsc7d.MPI" ) set(CTEST_CMAKE_GENERATOR "Ninja") list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") diff --git a/scripts/ci/cmake/ci-debian-sid.cmake b/scripts/ci/cmake/ci-debian-sid.cmake index 08f8ab23c5..a94eb99b92 100644 --- a/scripts/ci/cmake/ci-debian-sid.cmake +++ b/scripts/ci/cmake/ci-debian-sid.cmake @@ -6,6 +6,9 @@ execute_process( OUTPUT_STRIP_TRAILING_WHITESPACE ) +set(ENV{CFLAGS} "-Wno-deprecated -Wno-deprecated-declarations") +set(ENV{CXXFLAGS} "-Wno-deprecated -Wno-deprecated-declarations") + set(dashboard_cache " ADIOS2_USE_EXTERNAL_DEPENDENCIES:BOOL=ON ADIOS2_USE_EXTERNAL_EVPATH:BOOL=OFF diff --git a/scripts/ci/cmake/ci-el7-gnu8-ohpc.cmake b/scripts/ci/cmake/ci-el7-gnu8-ohpc.cmake index 6b36737ba4..6bbc905f82 100644 --- a/scripts/ci/cmake/ci-el7-gnu8-ohpc.cmake +++ b/scripts/ci/cmake/ci-el7-gnu8-ohpc.cmake @@ -25,8 +25,6 @@ ADIOS2_USE_Python:BOOL=ON ADIOS2_USE_SZ:BOOL=ON ADIOS2_USE_ZeroMQ:STRING=ON ADIOS2_USE_ZFP:BOOL=ON - -PYTHON_EXECUTABLE:FILEPATH=/usr/bin/python3.4 ") set(CTEST_CMAKE_GENERATOR "Unix Makefiles") diff --git a/scripts/ci/cmake/ci-el7-gnu8-openmpi-ohpc-static.cmake b/scripts/ci/cmake/ci-el7-gnu8-openmpi-ohpc-static.cmake index 6097378c81..581be161aa 100644 --- a/scripts/ci/cmake/ci-el7-gnu8-openmpi-ohpc-static.cmake +++ b/scripts/ci/cmake/ci-el7-gnu8-openmpi-ohpc-static.cmake @@ -8,10 +8,10 @@ find_package(EnvModules REQUIRED) env_module(purge) env_module(load gnu8) -env_module(load py2-numpy) +env_module(load py3-numpy) env_module(load openmpi3) env_module(load phdf5) -env_module(load py2-mpi4py) +env_module(load py3-mpi4py) set(ENV{CC} gcc) set(ENV{CXX} g++) @@ -42,9 +42,6 @@ MPIEXEC_MAX_NUMPROCS:STRING=${N2CPUS} set(CTEST_TEST_ARGS PARALLEL_LEVEL 1 - - # Unclear why this test currently dies. Disabling until it can be addressed. - EXCLUDE "Install.Make.Fortran" ) set(CTEST_CMAKE_GENERATOR "Unix Makefiles") list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") diff --git a/scripts/ci/cmake/ci-el7-gnu8-openmpi-ohpc.cmake b/scripts/ci/cmake/ci-el7-gnu8-openmpi-ohpc.cmake index e78dd52fe8..16f5ccf725 100644 --- a/scripts/ci/cmake/ci-el7-gnu8-openmpi-ohpc.cmake +++ b/scripts/ci/cmake/ci-el7-gnu8-openmpi-ohpc.cmake @@ -8,10 +8,10 @@ find_package(EnvModules REQUIRED) env_module(purge) env_module(load gnu8) -env_module(load py2-numpy) +env_module(load py3-numpy) env_module(load openmpi3) env_module(load phdf5) -env_module(load py2-mpi4py) +env_module(load py3-mpi4py) set(ENV{CC} gcc) set(ENV{CXX} g++) @@ -34,7 +34,6 @@ ADIOS2_USE_ZFP:BOOL=ON MPIEXEC_EXTRA_FLAGS:STRING=--allow-run-as-root --oversubscribe MPIEXEC_MAX_NUMPROCS:STRING=${N2CPUS} -PYTHON_EXECUTABLE:FILEPATH=/usr/bin/python2.7 ") set(CTEST_TEST_ARGS PARALLEL_LEVEL 1) diff --git a/scripts/ci/cmake/ci-el7-intel-ohpc.cmake b/scripts/ci/cmake/ci-el7-intel-ohpc.cmake index 6e2af45bad..6aec07e815 100644 --- a/scripts/ci/cmake/ci-el7-intel-ohpc.cmake +++ b/scripts/ci/cmake/ci-el7-intel-ohpc.cmake @@ -4,7 +4,7 @@ find_package(EnvModules REQUIRED) env_module(purge) env_module(load intel) -env_module(load py2-numpy) +env_module(load py3-numpy) env_module(load hdf5) set(ENV{CC} icc) @@ -25,8 +25,6 @@ ADIOS2_USE_Python:BOOL=ON ADIOS2_USE_SZ:BOOL=ON ADIOS2_USE_ZeroMQ:STRING=ON ADIOS2_USE_ZFP:STRING=ON - -PYTHON_EXECUTABLE:FILEPATH=/usr/bin/python2.7 ") set(CTEST_CMAKE_GENERATOR "Unix Makefiles") diff --git a/scripts/ci/cmake/ci-el7-intel-openmpi-ohpc.cmake b/scripts/ci/cmake/ci-el7-intel-openmpi-ohpc.cmake index 0eeef5c7d7..696ca55aad 100644 --- a/scripts/ci/cmake/ci-el7-intel-openmpi-ohpc.cmake +++ b/scripts/ci/cmake/ci-el7-intel-openmpi-ohpc.cmake @@ -34,7 +34,6 @@ ADIOS2_USE_ZFP:STRING=ON MPIEXEC_EXTRA_FLAGS:STRING=--allow-run-as-root --oversubscribe MPIEXEC_MAX_NUMPROCS:STRING=${N2CPUS} -PYTHON_EXECUTABLE:FILEPATH=/usr/bin/python3.4 ") set(CTEST_TEST_ARGS PARALLEL_LEVEL 1) diff --git a/scripts/ci/cmake/ci-el7-intel18-ohpc.cmake b/scripts/ci/cmake/ci-el7-intel18-ohpc.cmake index 6e2af45bad..6aec07e815 100644 --- a/scripts/ci/cmake/ci-el7-intel18-ohpc.cmake +++ b/scripts/ci/cmake/ci-el7-intel18-ohpc.cmake @@ -4,7 +4,7 @@ find_package(EnvModules REQUIRED) env_module(purge) env_module(load intel) -env_module(load py2-numpy) +env_module(load py3-numpy) env_module(load hdf5) set(ENV{CC} icc) @@ -25,8 +25,6 @@ ADIOS2_USE_Python:BOOL=ON ADIOS2_USE_SZ:BOOL=ON ADIOS2_USE_ZeroMQ:STRING=ON ADIOS2_USE_ZFP:STRING=ON - -PYTHON_EXECUTABLE:FILEPATH=/usr/bin/python2.7 ") set(CTEST_CMAKE_GENERATOR "Unix Makefiles") diff --git a/scripts/ci/cmake/ci-el7-intel18-openmpi-ohpc.cmake b/scripts/ci/cmake/ci-el7-intel18-openmpi-ohpc.cmake index 0eeef5c7d7..696ca55aad 100644 --- a/scripts/ci/cmake/ci-el7-intel18-openmpi-ohpc.cmake +++ b/scripts/ci/cmake/ci-el7-intel18-openmpi-ohpc.cmake @@ -34,7 +34,6 @@ ADIOS2_USE_ZFP:STRING=ON MPIEXEC_EXTRA_FLAGS:STRING=--allow-run-as-root --oversubscribe MPIEXEC_MAX_NUMPROCS:STRING=${N2CPUS} -PYTHON_EXECUTABLE:FILEPATH=/usr/bin/python3.4 ") set(CTEST_TEST_ARGS PARALLEL_LEVEL 1) diff --git a/scripts/ci/cmake/ci-el7-spack.cmake b/scripts/ci/cmake/ci-el7-spack.cmake new file mode 100644 index 0000000000..6fcd590243 --- /dev/null +++ b/scripts/ci/cmake/ci-el7-spack.cmake @@ -0,0 +1,36 @@ +# Client maintainer: chuck.atkins@kitware.com + +include(ProcessorCount) +ProcessorCount(NCPUS) +math(EXPR N2CPUS "${NCPUS}*2") + +set(ENV{CC} gcc) +set(ENV{CXX} g++) +set(ENV{FC} gfortran) + +set(dashboard_cache " +ADIOS2_USE_BZip2:BOOL=ON +ADIOS2_USE_Blosc:BOOL=ON +ADIOS2_USE_DataMan:BOOL=ON +ADIOS2_USE_Fortran:BOOL=ON +ADIOS2_USE_HDF5:BOOL=ON +ADIOS2_USE_MPI:BOOL=ON +ADIOS2_USE_Python:BOOL=ON +ADIOS2_USE_SZ:BOOL=ON +ADIOS2_USE_ZeroMQ:STRING=ON +ADIOS2_USE_ZFP:BOOL=ON +ADIOS2_USE_Blosc:BOOL=ON +ADIOS2_USE_DataSpaces:BOOL=OFF + +ADIOS2_USE_EXTERNAL_DEPENDENCIES:BOOL=ON +ADIOS2_USE_EXTERNAL_GTEST:BOOL=OFF +ADIOS2_USE_EXTERNAL_PUGIXML:BOOL=OFF + +MPIEXEC_EXTRA_FLAGS:STRING=--allow-run-as-root --oversubscribe +MPIEXEC_MAX_NUMPROCS:STRING=${N2CPUS} +") + +set(CTEST_TEST_ARGS PARALLEL_LEVEL 1) +set(CTEST_CMAKE_GENERATOR "Unix Makefiles") +list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") +include(${CMAKE_CURRENT_LIST_DIR}/ci-common.cmake) diff --git a/scripts/ci/cmake/ci-fedora-asan.cmake b/scripts/ci/cmake/ci-fedora-asan.cmake index acb8f27c66..53924bc867 100644 --- a/scripts/ci/cmake/ci-fedora-asan.cmake +++ b/scripts/ci/cmake/ci-fedora-asan.cmake @@ -2,10 +2,10 @@ set(ENV{CC} clang) set(ENV{CXX} clang++) -set(ASAN_FLAGS "-fsanitize=address -fno-omit-frame-pointer -pthread") +set(ASAN_FLAGS "-fsanitize=address -fno-omit-frame-pointer -pthread -mllvm -asan-use-private-alias=1 -Wno-unused-command-line-argument") +set(ENV{ASAN_OPTIONS} "use_odr_indicator=1") set(ENV{CFLAGS} "${ASAN_FLAGS}") set(ENV{CXXFLAGS} "${ASAN_FLAGS}") -set(ENV{FFLAGS} "${ASAN_FLAGS}") set(dashboard_cache " ADIOS2_USE_Fortran:STRING=OFF @@ -18,6 +18,7 @@ ADIOS2_USE_ZFP:STRING=ON set(dashboard_track "Analysis") set(CTEST_CMAKE_GENERATOR "Unix Makefiles") set(CTEST_BUILD_FLAGS "-k -j4") +set(CTEST_MEMORYCHECK_TYPE "AddressSanitizer") set(ADIOS_TEST_REPEAT 0) list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") diff --git a/scripts/ci/cmake/ci-fedora-msan.cmake b/scripts/ci/cmake/ci-fedora-msan.cmake index 289dc751f9..7aacf9e9d4 100644 --- a/scripts/ci/cmake/ci-fedora-msan.cmake +++ b/scripts/ci/cmake/ci-fedora-msan.cmake @@ -14,6 +14,7 @@ HDF5_DIFF_EXECUTABLE:FILEPATH=/opt/msan/bin/h5diff set(dashboard_track "Analysis") set(CTEST_CMAKE_GENERATOR "Unix Makefiles") set(CTEST_BUILD_FLAGS "-k -j4") +set(CTEST_MEMORYCHECK_TYPE "MemorySanitizer") set(ADIOS_TEST_REPEAT 0) list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") diff --git a/scripts/ci/cmake/ci-fedora-openmpi-ubsan.cmake b/scripts/ci/cmake/ci-fedora-openmpi-ubsan.cmake index 929d8ba277..2ea456b50b 100644 --- a/scripts/ci/cmake/ci-fedora-openmpi-ubsan.cmake +++ b/scripts/ci/cmake/ci-fedora-openmpi-ubsan.cmake @@ -26,6 +26,7 @@ MPIEXEC_MAX_NUMPROCS:STRING=4 set(dashboard_track "Analysis") set(CTEST_CMAKE_GENERATOR "Unix Makefiles") set(CTEST_BUILD_FLAGS "-k -j2") +set(CTEST_MEMORYCHECK_TYPE "UndefinedBehaviorSanitizer") list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") include(${CMAKE_CURRENT_LIST_DIR}/ci-common.cmake) diff --git a/scripts/ci/cmake/ci-fedora-tsan.cmake b/scripts/ci/cmake/ci-fedora-tsan.cmake index 002ba4a2a3..2c849584f1 100644 --- a/scripts/ci/cmake/ci-fedora-tsan.cmake +++ b/scripts/ci/cmake/ci-fedora-tsan.cmake @@ -14,6 +14,7 @@ HDF5_DIFF_EXECUTABLE:FILEPATH=/opt/tsan/bin/h5diff set(dashboard_track "Analysis") set(CTEST_CMAKE_GENERATOR "Unix Makefiles") set(CTEST_BUILD_FLAGS "-k -j4") +set(CTEST_MEMORYCHECK_TYPE "ThreadSanitizer") set(ADIOS_TEST_REPEAT 0) list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") diff --git a/scripts/ci/cmake/ci-fedora-ubsan.cmake b/scripts/ci/cmake/ci-fedora-ubsan.cmake index ee9073b123..f2c40b3727 100644 --- a/scripts/ci/cmake/ci-fedora-ubsan.cmake +++ b/scripts/ci/cmake/ci-fedora-ubsan.cmake @@ -20,6 +20,7 @@ HDF5_DIFF_EXECUTABLE:FILEPATH=/usr/bin/h5diff set(dashboard_track "Analysis") set(CTEST_CMAKE_GENERATOR "Unix Makefiles") set(CTEST_BUILD_FLAGS "-k -j2") +set(CTEST_MEMORYCHECK_TYPE "UndefinedBehaviorSanitizer") set(ADIOS_TEST_REPEAT 0) list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") diff --git a/scripts/ci/cmake/ci-suse-pgi-openmpi.cmake b/scripts/ci/cmake/ci-suse-pgi-openmpi.cmake index 0fabfbe4ab..ae4a2852e2 100644 --- a/scripts/ci/cmake/ci-suse-pgi-openmpi.cmake +++ b/scripts/ci/cmake/ci-suse-pgi-openmpi.cmake @@ -7,11 +7,8 @@ math(EXPR N2CPUS "${NCPUS}*2") find_package(EnvModules REQUIRED) env_module(purge) -env_module(load pgi) +env_module(load nvhpc) -set(ENV{CC} pgcc) -set(ENV{CXX} pgc++) -set(ENV{FC} pgfortran) #set(ENV{CFLAGS} -Werror) set(ENV{CXXFLAGS} --brief_diagnostics) #set(ENV{FFLAGS} "-warn errors") diff --git a/scripts/ci/cmake/ci-suse-pgi.cmake b/scripts/ci/cmake/ci-suse-pgi.cmake index 515c667bb6..716a4d36a2 100644 --- a/scripts/ci/cmake/ci-suse-pgi.cmake +++ b/scripts/ci/cmake/ci-suse-pgi.cmake @@ -3,11 +3,8 @@ find_package(EnvModules REQUIRED) env_module(purge) -env_module(load pgi) +env_module(load nvhpc-nompi) -set(ENV{CC} pgcc) -set(ENV{CXX} pgc++) -set(ENV{FC} pgfortran) #set(ENV{CFLAGS} -Werror) set(ENV{CXXFLAGS} --brief_diagnostics) #set(ENV{FFLAGS} "-warn errors") diff --git a/scripts/ci/cmake/ci-ubuntu1804-spack.cmake b/scripts/ci/cmake/ci-ubuntu1804-spack.cmake new file mode 100644 index 0000000000..2b833980d4 --- /dev/null +++ b/scripts/ci/cmake/ci-ubuntu1804-spack.cmake @@ -0,0 +1,39 @@ +# Client maintainer: chuck.atkins@kitware.com + +include(ProcessorCount) +ProcessorCount(NCPUS) +math(EXPR N2CPUS "${NCPUS}*2") + +set(ENV{CC} gcc) +set(ENV{CXX} g++) +set(ENV{FC} gfortran) +set(ENV{CFLAGS} "-Werror -Wno-error=builtin-declaration-mismatch") +set(ENV{CXXFLAGS} "-Werror -Wno-error=builtin-declaration-mismatch") +set(ENV{FFLAGS} "-Werror -Wno-error=builtin-declaration-mismatch") + +set(dashboard_cache " +ADIOS2_USE_BZip2:BOOL=ON +ADIOS2_USE_Blosc:BOOL=ON +ADIOS2_USE_DataMan:BOOL=ON +ADIOS2_USE_Fortran:BOOL=ON +ADIOS2_USE_HDF5:BOOL=ON +ADIOS2_USE_MPI:BOOL=ON +ADIOS2_USE_Python:BOOL=ON +ADIOS2_USE_SZ:BOOL=ON +ADIOS2_USE_ZeroMQ:STRING=ON +ADIOS2_USE_ZFP:BOOL=ON +ADIOS2_USE_Blosc:BOOL=ON +ADIOS2_USE_DataSpaces:BOOL=OFF + +ADIOS2_USE_EXTERNAL_DEPENDENCIES:BOOL=ON +ADIOS2_USE_EXTERNAL_GTEST:BOOL=OFF +ADIOS2_USE_EXTERNAL_PUGIXML:BOOL=OFF + +MPIEXEC_EXTRA_FLAGS:STRING=--allow-run-as-root --oversubscribe +MPIEXEC_MAX_NUMPROCS:STRING=${N2CPUS} +") + +set(CTEST_TEST_ARGS PARALLEL_LEVEL 1) +set(CTEST_CMAKE_GENERATOR "Unix Makefiles") +list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") +include(${CMAKE_CURRENT_LIST_DIR}/ci-common.cmake) diff --git a/scripts/ci/gh-actions/run.sh b/scripts/ci/gh-actions/run.sh index aa40a9b3ec..c71f2be119 100755 --- a/scripts/ci/gh-actions/run.sh +++ b/scripts/ci/gh-actions/run.sh @@ -75,6 +75,8 @@ then export OMPI_MCA_opal_warn_on_missing_libcuda=0 fi +# Make sure staging tests use localhost +export ADIOS2_IP=127.0.0.1 echo "**********Env Begin**********" env | sort diff --git a/scripts/ci/gitlab-ci/pr-sync.sh b/scripts/ci/gitlab-ci/pr-sync.sh deleted file mode 100755 index 9e093daf98..0000000000 --- a/scripts/ci/gitlab-ci/pr-sync.sh +++ /dev/null @@ -1,116 +0,0 @@ -#!/usr/bin/env bash - -set -e - -if [ -z "${SSH_KEY_BASE64}" ] -then - echo "Error: SSH_KEY_BASE64 is empty" - exit 1 -fi - -git --version - -export GIT_SSH_COMMAND="ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" - -# Start the ssh agent -eval $(ssh-agent -s) - -# Add the necessary key -echo "${SSH_KEY_BASE64}" | base64 -d | tr -d '\r' | ssh-add - - -# Setup push access -git config remote.origin.pushurl $(echo ${CI_REPOSITORY_URL} | sed 's|.*@\([^\/]*\)\/|git@\1:|') - -# Setup the github upstream remote -if git remote | grep -q upstream -then - git remote rm upstream -fi -git remote add upstream https://github.com/ornladios/ADIOS2.git -git fetch -p upstream - -# Retrieve open PRs -OPEN_PR_BRANCHES="$(python3 -c 'from github import Github; print(" ".join(["pr%d_%s" % (pr.number, pr.head.ref) for pr in Github().get_repo("ornladios/ADIOS2").get_pulls(state="open")]))')" -echo "Open PRs:" -for PR in ${OPEN_PR_BRANCHES} -do - echo " ${PR}" -done - -# Retrieve sync'd PRs -SYNCD_PR_BRANCHES="$(echo $(git ls-remote origin github/pr* | sed -n 's|.*/github/\(pr[0-9].*\)|\1|p'))" -echo "Syncd PRs:" -for PR in ${SYNCD_PR_BRANCHES} -do - echo " ${PR}" -done - -# Determine any closed PRs that are currently sync'd -SYNCD_CLOSED_PR_BRANCHES="" -if [ -n "${SYNCD_PR_BRANCHES}" ] -then - for SPR in ${SYNCD_PR_BRANCHES} - do - if [ -n "${OPEN_PR_BRANCHES}" ] - then - IS_OPEN=0 - for OPR in ${OPEN_PR_BRANCHES} - do - if [ "${SPR}" = "${OPR}" ] - then - IS_OPEN=1 - break - fi - done - if [ ${IS_OPEN} -eq 0 ] - then - SYNCD_CLOSED_PR_BRANCHES="${SYNCD_CLOSED_PR_BRANCHES} ${SPR}" - fi - fi - done -fi -echo "Syncd Closed PRs:" -for PR in ${SYNCD_CLOSED_PR_BRANCHES} -do - echo " ${PR}" -done - -# Delete any sync'd PRs -if [ -n "${SYNCD_CLOSED_PR_BRANCHES}" ] -then - CLOSED_REFSPECS="" - for PR in ${SYNCD_CLOSED_PR_BRANCHES} - do - echo "Adding respec for closed ${PR}" - CLOSED_REFSPECS="${CLOSED_REFSPECS} :github/${PR}" - done - - echo "Removing closed PRs" - git push -f origin ${CLOSED_REFSPECS} -fi - -# Sync open PRs to OLCF -if [ -n "${OPEN_PR_BRANCHES}" ] -then - FETCH_REFSPECS="" - PUSH_REFSPECS="" - for PR in ${OPEN_PR_BRANCHES} - do - PR_NUM=$(expr "${PR}" : 'pr\([0-9]\+\)') - echo "Adding refspecs for ${PR}" - FETCH_REFSPECS="${FETCH_REFSPECS} +refs/pull/${PR_NUM}/head:refs/remotes/upstream/github/${PR}" - PUSH_REFSPECS="${PUSH_REFSPECS} github/${PR}:github/${PR}" - done - - echo "Fetching upstream refs for open PRs" - git fetch upstream ${FETCH_REFSPECS} - - for PR in ${OPEN_PR_BRANCHES} - do - echo "Building local branch for ${PR}" - git branch github/${PR} upstream/github/${PR} - done - - echo "Pushing branches for open PRs" - git push -f origin ${PUSH_REFSPECS} -fi diff --git a/scripts/ci/images/build-emu-power8-images.sh b/scripts/ci/images/build-emu-power8-images.sh index ae60bda032..f138e55ee1 100755 --- a/scripts/ci/images/build-emu-power8-images.sh +++ b/scripts/ci/images/build-emu-power8-images.sh @@ -3,7 +3,7 @@ ######################################## # ppc64le CentOS 7 emulation base image ######################################## -docker build --rm --squash \ +docker build --squash \ --build-arg TARGET_ARCH_SYSTEM=ppc64le \ --build-arg TARGET_ARCH_DOCKER=ppc64le \ --build-arg TARGET_CPU=power8 \ @@ -17,7 +17,7 @@ docker push ornladios/adios2:ci-x86_64-power8-el7 ######################################## # ppc64le CI base image ######################################## -docker build --rm --squash \ +docker build --squash \ --build-arg TARGET_CPU=power8 \ -t ornladios/adios2:ci-x86_64-power8-el7-base \ emu-el7-base @@ -33,7 +33,7 @@ docker push ornladios/adios2:ci-x86_64-power8-el7-base ######################################## # XL base image ######################################## -docker build --rm \ +docker build \ -t ornladios/adios2:ci-x86_64-power8-el7-xl-base \ power8-el7-xl-base docker-squash \ @@ -48,7 +48,7 @@ docker push ornladios/adios2:ci-x86_64-power8-el7-xl-base ######################################## # XL builder image ######################################## -docker build --rm \ +docker build \ -t ornladios/adios2:ci-x86_64-power8-el7-xl \ --build-arg COMPILER=xl \ power8-el7-leaf @@ -64,7 +64,7 @@ docker push ornladios/adios2:ci-x86_64-power8-el7-xl ######################################## # XL + MPI builder image ######################################## -docker build --rm \ +docker build \ -t ornladios/adios2:ci-x86_64-power8-el7-xl-smpi \ --build-arg COMPILER=xl \ power8-el7-leaf-smpi @@ -80,7 +80,7 @@ docker push ornladios/adios2:ci-x86_64-power8-el7-xl-smpi ######################################## # PGI base image ######################################## -docker build --rm \ +docker build \ -t ornladios/adios2:ci-x86_64-power8-el7-pgi-base \ power8-el7-pgi-base docker-squash \ @@ -95,7 +95,7 @@ docker push ornladios/adios2:ci-x86_64-power8-el7-pgi-base ######################################## # PGI builder image ######################################## -docker build --rm \ +docker build \ -t ornladios/adios2:ci-x86_64-power8-el7-pgi \ --build-arg COMPILER=pgi \ power8-el7-leaf @@ -111,7 +111,7 @@ docker push ornladios/adios2:ci-x86_64-power8-el7-pgi ######################################## # PGI + MPI builder image ######################################## -docker build --rm \ +docker build \ -t ornladios/adios2:ci-x86_64-power8-el7-pgi-smpi \ --build-arg COMPILER=pgi --build-arg HDF5_ARGS="-DMPI_C_COMPILER=mpipgicc" \ power8-el7-leaf-smpi diff --git a/scripts/ci/images/build-native-images.sh b/scripts/ci/images/build-native-images.sh index 21e5cea44c..f84eec0e9a 100755 --- a/scripts/ci/images/build-native-images.sh +++ b/scripts/ci/images/build-native-images.sh @@ -14,7 +14,8 @@ function build_partially_squashed_image() local IMAGE_FROM=$1 local IMAGE_TO=$2 - docker build --rm -t ornladios/adios2:ci-${IMAGE_TO} ${IMAGE_TO} + echo "${IMAGE_TO}" + docker build -t ornladios/adios2:ci-${IMAGE_TO} ${IMAGE_TO} docker-squash \ -f ornladios/adios2:ci-${IMAGE_FROM} \ @@ -22,14 +23,17 @@ function build_partially_squashed_image() ornladios/adios2:ci-${IMAGE_TO} } +if [ "${ADIOS2_DOCKER_BUILD}" != "0" ] +then + echo "************************************************************" echo "* Building fully squashed root base images *" echo "************************************************************" -ROOT_BASE_IMAGES="el7-base suse-pgi-base fedora-sanitizers-base debian-sid" +ROOT_BASE_IMAGES="el7-base suse-nvphcsdk-base fedora-sanitizers-base debian-sid" for IMAGE in ${ROOT_BASE_IMAGES} do echo "${IMAGE}" - docker build --squash --rm -t ornladios/adios2:ci-${IMAGE} ${IMAGE} + docker build --no-cache --squash -t ornladios/adios2:ci-${IMAGE} ${IMAGE} echo done @@ -47,7 +51,7 @@ done echo "************************************************************" echo "* Building partially squashed final images *" echo "************************************************************" -LEAF_IMAGES="el7-base,el7 el7-gnu8-ohpc-base,el7-gnu8-ohpc el7-gnu8-ohpc-base,el7-gnu8-openmpi-ohpc el7-intel-ohpc-base,el7-intel-ohpc el7-intel-ohpc-base,el7-intel-openmpi-ohpc suse-pgi-base,suse-pgi suse-pgi-base,suse-pgi-openmpi fedora-sanitizers-base,fedora-ubsan" +LEAF_IMAGES="el7-base,el7 el7-gnu8-ohpc-base,el7-gnu8-ohpc el7-gnu8-ohpc-base,el7-gnu8-openmpi-ohpc el7-intel-ohpc-base,el7-intel-ohpc el7-intel-ohpc-base,el7-intel-openmpi-ohpc suse-nvphcsdk-base,suse-pgi suse-nvphcsdk-base,suse-nvphcsdk-openmpi fedora-sanitizers-base,fedora-asan fedora-sanitizers-base,fedora-ubsan" for IMAGE_PAIR in ${LEAF_IMAGES} do echo "${IMAGE_PAIR%,*} -> ${IMAGE_PAIR#*,}" @@ -55,13 +59,21 @@ do echo done +fi + + +if [ "${ADIOS2_DOCKER_PUSH}" != "0" ] +then + echo "************************************************************" echo "* Push all images *" echo "************************************************************" -ALL_IMAGES="el7-base el7 el7-gnu8-ohpc-base el7-gnu8-ohpc el7-gnu8-openmpi-ohpc el7-intel-ohpc-base el7-intel-ohpc el7-intel-openmpi-ohpc suse-pgi-base suse-pgi suse-pgi-openmpi fedora-sanitizers-base fedora-ubsan debian-sid" +ALL_IMAGES="el7-base el7 el7-gnu8-ohpc-base el7-gnu8-ohpc el7-gnu8-openmpi-ohpc el7-intel-ohpc-base el7-intel-ohpc el7-intel-openmpi-ohpc suse-nvphcsdk-base suse-pgi suse-nvphcsdk-openmpi fedora-sanitizers-base fedora-asan fedora-ubsan debian-sid" for IMAGE in ${ALL_IMAGES} do echo "${IMAGE}" docker push ornladios/adios2:ci-${IMAGE} echo done + +fi diff --git a/scripts/ci/images/build-spack-images.sh b/scripts/ci/images/build-spack-images.sh new file mode 100755 index 0000000000..d76149722b --- /dev/null +++ b/scripts/ci/images/build-spack-images.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +######################################## +# CentOS 7 +######################################## +if [ "${ADIOS2_DOCKER_BUILD}" != "0" ] +then + docker build --rm \ + --build-arg DISTRO=centos7 \ + -t ornladios/adios2:ci-el7-spack \ + spack + docker-squash \ + -f spack/centos7 \ + -t ornladios/adios2:ci-el7-spack \ + ornladios/adios2:ci-el7-spack +fi +if [ "${ADIOS2_DOCKER_PUSH}" != "0" ] +then + echo "" + echo "Pushing ornladios/adios2:ci-el7-spack" + echo "" + docker push ornladios/adios2:ci-el7-spack +fi + +######################################## +# Ubuntu 18.04 +######################################## +if [ "${ADIOS2_DOCKER_BUILD}" != "0" ] +then + docker build --rm \ + --build-arg DISTRO=ubuntu-bionic \ + -t ornladios/adios2:ci-ubuntu1804-spack \ + spack + docker-squash \ + -f spack/ubuntu-bionic \ + -t ornladios/adios2:ci-ubuntu1804-spack \ + ornladios/adios2:ci-ubuntu1804-spack +fi +if [ "${ADIOS2_DOCKER_PUSH}" != "0" ] +then + echo "" + echo "Pushing ornladios/adios2:ci-ubuntu1804-spack" + echo "" + docker push ornladios/adios2:ci-ubuntu1804-spack +fi diff --git a/scripts/ci/images/debian-sid/Dockerfile b/scripts/ci/images/debian-sid/Dockerfile index a01b276468..764b65e833 100644 --- a/scripts/ci/images/debian-sid/Dockerfile +++ b/scripts/ci/images/debian-sid/Dockerfile @@ -1,10 +1,9 @@ FROM debian:sid -RUN apt-get update && \ - apt-get dist-upgrade -y --no-install-recommends && \ - apt-get install -y --no-install-recommends \ +RUN apt update && \ + apt full-upgrade -y --no-install-recommends && \ + apt install -y --no-install-recommends \ curl \ - python3-all \ ca-certificates \ devscripts \ git \ @@ -19,7 +18,11 @@ RUN apt-get update && \ pybind11-dev \ libgtest-dev \ nlohmann-json3-dev \ - libpython3-dev \ + python3.8-dev \ + libpython3.8-dev \ + python3.9-dev \ + libpython3.9-dev \ + python3-distutils \ python3-numpy \ python3-mpi4py \ libblosc-dev \ @@ -30,4 +33,5 @@ RUN apt-get update && \ libhdf5-serial-dev \ libhdf5-openmpi-dev \ libfabric-dev \ - libffi-dev + libffi-dev && \ + apt autoremove -y diff --git a/scripts/ci/images/el7-base/Dockerfile b/scripts/ci/images/el7-base/Dockerfile index 649ed6efde..fea462b765 100644 --- a/scripts/ci/images/el7-base/Dockerfile +++ b/scripts/ci/images/el7-base/Dockerfile @@ -5,13 +5,14 @@ RUN yum upgrade -y && \ yum -y install make curl file valgrind vim bison flex sudo gdb \ pkgconfig bison flex pkgconfig gcc gcc-c++ gcc-gfortran \ zlib zlib-devel bzip2 bzip2-libs bzip2-devel libpng-devel \ - libfabric-devel libffi-devel + libfabric-devel libffi-devel python3 python3-devel RUN yum -y install epel-release && \ yum -y install zeromq-devel blosc-devel libzstd-devel # Install and setup newer version of git -RUN yum install -y https://centos7.iuscommunity.org/ius-release.rpm && \ - yum -y install git222 +RUN yum install -y https://repo.ius.io/ius-release-el7.rpm && \ + yum -y install git224 && \ + yum remove -y ius-release # Install the most recent CMake nightly binary WORKDIR /opt/cmake diff --git a/scripts/ci/images/el7-gnu8-ohpc-base/Dockerfile b/scripts/ci/images/el7-gnu8-ohpc-base/Dockerfile index ef1ae65299..4da24cdf62 100644 --- a/scripts/ci/images/el7-gnu8-ohpc-base/Dockerfile +++ b/scripts/ci/images/el7-gnu8-ohpc-base/Dockerfile @@ -2,7 +2,8 @@ FROM ornladios/adios2:ci-el7-base # Install OpenHPC packages RUN yum -y install https://github.com/openhpc/ohpc/releases/download/v1.3.GA/ohpc-release-1.3-1.el7.x86_64.rpm && \ - yum -y install lmod-ohpc gnu8-compilers-ohpc + yum -y install lmod-ohpc gnu8-compilers-ohpc python3-numpy-gnu8-ohpc && \ + sed 's|python3\.4|python3.6|g' -i /opt/ohpc/pub/moduledeps/*/py3-numpy/* # Install ZFP WORKDIR /opt/zfp @@ -22,19 +23,19 @@ ENV PATH=/opt/zfp/0.5.5/bin:${PATH} \ # Install SZ WORKDIR /opt/sz -RUN curl -L https://github.com/disheng222/SZ/archive/v2.1.8.0.tar.gz | tar -xvz && \ +RUN curl -L https://github.com/disheng222/SZ/archive/v2.1.8.3.tar.gz | tar -xvz && \ mkdir build && \ cd build && \ source /etc/profile && \ module load gnu8 && \ export CC=gcc CXX=g++ FC=gfortran && \ - cmake -DBUILD_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/opt/sz/2.1.8.0 ../SZ-2.1.8.0 && \ + cmake -DBUILD_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/opt/sz/2.1.8.3 ../SZ-2.1.8.3 && \ make -j$(grep -c '^processor' /proc/cpuinfo) install && \ cd .. && \ - rm -rf SZ-2.1.8.0 build -ENV PATH=/opt/sz/2.1.8.0/bin:${PATH} \ - LD_LIBRARY_PATH=/opt/sz/2.1.8.0/lib64:${LD_LIBRARY_PATH} \ - CMAKE_PREFIX_PATH=/opt/sz/2.1.8.0:${CMAKE_PREFIX_PATH} + rm -rf SZ-2.1.8.3 build +ENV PATH=/opt/sz/2.1.8.3/bin:${PATH} \ + LD_LIBRARY_PATH=/opt/sz/2.1.8.3/lib64:${LD_LIBRARY_PATH} \ + CMAKE_PREFIX_PATH=/opt/sz/2.1.8.3:${CMAKE_PREFIX_PATH} # Misc cleanup of unneeded files RUN rm -rfv \ diff --git a/scripts/ci/images/el7-gnu8-ohpc/Dockerfile b/scripts/ci/images/el7-gnu8-ohpc/Dockerfile index 360b646e82..905e26df46 100644 --- a/scripts/ci/images/el7-gnu8-ohpc/Dockerfile +++ b/scripts/ci/images/el7-gnu8-ohpc/Dockerfile @@ -1,7 +1,7 @@ FROM ornladios/adios2:ci-el7-gnu8-ohpc-base # Install OpenHPC packages -RUN yum -y install python34-devel python34-numpy-gnu8-ohpc hdf5-gnu8-ohpc +RUN yum -y install hdf5-gnu8-ohpc # Misc cleanup of unneeded files RUN yum clean all && \ diff --git a/scripts/ci/images/el7-gnu8-openmpi-ohpc/Dockerfile b/scripts/ci/images/el7-gnu8-openmpi-ohpc/Dockerfile index 76ff59334a..3b81982bdb 100644 --- a/scripts/ci/images/el7-gnu8-openmpi-ohpc/Dockerfile +++ b/scripts/ci/images/el7-gnu8-openmpi-ohpc/Dockerfile @@ -1,8 +1,10 @@ FROM ornladios/adios2:ci-el7-gnu8-ohpc-base # Install OpenHPC packages -RUN yum -y install python-devel python-numpy-gnu8-ohpc openmpi3-gnu8-ohpc \ - phdf5-gnu8-openmpi3-ohpc python-mpi4py-gnu8-openmpi3-ohpc +RUN yum -y install openmpi3-gnu8-ohpc phdf5-gnu8-openmpi3-ohpc \ + python3-mpi4py-gnu8-openmpi3-ohpc && \ + sed 's|python3\.4|python3.6|g' -i /opt/ohpc/pub/moduledeps/*/py3-mpi4py/* + # Misc cleanup of unneeded files RUN yum clean all && \ diff --git a/scripts/ci/images/el7-intel-ohpc-base/Dockerfile b/scripts/ci/images/el7-intel-ohpc-base/Dockerfile index 560417479e..fcd817a38b 100644 --- a/scripts/ci/images/el7-intel-ohpc-base/Dockerfile +++ b/scripts/ci/images/el7-intel-ohpc-base/Dockerfile @@ -33,7 +33,7 @@ FROM ornladios/adios2:ci-el7-base # Install Intel C++ and Fortran compilers ARG LICENSE_FILE=CI.lic -ARG INTEL_VERSION=2020_update1 +ARG INTEL_VERSION=2020_update2 WORKDIR /tmp COPY ${LICENSE_FILE} /opt/intel/licenses/license.lic COPY silent-cpp.cfg /tmp @@ -46,8 +46,11 @@ RUN tar -xzf parallel_studio_xe_${INTEL_VERSION}_composer_edition_for_cpp.tgz && ./parallel_studio_xe_${INTEL_VERSION}_composer_edition_for_fortran/install.sh -s ./silent-fortran.cfg # Install OpenHPC packages -RUN yum -y install https://github.com/openhpc/ohpc/releases/download/v1.3.GA/ohpc-release-1.3-1.el7.x86_64.rpm && \ - yum -y install lmod-ohpc intel-compilers-devel-ohpc +RUN yum install -y /tmp/*_for_cpp/rpm/intel-{comp,ps}xe-doc-*.rpm && \ + yum -y install https://github.com/openhpc/ohpc/releases/download/v1.3.GA/ohpc-release-1.3-1.el7.x86_64.rpm && \ + yum -y install lmod-ohpc intel-compilers-devel-ohpc \ + python3-numpy-intel-ohpc && \ + sed 's|python3\.4|python3.6|g' -i /opt/ohpc/pub/moduledeps/*/py3-numpy/* # Install ZFP WORKDIR /opt/zfp @@ -67,19 +70,19 @@ ENV PATH=/opt/zfp/0.5.5/bin:${PATH} \ # Install SZ WORKDIR /opt/sz -RUN curl -L https://github.com/disheng222/SZ/archive/v2.1.8.0.tar.gz | tar -xvz && \ +RUN curl -L https://github.com/disheng222/SZ/archive/v2.1.8.3.tar.gz | tar -xvz && \ mkdir build && \ cd build && \ source /etc/profile && \ module load intel && \ export CC=icc CXX=icpc FC=ifort && \ - cmake -DBUILD_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/opt/sz/2.1.8.0 ../SZ-2.1.8.0 && \ + cmake -DBUILD_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/opt/sz/2.1.8.3 ../SZ-2.1.8.3 && \ make -j$(grep -c '^processor' /proc/cpuinfo) install && \ cd .. && \ - rm -rf SZ-2.1.8.0 build -ENV PATH=/opt/sz/2.1.8.0/bin:${PATH} \ - LD_LIBRARY_PATH=/opt/sz/2.1.8.0/lib64:${LD_LIBRARY_PATH} \ - CMAKE_PREFIX_PATH=/opt/sz/2.1.8.0:${CMAKE_PREFIX_PATH} + rm -rf SZ-2.1.8.3 build +ENV PATH=/opt/sz/2.1.8.3/bin:${PATH} \ + LD_LIBRARY_PATH=/opt/sz/2.1.8.3/lib64:${LD_LIBRARY_PATH} \ + CMAKE_PREFIX_PATH=/opt/sz/2.1.8.3:${CMAKE_PREFIX_PATH} # Misc cleanup of unneeded files RUN rm -rfv \ diff --git a/scripts/ci/images/el7-intel-ohpc/Dockerfile b/scripts/ci/images/el7-intel-ohpc/Dockerfile index 09ed2189f1..0b12609af9 100644 --- a/scripts/ci/images/el7-intel-ohpc/Dockerfile +++ b/scripts/ci/images/el7-intel-ohpc/Dockerfile @@ -1,7 +1,7 @@ FROM ornladios/adios2:ci-el7-intel-ohpc-base # Install OpenHPC packages -RUN yum -y install python-devel python-numpy-intel-ohpc hdf5-intel-ohpc +RUN yum -y install hdf5-intel-ohpc # Misc cleanup of unneeded files RUN yum clean all && \ diff --git a/scripts/ci/images/el7-intel-openmpi-ohpc/Dockerfile b/scripts/ci/images/el7-intel-openmpi-ohpc/Dockerfile index 1f0b46808e..752fd0224c 100644 --- a/scripts/ci/images/el7-intel-openmpi-ohpc/Dockerfile +++ b/scripts/ci/images/el7-intel-openmpi-ohpc/Dockerfile @@ -1,9 +1,9 @@ FROM ornladios/adios2:ci-el7-intel-ohpc-base # Install OpenHPC packages -RUN yum -y install python34-devel python34-numpy-intel-ohpc \ - openmpi3-intel-ohpc phdf5-intel-openmpi3-ohpc \ - python34-mpi4py-intel-openmpi3-ohpc +RUN yum -y install openmpi3-intel-ohpc phdf5-intel-openmpi3-ohpc \ + python3-mpi4py-intel-openmpi3-ohpc && \ + sed 's|python3\.4|python3.6|g' -i /opt/ohpc/pub/moduledeps/*/py3-mpi4py/* # Misc cleanup of unneeded files RUN yum clean all && \ diff --git a/scripts/ci/images/el7/Dockerfile b/scripts/ci/images/el7/Dockerfile index f9f0e55de2..b6476d3089 100644 --- a/scripts/ci/images/el7/Dockerfile +++ b/scripts/ci/images/el7/Dockerfile @@ -1,7 +1,7 @@ FROM ornladios/adios2:ci-el7-base # Install extra system dev packages -RUN yum -y install hdf5-devel python-devel numpy +RUN yum -y install hdf5-devel python36-numpy # Install ZFP WORKDIR /opt/zfp @@ -18,16 +18,16 @@ ENV PATH=/opt/zfp/0.5.5/bin:${PATH} \ # Install SZ WORKDIR /opt/sz -RUN curl -L https://github.com/disheng222/SZ/archive/v2.1.8.0.tar.gz | tar -xvz && \ +RUN curl -L https://github.com/disheng222/SZ/archive/v2.1.8.3.tar.gz | tar -xvz && \ mkdir build && \ cd build && \ - cmake -DBUILD_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/opt/sz/2.1.8.0 ../SZ-2.1.8.0 && \ + cmake -DBUILD_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/opt/sz/2.1.8.3 ../SZ-2.1.8.3 && \ make -j$(grep -c '^processor' /proc/cpuinfo) install && \ cd .. && \ - rm -rf SZ-2.1.8.0 build -ENV PATH /opt/sz/2.1.8.0/bin:${PATH} -ENV LD_LIBRARY_PATH /opt/sz/2.1.8.0/lib64:${LD_LIBRARY_PATH} -ENV CMAKE_PREFIX_PATH /opt/sz/2.1.8.0:${CMAKE_PREFIX_PATH} + rm -rf SZ-2.1.8.3 build +ENV PATH /opt/sz/2.1.8.3/bin:${PATH} +ENV LD_LIBRARY_PATH /opt/sz/2.1.8.3/lib64:${LD_LIBRARY_PATH} +ENV CMAKE_PREFIX_PATH /opt/sz/2.1.8.3:${CMAKE_PREFIX_PATH} # Misc cleanup of unneeded files RUN yum clean all && \ diff --git a/scripts/ci/images/emu-el7-base/Dockerfile b/scripts/ci/images/emu-el7-base/Dockerfile index 1dc5ea759f..370815c529 100644 --- a/scripts/ci/images/emu-el7-base/Dockerfile +++ b/scripts/ci/images/emu-el7-base/Dockerfile @@ -30,7 +30,7 @@ RUN yum install -y \ xz-devel rhash-devel zlib-devel libzstd-devel && \ mkdir tmp && \ cd tmp && \ - curl -L https://github.com/Kitware/CMake/releases/download/v3.17.0/cmake-3.17.0.tar.gz | \ + curl -L https://github.com/Kitware/CMake/releases/download/v3.18.0/cmake-3.18.0.tar.gz | \ tar -xz && \ mkdir build && \ cd build && \ @@ -40,12 +40,12 @@ RUN yum install -y \ --no-system-libarchive \ --no-system-libuv \ --no-system-jsoncpp \ - --prefix=/opt/cmake/3.17.0 \ + --prefix=/opt/cmake/3.18.0 \ --parallel=$(grep -c '^processor' /proc/cpuinfo) && \ make -j$(grep -c '^processor' /proc/cpuinfo) install && \ cd ../.. && \ rm -rf tmp -ENV PATH=/opt/cmake/3.17.0/bin:${PATH} +ENV PATH=/opt/cmake/3.18.0/bin:${PATH} # Misc cleanup of unneeded files RUN yum clean all && \ diff --git a/scripts/ci/images/formatting/Dockerfile b/scripts/ci/images/formatting/Dockerfile new file mode 100644 index 0000000000..b5a35a7106 --- /dev/null +++ b/scripts/ci/images/formatting/Dockerfile @@ -0,0 +1,10 @@ +FROM ubuntu:20.04 + +RUN apt-get update && \ + apt-get install -y apt-utils && \ + apt-get install -y curl git flake8 libtinfo5 && \ + apt-get clean + +RUN curl -L https://github.com/llvm/llvm-project/releases/download/llvmorg-7.1.0/clang+llvm-7.1.0-x86_64-linux-gnu-ubuntu-14.04.tar.xz | tar -C /opt -xJv && \ + mv /opt/clang* /opt/llvm-7.1.0 +ENV PATH=/opt/llvm-7.1.0/bin:$PATH diff --git a/scripts/ci/images/power8-el7-pgi-base/Dockerfile b/scripts/ci/images/power8-el7-pgi-base/Dockerfile index 1c803c90fd..554b7bf0ba 100644 --- a/scripts/ci/images/power8-el7-pgi-base/Dockerfile +++ b/scripts/ci/images/power8-el7-pgi-base/Dockerfile @@ -48,20 +48,20 @@ ENV PATH=/opt/zfp/0.5.5/bin:${PATH} \ # Install SZ WORKDIR /opt/sz -RUN curl -L https://github.com/disheng222/SZ/archive/v2.1.8.0.tar.gz | tar -xvz && \ +RUN curl -L https://github.com/disheng222/SZ/archive/v2.1.8.3.tar.gz | tar -xvz && \ mkdir build && \ cd build && \ cmake \ -DBUILD_SHARED_LIBS=ON \ -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_INSTALL_PREFIX=/opt/sz/2.1.8.0 \ - ../SZ-2.1.8.0 && \ + -DCMAKE_INSTALL_PREFIX=/opt/sz/2.1.8.3 \ + ../SZ-2.1.8.3 && \ make -j$(grep -c '^processor' /proc/cpuinfo) install && \ cd .. && \ - rm -rf SZ-2.1.8.0 build -ENV PATH=/opt/sz/2.1.8.0/bin:${PATH} \ - LD_LIBRARY_PATH=/opt/sz/2.1.8.0/lib64:${LD_LIBRARY_PATH} \ - CMAKE_PREFIX_PATH=/opt/sz/2.1.8.0:${CMAKE_PREFIX_PATH} + rm -rf SZ-2.1.8.3 build +ENV PATH=/opt/sz/2.1.8.3/bin:${PATH} \ + LD_LIBRARY_PATH=/opt/sz/2.1.8.3/lib64:${LD_LIBRARY_PATH} \ + CMAKE_PREFIX_PATH=/opt/sz/2.1.8.3:${CMAKE_PREFIX_PATH} # Misc cleanup of unneeded files RUN rm -rf /tmp/* diff --git a/scripts/ci/images/power8-el7-xl-base/Dockerfile b/scripts/ci/images/power8-el7-xl-base/Dockerfile index 7395debf22..fe70270954 100644 --- a/scripts/ci/images/power8-el7-xl-base/Dockerfile +++ b/scripts/ci/images/power8-el7-xl-base/Dockerfile @@ -39,20 +39,20 @@ ENV PATH=/opt/zfp/0.5.5/bin:${PATH} \ # Install SZ WORKDIR /opt/sz -RUN curl -L https://github.com/disheng222/SZ/archive/v2.1.8.0.tar.gz | tar -xvz && \ +RUN curl -L https://github.com/disheng222/SZ/archive/v2.1.8.3.tar.gz | tar -xvz && \ mkdir build && \ cd build && \ cmake \ -DBUILD_SHARED_LIBS=ON \ -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_INSTALL_PREFIX=/opt/sz/2.1.8.0 \ - ../SZ-2.1.8.0 && \ + -DCMAKE_INSTALL_PREFIX=/opt/sz/2.1.8.3 \ + ../SZ-2.1.8.3 && \ make -j$(grep -c '^processor' /proc/cpuinfo) install && \ cd .. && \ - rm -rf SZ-2.1.8.0 build -ENV PATH=/opt/sz/2.1.8.0/bin:${PATH} \ - LD_LIBRARY_PATH=/opt/sz/2.1.8.0/lib64:${LD_LIBRARY_PATH} \ - CMAKE_PREFIX_PATH=/opt/sz/2.1.8.0:${CMAKE_PREFIX_PATH} + rm -rf SZ-2.1.8.3 build +ENV PATH=/opt/sz/2.1.8.3/bin:${PATH} \ + LD_LIBRARY_PATH=/opt/sz/2.1.8.3/lib64:${LD_LIBRARY_PATH} \ + CMAKE_PREFIX_PATH=/opt/sz/2.1.8.3:${CMAKE_PREFIX_PATH} # Misc cleanup of unneeded files RUN rm -rf /tmp/* diff --git a/scripts/ci/images/spack/Dockerfile b/scripts/ci/images/spack/Dockerfile new file mode 100644 index 0000000000..d7e59b0034 --- /dev/null +++ b/scripts/ci/images/spack/Dockerfile @@ -0,0 +1,9 @@ +ARG DISTRO=centos7 +FROM spack/${DISTRO} + +COPY adios2-ci-env.yml /root/adios2-ci-env.yml +RUN spack env create adios2-ci /root/adios2-ci-env.yml && \ + spack env activate adios2-ci && \ + spack install -v -j$(grep -c '^processor' /proc/cpuinfo) && \ + spack clean -a +RUN rm -f /root/adios2-ci-env.yml diff --git a/scripts/ci/images/spack/adios2-ci-env.yml b/scripts/ci/images/spack/adios2-ci-env.yml new file mode 100644 index 0000000000..1dc6428728 --- /dev/null +++ b/scripts/ci/images/spack/adios2-ci-env.yml @@ -0,0 +1,26 @@ +# A Spack environment to be used for ADIOS 2.x CI testing +# It contains every possible dependency for ADIOS +spack: + packages: + all: + target: [x86_64] + specs: + - git + - cmake~ownlibs + - openmpi + - hdf5+mpi + - libfabric + - zfp + - sz + - bzip2 + - python + - py-numpy + - py-mpi4py + - c-blosc + - libzmq + - py-pybind11 + - yaml-cpp + - pugixml + - nlohmann-json + - libpng + - dataspaces diff --git a/scripts/ci/images/suse-nvhpcsdk-base/.gitignore b/scripts/ci/images/suse-nvhpcsdk-base/.gitignore new file mode 100644 index 0000000000..a8ab452162 --- /dev/null +++ b/scripts/ci/images/suse-nvhpcsdk-base/.gitignore @@ -0,0 +1 @@ +nvhpc*.tar.gz diff --git a/scripts/ci/images/suse-nvhpcsdk-base/Dockerfile b/scripts/ci/images/suse-nvhpcsdk-base/Dockerfile new file mode 100644 index 0000000000..3a5b6854cb --- /dev/null +++ b/scripts/ci/images/suse-nvhpcsdk-base/Dockerfile @@ -0,0 +1,84 @@ +FROM opensuse/leap:15.2 + +# Install core dev packages +RUN zypper addrepo -fc https://download.opensuse.org/repositories/devel:/tools:/scm/openSUSE_Leap_15.2/devel:tools:scm.repo +RUN zypper --gpg-auto-import-keys ref +RUN zypper in -y gcc gcc-c++ gcc-fortran git make curl tar f2c glibc-locale \ + glibc-devel libbz2-devel pkg-config zeromq-devel zlib-devel gdb vim valgrind \ + bzip2 gzip blosc-devel libzstd-devel libopenssl-devel Modules + +# Workaround so pgi can find g77 +WORKDIR /usr/bin +RUN ln -s gfortran g77 + +# Install NVidia HPC SDK +WORKDIR /tmp/nvhpcsdk-install +COPY nvhpc_2020_209_Linux_x86_64_cuda_11.0.tar.gz . +RUN tar -xzf nvhpc_2020_209_Linux_x86_64_cuda_11.0.tar.gz && \ + cd nvhpc_2020_209_Linux_x86_64_cuda_11.0 && \ + export \ + NVHPC_SILENT=true \ + NVHPC_INSTALL_DIR=/opt/nvidia/hpc_sdk \ + NVHPC_INSTALL_TYPE=single && \ + ./install && \ + echo 'export MODULEPATH=/opt/nvidia/hpc_sdk/modulefiles:${MODULEPATH}' > /etc/profile.d/nvhpc-modules.sh && \ + echo 'setenv MODULEPATH /opt/nvidia/hpc_sdk/modulefiles:${MODULEPATH}' > /etc/profile.d/pgi-modules.csh + +# Remove all the CUDA components since we don't need them for CI images +WORKDIR /opt/nvidia/hpc_sdk +RUN rm -rf \ + Linux_x86_64/2020 \ + Linux_x86_64/20.9/cuda \ + Linux_x86_64/20.9/math \ + Linux_x86_64/20.9/profilers \ + Linux_x86_64/20.9/examples \ + Linux_x86_64/20.9/REDIST \ + modulefiles/nvhpc-byo-compiler && \ + sed -e '/cuda/d' -e '/math/d' -i modulefiles/*/* + +# Install the most recent CMake nightly binary +WORKDIR /opt/cmake +RUN curl -L https://cmake.org/files/dev/$(curl https://cmake.org/files/dev/ | sed -n '/Linux-x86_64.tar.gz/s/.*>\(cmake[^<]*\)<.*/\1/p' | sort | tail -1) | tar --strip-components=1 -xzv +ENV PATH=/opt/cmake/bin:${PATH} + +# Install ZFP +# Note that ZFP needs to be built with GCC at the moment as the results +# are broken when built with PGI +WORKDIR /opt/zfp +RUN curl -L https://github.com/LLNL/zfp/releases/download/0.5.5/zfp-0.5.5.tar.gz | tar -xvz && \ + mkdir build && \ + cd build && \ + /opt/cmake/bin/cmake \ + -DBUILD_SHARED_LIBS=ON \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=/opt/zfp/0.5.5 \ + ../zfp-0.5.5 && \ + make -j$(grep -c '^processor' /proc/cpuinfo) install && \ + cd .. && \ + rm -rf zfp-0.5.5 build +ENV PATH=/opt/zfp/0.5.5/bin:${PATH} \ + LD_LIBRARY_PATH=/opt/zfp/0.5.5/lib64:${LD_LIBRARY_PATH} \ + CMAKE_PREFIX_PATH=/opt/zfp/0.5.5:${CMAKE_PREFIX_PATH} + +# Install SZ +WORKDIR /opt/sz +RUN curl -L https://github.com/szcompressor/SZ/releases/download/v2.1.11/SZ-2.1.11.tar.gz | tar -xvz && \ + mkdir build && \ + cd build && \ + source /etc/profile && \ + module load nvhpc-nompi && \ + /opt/cmake/bin/cmake \ + -DBUILD_SHARED_LIBS=ON \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=/opt/sz/2.1.11 \ + ../SZ-2.1.11 && \ + make -j$(grep -c '^processor' /proc/cpuinfo) install && \ + cd .. && \ + rm -rf SZ-2.1.11 build +ENV PATH=/opt/sz/2.1.11/bin:${PATH} \ + LD_LIBRARY_PATH=/opt/sz/2.1.11/lib64:${LD_LIBRARY_PATH} \ + CMAKE_PREFIX_PATH=/opt/sz/2.1.11:${CMAKE_PREFIX_PATH} + +# Misc cleanup of unneeded files +RUN rm -rf /tmp/* && \ + zypper clean diff --git a/scripts/ci/images/suse-nvhpcsdk-openmpi/Dockerfile b/scripts/ci/images/suse-nvhpcsdk-openmpi/Dockerfile new file mode 100644 index 0000000000..d7a6c59bf9 --- /dev/null +++ b/scripts/ci/images/suse-nvhpcsdk-openmpi/Dockerfile @@ -0,0 +1,31 @@ +FROM ornladios/adios2:ci-suse-nvhpcsdk-base + +# Install HDF5 1.12.0 +WORKDIR /opt/hdf5 +RUN curl -L https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.12/hdf5-1.12.0/src/hdf5-1.12.0.tar.bz2 | \ + tar -xvj && \ + mkdir build && \ + cd build && \ + source /etc/profile && \ + module load nvhpc && \ + /opt/cmake/bin/cmake \ + -DCMAKE_INSTALL_PREFIX=/opt/hdf5/1.12.0 \ + -DBUILD_SHARED_LIBS=ON \ + -DBUILD_STATIC_LIBS=OFF \ + -DCMAKE_BUILD_TYPE=Release \ + -DHDF5_ENABLE_PARALLEL=ON \ + -DHDF5_BUILD_CPP_LIB=OFF\ + -DHDF5_BUILD_EXAMPLES=OFF \ + -DBUILD_TESTING=OFF \ + -DHDF5_BUILD_TOOLS=OFF \ + ../hdf5-1.12.0 && \ + make -j$(grep -c '^processor' /proc/cpuinfo) install && \ + cd .. && \ + rm -rf hdf5-1.12.0 build +ENV PATH=/opt/hdf5/1.12.0/bin:${PATH} \ + LD_LIBRARY_PATH=/opt/hdf5/1.12.0/lib:${LD_LIBRARY_PATH} \ + CMAKE_PREFIX_PATH=/opt/hdf5/1.12.0:${CMAKE_PREFIX_PATH} + +# Misc cleanup of unneeded files +RUN rm -rf /tmp/* && \ + zypper clean diff --git a/scripts/ci/images/suse-pgi/Dockerfile b/scripts/ci/images/suse-nvhpcsdk/Dockerfile similarity index 53% rename from scripts/ci/images/suse-pgi/Dockerfile rename to scripts/ci/images/suse-nvhpcsdk/Dockerfile index 75b9858680..294cdbab84 100644 --- a/scripts/ci/images/suse-pgi/Dockerfile +++ b/scripts/ci/images/suse-nvhpcsdk/Dockerfile @@ -1,31 +1,30 @@ -FROM ornladios/adios2:ci-suse-pgi-base +FROM ornladios/adios2:ci-suse-nvhpcsdk-base -# Install HDF5 1.10.4 (the current 1.10.5 has a parallel close bug affecting -# the tests +# Install HDF5 WORKDIR /opt/hdf5 -RUN curl -L https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.10/hdf5-1.10.4/src/hdf5-1.10.4.tar.bz2 | \ +RUN curl -L https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.12/hdf5-1.12.0/src/hdf5-1.12.0.tar.bz2 | \ tar -xvj && \ mkdir build && \ cd build && \ source /etc/profile && \ - module load pgi openmpi && \ - export CC=pgcc CXX=pgc++ FC=pgfortran && \ + module load nvhpc-nompi && \ /opt/cmake/bin/cmake \ - -DCMAKE_INSTALL_PREFIX=/opt/hdf5/1.10.4 \ + -DCMAKE_INSTALL_PREFIX=/opt/hdf5/1.12.0 \ -DBUILD_SHARED_LIBS=ON \ + -DBUILD_STATIC_LIBS=OFF \ -DCMAKE_BUILD_TYPE=Release \ -DHDF5_ENABLE_PARALLEL=OFF \ -DHDF5_BUILD_CPP_LIB=OFF\ -DHDF5_BUILD_EXAMPLES=OFF \ -DBUILD_TESTING=OFF \ -DHDF5_BUILD_TOOLS=OFF \ - ../hdf5-1.10.4 && \ + ../hdf5-1.12.0 && \ make -j$(grep -c '^processor' /proc/cpuinfo) install && \ cd .. && \ - rm -rf hdf5-1.10.4 build -ENV PATH=/opt/hdf5/1.10.4/bin:${PATH} \ - LD_LIBRARY_PATH=/opt/hdf5/1.10.4/lib:${LD_LIBRARY_PATH} \ - CMAKE_PREFIX_PATH=/opt/hdf5/1.10.4:${CMAKE_PREFIX_PATH} + rm -rf hdf5-1.12.0 build +ENV PATH=/opt/hdf5/1.12.0/bin:${PATH} \ + LD_LIBRARY_PATH=/opt/hdf5/1.12.0/lib:${LD_LIBRARY_PATH} \ + CMAKE_PREFIX_PATH=/opt/hdf5/1.12.0:${CMAKE_PREFIX_PATH} # Misc cleanup of unneeded files RUN rm -rf /tmp/* && \ diff --git a/scripts/ci/images/suse-pgi-base/Dockerfile b/scripts/ci/images/suse-pgi-base/Dockerfile deleted file mode 100644 index 3fae6328e4..0000000000 --- a/scripts/ci/images/suse-pgi-base/Dockerfile +++ /dev/null @@ -1,77 +0,0 @@ -FROM opensuse/leap:15.2 - -# Install core dev packages -RUN zypper addrepo -fc https://download.opensuse.org/repositories/devel:/tools:/scm/openSUSE_Leap_15.2/devel:tools:scm.repo -RUN zypper --gpg-auto-import-keys ref -RUN zypper in -y gcc gcc-c++ gcc-fortran git make curl tar f2c glibc-locale \ - glibc-devel libbz2-devel pkg-config libzmq-devel zlib-devel gdb vim valgrind \ - bzip2 gzip blosc-devel libzstd-devel openssl-devel environment-modules - -# Workaround so pgi can find g77 -WORKDIR /usr/bin -RUN ln -s gfortran g77 - -# Install PGI compiler -WORKDIR /tmp/pgi-install -RUN curl -L https://data.kitware.com/api/v1/file/5defe4f7af2e2eed35763524/download | tar -xz -RUN export \ - PGI_SILENT=true \ - PGI_ACCEPT_EULA=accept \ - PGI_INSTALL_DIR=/opt/pgi \ - PGI_INSTALL_NVIDIA=false \ - PGI_INSTALL_JAVA=false \ - PGI_INSTALL_MPI=false \ - PGI_MPI_GPU_SUPPORT=false \ - && ./install - -RUN echo 'export MODULEPATH=/opt/pgi/modulefiles:${MODULEPATH}' > /etc/profile.d/pgi-modules.sh \ - && echo 'setenv MODULEPATH /opt/pgi/modulefiles:${MODULEPATH}' > /etc/profile.d/pgi-modules.csh - -# Install the most recent CMake nightly binary -WORKDIR /opt/cmake -RUN curl -L https://cmake.org/files/dev/$(curl https://cmake.org/files/dev/ | sed -n '/Linux-x86_64.tar.gz/s/.*>\(cmake[^<]*\)<.*/\1/p' | sort | tail -1) | tar --strip-components=1 -xzv -ENV PATH=/opt/cmake/bin:${PATH} - -# Install ZFP -WORKDIR /opt/zfp -RUN curl -L https://github.com/LLNL/zfp/releases/download/0.5.5/zfp-0.5.5.tar.gz | tar -xvz && \ - mkdir build && \ - cd build && \ - source /etc/profile && \ - module load pgi && \ - export CC=pgcc CXX=pgc++ FC=pgffort && \ - /opt/cmake/bin/cmake \ - -DBUILD_SHARED_LIBS=ON \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_INSTALL_PREFIX=/opt/zfp/0.5.5 \ - ../zfp-0.5.5 && \ - make -j$(grep -c '^processor' /proc/cpuinfo) install && \ - cd .. && \ - rm -rf zfp-0.5.5 build -ENV PATH=/opt/zfp/0.5.5/bin:${PATH} \ - LD_LIBRARY_PATH=/opt/zfp/0.5.5/lib64:${LD_LIBRARY_PATH} \ - CMAKE_PREFIX_PATH=/opt/zfp/0.5.5:${CMAKE_PREFIX_PATH} - -# Install SZ -WORKDIR /opt/sz -RUN curl -L https://github.com/disheng222/SZ/archive/v2.1.8.0.tar.gz | tar -xvz && \ - mkdir build && \ - cd build && \ - source /etc/profile && \ - module load pgi && \ - export CC=pgcc CXX=pgc++ FC=pgffort && \ - /opt/cmake/bin/cmake \ - -DBUILD_SHARED_LIBS=ON \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_INSTALL_PREFIX=/opt/sz/2.1.8.0 \ - ../SZ-2.1.8.0 && \ - make -j$(grep -c '^processor' /proc/cpuinfo) install && \ - cd .. && \ - rm -rf SZ-2.1.8.0 build -ENV PATH=/opt/sz/2.1.8.0/bin:${PATH} \ - LD_LIBRARY_PATH=/opt/sz/2.1.8.0/lib64:${LD_LIBRARY_PATH} \ - CMAKE_PREFIX_PATH=/opt/sz/2.1.8.0:${CMAKE_PREFIX_PATH} - -# Misc cleanup of unneeded files -RUN rm -rf /tmp/* && \ - zypper clean diff --git a/scripts/ci/images/suse-pgi-openmpi/Dockerfile b/scripts/ci/images/suse-pgi-openmpi/Dockerfile deleted file mode 100644 index 073f59be25..0000000000 --- a/scripts/ci/images/suse-pgi-openmpi/Dockerfile +++ /dev/null @@ -1,53 +0,0 @@ -FROM ornladios/adios2:ci-suse-pgi-base - -# Install OpenMPI -WORKDIR /opt/openmpi -RUN curl -L https://download.open-mpi.org/release/open-mpi/v4.0/openmpi-4.0.2.tar.bz2 | \ - tar -xvj && \ - cd openmpi-4.0.2 && \ - source /etc/profile && \ - module load pgi && \ - export CC=pgcc CXX=pgc++ FC=pgfortran CPP=/usr/bin/cpp && \ - ./configure \ - --prefix=/opt/openmpi/4.0.2 \ - --enable-mpi-cxx \ - --enable-shared \ - --disable-static && \ - make -j$(grep -c '^processor' /proc/cpuinfo) all && \ - make install && \ - cd .. && \ - rm -rf openmpi-4.0.2 -ENV PATH=/opt/openmpi/4.0.2/bin:${PATH} \ - LD_LIBRARY_PATH=/opt/openmpi/4.0.2/lib:${LD_LIBRARY_PATH} \ - CMAKE_PREFIX_PATH=/opt/openmpi/4.0.2:${CMAKE_PREFIX_PATH} - -# Install HDF5 1.10.4 (the current 1.10.5 has a parallel close bug affecting -# the tests -WORKDIR /opt/hdf5 -RUN curl -L https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.10/hdf5-1.10.4/src/hdf5-1.10.4.tar.bz2 | \ - tar -xvj && \ - mkdir build && \ - cd build && \ - source /etc/profile && \ - module load pgi openmpi && \ - export CC=pgcc CXX=pgc++ FC=pgfortran && \ - /opt/cmake/bin/cmake \ - -DCMAKE_INSTALL_PREFIX=/opt/hdf5/1.10.4 \ - -DBUILD_SHARED_LIBS=ON \ - -DCMAKE_BUILD_TYPE=Release \ - -DHDF5_ENABLE_PARALLEL=ON \ - -DHDF5_BUILD_CPP_LIB=OFF\ - -DHDF5_BUILD_EXAMPLES=OFF \ - -DBUILD_TESTING=OFF \ - -DHDF5_BUILD_TOOLS=OFF \ - ../hdf5-1.10.4 && \ - make -j$(grep -c '^processor' /proc/cpuinfo) install && \ - cd .. && \ - rm -rf hdf5-1.10.4 build -ENV PATH=/opt/hdf5/1.10.4/bin:${PATH} \ - LD_LIBRARY_PATH=/opt/hdf5/1.10.4/lib:${LD_LIBRARY_PATH} \ - CMAKE_PREFIX_PATH=/opt/hdf5/1.10.4:${CMAKE_PREFIX_PATH} - -# Misc cleanup of unneeded files -RUN rm -rf /tmp/* && \ - zypper clean diff --git a/scripts/ci/scripts/github-list-prs.py b/scripts/ci/scripts/github-list-prs.py new file mode 100755 index 0000000000..4be8954f3e --- /dev/null +++ b/scripts/ci/scripts/github-list-prs.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 + +import sys +import argparse +import urllib.request +import json + +parser = argparse.ArgumentParser(description="List open PRs") +parser.add_argument("-t", "--token", help="GitHub access token") +group = parser.add_mutually_exclusive_group() +group.add_argument("-o", "--open", help="List open PRs", + action="store_true", default=False) +group.add_argument("-c", "--closed", help="List closed PRs", + action="store_true", default=False) +group.add_argument("-a", "--all", help="List all PRs", + action="store_true", default=False) +parser.add_argument("repo", help="GitHub repo (org/repo or user/repo)") +args = parser.parse_args() + +if args.open: + state = "open" +elif args.closed: + state = "closed" +elif args.all: + state = "all" +else: + state = "open" + +try: + request = urllib.request.Request( + 'https://api.github.com/repos/%s/pulls?state=%s' % (args.repo, state)) + if args.token: + request.add_header('Authorization', 'token %s' % args.token) + response = urllib.request.urlopen(request) +except OSError: + sys.exit(1) + +prs = json.loads(response.read()) + +for pr in prs: + print("%d %s" % (pr['number'], pr['head']['ref'])) diff --git a/scripts/ci/scripts/github-prs-to-gitlab.sh b/scripts/ci/scripts/github-prs-to-gitlab.sh new file mode 100755 index 0000000000..fcabbfde86 --- /dev/null +++ b/scripts/ci/scripts/github-prs-to-gitlab.sh @@ -0,0 +1,153 @@ +#!/usr/bin/env bash + +set -e + +CLEANUP_SSH_AGENT=0 +CLEANUP_WORKDIR=0 + +function cleanup() +{ + echo "Cleanup:" + if [ ${CLEANUP_WORKDIR} -eq 1 ] + then + echo " Removing ${TMP_WORKDIR}" + rm -rf "${TMP_WORKDIR}" + fi + if [ ${CLEANUP_SSH_AGENT} -eq 1 ] + then + echo " Shutting down ssh-agent(${SSH_AGENT_PID})" + eval $(ssh-agent -k) + fi +} + + +if [ $# -ne 3 ] +then + echo "Usage: $0 " + exit 1 +fi + +GITHUB_REPO=$1 +GITLAB_HOST=$2 +GITLAB_REPO=$3 +GITLAB_URL="git@${GITLAB_HOST}:${GITLAB_REPO}.git" +BASEDIR="$(cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +if [ -z "${GITLAB_SSH_KEY_BASE64}" ] +then + echo "Error: GITLAB_SSH_KEY_BASE64 is empty" + exit 1 +fi + +trap cleanup EXIT + +# Start the ssh agent +eval $(ssh-agent -s) +CLEANUP_SSH_AGENT=1 + +# Add the necessary key +echo "${GITLAB_SSH_KEY_BASE64}" | base64 -d | tr -d '\r' | ssh-add - + +git --version + +export GIT_SSH_COMMAND="ssh -F /dev/null -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" + +# Setup the local repo with two remotes +TMP_WORKDIR=$(mktemp -d) +CLEANUP_WORKDIR=1 +echo "Initializing local repo in ${TMP_WORKDIR}" +cd ${TMP_WORKDIR} +git init +git remote add github https://github.com/${GITHUB_REPO}.git +git remote add gitlab ${GITLAB_URL} + +# Retrieve open PRs +OPEN_PR_BRANCHES="$(python3 ${BASEDIR}/github-list-prs.py -o ${GITHUB_REPO} | awk '{print "pr"$1"_"$2}' | sort -r)" +echo "Open PRs:" +for PR in ${OPEN_PR_BRANCHES} +do + echo " ${PR}" +done +echo + +# Retrieve sync'd PRs +SYNCD_PR_BRANCHES="$(echo $(git ls-remote gitlab github/pr* | awk '{print $2}' | sed 's|^refs/heads/github/||' | sort -r))" +echo "Syncd PRs:" +for PR in ${SYNCD_PR_BRANCHES} +do + echo " ${PR}" +done +echo + +# Determine any closed PRs that are currently sync'd +SYNCD_CLOSED_PR_BRANCHES="" +if [ -n "${SYNCD_PR_BRANCHES}" ] +then + for SPR in ${SYNCD_PR_BRANCHES} + do + if [ -n "${OPEN_PR_BRANCHES}" ] + then + IS_OPEN=0 + for OPR in ${OPEN_PR_BRANCHES} + do + if [ "${SPR}" = "${OPR}" ] + then + IS_OPEN=1 + break + fi + done + if [ ${IS_OPEN} -eq 0 ] + then + SYNCD_CLOSED_PR_BRANCHES="${SYNCD_CLOSED_PR_BRANCHES} ${SPR}" + fi + fi + done +fi +echo "Syncd Closed PRs:" +for PR in ${SYNCD_CLOSED_PR_BRANCHES} +do + echo " ${PR}" +done +echo + +# Delete any sync'd PRs +if [ -n "${SYNCD_CLOSED_PR_BRANCHES}" ] +then + CLOSED_REFSPECS="" + for PR in ${SYNCD_CLOSED_PR_BRANCHES} + do + echo "Adding respec for closed ${PR}" + CLOSED_REFSPECS="${CLOSED_REFSPECS} :github/${PR}" + done +fi + +# Sync open PRs to OLCF +if [ -n "${OPEN_PR_BRANCHES}" ] +then + FETCH_REFSPECS="" + OPEN_REFSPECS="" + for PR in ${OPEN_PR_BRANCHES} + do + PR_NUM=$(expr "${PR}" : 'pr\([0-9]\+\)') + FETCH_REFSPECS="${FETCH_REFSPECS} +refs/pull/${PR_NUM}/head:refs/remotes/github/${PR}" + OPEN_REFSPECS="${OPEN_REFSPECS} github/${PR}:github/${PR}" + done + + echo "Fetching GitHub refs for open PRs" + git fetch -q github ${FETCH_REFSPECS} + echo + + echo "Building local branches for open PRs" + for PR in ${OPEN_PR_BRANCHES} + do + git branch -q github/${PR} github/${PR} + done + echo +fi + +if [ -n "${CLOSED_REFSPECS}" -o -n "${OPEN_REFSPECS}" ] +then + echo "Syncing PRs to GitLab" + git push --porcelain -f gitlab ${CLOSED_REFSPECS} ${OPEN_REFSPECS} + echo +fi diff --git a/scripts/ci/scripts/run-clang-format.sh b/scripts/ci/scripts/run-clang-format.sh new file mode 100755 index 0000000000..f011792ae1 --- /dev/null +++ b/scripts/ci/scripts/run-clang-format.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +echo "---------- Begin ENV ----------" +env | sort +echo "---------- End ENV ----------" + +# If a source dir is given, then change to it. Otherwise assume we're already +# in it +if [ -n "${SOURCE_DIR}" ] +then + cd ${SOURCE_DIR} +fi + +# Check C and C++ code with clang-format +find source testing examples bindings -iname '*.h' -o -iname '*.c' -o -iname '*.cpp' -o -iname '*.tcc' | xargs clang-format -i +DIFF="$(git diff)" +if [ -n "${DIFF}" ] +then + echo "clang-format:" + echo " Code format checks failed." + echo " Please run clang-format v7.1.0 on your changes before committing." + echo " The following changes are suggested:" + echo "${DIFF}" + exit 1 +fi + +exit 0 diff --git a/scripts/ci/scripts/run-flake8.sh b/scripts/ci/scripts/run-flake8.sh new file mode 100755 index 0000000000..1c5a1ad354 --- /dev/null +++ b/scripts/ci/scripts/run-flake8.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +echo "---------- Begin ENV ----------" +env | sort +echo "---------- End ENV ----------" + +# If a source dir is given, then change to it. Otherwise assume we're already +# in it +if [ -n "${SOURCE_DIR}" ] +then + cd ${SOURCE_DIR} +fi + +exec flake8 --config=flake8.cfg . diff --git a/scripts/conda/adios2/superbuild/CMakeLists.txt b/scripts/conda/adios2/superbuild/CMakeLists.txt index 555268604f..75e2ad9840 100644 --- a/scripts/conda/adios2/superbuild/CMakeLists.txt +++ b/scripts/conda/adios2/superbuild/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.6) +cmake_minimum_required(VERSION 3.12) project(ADIOS2_SUPERBUILD C CXX) include(ExternalProject) diff --git a/scripts/dashboard/common.cmake b/scripts/dashboard/common.cmake index ffb839c72d..e4284afa81 100644 --- a/scripts/dashboard/common.cmake +++ b/scripts/dashboard/common.cmake @@ -65,7 +65,7 @@ # set(ENV{FC} /path/to/fc) # Fortran compiler (optional) # set(ENV{LD_LIBRARY_PATH} /path/to/vendor/lib) # (if necessary) -cmake_minimum_required(VERSION 2.8.2 FATAL_ERROR) +cmake_minimum_required(VERSION 2.8.12 FATAL_ERROR) if(NOT DEFINED dashboard_full) set(dashboard_full TRUE) @@ -490,7 +490,10 @@ if(dashboard_do_memcheck) dashboard_hook_memcheck() endif() message("Calling ctest_memcheck") - ctest_memcheck() + ctest_memcheck(RETURN_VALUE MEMCHECK_RETURN DEFECT_COUNT MEMCHECK_DEFECTS) + if(NOT (MEMCHECK_DEFECTS EQUAL 0)) + message(SEND_ERROR "ctest memcheck defects found: ${MEMCHECK_DEFECTS}") + endif() endif() if(dashboard_do_submit) ctest_submit(PARTS MemCheck) diff --git a/scripts/travis/run-format.sh b/scripts/travis/run-format.sh index 8395840ae9..619664e482 100755 --- a/scripts/travis/run-format.sh +++ b/scripts/travis/run-format.sh @@ -20,7 +20,7 @@ git config --global clangformat.binary ${HOME}/clang+llvm-7.0.1-x86_64-linux-gnu popd # Install flake8 -pip install --user flake8 +pip3 install --user flake8 cd ${SOURCE_DIR} diff --git a/source/adios2/CMakeLists.txt b/source/adios2/CMakeLists.txt index e39556d70d..2b3a844474 100644 --- a/source/adios2/CMakeLists.txt +++ b/source/adios2/CMakeLists.txt @@ -29,6 +29,7 @@ add_library(adios2_core core/Variable.cpp core/Variable.tcc core/VariableBase.cpp core/VariableCompound.cpp core/VariableCompound.tcc + core/Group.cpp core/Group.tcc #operator callback operator/callback/Signature1.cpp @@ -175,6 +176,7 @@ endif() if(ADIOS2_HAVE_DataMan) target_sources(adios2_core PRIVATE + engine/dataman/DataManMonitor.cpp engine/dataman/DataManReader.cpp engine/dataman/DataManReader.tcc engine/dataman/DataManWriter.cpp diff --git a/source/adios2/common/ADIOSMacros.h b/source/adios2/common/ADIOSMacros.h index 340fe03c45..f3545c05d5 100644 --- a/source/adios2/common/ADIOSMacros.h +++ b/source/adios2/common/ADIOSMacros.h @@ -169,21 +169,6 @@ MACRO(std::complex) \ MACRO(std::complex) -#define ADIOS2_FOREACH_NUMERIC_ATTRIBUTE_TYPE_1ARG(MACRO) \ - MACRO(short) \ - MACRO(unsigned short) \ - MACRO(int) \ - MACRO(unsigned int) \ - MACRO(long int) \ - MACRO(long long int) \ - MACRO(unsigned long int) \ - MACRO(unsigned long long int) \ - MACRO(float) \ - MACRO(double) \ - MACRO(long double) \ - MACRO(std::complex) \ - MACRO(std::complex) - /** 
  The ADIOS2_FOREACH_STDTYPE_2ARGS macro assumes the given argument is a macro
@@ -242,14 +227,6 @@
 #define ADIOS2_FOREACH_STDTYPE_2ARGS(MACRO)                                    \
     ADIOS2_FOREACH_ATTRIBUTE_STDTYPE_2ARGS(MACRO)
 
-#define ADIOS2_FOREACH_COMPLEX_TYPE_2ARGS(MACRO)                               \
-    MACRO(std::complex, CFloat)                                         \
-    MACRO(std::complex, CDouble)
-
-#define ADIOS2_FOREACH_LAUNCH_MODE(MACRO)                                      \
-    MACRO(Sync)                                                                \
-    MACRO(Deferred)
-
 #define ADIOS2_CLASS_iterator                                                  \
     class iterator                                                             \
     {                                                                          \
diff --git a/source/adios2/common/ADIOSTypes.cpp b/source/adios2/common/ADIOSTypes.cpp
index 4b21a8dc1d..7fec940306 100644
--- a/source/adios2/common/ADIOSTypes.cpp
+++ b/source/adios2/common/ADIOSTypes.cpp
@@ -180,4 +180,45 @@ std::string ToString(SelectionType value)
     }
 }
 
+std::string ToString(DataType type)
+{
+    // Keep in sync with helper::GetDataTypeFromString
+    switch (type)
+    {
+    case DataType::None:
+        break;
+    case DataType::Int8:
+        return "int8_t";
+    case DataType::Int16:
+        return "int16_t";
+    case DataType::Int32:
+        return "int32_t";
+    case DataType::Int64:
+        return "int64_t";
+    case DataType::UInt8:
+        return "uint8_t";
+    case DataType::UInt16:
+        return "uint16_t";
+    case DataType::UInt32:
+        return "uint32_t";
+    case DataType::UInt64:
+        return "uint64_t";
+    case DataType::Float:
+        return "float";
+    case DataType::Double:
+        return "double";
+    case DataType::LongDouble:
+        return "long double";
+    case DataType::FloatComplex:
+        return "float complex";
+    case DataType::DoubleComplex:
+        return "double complex";
+    case DataType::String:
+        return "string";
+    case DataType::Compound:
+        return "compound";
+    }
+    return std::string();
+}
+
 } // end namespace adios2
diff --git a/source/adios2/common/ADIOSTypes.h b/source/adios2/common/ADIOSTypes.h
index bf3b23131d..a36b923e7a 100644
--- a/source/adios2/common/ADIOSTypes.h
+++ b/source/adios2/common/ADIOSTypes.h
@@ -119,6 +119,27 @@ enum class SelectionType
     Auto         ///< Let the engine decide what to return
 };
 
+// Data types.
+enum class DataType
+{
+    None,
+    Int8,
+    Int16,
+    Int32,
+    Int64,
+    UInt8,
+    UInt16,
+    UInt32,
+    UInt64,
+    Float,
+    Double,
+    LongDouble,
+    FloatComplex,
+    DoubleComplex,
+    String,
+    Compound
+};
+
 // Types
 using std::size_t;
 
@@ -215,6 +236,7 @@ std::string ToString(StepMode value);
 std::string ToString(StepStatus value);
 std::string ToString(TimeUnit value);
 std::string ToString(SelectionType value);
+std::string ToString(DataType type);
 
 /**
  * os << [adios2_type] enables output of adios2 enums/classes directly
diff --git a/source/adios2/core/ADIOS.cpp b/source/adios2/core/ADIOS.cpp
index 5d773feabd..f0341919bb 100644
--- a/source/adios2/core/ADIOS.cpp
+++ b/source/adios2/core/ADIOS.cpp
@@ -123,7 +123,9 @@ IO &ADIOS::DeclareIO(const std::string name)
         }
     }
 
-    auto ioPair = m_IOs.emplace(name, IO(*this, name, false, m_HostLanguage));
+    auto ioPair = m_IOs.emplace(
+        std::piecewise_construct, std::forward_as_tuple(name),
+        std::forward_as_tuple(*this, name, false, m_HostLanguage));
     IO &io = ioPair.first->second;
     io.SetDeclared();
     return io;
diff --git a/source/adios2/core/Attribute.cpp b/source/adios2/core/Attribute.cpp
index 7ca0d400ba..5a087435ce 100644
--- a/source/adios2/core/Attribute.cpp
+++ b/source/adios2/core/Attribute.cpp
@@ -12,7 +12,7 @@
 #include "Attribute.tcc"
 
 #include "adios2/common/ADIOSMacros.h"
-#include "adios2/helper/adiosFunctions.h" //GetType
+#include "adios2/helper/adiosFunctions.h" //GetDataType
 
 #include 
 
@@ -33,7 +33,6 @@ template <>
 struct RequiresZeroPadding : std::true_type
 {
 };
-
 }
 
 #define declare_type(T)                                                        \
@@ -50,7 +49,7 @@ struct RequiresZeroPadding : std::true_type
     template <>                                                                \
     Attribute::Attribute(const std::string &name, const T *array,           \
                             const size_t elements)                             \
-    : AttributeBase(name, helper::GetType(), elements)                      \
+    : AttributeBase(name, helper::GetDataType(), elements)                  \
     {                                                                          \
         if (RequiresZeroPadding::value)                                     \
             std::memset(&m_DataSingleValue, 0, sizeof(m_DataSingleValue));     \
@@ -59,17 +58,11 @@ struct RequiresZeroPadding : std::true_type
                                                                                \
     template <>                                                                \
     Attribute::Attribute(const std::string &name, const T &value)           \
-    : AttributeBase(name, helper::GetType())                                \
+    : AttributeBase(name, helper::GetDataType())                            \
     {                                                                          \
         if (RequiresZeroPadding::value)                                     \
             std::memset(&m_DataSingleValue, 0, sizeof(m_DataSingleValue));     \
         m_DataSingleValue = value;                                             \
-    }                                                                          \
-                                                                               \
-    template <>                                                                \
-    Params Attribute::GetInfo() const noexcept                              \
-    {                                                                          \
-        return DoGetInfo();                                                    \
     }
 
 ADIOS2_FOREACH_ATTRIBUTE_STDTYPE_1ARG(declare_type)
diff --git a/source/adios2/core/Attribute.h b/source/adios2/core/Attribute.h
index 178ae60b50..83fb5fcf1b 100644
--- a/source/adios2/core/Attribute.h
+++ b/source/adios2/core/Attribute.h
@@ -51,10 +51,8 @@ class Attribute : public AttributeBase
 
     ~Attribute() = default;
 
-    Params GetInfo() const noexcept;
-
 private:
-    Params DoGetInfo() const noexcept;
+    std::string DoGetInfoValue() const noexcept override;
 };
 
 } // end namespace core
diff --git a/source/adios2/core/Attribute.tcc b/source/adios2/core/Attribute.tcc
index a830b6b4a8..a047f64f96 100644
--- a/source/adios2/core/Attribute.tcc
+++ b/source/adios2/core/Attribute.tcc
@@ -13,7 +13,7 @@
 
 #include "Attribute.h"
 
-#include "adios2/helper/adiosFunctions.h"
+#include "adios2/helper/adiosType.h"
 
 namespace adios2
 {
@@ -21,21 +21,18 @@ namespace core
 {
 
 template 
-Params Attribute::DoGetInfo() const noexcept
+std::string Attribute::DoGetInfoValue() const noexcept
 {
-    Params info;
-    info["Type"] = m_Type;
-    info["Elements"] = std::to_string(m_Elements);
-
+    std::string value;
     if (m_IsSingleValue)
     {
-        info["Value"] = helper::ValueToString(m_DataSingleValue);
+        value = helper::ValueToString(m_DataSingleValue);
     }
     else
     {
-        info["Value"] = "{ " + helper::VectorToCSV(m_DataArray) + " }";
+        value = "{ " + helper::VectorToCSV(m_DataArray) + " }";
     }
-    return info;
+    return value;
 }
 
 } // end namespace core
diff --git a/source/adios2/core/AttributeBase.cpp b/source/adios2/core/AttributeBase.cpp
index c13b8f11fa..f77ccdb4c5 100644
--- a/source/adios2/core/AttributeBase.cpp
+++ b/source/adios2/core/AttributeBase.cpp
@@ -15,16 +15,25 @@ namespace adios2
 namespace core
 {
 
-AttributeBase::AttributeBase(const std::string &name, const std::string type)
+AttributeBase::AttributeBase(const std::string &name, const DataType type)
 : m_Name(name), m_Type(type), m_Elements(1), m_IsSingleValue(true)
 {
 }
 
-AttributeBase::AttributeBase(const std::string &name, const std::string type,
+AttributeBase::AttributeBase(const std::string &name, const DataType type,
                              const size_t elements)
 : m_Name(name), m_Type(type), m_Elements(elements), m_IsSingleValue(false)
 {
 }
 
+Params AttributeBase::GetInfo() const noexcept
+{
+    Params info;
+    info["Type"] = ToString(m_Type);
+    info["Elements"] = std::to_string(m_Elements);
+    info["Value"] = this->DoGetInfoValue();
+    return info;
+}
+
 } // end namespace core
 } // end namespace adios2
diff --git a/source/adios2/core/AttributeBase.h b/source/adios2/core/AttributeBase.h
index 9b85d0001d..94d2916f5c 100644
--- a/source/adios2/core/AttributeBase.h
+++ b/source/adios2/core/AttributeBase.h
@@ -28,7 +28,7 @@ class AttributeBase
 
 public:
     const std::string m_Name;
-    const std::string m_Type;
+    const DataType m_Type;
     const size_t m_Elements;
     const bool m_IsSingleValue;
 
@@ -37,7 +37,7 @@ class AttributeBase
      * @param name
      * @param type
      */
-    AttributeBase(const std::string &name, const std::string type);
+    AttributeBase(const std::string &name, const DataType type);
 
     /**
      * Array constructor used by Attribute derived class
@@ -45,10 +45,15 @@ class AttributeBase
      * @param type
      * @param elements
      */
-    AttributeBase(const std::string &name, const std::string type,
+    AttributeBase(const std::string &name, const DataType type,
                   const size_t elements);
 
     virtual ~AttributeBase() = default;
+
+    Params GetInfo() const noexcept;
+
+private:
+    virtual std::string DoGetInfoValue() const noexcept = 0;
 };
 
 } // end namespace core
diff --git a/source/adios2/core/Engine.cpp b/source/adios2/core/Engine.cpp
index fe7c94fe89..061555e5d4 100644
--- a/source/adios2/core/Engine.cpp
+++ b/source/adios2/core/Engine.cpp
@@ -89,6 +89,12 @@ void Engine::LockReaderSelections() noexcept
     m_ReaderSelectionsLocked = true;
 }
 
+size_t Engine::DebugGetDataBufferSize() const
+{
+    ThrowUp("DebugGetDataBufferSize");
+    return 0;
+}
+
 // PROTECTED
 void Engine::Init() {}
 void Engine::InitParameters() {}
@@ -153,6 +159,12 @@ ADIOS2_FOREACH_STDTYPE_1ARG(declare_type)
     {                                                                          \
         ThrowUp("DoBlocksInfo");                                               \
         return std::vector::Info>();                      \
+    }                                                                          \
+    std::vector Engine::DoGetAbsoluteSteps(                            \
+        const Variable &variable) const                                     \
+    {                                                                          \
+        ThrowUp("DoGetAbsoluteSteps");                                         \
+        return std::vector();                                          \
     }
 
 ADIOS2_FOREACH_STDTYPE_1ARG(declare_type)
@@ -171,7 +183,7 @@ ADIOS2_FOREACH_PRIMITVE_STDTYPE_2ARGS(declare_type)
 
 size_t Engine::DoSteps() const
 {
-    ThrowUp("DoPut");
+    ThrowUp("DoSteps");
     return MaxSizeT;
 }
 
@@ -227,7 +239,9 @@ void Engine::CheckOpenModes(const std::set &modes,
     Engine::AllRelativeStepsBlocksInfo(const Variable &) const;             \
                                                                                \
     template std::vector::Info> Engine::BlocksInfo(       \
-        const Variable &, const size_t) const;
+        const Variable &, const size_t) const;                              \
+    template std::vector Engine::GetAbsoluteSteps(const Variable &) \
+        const;
 
 ADIOS2_FOREACH_STDTYPE_1ARG(declare_template_instantiation)
 #undef declare_template_instantiation
diff --git a/source/adios2/core/Engine.h b/source/adios2/core/Engine.h
index eff087fe1a..fed0c231f5 100644
--- a/source/adios2/core/Engine.h
+++ b/source/adios2/core/Engine.h
@@ -413,6 +413,14 @@ class Engine
     std::vector::Info>
     BlocksInfo(const Variable &variable, const size_t step) const;
 
+    /**
+     * Get the absolute steps of a variable in a file. This is for
+     * information purposes only, because absolute steps cannot be used
+     * in any ADIOS2 calls.
+     */
+    template 
+    std::vector GetAbsoluteSteps(const Variable &variable) const;
+
     template 
     T *BufferData(const size_t payloadOffset,
                   const size_t bufferID = 0) noexcept;
@@ -436,6 +444,9 @@ class Engine
      */
     void LockReaderSelections() noexcept;
 
+    /* for adios2 internal testing */
+    virtual size_t DebugGetDataBufferSize() const;
+
 protected:
     /** from ADIOS class passed to Engine created with Open
      *  if no communicator is passed */
@@ -506,7 +517,9 @@ class Engine
     DoAllRelativeStepsBlocksInfo(const Variable &variable) const;           \
                                                                                \
     virtual std::vector::Info> DoBlocksInfo(              \
-        const Variable &variable, const size_t step) const;
+        const Variable &variable, const size_t step) const;                 \
+    virtual std::vector DoGetAbsoluteSteps(                            \
+        const Variable &variable) const;
 
     ADIOS2_FOREACH_STDTYPE_1ARG(declare_type)
 #undef declare_type
@@ -592,7 +605,10 @@ class Engine
     Engine::AllRelativeStepsBlocksInfo(const Variable &) const;             \
                                                                                \
     extern template std::vector::Info>                    \
-    Engine::BlocksInfo(const Variable &, const size_t) const;
+    Engine::BlocksInfo(const Variable &, const size_t) const;               \
+                                                                               \
+    extern template std::vector Engine::GetAbsoluteSteps(              \
+        const Variable &) const;
 
 ADIOS2_FOREACH_STDTYPE_1ARG(declare_template_instantiation)
 #undef declare_template_instantiation
diff --git a/source/adios2/core/Engine.tcc b/source/adios2/core/Engine.tcc
index 9ec69adbc6..f2540258ca 100644
--- a/source/adios2/core/Engine.tcc
+++ b/source/adios2/core/Engine.tcc
@@ -192,6 +192,19 @@ Engine::BlocksInfo(const Variable &variable, const size_t step) const
     return DoBlocksInfo(variable, step);
 }
 
+template 
+std::vector Engine::GetAbsoluteSteps(const Variable &variable) const
+{
+    const auto &m = variable.m_AvailableStepBlockIndexOffsets;
+    std::vector keys;
+    keys.reserve(m.size());
+    for (auto it = m.cbegin(); it != m.cend(); ++it)
+    {
+        keys.push_back(it->first - 1);
+    }
+    return keys;
+}
+
 #define declare_type(T, L)                                                     \
     template <>                                                                \
     T *Engine::BufferData(const size_t payloadPosition,                        \
diff --git a/source/adios2/core/Group.cpp b/source/adios2/core/Group.cpp
new file mode 100644
index 0000000000..216377b955
--- /dev/null
+++ b/source/adios2/core/Group.cpp
@@ -0,0 +1,223 @@
+/*
+ * Distributed under the OSI-approved Apache License, Version 2.0.  See
+ * accompanying file Copyright.txt for details.
+ *
+ * Group.cpp :
+ *
+ *  Created on: August 25, 2020
+ *      Author: Dmitry Ganyushin ganyushindi@ornl.gov
+ */
+#include "Group.h"
+#include "Group.tcc"
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "adios2/common/ADIOSMacros.h"
+#include "adios2/core/IO.h"
+namespace adios2
+{
+namespace core
+{
+std::vector split(const std::string &s, char delimiter)
+{
+    std::vector tokens;
+    std::string token;
+    std::istringstream tokenStream(s);
+    while (std::getline(tokenStream, token, delimiter))
+    {
+        tokens.push_back(token);
+    }
+    return tokens;
+}
+void Group::setPath(std::string path) { currentPath = path; }
+void Group::setDelimiter(char delimiter) { groupDelimiter = delimiter; }
+
+Group::Group(std::string path, char delimiter, IO &io)
+: currentPath(path), groupDelimiter(delimiter), m_IO(io)
+{
+    if (mapPtr == nullptr)
+    {
+        mapPtr = std::shared_ptr(new TreeMap());
+    }
+}
+// copy constructor
+Group::Group(const Group &G)
+: currentPath(G.currentPath), groupDelimiter(G.groupDelimiter), m_IO(G.m_IO)
+{
+    mapPtr = G.mapPtr;
+}
+Group *Group::InquireGroup(std::string groupName)
+{
+    Group *g_out = new Group(currentPath + groupDelimiter + groupName,
+                             this->groupDelimiter, this->m_IO);
+    g_out->mapPtr = this->mapPtr;
+    return g_out;
+}
+void Group::PrintTree()
+{
+    for (auto k : mapPtr->treeMap)
+    {
+        std::cout << k.first << "=>";
+        for (auto v : k.second)
+            std::cout << v << " ";
+        std::cout << std::endl;
+    }
+}
+
+void Group::BuildTree()
+{
+    const core::VarMap &variables = m_IO.GetVariables();
+    for (const auto &variablePair : variables)
+    {
+        std::vector tokens =
+            split(variablePair.first, groupDelimiter);
+
+        if (tokens.size() == 0)
+        {
+            // record = "group". Handled by default case
+        }
+        else if (tokens.size() == 1)
+        {
+            // case record = "/group1" or "group/"
+        }
+        if (tokens.size() > 1)
+        {
+            std::string key = tokens[0];
+            for (int level = 1; level < tokens.size(); level++)
+            {
+                std::string value = tokens[level];
+                // get previous vector
+                std::set val = mapPtr->treeMap[key];
+                // modify it
+                val.insert(value);
+                mapPtr->treeMap[key] = val;
+                key += groupDelimiter + tokens[level];
+            }
+        }
+    }
+    const core::AttrMap &attributes = m_IO.GetAttributes();
+    for (const auto &attributePair : attributes)
+    {
+        std::vector tokens =
+            split(attributePair.first, groupDelimiter);
+
+        if (tokens.size() == 0)
+        {
+            // record = "group". Handled by default case
+        }
+        else if (tokens.size() == 1)
+        {
+            // case record = "/group1" or "group/"
+        }
+        if (tokens.size() > 1)
+        {
+            std::string key = tokens[0];
+            for (int level = 1; level < tokens.size(); level++)
+            {
+                std::string value = tokens[level];
+                // get previous vector
+                std::set val = mapPtr->treeMap[key];
+                // modify it
+                val.insert(value);
+                mapPtr->treeMap[key] = val;
+                key += groupDelimiter + tokens[level];
+            }
+        }
+    }
+}
+std::vector Group::AvailableVariables()
+{
+    // look into map
+    std::set val = mapPtr->treeMap[currentPath];
+    // TODODG check that currentPath exists
+    std::vector available_variables;
+    for (auto v : val)
+    {
+        if (mapPtr->treeMap.find(currentPath + groupDelimiter + v) ==
+            mapPtr->treeMap.end())
+        {
+            const core::VarMap &variables = m_IO.GetVariables();
+
+            if (variables.find(currentPath + groupDelimiter + v) !=
+                variables.end())
+            {
+                available_variables.push_back(v);
+            }
+        }
+    }
+
+    return available_variables;
+}
+
+std::vector Group::AvailableAttributes()
+{
+    // look into map
+    std::set val = mapPtr->treeMap[currentPath];
+    // TODODG check that currentPath exists
+    std::vector available_attributes;
+    for (auto v : val)
+    {
+        if (mapPtr->treeMap.find(currentPath + groupDelimiter + v) ==
+            mapPtr->treeMap.end())
+        {
+            const core::AttrMap &attributes = m_IO.GetAttributes();
+            if (attributes.find(currentPath + groupDelimiter + v) !=
+                attributes.end())
+            {
+                available_attributes.push_back(v);
+            }
+        }
+    }
+
+    return available_attributes;
+}
+
+std::vector Group::AvailableGroups()
+{
+
+    std::vector available_groups;
+    std::set val = mapPtr->treeMap[currentPath];
+
+    for (auto v : val)
+    {
+        if (mapPtr->treeMap.find(currentPath + groupDelimiter + v) !=
+            mapPtr->treeMap.end())
+            available_groups.push_back(v);
+    }
+
+    return available_groups;
+}
+
+std::map> &Group::getTreeMap()
+{
+    std::map> &tree = mapPtr->treeMap;
+    return tree;
+}
+
+std::string Group::InquirePath() { return currentPath; }
+Group::~Group() = default;
+DataType Group::InquireVariableType(const std::string &name) const noexcept
+{
+
+    return m_IO.InquireVariableType(currentPath + groupDelimiter + name);
+}
+
+DataType Group::InquireAttributeType(const std::string &name,
+                                     const std::string &variableName,
+                                     const std::string separator) const noexcept
+{
+    return m_IO.InquireAttributeType(name, variableName, separator);
+}
+// Explicitly instantiate the necessary public template implementations
+#define define_template_instantiation(T)                                       \
+    template Variable *Group::InquireVariable(                           \
+        const std::string &) noexcept;
+
+ADIOS2_FOREACH_STDTYPE_1ARG(define_template_instantiation)
+#undef define_template_instatiation
+
+} // end namespace core
+} // end namespace adios2
diff --git a/source/adios2/core/Group.h b/source/adios2/core/Group.h
new file mode 100644
index 0000000000..c81735b1e8
--- /dev/null
+++ b/source/adios2/core/Group.h
@@ -0,0 +1,160 @@
+/*
+ * Distributed under the OSI-approved Apache License, Version 2.0.  See
+ * accompanying file Copyright.txt for details.
+ *
+ * Group.h template implementations with fix types and specializations
+ *
+ *  Created on: August 25, 2020
+ *      Author: Dmitry Ganyushin ganyushindi@ornl.gov
+ */
+#ifndef ADIOS2_CORE_GROUP_H
+#define ADIOS2_CORE_GROUP_H
+#include "adios2/core/IO.h"
+#include "adios2/core/Variable.h"
+#include 
+#include 
+#include 
+#include 
+namespace adios2
+{
+namespace core
+{
+/** used for Variables and Attributes, name, type, type-index */
+using DataMap =
+    std::unordered_map>;
+class Group
+{
+private:
+    struct TreeMap
+    {
+        std::map> treeMap;
+    };
+    /** current path of the Group object */
+    std::string currentPath;
+    /** delimiter symbol between groups */
+    char groupDelimiter;
+    /** shared pointer to a map representing the tree structure */
+    std::shared_ptr mapPtr = nullptr;
+
+public:
+    /**
+     * @brief Constructor called from IO factory class InquireGroup function.
+     * Not to be used directly in applications.
+     * @param current path
+     * @param a separate symbol
+     * @param IO reference object to IO object that owns the current Group
+     * object
+     */
+    Group(std::string path, char delimiter, IO &m_IO);
+    /** copy constructor */
+    Group(const Group &G);
+    /** destructor */
+    ~Group();
+    /**
+     * @brief Builds map that represents tree structure from m_Variable and
+     * m_Attributes from IO class
+     * @param
+     */
+    void BuildTree();
+    /**
+     * @brief Prints map that represents tree structure
+     * @param
+     */
+    void PrintTree();
+    /**
+     * @brief returns available groups on the path set
+     * @param
+     * @return vector of strings
+     */
+    std::vector AvailableGroups();
+    /**
+     * @brief returns available variables on the path set
+     * @param
+     * @return vector of strings
+     */
+    std::vector AvailableVariables();
+    /**
+     * @brief returns available attributes on the path set
+     * @param
+     * @return vector of strings
+     */
+    std::vector AvailableAttributes();
+    /**
+     * @brief returns the current path
+     * @param
+     * @return current path as a string
+     */
+    std::string InquirePath();
+    /**
+     * @brief returns a new group object
+     * @param name of the group
+     * @return new group object
+     */
+    Group *InquireGroup(std::string groupName);
+    /**
+     * @brief set the path, points to a particular node on the tree
+     * @param next possible path extension
+     */
+    void setPath(std::string path);
+    /**
+     * @brief set the delimiter for group connection in a string representation
+     * @param delimiter symbol
+     */
+    void setDelimiter(char delimiter);
+    /**
+     * @brief returns  a reference to the map representing the tree stucture
+     * @param delimiter symbol
+     */
+    std::map> &getTreeMap();
+    /** reference to object that created current Group */
+    IO &m_IO;
+    /**
+     * @brief Gets an existing variable of primitive type by name. A wrapper for
+     * the corresponding function of the IO class
+     * @param name of variable to be retrieved
+     * @return pointer to an existing variable in current IO, nullptr if not
+     * found
+     */
+    template 
+    Variable *InquireVariable(const std::string &name) noexcept;
+    /**
+     * Gets an existing attribute of primitive type by name. A wrapper for
+     * the corresponding function of the IO class
+     * @param name of attribute to be retrieved
+     * @return pointer to an existing attribute in current IO, nullptr if not
+     * found
+     */
+    template 
+    Attribute *InquireAttribute(const std::string &name,
+                                   const std::string &variableName = "",
+                                   const std::string separator = "/") noexcept;
+    /**
+     * @brief Returns the type of an existing variable as an string. A wrapper
+     * for the corresponding function of the IO class
+     * @param name input variable name
+     * @return type primitive type
+     */
+    DataType InquireVariableType(const std::string &name) const noexcept;
+
+    /**
+     * @brief Returns the type of an existing attribute as an string. A wrapper
+     * for the corresponding function of the IO class
+     * @param name input attribute name
+     * @return type if found returns type as string, otherwise an empty string
+     */
+    DataType InquireAttributeType(const std::string &name,
+                                  const std::string &variableName = "",
+                                  const std::string separator = "/") const
+        noexcept;
+};
+// Explicit declaration of the public template methods
+#define declare_template_instantiation(T)                                      \
+    extern template Variable *Group::InquireVariable(                    \
+        const std::string &name) noexcept;
+
+ADIOS2_FOREACH_STDTYPE_1ARG(declare_template_instantiation)
+#undef declare_template_instantiation
+} // end namespace core
+} // end namespace adios2
+
+#endif // ADIOS2_CORE_GROUP_H
diff --git a/source/adios2/core/Group.tcc b/source/adios2/core/Group.tcc
new file mode 100644
index 0000000000..da5c883115
--- /dev/null
+++ b/source/adios2/core/Group.tcc
@@ -0,0 +1,40 @@
+/*
+ * Distributed under the OSI-approved Apache License, Version 2.0.  See
+ * accompanying file Copyright.txt for details.
+ *
+ * Group.tcc template implementations with fix types and specializations
+ *
+ *  Created on: August 25, 2020
+ *      Author: Dmitry Ganyushin ganyushindi@ornl.gov
+ */
+#ifndef ADIOS2_CORE_GROUP_TCC_
+#define ADIOS2_CORE_GROUP_TCC_
+
+#include "Group.h"
+
+namespace adios2
+{
+namespace core
+{
+
+template 
+Variable *Group::InquireVariable(const std::string &name) noexcept
+{
+    Variable &variable =
+        *m_IO.InquireVariable(currentPath + groupDelimiter + name);
+    return &variable;
+}
+
+template 
+Attribute *Group::InquireAttribute(const std::string &name,
+                                      const std::string &variableName,
+                                      const std::string separator) noexcept
+{
+    Attribute &attribute = m_IO.InquireAttribute(
+        currentPath + groupDelimiter + name, variableName, separator);
+    return &attribute;
+}
+} // end namespace core
+} // end namespace adios2
+
+#endif /* ADIOS2_CORE_GROUP_TCC_ */
diff --git a/source/adios2/core/IO.cpp b/source/adios2/core/IO.cpp
index 1d2b90c724..af026ab02e 100644
--- a/source/adios2/core/IO.cpp
+++ b/source/adios2/core/IO.cpp
@@ -11,6 +11,7 @@
 #include "IO.h"
 #include "IO.tcc"
 
+#include 
 #include 
 #include 
 #include  // std::pair
@@ -179,6 +180,8 @@ IO::IO(ADIOS &adios, const std::string name, const bool inConfigFile,
 {
 }
 
+IO::~IO() = default;
+
 void IO::SetEngine(const std::string engineType) noexcept
 {
     auto lf_InsertParam = [&](const std::string &key,
@@ -222,6 +225,7 @@ void IO::SetEngine(const std::string engineType) noexcept
     {
         finalEngineType = "BP4";
         lf_InsertParam("OpenTimeoutSecs", "3600");
+        lf_InsertParam("StreamReader", "true");
     }
     /* "file" is handled entirely in IO::Open() as it needs the name */
     else
@@ -299,12 +303,9 @@ void IO::SetTransportParameter(const size_t transportIndex,
     m_TransportsParameters[transportIndex][key] = value;
 }
 
-const DataMap &IO::GetVariablesDataMap() const noexcept { return m_Variables; }
+const VarMap &IO::GetVariables() const noexcept { return m_Variables; }
 
-const DataMap &IO::GetAttributesDataMap() const noexcept
-{
-    return m_Attributes;
-}
+const AttrMap &IO::GetAttributes() const noexcept { return m_Attributes; }
 
 bool IO::InConfigFile() const noexcept { return m_InConfigFile; }
 
@@ -320,31 +321,9 @@ bool IO::RemoveVariable(const std::string &name) noexcept
     // variable exists
     if (itVariable != m_Variables.end())
     {
-        // first remove the Variable object
-        const std::string type(itVariable->second.first);
-        const unsigned int index(itVariable->second.second);
-
-        if (type == "compound")
-        {
-            auto variableMap = m_Compound;
-            variableMap.erase(index);
-        }
-#define declare_type(T)                                                        \
-    else if (type == helper::GetType())                                     \
-    {                                                                          \
-        auto &variableMap = GetVariableMap();                               \
-        variableMap.erase(index);                                              \
-        isRemoved = true;                                                      \
-    }
-        ADIOS2_FOREACH_STDTYPE_1ARG(declare_type)
-#undef declare_type
-    }
-
-    if (isRemoved)
-    {
-        m_Variables.erase(name);
+        m_Variables.erase(itVariable);
+        isRemoved = true;
     }
-
     return isRemoved;
 }
 
@@ -352,10 +331,6 @@ void IO::RemoveAllVariables() noexcept
 {
     TAU_SCOPED_TIMER("IO::RemoveAllVariables");
     m_Variables.clear();
-#define declare_type(T) GetVariableMap().clear();
-    ADIOS2_FOREACH_STDTYPE_1ARG(declare_type)
-#undef declare_type
-    m_Compound.clear();
 }
 
 bool IO::RemoveAttribute(const std::string &name) noexcept
@@ -367,27 +342,17 @@ bool IO::RemoveAttribute(const std::string &name) noexcept
     if (itAttribute != m_Attributes.end())
     {
         // first remove the Attribute object
-        const std::string type(itAttribute->second.first);
-        const unsigned int index(itAttribute->second.second);
+        const DataType type(itAttribute->second->m_Type);
 
-        if (type.empty())
+        if (type == DataType::None)
         {
             // nothing to do
         }
-#define declare_type(T)                                                        \
-    else if (type == helper::GetType())                                     \
-    {                                                                          \
-        auto &attributeMap = GetAttributeMap();                             \
-        attributeMap.erase(index);                                             \
-        isRemoved = true;                                                      \
-    }
-        ADIOS2_FOREACH_ATTRIBUTE_STDTYPE_1ARG(declare_type)
-#undef declare_type
-    }
-
-    if (isRemoved)
-    {
-        m_Attributes.erase(name);
+        else
+        {
+            m_Attributes.erase(itAttribute);
+            isRemoved = true;
+        }
     }
 
     return isRemoved;
@@ -397,10 +362,6 @@ void IO::RemoveAllAttributes() noexcept
 {
     TAU_SCOPED_TIMER("IO::RemoveAllAttributes");
     m_Attributes.clear();
-
-#define declare_type(T) GetAttributeMap().clear();
-    ADIOS2_FOREACH_ATTRIBUTE_STDTYPE_1ARG(declare_type)
-#undef declare_type
 }
 
 std::map
@@ -412,13 +373,13 @@ IO::GetAvailableVariables(const std::set &keys) noexcept
     for (const auto &variablePair : m_Variables)
     {
         const std::string variableName = variablePair.first;
-        const std::string type = InquireVariableType(variableName);
+        const DataType type = InquireVariableType(variableName);
 
-        if (type == "compound")
+        if (type == DataType::Compound)
         {
         }
 #define declare_template_instantiation(T)                                      \
-    else if (type == helper::GetType())                                     \
+    else if (type == helper::GetDataType())                                 \
     {                                                                          \
         variablesInfo[variableName] = GetVariableInfo(variableName, keys);  \
     }
@@ -440,22 +401,16 @@ IO::GetAvailableAttributes(const std::string &variableName,
     if (!variableName.empty())
     {
         auto itVariable = m_Variables.find(variableName);
-        const std::string type = InquireVariableType(itVariable);
+        const DataType type = InquireVariableType(itVariable);
 
-        if (type == "compound")
+        if (type == DataType::Compound)
         {
         }
-#define declare_template_instantiation(T)                                      \
-    else if (type == helper::GetType())                                     \
-    {                                                                          \
-        Variable &variable =                                                \
-            GetVariableMap().at(itVariable->second.second);                 \
-        attributesInfo =                                                       \
-            variable.GetAttributesInfo(*this, separator, fullNameKeys);        \
-    }
-        ADIOS2_FOREACH_STDTYPE_1ARG(declare_template_instantiation)
-#undef declare_template_instantiation
-
+        else
+        {
+            attributesInfo = itVariable->second->GetAttributesInfo(
+                *this, separator, fullNameKeys);
+        }
         return attributesInfo;
     }
 
@@ -463,67 +418,55 @@ IO::GetAvailableAttributes(const std::string &variableName,
     for (const auto &attributePair : m_Attributes)
     {
         const std::string &name = attributePair.first;
-        const std::string type = attributePair.second.first;
 
-        if (type == "compound")
+        if (attributePair.second->m_Type == DataType::Compound)
         {
         }
-#define declare_template_instantiation(T)                                      \
-    else if (type == helper::GetType())                                     \
-    {                                                                          \
-        Attribute &attribute =                                              \
-            GetAttributeMap().at(attributePair.second.second);              \
-        attributesInfo[name] = attribute.GetInfo();                            \
-    }
-        ADIOS2_FOREACH_ATTRIBUTE_STDTYPE_1ARG(declare_template_instantiation)
-#undef declare_template_instantiation
+        else
+        {
+            attributesInfo[name] = attributePair.second->GetInfo();
+        }
     }
     return attributesInfo;
 }
 
-std::string IO::InquireVariableType(const std::string &name) const noexcept
+DataType IO::InquireVariableType(const std::string &name) const noexcept
 {
     TAU_SCOPED_TIMER("IO::other");
     auto itVariable = m_Variables.find(name);
     return InquireVariableType(itVariable);
 }
 
-std::string
-IO::InquireVariableType(const DataMap::const_iterator itVariable) const noexcept
+DataType IO::InquireVariableType(const VarMap::const_iterator itVariable) const
+    noexcept
 {
     if (itVariable == m_Variables.end())
     {
-        return "";
+        return DataType::None;
     }
 
-    const std::string type = itVariable->second.first;
+    const DataType type = itVariable->second->m_Type;
 
     if (m_ReadStreaming)
     {
-        if (type == "compound")
+        if (type == DataType::Compound)
         {
         }
-#define declare_template_instantiation(T)                                      \
-    else if (type == helper::GetType())                                     \
-    {                                                                          \
-        const Variable &variable =                                          \
-            const_cast(this)->GetVariableMap().at(                    \
-                itVariable->second.second);                                    \
-        if (!variable.IsValidStep(m_EngineStep + 1))                           \
-        {                                                                      \
-            return "";                                                         \
-        }                                                                      \
-    }
-        ADIOS2_FOREACH_STDTYPE_1ARG(declare_template_instantiation)
-#undef declare_template_instantiation
+        else
+        {
+            if (!itVariable->second->IsValidStep(m_EngineStep + 1))
+            {
+                return DataType::None;
+            }
+        }
     }
 
     return type;
 }
 
-std::string IO::InquireAttributeType(const std::string &name,
-                                     const std::string &variableName,
-                                     const std::string separator) const noexcept
+DataType IO::InquireAttributeType(const std::string &name,
+                                  const std::string &variableName,
+                                  const std::string separator) const noexcept
 {
     TAU_SCOPED_TIMER("IO::other");
     const std::string globalName =
@@ -532,10 +475,10 @@ std::string IO::InquireAttributeType(const std::string &name,
     auto itAttribute = m_Attributes.find(globalName);
     if (itAttribute == m_Attributes.end())
     {
-        return "";
+        return DataType::None;
     }
 
-    return itAttribute->second.first;
+    return itAttribute->second->m_Type;
 }
 
 size_t IO::AddOperation(Operator &op, const Params ¶meters) noexcept
@@ -631,6 +574,51 @@ Engine &IO::Open(const std::string &name, const Mode mode, helper::Comm comm)
         }
     }
 
+    // For the inline engine, there must be exactly 1 reader, and exactly 1
+    // writer.
+    if (engineTypeLC == "inline")
+    {
+        if (mode == Mode::Append)
+        {
+            throw std::runtime_error(
+                "Append mode is not supported for the inline engine.");
+        }
+
+        // See inline.rst:44
+        if (mode == Mode::Sync)
+        {
+            throw std::runtime_error(
+                "Sync mode is not supported for the inline engine.");
+        }
+
+        if (m_Engines.size() >= 2)
+        {
+            std::string msg =
+                "Failed to add engine " + name + " to IO \'" + m_Name + "\'. ";
+            msg += "An inline engine must have exactly one writer, and one "
+                   "reader. ";
+            msg += "There are already two engines declared, so no more can be "
+                   "added.";
+            throw std::runtime_error(msg);
+        }
+        // Now protect against declaration of two writers, or declaration of two
+        // readers:
+        if (m_Engines.size() == 1)
+        {
+            auto engine_ptr = m_Engines.begin()->second;
+            if (engine_ptr->OpenMode() == mode)
+            {
+                std::string msg =
+                    "The previously added engine " + engine_ptr->m_Name +
+                    " is already opened in same mode requested for " + name +
+                    ". ";
+                msg += "The inline engine requires exactly one writer and one "
+                       "reader.";
+                throw std::runtime_error(msg);
+            }
+        }
+    }
+
     auto f = FactoryLookup(engineTypeLC);
     if (f != Factory.end())
     {
@@ -667,7 +655,13 @@ Engine &IO::Open(const std::string &name, const Mode mode)
 {
     return Open(name, mode, m_ADIOS.GetComm().Duplicate());
 }
+Group &IO::CreateGroup(const std::string &path, char delimiter)
+{
 
+    m_Gr = std::make_shared(path, delimiter, *this);
+    m_Gr->BuildTree();
+    return *m_Gr;
+}
 Engine &IO::GetEngine(const std::string &name)
 {
     TAU_SCOPED_TIMER("IO::other");
@@ -712,28 +706,23 @@ void IO::ResetVariablesStepSelection(const bool zeroStart,
          ++itVariable)
     {
         const std::string &name = itVariable->first;
-        const std::string type = InquireVariableType(itVariable);
+        const DataType type = InquireVariableType(itVariable);
 
-        if (type.empty())
+        if (type == DataType::None)
         {
             continue;
         }
 
-        if (type == "compound")
+        if (type == DataType::Compound)
         {
         }
-// using relative start
-#define declare_type(T)                                                        \
-    else if (type == helper::GetType())                                     \
-    {                                                                          \
-        Variable &variable =                                                \
-            GetVariableMap().at(itVariable->second.second);                 \
-        variable.CheckRandomAccessConflict(hint);                              \
-        variable.ResetStepsSelection(zeroStart);                               \
-        variable.m_RandomAccess = false;                                       \
-    }
-        ADIOS2_FOREACH_STDTYPE_1ARG(declare_type)
-#undef declare_type
+        else
+        {
+            VariableBase &variable = *itVariable->second;
+            variable.CheckRandomAccessConflict(hint);
+            variable.ResetStepsSelection(zeroStart);
+            variable.m_RandomAccess = false;
+        }
     }
 }
 
@@ -748,50 +737,35 @@ void IO::SetPrefixedNames(const bool isStep) noexcept
         const std::string &name = itVariable->first;
         // if for each step (BP4), check if variable type is not empty (means
         // variable exist in that step)
-        const std::string type =
-            isStep ? InquireVariableType(itVariable) : itVariable->second.first;
+        const DataType type = isStep ? InquireVariableType(itVariable)
+                                     : itVariable->second->m_Type;
 
-        if (type.empty())
+        if (type == DataType::None)
         {
             continue;
         }
 
-        if (type == "compound")
+        if (type == DataType::Compound)
         {
         }
-#define declare_type(T)                                                        \
-    else if (type == helper::GetType())                                     \
-    {                                                                          \
-        Variable &variable =                                                \
-            GetVariableMap().at(itVariable->second.second);                 \
-        variable.m_PrefixedVariables =                                         \
-            helper::PrefixMatches(variable.m_Name, variables);                 \
-        variable.m_PrefixedAttributes =                                        \
-            helper::PrefixMatches(variable.m_Name, attributes);                \
-    }
-        ADIOS2_FOREACH_STDTYPE_1ARG(declare_type)
-#undef declare_type
+        else
+        {
+            VariableBase &variable = *itVariable->second;
+            variable.m_PrefixedVariables =
+                helper::PrefixMatches(variable.m_Name, variables);
+            variable.m_PrefixedAttributes =
+                helper::PrefixMatches(variable.m_Name, attributes);
+        }
     }
 
     m_IsPrefixedNames = true;
 }
 
 // PRIVATE
-int IO::GetMapIndex(const std::string &name, const DataMap &dataMap) const
-    noexcept
-{
-    auto itName = dataMap.find(name);
-    if (itName == dataMap.end())
-    {
-        return -1;
-    }
-    return itName->second.second;
-}
-
 void IO::CheckAttributeCommon(const std::string &name) const
 {
     auto itAttribute = m_Attributes.find(name);
-    if (!IsEnd(itAttribute, m_Attributes))
+    if (itAttribute != m_Attributes.end())
     {
         throw std::invalid_argument("ERROR: attribute " + name +
                                     " exists in IO object " + m_Name +
@@ -799,15 +773,6 @@ void IO::CheckAttributeCommon(const std::string &name) const
     }
 }
 
-bool IO::IsEnd(DataMap::const_iterator itDataMap, const DataMap &dataMap) const
-{
-    if (itDataMap == dataMap.end())
-    {
-        return true;
-    }
-    return false;
-}
-
 void IO::CheckTransportType(const std::string type) const
 {
     if (type.empty() || type.find("=") != type.npos)
diff --git a/source/adios2/core/IO.h b/source/adios2/core/IO.h
index 234cda75ab..b63f136152 100644
--- a/source/adios2/core/IO.h
+++ b/source/adios2/core/IO.h
@@ -25,6 +25,7 @@
 #include "adios2/common/ADIOSTypes.h"
 #include "adios2/core/ADIOS.h"
 #include "adios2/core/Attribute.h"
+#include "adios2/core/Group.h"
 #include "adios2/core/Variable.h"
 #include "adios2/core/VariableCompound.h"
 
@@ -34,13 +35,13 @@ namespace adios2
 namespace core
 {
 
-/** used for Variables and Attributes, name, type, type-index */
-using DataMap =
-    std::unordered_map>;
+using VarMap = std::unordered_map>;
+using AttrMap = std::unordered_map>;
 
 // forward declaration needed as IO is passed to Engine derived
 // classes
 class Engine;
+class Group;
 
 /** Factory class IO for settings, variables, and transports to an engine */
 class IO
@@ -49,6 +50,8 @@ class IO
 public:
     /** reference to object that created current IO */
     ADIOS &m_ADIOS;
+    /** a pointer to a Group Object created from IO */
+    std::shared_ptr m_Gr;
 
     /** unique identifier */
     const std::string m_Name;
@@ -56,27 +59,6 @@ class IO
     /** from ADIOS class passed to Engine created with Open */
     const std::string m_HostLanguage = "C++";
 
-    /**
-     * Map holding variable identifiers
-     * 
-     * key: unique variable name,
-     * value: pair.first = type as string GetType from adiosTemplates.h
-     *        pair.second = index in fixed size map (e.g. m_Int8, m_Double)
-     * 

-     */
-    DataMap m_Variables;
-
-    /**
-     * Map holding attribute identifiers
-     * 
-     * key: unique attribute name,
-     * value: pair.first = type as string GetType from
-     *                     helper/adiosTemplates.h
-     *        pair.second = index in fixed size map (e.g. m_Int8, m_Double)
-     * 

-     */
-    DataMap m_Attributes;
-
     /** From SetParameter, parameters for a particular engine from m_Type */
     Params m_Parameters;
 
@@ -123,7 +105,10 @@ class IO
     IO(ADIOS &adios, const std::string name, const bool inConfigFile,
        const std::string hostLanguage);
 
-    ~IO() = default;
+    ~IO();
+
+    IO(IO const &) = delete;
+    IO &operator=(IO const &) = delete;
 
     /**
      * @brief Sets the engine type for this IO class object
@@ -294,15 +279,14 @@ class IO
      * @param name input variable name
      * @return type primitive type
      */
-    std::string InquireVariableType(const std::string &name) const noexcept;
+    DataType InquireVariableType(const std::string &name) const noexcept;
 
     /**
      * Overload that accepts a const iterator into the m_Variables map if found
      * @param itVariable
      * @return type primitive type
      */
-    std::string
-    InquireVariableType(const DataMap::const_iterator itVariable) const
+    DataType InquireVariableType(const VarMap::const_iterator itVariable) const
         noexcept;
 
     /**
@@ -310,22 +294,20 @@ class IO
      * @return
      * 
      * key: unique variable name,
-     * value: pair.first = string type
-     *        pair.second = order in the type bucket
+     * value: pointer to VariableBase
      * 

      */
-    const DataMap &GetVariablesDataMap() const noexcept;
+    const VarMap &GetVariables() const noexcept;
 
     /**
      * Retrieves hash holding internal Attributes identifiers
      * @return
      * 
      * key: unique attribute name,
-     * value: pair.first = string type
-     *        pair.second = order in the type bucket
+     * value: pointer to AttributeBase
      * 

      */
-    const DataMap &GetAttributesDataMap() const noexcept;
+    const AttrMap &GetAttributes() const noexcept;
 
     /**
      * Gets an existing attribute of primitive type by name
@@ -343,9 +325,9 @@ class IO
      * @param name input attribute name
      * @return type if found returns type as string, otherwise an empty string
      */
-    std::string InquireAttributeType(const std::string &name,
-                                     const std::string &variableName = "",
-                                     const std::string separator = "/") const
+    DataType InquireAttributeType(const std::string &name,
+                                  const std::string &variableName = "",
+                                  const std::string separator = "/") const
         noexcept;
 
     /**
@@ -430,6 +412,8 @@ class IO
      */
     void FlushAll();
 
+    Group &CreateGroup(const std::string &path, char delimiter);
+
     // READ FUNCTIONS, not yet implemented:
     /**
      * not yet implented
@@ -452,14 +436,6 @@ class IO
 
     void SetPrefixedNames(const bool isStep) noexcept;
 
-    /** Gets the internal reference to a variable map for type T */
-    template 
-    std::map> &GetVariableMap() noexcept;
-
-    /** Gets the internal reference to an attribute map for type T */
-    template 
-    std::map> &GetAttributeMap() noexcept;
-
     using MakeEngineFunc = std::function(
         IO &, const std::string &, const Mode, helper::Comm)>;
     struct EngineFactoryEntry
@@ -501,6 +477,15 @@ class IO
     static void RegisterEngine(const std::string &engineType,
                                EngineFactoryEntry entry);
 
+    /*
+     * Return list of all engines associated with this IO.
+     */
+
+    const std::map> &GetEngines() const
+    {
+        return m_Engines;
+    }
+
 private:
     /** true: exist in config file (XML) */
     const bool m_InConfigFile = false;
@@ -510,40 +495,16 @@ class IO
     /** Independent (default) or Collective */
     adios2::IOMode m_IOMode = adios2::IOMode::Independent;
 
-/** Variable containers based on fixed-size type */
-#define declare_map(T, NAME) std::map> m_##NAME;
-    ADIOS2_FOREACH_STDTYPE_2ARGS(declare_map)
-#undef declare_map
+    VarMap m_Variables;
 
-#define declare_map(T, NAME) std::map> m_##NAME##A;
-    ADIOS2_FOREACH_ATTRIBUTE_STDTYPE_2ARGS(declare_map)
-#undef declare_map
-
-    std::map m_Compound;
+    AttrMap m_Attributes;
 
     std::map> m_Engines;
 
-    /**
-     * Gets map index for Variables or Attributes
-     * @param name
-     * @param dataMap m_Variables or m_Attributes
-     * @return index in type map, -1 if not found
-     */
-    int GetMapIndex(const std::string &name, const DataMap &dataMap) const
-        noexcept;
-
     /** Checks if attribute exists, called from DefineAttribute different
      *  signatures */
     void CheckAttributeCommon(const std::string &name) const;
 
-    /**
-     * Checks if iterator points to end. Used for Variables and Attributes.
-     * @param itDataMap iterator to be tested
-     * @param dataMap map
-     * @return true: itDataMap == dataMap.end(), false otherwise
-     */
-    bool IsEnd(DataMap::const_iterator itDataMap, const DataMap &dataMap) const;
-
     void CheckTransportType(const std::string type) const;
 
     template 
diff --git a/source/adios2/core/IO.tcc b/source/adios2/core/IO.tcc
index 7e82f27dab..fc5f3440a9 100644
--- a/source/adios2/core/IO.tcc
+++ b/source/adios2/core/IO.tcc
@@ -15,11 +15,13 @@
 
 /// \cond EXCLUDE_FROM_DOXYGEN
 #include 
+#include 
 #include  //std::invalid_argument
 /// \endcond
 
 #include "adios2/common/ADIOSMacros.h"
-#include "adios2/helper/adiosFunctions.h" //helper::GetType
+#include "adios2/helper/adiosFunctions.h"
+#include "adios2/helper/adiosType.h"
 #include "adios2/toolkit/profiling/taustubs/tautimer.hpp"
 
 namespace adios2
@@ -36,7 +38,7 @@ Variable &IO::DefineVariable(const std::string &name, const Dims &shape,
 
     {
         auto itVariable = m_Variables.find(name);
-        if (!IsEnd(itVariable, m_Variables))
+        if (itVariable != m_Variables.end())
         {
             throw std::invalid_argument("ERROR: variable " + name +
                                         " exists in IO object " + m_Name +
@@ -44,15 +46,12 @@ Variable &IO::DefineVariable(const std::string &name, const Dims &shape,
         }
     }
 
-    auto &variableMap = GetVariableMap();
-    const unsigned int newIndex =
-        variableMap.empty() ? 0 : variableMap.rbegin()->first + 1;
+    auto itVariablePair = m_Variables.emplace(
+        name, std::unique_ptr(
+                  new Variable(name, shape, start, count, constantDims)));
 
-    auto itVariablePair = variableMap.emplace(
-        newIndex, Variable(name, shape, start, count, constantDims));
-    m_Variables.emplace(name, std::make_pair(helper::GetType(), newIndex));
-
-    Variable &variable = itVariablePair.first->second;
+    Variable &variable =
+        static_cast &>(*itVariablePair.first->second);
 
     // check IO placeholder for variable operations
     auto itOperations = m_VarOpsPlaceholder.find(name);
@@ -80,12 +79,13 @@ Variable *IO::InquireVariable(const std::string &name) noexcept
         return nullptr;
     }
 
-    if (itVariable->second.first != helper::GetType())
+    if (itVariable->second->m_Type != helper::GetDataType())
     {
         return nullptr;
     }
 
-    Variable *variable = &GetVariableMap().at(itVariable->second.second);
+    Variable *variable =
+        static_cast *>(itVariable->second.get());
     if (m_ReadStreaming)
     {
         if (!variable->IsValidStep(m_EngineStep + 1))
@@ -102,7 +102,8 @@ Attribute &IO::DefineAttribute(const std::string &name, const T &value,
                                   const std::string separator)
 {
     TAU_SCOPED_TIMER("IO::DefineAttribute");
-    if (!variableName.empty() && InquireVariableType(variableName).empty())
+    if (!variableName.empty() &&
+        InquireVariableType(variableName) == DataType::None)
     {
         throw std::invalid_argument(
             "ERROR: variable " + variableName +
@@ -113,15 +114,13 @@ Attribute &IO::DefineAttribute(const std::string &name, const T &value,
     const std::string globalName =
         helper::GlobalName(name, variableName, separator);
 
-    auto &attributeMap = GetAttributeMap();
     auto itExistingAttribute = m_Attributes.find(globalName);
-    if (!IsEnd(itExistingAttribute, m_Attributes))
+    if (itExistingAttribute != m_Attributes.end())
     {
         if (helper::ValueToString(value) ==
-            attributeMap.at(itExistingAttribute->second.second)
-                .GetInfo()["Value"])
+            itExistingAttribute->second->GetInfo()["Value"])
         {
-            return attributeMap.at(itExistingAttribute->second.second);
+            return static_cast &>(*itExistingAttribute->second);
         }
         else
         {
@@ -131,15 +130,12 @@ Attribute &IO::DefineAttribute(const std::string &name, const T &value,
                 "DefineAttribute\n");
         }
     }
-    const unsigned int newIndex =
-        attributeMap.empty() ? 0 : attributeMap.rbegin()->first + 1;
 
-    auto itAttributePair =
-        attributeMap.emplace(newIndex, Attribute(globalName, value));
-    m_Attributes.emplace(globalName,
-                         std::make_pair(helper::GetType(), newIndex));
+    auto itAttributePair = m_Attributes.emplace(
+        globalName,
+        std::unique_ptr(new Attribute(globalName, value)));
 
-    return itAttributePair.first->second;
+    return static_cast &>(*itAttributePair.first->second);
 }
 
 template 
@@ -149,7 +145,8 @@ Attribute &IO::DefineAttribute(const std::string &name, const T *array,
                                   const std::string separator)
 {
     TAU_SCOPED_TIMER("IO::DefineAttribute");
-    if (!variableName.empty() && InquireVariableType(variableName).empty())
+    if (!variableName.empty() &&
+        InquireVariableType(variableName) == DataType::None)
     {
         throw std::invalid_argument(
             "ERROR: variable " + variableName +
@@ -160,19 +157,17 @@ Attribute &IO::DefineAttribute(const std::string &name, const T *array,
     const std::string globalName =
         helper::GlobalName(name, variableName, separator);
 
-    auto &attributeMap = GetAttributeMap();
     auto itExistingAttribute = m_Attributes.find(globalName);
-    if (!IsEnd(itExistingAttribute, m_Attributes))
+    if (itExistingAttribute != m_Attributes.end())
     {
         const std::string arrayValues(
             "{ " +
             helper::VectorToCSV(std::vector(array, array + elements)) +
             " }");
 
-        if (attributeMap.at(itExistingAttribute->second.second)
-                .GetInfo()["Value"] == arrayValues)
+        if (itExistingAttribute->second->GetInfo()["Value"] == arrayValues)
         {
-            return attributeMap.at(itExistingAttribute->second.second);
+            return static_cast &>(*itExistingAttribute->second);
         }
         else
         {
@@ -182,15 +177,11 @@ Attribute &IO::DefineAttribute(const std::string &name, const T *array,
                 "DefineAttribute\n");
         }
     }
-    const unsigned int newIndex =
-        attributeMap.empty() ? 0 : attributeMap.rbegin()->first + 1;
-
-    auto itAttributePair = attributeMap.emplace(
-        newIndex, Attribute(globalName, array, elements));
-    m_Attributes.emplace(globalName,
-                         std::make_pair(helper::GetType(), newIndex));
 
-    return itAttributePair.first->second;
+    auto itAttributePair = m_Attributes.emplace(
+        globalName, std::unique_ptr(
+                        new Attribute(globalName, array, elements)));
+    return static_cast &>(*itAttributePair.first->second);
 }
 
 template 
@@ -208,36 +199,16 @@ Attribute *IO::InquireAttribute(const std::string &name,
         return nullptr;
     }
 
-    if (itAttribute->second.first != helper::GetType())
+    if (itAttribute->second->m_Type != helper::GetDataType())
     {
         return nullptr;
     }
 
-    return &GetAttributeMap().at(itAttribute->second.second);
+    return static_cast *>(itAttribute->second.get());
 }
 
 // PRIVATE
 
-// GetVariableMap
-#define make_GetVariableMap(T, NAME)                                           \
-    template <>                                                                \
-    std::map> &IO::GetVariableMap() noexcept         \
-    {                                                                          \
-        return m_##NAME;                                                       \
-    }
-ADIOS2_FOREACH_STDTYPE_2ARGS(make_GetVariableMap)
-#undef make_GetVariableMap
-
-// GetAttributeMap
-#define make_GetAttributeMap(T, NAME)                                          \
-    template <>                                                                \
-    std::map> &IO::GetAttributeMap() noexcept       \
-    {                                                                          \
-        return m_##NAME##A;                                                    \
-    }
-ADIOS2_FOREACH_ATTRIBUTE_STDTYPE_2ARGS(make_GetAttributeMap)
-#undef make_GetAttributeMap
-
 template 
 Params IO::GetVariableInfo(const std::string &variableName,
                            const std::set &keys)
@@ -256,7 +227,7 @@ Params IO::GetVariableInfo(const std::string &variableName,
 
     if (keys.empty() || keysLC.count("type") == 1)
     {
-        info["Type"] = variable.m_Type;
+        info["Type"] = ToString(variable.m_Type);
     }
 
     if (keys.empty() || keysLC.count("availablestepscount") == 1)
diff --git a/source/adios2/core/Operator.cpp b/source/adios2/core/Operator.cpp
index 9d6d7a9927..6e7bebf165 100644
--- a/source/adios2/core/Operator.cpp
+++ b/source/adios2/core/Operator.cpp
@@ -63,16 +63,16 @@ size_t Operator::BufferMaxSize(const size_t sizeIn) const
     size_t Operator::BufferMaxSize(const T *dataIn, const Dims &dimensions, \
                                       const Params ¶meters) const          \
     {                                                                          \
-        return DoBufferMaxSize(dataIn, dimensions, helper::GetType(),       \
+        return DoBufferMaxSize(dataIn, dimensions, helper::GetDataType(),   \
                                parameters);                                    \
     }
 ADIOS2_FOREACH_ZFP_TYPE_1ARG(declare_type)
 #undef declare_type
 
 size_t Operator::Compress(const void * /*dataIn*/, const Dims & /*dimensions*/,
-                          const size_t /*elementSize*/,
-                          const std::string /*type*/, void * /*bufferOut*/,
-                          const Params & /*params*/, Params & /*info*/) const
+                          const size_t /*elementSize*/, DataType /*type*/,
+                          void * /*bufferOut*/, const Params & /*params*/,
+                          Params & /*info*/) const
 {
     throw std::invalid_argument("ERROR: signature (const void*, const "
                                 "Dims, const size_t, const std::string, "
@@ -93,7 +93,7 @@ size_t Operator::Decompress(const void *bufferIn, const size_t sizeIn,
 
 size_t Operator::Decompress(const void * /*bufferIn*/, const size_t /*sizeIn*/,
                             void * /*dataOut*/, const Dims & /*dimensions*/,
-                            const std::string /*type*/,
+                            DataType /*type*/,
                             const Params & /*parameters*/) const
 {
     throw std::invalid_argument("ERROR: signature (const void*, const "
@@ -105,8 +105,7 @@ size_t Operator::Decompress(const void * /*bufferIn*/, const size_t /*sizeIn*/,
 
 // PROTECTED
 size_t Operator::DoBufferMaxSize(const void *dataIn, const Dims &dimensions,
-                                 const std::string type,
-                                 const Params ¶meters) const
+                                 DataType type, const Params ¶meters) const
 {
     throw std::invalid_argument("ERROR: signature (const void*, const Dims& "
                                 "std::string ) not supported "
diff --git a/source/adios2/core/Operator.h b/source/adios2/core/Operator.h
index 6cec1347b7..0794eb22b5 100644
--- a/source/adios2/core/Operator.h
+++ b/source/adios2/core/Operator.h
@@ -86,7 +86,7 @@ class Operator
      * @return size of compressed buffer
      */
     virtual size_t Compress(const void *dataIn, const Dims &dimensions,
-                            const size_t elementSize, const std::string type,
+                            const size_t elementSize, DataType type,
                             void *bufferOut, const Params ¶meters,
                             Params &info) const;
 
@@ -105,8 +105,7 @@ class Operator
      */
     virtual size_t Decompress(const void *bufferIn, const size_t sizeIn,
                               void *dataOut, const Dims &dimensions,
-                              const std::string type,
-                              const Params ¶meters) const;
+                              DataType type, const Params ¶meters) const;
 
 protected:
     /** Parameters associated with a particular Operator */
@@ -121,7 +120,7 @@ class Operator
      * @return conservative buffer size for allocation
      */
     virtual size_t DoBufferMaxSize(const void *dataIn, const Dims &dimensions,
-                                   const std::string type,
+                                   DataType type,
                                    const Params ¶meters) const;
 
 private:
diff --git a/source/adios2/core/Variable.cpp b/source/adios2/core/Variable.cpp
index 9597db5919..a37c60c7f7 100644
--- a/source/adios2/core/Variable.cpp
+++ b/source/adios2/core/Variable.cpp
@@ -13,7 +13,7 @@
 
 #include "adios2/common/ADIOSMacros.h"
 #include "adios2/core/Engine.h"
-#include "adios2/helper/adiosFunctions.h" //helper::GetType
+#include "adios2/helper/adiosFunctions.h" //helper::GetDataType
 
 namespace adios2
 {
@@ -26,8 +26,8 @@ namespace core
     Variable::Variable(const std::string &name, const Dims &shape,          \
                           const Dims &start, const Dims &count,                \
                           const bool constantDims)                             \
-    : VariableBase(name, helper::GetType(), sizeof(T), shape, start, count, \
-                   constantDims)                                               \
+    : VariableBase(name, helper::GetDataType(), sizeof(T), shape, start,    \
+                   count, constantDims)                                        \
     {                                                                          \
         m_BlocksInfo.reserve(1);                                               \
     }                                                                          \
diff --git a/source/adios2/core/Variable.tcc b/source/adios2/core/Variable.tcc
index 06c2579527..edf7e9b5c5 100644
--- a/source/adios2/core/Variable.tcc
+++ b/source/adios2/core/Variable.tcc
@@ -26,7 +26,7 @@ Dims Variable::DoShape(const size_t step) const
 {
     CheckRandomAccess(step, "Shape");
 
-    if (m_FirstStreamingStep && step == DefaultSizeT)
+    if (m_FirstStreamingStep && step == adios2::EngineCurrentStep)
     {
         return m_Shape;
     }
@@ -36,21 +36,11 @@ Dims Variable::DoShape(const size_t step) const
         const size_t stepInput =
             !m_FirstStreamingStep ? m_Engine->CurrentStep() : step;
 
-        const std::vector::Info> blocksInfo =
-            m_Engine->BlocksInfo(*this, stepInput);
-
-        if (blocksInfo.size() == 0)
+        const auto it = m_AvailableShapes.find(stepInput + 1);
+        if (it != m_AvailableShapes.end())
         {
-            return Dims();
+            return it->second;
         }
-
-        if (blocksInfo.front().Shape.size() == 1 &&
-            blocksInfo.front().Shape.front() == LocalValueDim)
-        {
-            return Dims{blocksInfo.size()};
-        }
-
-        return blocksInfo.front().Shape;
     }
     return m_Shape;
 }
diff --git a/source/adios2/core/VariableBase.cpp b/source/adios2/core/VariableBase.cpp
index 1eceb14ec8..17398ff092 100644
--- a/source/adios2/core/VariableBase.cpp
+++ b/source/adios2/core/VariableBase.cpp
@@ -28,7 +28,7 @@ namespace adios2
 namespace core
 {
 
-VariableBase::VariableBase(const std::string &name, const std::string type,
+VariableBase::VariableBase(const std::string &name, const DataType type,
                            const size_t elementSize, const Dims &shape,
                            const Dims &start, const Dims &count,
                            const bool constantDims)
@@ -45,7 +45,7 @@ size_t VariableBase::TotalSize() const noexcept
 
 void VariableBase::SetShape(const adios2::Dims &shape)
 {
-    if (m_Type == helper::GetType())
+    if (m_Type == helper::GetDataType())
     {
         throw std::invalid_argument("ERROR: string variable " + m_Name +
                                     " is always LocalValue, can't change "
@@ -87,7 +87,7 @@ void VariableBase::SetSelection(const Box &boxDims)
     const Dims &start = boxDims.first;
     const Dims &count = boxDims.second;
 
-    if (m_Type == helper::GetType() &&
+    if (m_Type == helper::GetDataType() &&
         m_ShapeID != ShapeID::GlobalArray)
     {
         throw std::invalid_argument("ERROR: string variable " + m_Name +
@@ -203,6 +203,15 @@ void VariableBase::SetStepSelection(const Box &boxSteps)
     m_StepsStart = boxSteps.first;
     m_StepsCount = boxSteps.second;
     m_RandomAccess = true;
+    if (m_ShapeID == ShapeID::GlobalArray)
+    {
+        /* Handle Global Array with changing shape over steps */
+        const auto it = m_AvailableShapes.find(m_StepsStart + 1);
+        if (it != m_AvailableShapes.end())
+        {
+            m_Shape = it->second;
+        }
+    }
 }
 
 size_t VariableBase::AddOperation(Operator &op,
@@ -304,24 +313,18 @@ VariableBase::GetAttributesInfo(core::IO &io, const std::string separator,
             return;
         }
 
-        auto itAttribute = io.m_Attributes.find(attributeName);
-        const std::string type = itAttribute->second.first;
+        auto itAttribute = io.GetAttributes().find(attributeName);
 
         const std::string key =
             fullNameKeys ? attributeName : attributeName.substr(prefix.size());
 
-        if (type == "compound")
+        if (itAttribute->second->m_Type == DataType::Compound)
         {
         }
-#define declare_template_instantiation(T)                                      \
-    else if (type == helper::GetType())                                     \
-    {                                                                          \
-        Attribute &attribute =                                              \
-            io.GetAttributeMap().at(itAttribute->second.second);            \
-        attributesInfo[key] = attribute.GetInfo();                             \
-    }
-        ADIOS2_FOREACH_ATTRIBUTE_STDTYPE_1ARG(declare_template_instantiation)
-#undef declare_template_instantiation
+        else
+        {
+            attributesInfo[key] = itAttribute->second->GetInfo();
+        }
     };
 
     // BODY OF FUNCTION STARTS HERE
@@ -339,7 +342,7 @@ VariableBase::GetAttributesInfo(core::IO &io, const std::string separator,
     }
     else
     { // get prefixed attributes on-the-fly (expensive)
-        for (const auto &attributePair : io.m_Attributes)
+        for (const auto &attributePair : io.GetAttributes())
         {
             const std::string &attributeName = attributePair.first;
             lf_GetAttributeInfo(prefix, attributeName, io, attributesInfo,
@@ -353,7 +356,7 @@ VariableBase::GetAttributesInfo(core::IO &io, const std::string separator,
 // PRIVATE
 void VariableBase::InitShapeType()
 {
-    if (m_Type == helper::GetType())
+    if (m_Type == helper::GetDataType())
     {
         if (m_Shape.empty())
         {
@@ -521,12 +524,12 @@ void VariableBase::CheckDimensionsCommon(const std::string hint) const
 
 Dims VariableBase::GetShape(const size_t step)
 {
-    if (m_Type == "compound")
+    if (m_Type == DataType::Compound)
     {
         // not supported
     }
 #define declare_template_instantiation(T)                                      \
-    else if (m_Type == helper::GetType())                                   \
+    else if (m_Type == helper::GetDataType())                               \
     {                                                                          \
         Variable *variable = dynamic_cast *>(this);             \
         m_Shape = variable->Shape(step);                                       \
diff --git a/source/adios2/core/VariableBase.h b/source/adios2/core/VariableBase.h
index 716debd6c9..150423c93c 100644
--- a/source/adios2/core/VariableBase.h
+++ b/source/adios2/core/VariableBase.h
@@ -41,7 +41,7 @@ class VariableBase
     const std::string m_Name;
 
     /** primitive from  or compound from struct */
-    const std::string m_Type;
+    const DataType m_Type;
 
     /** Variable -> sizeof(T),
      *  VariableCompound -> from constructor sizeof(struct) */
@@ -112,7 +112,7 @@ class VariableBase
     std::set m_PrefixedVariables;
     std::set m_PrefixedAttributes;
 
-    VariableBase(const std::string &name, const std::string type,
+    VariableBase(const std::string &name, const DataType type,
                  const size_t elementSize, const Dims &shape, const Dims &start,
                  const Dims &count, const bool constantShape);
 
diff --git a/source/adios2/core/VariableCompound.cpp b/source/adios2/core/VariableCompound.cpp
index 633be12147..b4e4bd5d19 100644
--- a/source/adios2/core/VariableCompound.cpp
+++ b/source/adios2/core/VariableCompound.cpp
@@ -22,7 +22,8 @@ VariableCompound::VariableCompound(const std::string &name,
                                    const size_t structSize, const Dims &shape,
                                    const Dims &start, const Dims &count,
                                    const bool constantDims)
-: VariableBase(name, "compound", structSize, shape, start, count, constantDims)
+: VariableBase(name, DataType::Compound, structSize, shape, start, count,
+               constantDims)
 {
 }
 
diff --git a/source/adios2/core/VariableCompound.h b/source/adios2/core/VariableCompound.h
index f1d99b21d0..ebe7e5b90f 100644
--- a/source/adios2/core/VariableCompound.h
+++ b/source/adios2/core/VariableCompound.h
@@ -14,6 +14,7 @@
 #include "VariableBase.h"
 
 #include "adios2/common/ADIOSMacros.h"
+#include "adios2/common/ADIOSTypes.h"
 
 namespace adios2
 {
@@ -34,8 +35,8 @@ class VariableCompound : public VariableBase
     struct Element
     {
         const std::string Name;
-        const std::string Type; ///< from GetType
-        const size_t Offset;    ///< element offset in struct
+        const adios2::DataType Type; ///< from GetDataType
+        const size_t Offset;         ///< element offset in struct
     };
 
     /** vector of primitve element types defining compound struct */
diff --git a/source/adios2/core/VariableCompound.tcc b/source/adios2/core/VariableCompound.tcc
index 62d2ad3559..9322c1989b 100644
--- a/source/adios2/core/VariableCompound.tcc
+++ b/source/adios2/core/VariableCompound.tcc
@@ -13,7 +13,7 @@
 
 #include "VariableCompound.h"
 
-#include "adios2/helper/adiosFunctions.h" //GetType
+#include "adios2/helper/adiosFunctions.h" //GetDataType
 
 namespace adios2
 {
@@ -24,7 +24,7 @@ template 
 void VariableCompound::InsertMember(const std::string &name,
                                     const size_t offset)
 {
-    m_Elements.push_back(Element{name, helper::GetType(), offset});
+    m_Elements.push_back(Element{name, helper::GetDataType(), offset});
 }
 
 } // end namespace core
diff --git a/source/adios2/engine/bp3/BP3Reader.cpp b/source/adios2/engine/bp3/BP3Reader.cpp
index 211196f495..d3fc51acfa 100644
--- a/source/adios2/engine/bp3/BP3Reader.cpp
+++ b/source/adios2/engine/bp3/BP3Reader.cpp
@@ -92,13 +92,13 @@ void BP3Reader::PerformGets()
 
     for (const std::string &name : m_BP3Deserializer.m_DeferredVariables)
     {
-        const std::string type = m_IO.InquireVariableType(name);
+        const DataType type = m_IO.InquireVariableType(name);
 
-        if (type == "compound")
+        if (type == DataType::Compound)
         {
         }
 #define declare_type(T)                                                        \
-    else if (type == helper::GetType())                                     \
+    else if (type == helper::GetDataType())                                 \
     {                                                                          \
         Variable &variable =                                                \
             FindVariable(name, "in call to PerformGets, EndStep or Close"); \
diff --git a/source/adios2/engine/bp3/BP3Writer.cpp b/source/adios2/engine/bp3/BP3Writer.cpp
index 03c26d133d..76be495611 100644
--- a/source/adios2/engine/bp3/BP3Writer.cpp
+++ b/source/adios2/engine/bp3/BP3Writer.cpp
@@ -62,13 +62,13 @@ void BP3Writer::PerformPuts()
 
     for (const std::string &variableName : m_BP3Serializer.m_DeferredVariables)
     {
-        const std::string type = m_IO.InquireVariableType(variableName);
-        if (type == "compound")
+        const DataType type = m_IO.InquireVariableType(variableName);
+        if (type == DataType::Compound)
         {
             // not supported
         }
 #define declare_template_instantiation(T)                                      \
-    else if (type == helper::GetType())                                     \
+    else if (type == helper::GetDataType())                                 \
     {                                                                          \
         Variable &variable = FindVariable(                               \
             variableName, "in call to PerformPuts, EndStep or Close");         \
@@ -79,6 +79,7 @@ void BP3Writer::PerformPuts()
 #undef declare_template_instantiation
     }
     m_BP3Serializer.m_DeferredVariables.clear();
+    m_BP3Serializer.m_DeferredVariablesDataSize = 0;
 }
 
 void BP3Writer::EndStep()
@@ -117,6 +118,12 @@ void BP3Writer::Flush(const int transportIndex)
 void BP3Writer::Init()
 {
     InitParameters();
+    if (m_BP3Serializer.m_Parameters.NumAggregators <
+        static_cast(m_BP3Serializer.m_SizeMPI))
+    {
+        m_BP3Serializer.m_Aggregator.Init(
+            m_BP3Serializer.m_Parameters.NumAggregators, m_Comm);
+    }
     InitTransports();
     InitBPBuffer();
 }
@@ -180,6 +187,7 @@ void BP3Writer::InitTransports()
 
     m_BP3Serializer.m_Profiler.Start("mkdir");
     m_FileDataManager.MkDirsBarrier(bpSubStreamNames,
+                                    m_IO.m_TransportsParameters,
                                     m_BP3Serializer.m_Parameters.NodeLocal);
     m_BP3Serializer.m_Profiler.Stop("mkdir");
 
@@ -202,8 +210,9 @@ void BP3Writer::InitBPBuffer()
 {
     if (m_OpenMode == Mode::Append)
     {
-        throw std::invalid_argument(
-            "ADIOS2: OpenMode Append hasn't been implemented, yet");
+        throw std::invalid_argument("ADIOS2: Mode::Append is only available in "
+                                    "BP4; it is not implemented "
+                                    "for BP3 files.");
         // TODO: Get last pg timestep and update timestep counter in
     }
     else
@@ -260,6 +269,17 @@ void BP3Writer::WriteProfilingJSONFile()
 {
     TAU_SCOPED_TIMER("BP3Writer::WriteProfilingJSONFile");
     auto transportTypes = m_FileDataManager.GetTransportsTypes();
+
+    // find first File type output, where we can write the profile
+    int fileTransportIdx = -1;
+    for (size_t i = 0; i < transportTypes.size(); ++i)
+    {
+        if (transportTypes[i].compare(0, 4, "File") == 0)
+        {
+            fileTransportIdx = static_cast(i);
+        }
+    }
+
     auto transportProfilers = m_FileDataManager.GetTransportsProfilers();
 
     auto transportTypesMD = m_FileMetadataManager.GetTransportsTypes();
@@ -282,9 +302,24 @@ void BP3Writer::WriteProfilingJSONFile()
     if (m_BP3Serializer.m_RankMPI == 0)
     {
         transport::FileFStream profilingJSONStream(m_Comm);
-        auto bpBaseNames = m_BP3Serializer.GetBPBaseNames({m_Name});
-        profilingJSONStream.Open(bpBaseNames[0] + "/profiling.json",
-                                 Mode::Write);
+        std::string profileFileName;
+        if (fileTransportIdx > -1)
+        {
+            // write profile to .dir/profiling.json
+            auto bpBaseNames = m_BP3Serializer.GetBPBaseNames({m_Name});
+            profileFileName = bpBaseNames[fileTransportIdx] + "/profiling.json";
+        }
+        else
+        {
+            // write profile to _profiling.json
+            auto transportsNames = m_FileMetadataManager.GetFilesBaseNames(
+                m_Name, m_IO.m_TransportsParameters);
+
+            auto bpMetadataFileNames =
+                m_BP3Serializer.GetBPMetadataFileNames(transportsNames);
+            profileFileName = bpMetadataFileNames[0] + "_profiling.json";
+        }
+        profilingJSONStream.Open(profileFileName, Mode::Write);
         profilingJSONStream.Write(profilingJSON.data(), profilingJSON.size());
         profilingJSONStream.Close();
     }
@@ -413,6 +448,10 @@ void BP3Writer::AggregateWriteData(const bool isFinal, const int transportIndex)
 ADIOS2_FOREACH_PRIMITVE_STDTYPE_2ARGS(declare_type)
 #undef declare_type
 
+size_t BP3Writer::DebugGetDataBufferSize() const
+{
+    return m_BP3Serializer.DebugGetDataBufferSize();
+}
 } // end namespace engine
 } // end namespace core
 } // end namespace adios2
diff --git a/source/adios2/engine/bp3/BP3Writer.h b/source/adios2/engine/bp3/BP3Writer.h
index 976e207a2e..6529eab47e 100644
--- a/source/adios2/engine/bp3/BP3Writer.h
+++ b/source/adios2/engine/bp3/BP3Writer.h
@@ -46,6 +46,8 @@ class BP3Writer : public core::Engine
     void EndStep() final;
     void Flush(const int transportIndex = -1) final;
 
+    size_t DebugGetDataBufferSize() const final;
+
 private:
     /** Single object controlling BP buffering */
     format::BP3Serializer m_BP3Serializer;
@@ -85,7 +87,8 @@ class BP3Writer : public core::Engine
 
     template 
     void PutSyncCommon(Variable &variable,
-                       const typename Variable::Info &blockInfo);
+                       const typename Variable::Info &blockInfo,
+                       const bool resize = true);
 
     template 
     void PutDeferredCommon(Variable &variable, const T *data);
diff --git a/source/adios2/engine/bp3/BP3Writer.tcc b/source/adios2/engine/bp3/BP3Writer.tcc
index 16ddd7b4d2..6ab0b8ef5f 100644
--- a/source/adios2/engine/bp3/BP3Writer.tcc
+++ b/source/adios2/engine/bp3/BP3Writer.tcc
@@ -63,15 +63,21 @@ void BP3Writer::PutCommon(Variable &variable,
 
 template 
 void BP3Writer::PutSyncCommon(Variable &variable,
-                              const typename Variable::Info &blockInfo)
+                              const typename Variable::Info &blockInfo,
+                              const bool resize)
 {
-    const size_t dataSize =
-        helper::PayloadSize(blockInfo.Data, blockInfo.Count) +
-        m_BP3Serializer.GetBPIndexSizeInData(variable.m_Name, blockInfo.Count);
+    format::BP3Base::ResizeResult resizeResult =
+        format::BP3Base::ResizeResult::Success;
+    if (resize)
+    {
+        const size_t dataSize =
+            helper::PayloadSize(blockInfo.Data, blockInfo.Count) +
+            m_BP3Serializer.GetBPIndexSizeInData(variable.m_Name,
+                                                 blockInfo.Count);
 
-    const format::BP3Base::ResizeResult resizeResult =
-        m_BP3Serializer.ResizeBuffer(dataSize, "in call to variable " +
-                                                   variable.m_Name + " Put");
+        resizeResult = m_BP3Serializer.ResizeBuffer(
+            dataSize, "in call to variable " + variable.m_Name + " Put");
+    }
 
     // if first timestep Write create a new pg index or in time aggregation
     if (!m_BP3Serializer.m_MetadataSet.DataPGIsOpen)
@@ -133,7 +139,7 @@ void BP3Writer::PerformPutCommon(Variable &variable)
         auto itSpanBlock = variable.m_BlocksSpan.find(b);
         if (itSpanBlock == variable.m_BlocksSpan.end())
         {
-            PutSyncCommon(variable, variable.m_BlocksInfo[b]);
+            PutSyncCommon(variable, variable.m_BlocksInfo[b], false);
         }
         else
         {
diff --git a/source/adios2/engine/bp4/BP4Reader.cpp b/source/adios2/engine/bp4/BP4Reader.cpp
index b9afaa2703..ed3a276bfe 100644
--- a/source/adios2/engine/bp4/BP4Reader.cpp
+++ b/source/adios2/engine/bp4/BP4Reader.cpp
@@ -115,13 +115,13 @@ void BP4Reader::PerformGets()
 
     for (const std::string &name : m_BP4Deserializer.m_DeferredVariables)
     {
-        const std::string type = m_IO.InquireVariableType(name);
+        const DataType type = m_IO.InquireVariableType(name);
 
-        if (type == "compound")
+        if (type == DataType::Compound)
         {
         }
 #define declare_type(T)                                                        \
-    else if (type == helper::GetType())                                     \
+    else if (type == helper::GetDataType())                                 \
     {                                                                          \
         Variable &variable =                                                \
             FindVariable(name, "in call to PerformGets, EndStep or Close"); \
@@ -157,28 +157,79 @@ void BP4Reader::Init()
     const Seconds timeoutSeconds =
         Seconds(m_BP4Deserializer.m_Parameters.OpenTimeoutSecs);
 
-    // set poll to 1/100 of timeout
-    Seconds pollSeconds = timeoutSeconds / 100.0;
-    static const auto pollSecondsMin = Seconds(1.0);
-    if (pollSeconds < pollSecondsMin)
-    {
-        pollSeconds = pollSecondsMin;
-    }
-    static const auto pollSecondsMax = Seconds(10.0);
-    if (pollSeconds > pollSecondsMax)
+    Seconds pollSeconds =
+        Seconds(m_BP4Deserializer.m_Parameters.BeginStepPollingFrequencySecs);
+    if (pollSeconds > timeoutSeconds)
     {
-        pollSeconds = pollSecondsMax;
+        pollSeconds = timeoutSeconds;
     }
 
-    const TimePoint timeoutInstant =
+    TimePoint timeoutInstant =
         std::chrono::steady_clock::now() + timeoutSeconds;
 
     OpenFiles(timeoutInstant, pollSeconds, timeoutSeconds);
-    InitBuffer(timeoutInstant, pollSeconds / 10, timeoutSeconds);
+    if (!m_BP4Deserializer.m_Parameters.StreamReader)
+    {
+        /* non-stream reader gets as much steps as available now */
+        InitBuffer(timeoutInstant, pollSeconds / 10, timeoutSeconds);
+    }
+}
+
+bool BP4Reader::SleepOrQuit(const TimePoint &timeoutInstant,
+                            const Seconds &pollSeconds)
+{
+    auto now = std::chrono::steady_clock::now();
+    if (now + pollSeconds >= timeoutInstant)
+    {
+        return false;
+    }
+    auto remainderTime = timeoutInstant - now;
+    auto sleepTime = pollSeconds;
+    if (remainderTime < sleepTime)
+    {
+        sleepTime = remainderTime;
+    }
+    std::this_thread::sleep_for(sleepTime);
+    return true;
 }
 
-void BP4Reader::OpenFiles(const TimePoint &timeoutInstant,
-                          const Seconds &pollSeconds,
+size_t BP4Reader::OpenWithTimeout(transportman::TransportMan &tm,
+                                  const std::vector &fileNames,
+                                  const TimePoint &timeoutInstant,
+                                  const Seconds &pollSeconds,
+                                  std::string &lasterrmsg /*INOUT*/)
+{
+    size_t flag = 1; // 0 = OK, opened file, 1 = timeout, 2 = error
+    do
+    {
+        try
+        {
+            errno = 0;
+            const bool profile = m_BP4Deserializer.m_Profiler.m_IsActive;
+            tm.OpenFiles(fileNames, adios2::Mode::Read,
+                         m_IO.m_TransportsParameters, profile);
+            flag = 0; // found file
+            break;
+        }
+        catch (std::ios_base::failure &e)
+        {
+            lasterrmsg =
+                std::string("errno=" + std::to_string(errno) + ": " + e.what());
+            if (errno == ENOENT)
+            {
+                flag = 1; // timeout
+            }
+            else
+            {
+                flag = 2; // fatal error
+                break;
+            }
+        }
+    } while (SleepOrQuit(timeoutInstant, pollSeconds));
+    return flag;
+}
+
+void BP4Reader::OpenFiles(TimePoint &timeoutInstant, const Seconds &pollSeconds,
                           const Seconds &timeoutSeconds)
 {
     /* Poll */
@@ -186,44 +237,36 @@ void BP4Reader::OpenFiles(const TimePoint &timeoutInstant,
     std::string lasterrmsg;
     if (m_BP4Deserializer.m_RankMPI == 0)
     {
-        do
+        /* Open the metadata index table */
+        const std::string metadataIndexFile(
+            m_BP4Deserializer.GetBPMetadataIndexFileName(m_Name));
+
+        flag = OpenWithTimeout(m_MDIndexFileManager, {metadataIndexFile},
+                               timeoutInstant, pollSeconds, lasterrmsg);
+        if (flag == 0)
         {
-            try
+            /* Open the metadata file */
+            const std::string metadataFile(
+                m_BP4Deserializer.GetBPMetadataFileName(m_Name));
+
+            /* We found md.idx. If we don't find md.0 immediately  we should
+             * wait a little bit hoping for the file system to catch up.
+             * This slows down finding the error in file reading mode but
+             * it will be more robust in streaming mode
+             */
+            if (timeoutSeconds == Seconds(0.0))
             {
-                errno = 0;
-                const bool profile = m_BP4Deserializer.m_Profiler.m_IsActive;
-                /* Open the metadata index table */
-                const std::string metadataIndexFile(
-                    m_BP4Deserializer.GetBPMetadataIndexFileName(m_Name));
-                m_MDIndexFileManager.OpenFiles(
-                    {metadataIndexFile}, adios2::Mode::Read,
-                    m_IO.m_TransportsParameters, profile);
-
-                /* Open the metadata file */
-                const std::string metadataFile(
-                    m_BP4Deserializer.GetBPMetadataFileName(m_Name));
-
-                m_MDFileManager.OpenFiles({metadataFile}, adios2::Mode::Read,
-                                          m_IO.m_TransportsParameters, profile);
-                flag = 0; // found file
-                break;
+                timeoutInstant += Seconds(5.0);
             }
-            catch (std::ios_base::failure &e)
+
+            flag = OpenWithTimeout(m_MDFileManager, {metadataFile},
+                                   timeoutInstant, pollSeconds, lasterrmsg);
+            if (flag != 0)
             {
-                lasterrmsg = std::string(e.what());
-                if (errno == ENOENT)
-                {
-                    flag = 1; // timeout
-                }
-                else
-                {
-                    flag = 2; // fatal error
-                    break;
-                }
+                /* Close the metadata index table */
+                m_MDIndexFileManager.CloseFiles();
             }
-
-            std::this_thread::sleep_for(pollSeconds);
-        } while (std::chrono::steady_clock::now() < timeoutInstant);
+        }
     }
 
     flag = m_Comm.BroadcastValue(flag, 0);
@@ -272,6 +315,71 @@ void BP4Reader::InitTransports()
     }
 }
 
+/* Count index records to minimum 1 and maximum of N records so that
+ * expected metadata size is less then a predetermined constant
+ */
+void MetadataCalculateMinFileSize(
+    const format::BP4Deserializer &m_BP4Deserializer,
+    const std::string &IdxFileName, char *buf, size_t idxsize, bool hasHeader,
+    const size_t mdStartPos, size_t &newIdxSize, size_t &expectedMinFileSize)
+{
+    newIdxSize = 0;
+    expectedMinFileSize = 0;
+
+    if (hasHeader && idxsize < m_BP4Deserializer.m_IndexRecordSize)
+    {
+        return;
+    }
+
+    /* eliminate header for now for only calculating with records */
+    if (hasHeader)
+    {
+        buf += m_BP4Deserializer.m_IndexRecordSize;
+        idxsize -= m_BP4Deserializer.m_IndexRecordSize;
+    }
+
+    if (idxsize % m_BP4Deserializer.m_IndexRecordSize != 0)
+    {
+        throw std::runtime_error(
+            "FATAL CODING ERROR: ADIOS Index file " + IdxFileName +
+            " is assumed to always contain n*" +
+            std::to_string(m_BP4Deserializer.m_IndexRecordSize) +
+            " byte-length records. "
+            "Right now the length of index buffer is " +
+            std::to_string(idxsize) + " bytes.");
+    }
+
+    const size_t nTotalRecords = idxsize / m_BP4Deserializer.m_IndexRecordSize;
+    if (nTotalRecords == 0)
+    {
+        // no (new) step entry in the index, so no metadata is expected
+        newIdxSize = 0;
+        expectedMinFileSize = 0;
+        return;
+    }
+
+    size_t nRecords = 1;
+    expectedMinFileSize = *(uint64_t *)&(
+        buf[nRecords * m_BP4Deserializer.m_IndexRecordSize - 24]);
+    while (nRecords < nTotalRecords)
+    {
+        const size_t n = nRecords + 1;
+        const uint64_t mdEndPos =
+            *(uint64_t *)&(buf[n * m_BP4Deserializer.m_IndexRecordSize - 24]);
+        if (mdEndPos - mdStartPos > 16777216)
+        {
+            break;
+        }
+        expectedMinFileSize = mdEndPos;
+        ++nRecords;
+    }
+    newIdxSize = nRecords * m_BP4Deserializer.m_IndexRecordSize;
+    if (hasHeader)
+    {
+        newIdxSize += m_BP4Deserializer.m_IndexRecordSize;
+    }
+}
+
 uint64_t
 MetadataExpectedMinFileSize(const format::BP4Deserializer &m_BP4Deserializer,
                             const std::string &IdxFileName, bool hasHeader)
@@ -285,7 +393,9 @@ MetadataExpectedMinFileSize(const format::BP4Deserializer &m_BP4Deserializer,
             "The file size now is " +
             std::to_string(idxsize) + " bytes.");
     }
-    if ((hasHeader && idxsize < 128) || idxsize < 64)
+    if ((hasHeader && idxsize < m_BP4Deserializer.m_IndexHeaderSize +
+                                    m_BP4Deserializer.m_IndexRecordSize) ||
+        idxsize < m_BP4Deserializer.m_IndexRecordSize)
     {
         // no (new) step entry in the index, so no metadata is expected
         return 0;
@@ -299,7 +409,7 @@ void BP4Reader::InitBuffer(const TimePoint &timeoutInstant,
                            const Seconds &pollSeconds,
                            const Seconds &timeoutSeconds)
 {
-    std::vector sizes(2, 0);
+    size_t newIdxSize = 0;
     // Put all metadata in buffer
     if (m_BP4Deserializer.m_RankMPI == 0)
     {
@@ -327,8 +437,7 @@ void BP4Reader::InitBuffer(const TimePoint &timeoutInstant,
                 {
                     break;
                 }
-                std::this_thread::sleep_for(pollSeconds);
-            } while (std::chrono::steady_clock::now() < timeoutInstant);
+            } while (SleepOrQuit(timeoutInstant, pollSeconds));
 
             if (fileSize >= expectedMinFileSize)
             {
@@ -339,10 +448,9 @@ void BP4Reader::InitBuffer(const TimePoint &timeoutInstant,
                 m_MDFileManager.ReadFile(
                     m_BP4Deserializer.m_Metadata.m_Buffer.data(),
                     expectedMinFileSize);
-                m_MDIndexFileProcessedSize = metadataIndexFileSize;
-
-                sizes[0] = metadataIndexFileSize;
-                sizes[1] = m_MDFileProcessedSize;
+                m_MDFileAlreadyReadSize = expectedMinFileSize;
+                m_MDIndexFileAlreadyReadSize = metadataIndexFileSize;
+                newIdxSize = metadataIndexFileSize;
             }
             else
             {
@@ -351,13 +459,18 @@ void BP4Reader::InitBuffer(const TimePoint &timeoutInstant,
                     " was found with an index file but md.0 "
                     "has not contained enough data within "
                     "the specified timeout of " +
-                    std::to_string(timeoutSeconds.count()) + " seconds.");
+                    std::to_string(timeoutSeconds.count()) +
+                    " seconds. index size = " +
+                    std::to_string(metadataIndexFileSize) +
+                    " metadata size = " + std::to_string(fileSize) +
+                    " expected size = " + std::to_string(expectedMinFileSize) +
+                    ". One reason could be if the reader finds old data while "
+                    "the writer is creating the new files.");
             }
         }
     }
 
-    m_Comm.BroadcastVector(sizes, 0);
-    size_t newIdxSize = sizes[0];
+    newIdxSize = m_Comm.BroadcastValue(newIdxSize, 0);
 
     if (newIdxSize > 0)
     {
@@ -369,7 +482,7 @@ void BP4Reader::InitBuffer(const TimePoint &timeoutInstant,
 
         /* Parse metadata index table */
         m_BP4Deserializer.ParseMetadataIndex(m_BP4Deserializer.m_MetadataIndex,
-                                             0, true);
+                                             0, true, false);
         // now we are sure the index header has been parsed, first step parsing
         // done
         m_IdxHeaderParsed = true;
@@ -394,29 +507,42 @@ void BP4Reader::InitBuffer(const TimePoint &timeoutInstant,
 size_t BP4Reader::UpdateBuffer(const TimePoint &timeoutInstant,
                                const Seconds &pollSeconds)
 {
-    std::vector sizes(2, 0);
+    std::vector sizes(3, 0);
     if (m_BP4Deserializer.m_RankMPI == 0)
     {
         const size_t idxFileSize = m_MDIndexFileManager.GetFileSize(0);
-        if (idxFileSize > m_MDIndexFileProcessedSize)
+        if (idxFileSize > m_MDIndexFileAlreadyReadSize)
         {
-            const size_t newIdxSize = idxFileSize - m_MDIndexFileProcessedSize;
+            const size_t maxIdxSize =
+                idxFileSize - m_MDIndexFileAlreadyReadSize;
+            std::vector idxbuf(maxIdxSize);
+            m_MDIndexFileManager.ReadFile(idxbuf.data(), maxIdxSize,
+                                          m_MDIndexFileAlreadyReadSize);
+            size_t newIdxSize;
+            size_t expectedMinFileSize;
+            char *buf = idxbuf.data();
+
+            MetadataCalculateMinFileSize(
+                m_BP4Deserializer, m_Name, buf, maxIdxSize, !m_IdxHeaderParsed,
+                m_MDFileAlreadyReadSize, newIdxSize, expectedMinFileSize);
+
+            // const uint64_t expectedMinFileSize = MetadataExpectedMinFileSize(
+            //    m_BP4Deserializer, m_Name, !m_IdxHeaderParsed);
+
             if (m_BP4Deserializer.m_MetadataIndex.m_Buffer.size() < newIdxSize)
             {
                 m_BP4Deserializer.m_MetadataIndex.Resize(
                     newIdxSize, "re-allocating metadata index buffer, in "
                                 "call to BP4Reader::BeginStep/UpdateBuffer");
             }
-            m_BP4Deserializer.m_MetadataIndex.m_Position = 0;
-            m_MDIndexFileManager.ReadFile(
-                m_BP4Deserializer.m_MetadataIndex.m_Buffer.data(), newIdxSize,
-                m_MDIndexFileProcessedSize);
+            m_BP4Deserializer.m_MetadataIndex.Reset(true, false);
+            std::copy(idxbuf.begin(), idxbuf.begin() + newIdxSize,
+                      m_BP4Deserializer.m_MetadataIndex.m_Buffer.begin());
 
             /* Wait until as much metadata arrives in the file as much
              * is indicated by the existing index entries
              */
-            uint64_t expectedMinFileSize = MetadataExpectedMinFileSize(
-                m_BP4Deserializer, m_Name, !m_IdxHeaderParsed);
+
             size_t fileSize = 0;
             do
             {
@@ -425,8 +551,7 @@ size_t BP4Reader::UpdateBuffer(const TimePoint &timeoutInstant,
                 {
                     break;
                 }
-                std::this_thread::sleep_for(pollSeconds);
-            } while (std::chrono::steady_clock::now() < timeoutInstant);
+            } while (SleepOrQuit(timeoutInstant, pollSeconds));
 
             if (fileSize >= expectedMinFileSize)
             {
@@ -437,20 +562,27 @@ size_t BP4Reader::UpdateBuffer(const TimePoint &timeoutInstant,
                  * the buffer now.
                  */
                 const size_t fileSize = m_MDFileManager.GetFileSize(0);
-                const size_t newMDSize = fileSize - m_MDFileProcessedSize;
+                const size_t newMDSize =
+                    expectedMinFileSize - m_MDFileAlreadyReadSize;
                 if (m_BP4Deserializer.m_Metadata.m_Buffer.size() < newMDSize)
                 {
                     m_BP4Deserializer.m_Metadata.Resize(
                         newMDSize, "allocating metadata buffer, in call to "
                                    "BP4Reader Open");
                 }
-                m_BP4Deserializer.m_Metadata.m_Position = 0;
+                m_BP4Deserializer.m_Metadata.Reset(true, false);
                 m_MDFileManager.ReadFile(
                     m_BP4Deserializer.m_Metadata.m_Buffer.data(), newMDSize,
-                    m_MDFileProcessedSize);
+                    m_MDFileAlreadyReadSize);
+
+                m_MDFileAbsolutePos = m_MDFileAlreadyReadSize;
+                m_MDFileAlreadyReadSize = expectedMinFileSize;
+
+                m_MDIndexFileAlreadyReadSize += newIdxSize;
 
                 sizes[0] = newIdxSize;
-                sizes[1] = m_MDFileProcessedSize;
+                sizes[1] = m_MDFileAlreadyReadSize;
+                sizes[2] = m_MDFileAbsolutePos;
             }
         }
     }
@@ -460,18 +592,21 @@ size_t BP4Reader::UpdateBuffer(const TimePoint &timeoutInstant,
 
     if (newIdxSize > 0)
     {
-        // broadcast buffer to all ranks from zero
-        m_Comm.BroadcastVector(m_BP4Deserializer.m_Metadata.m_Buffer);
-
-        // broadcast metadata index buffer to all ranks from zero
-        m_Comm.BroadcastVector(m_BP4Deserializer.m_MetadataIndex.m_Buffer);
-
         if (m_BP4Deserializer.m_RankMPI != 0)
         {
-            m_MDFileProcessedSize = sizes[1];
+            m_MDFileAlreadyReadSize = sizes[1];
+            m_MDFileAbsolutePos = sizes[2];
+            m_BP4Deserializer.m_MetadataIndex.Reset(true, false);
+            m_BP4Deserializer.m_Metadata.Reset(true, false);
             // we need this pointer in Metadata buffer on all processes
             // for parsing it correctly in ProcessMetadataForNewSteps()
         }
+
+        // broadcast buffer to all ranks from zero
+        m_Comm.BroadcastVector(m_BP4Deserializer.m_Metadata.m_Buffer);
+
+        // broadcast metadata index buffer to all ranks from zero
+        m_Comm.BroadcastVector(m_BP4Deserializer.m_MetadataIndex.m_Buffer);
     }
     return newIdxSize;
 }
@@ -486,8 +621,8 @@ void BP4Reader::ProcessMetadataForNewSteps(const size_t newIdxSize)
        size of the already-processed metadata because the memory buffer of
        new metadata starts from 0 */
     m_BP4Deserializer.ParseMetadataIndex(m_BP4Deserializer.m_MetadataIndex,
-                                         m_MDFileProcessedSize,
-                                         !m_IdxHeaderParsed);
+                                         m_MDFileAbsolutePos,
+                                         !m_IdxHeaderParsed, true);
     m_IdxHeaderParsed = true;
 
     // fills IO with Variables and Attributes
@@ -495,14 +630,11 @@ void BP4Reader::ProcessMetadataForNewSteps(const size_t newIdxSize)
         m_BP4Deserializer.m_Metadata, *this, false);
 
     // remember current end position in metadata and index table for next round
-    m_MDFileProcessedSize += newProcessedMDSize;
-    if (m_BP4Deserializer.m_RankMPI == 0)
-    {
-        m_MDIndexFileProcessedSize += newIdxSize;
-    }
-    size_t idxsize = m_BP4Deserializer.m_MetadataIndex.m_Buffer.size();
-    uint64_t lastpos = *(uint64_t *)&(
-        m_BP4Deserializer.m_MetadataIndex.m_Buffer[idxsize - 24]);
+    m_MDFileProcessedSize = m_MDFileAbsolutePos + newProcessedMDSize;
+    // if (m_BP4Deserializer.m_RankMPI == 0)
+    //{
+    //    m_MDIndexFileAlreadyReadSize += newIdxSize;
+    //}
 }
 
 bool BP4Reader::CheckWriterActive()
@@ -510,8 +642,9 @@ bool BP4Reader::CheckWriterActive()
     size_t flag = 0;
     if (m_BP4Deserializer.m_RankMPI == 0)
     {
-        std::vector header(64, '\0');
-        m_MDIndexFileManager.ReadFile(header.data(), 64, 0, 0);
+        std::vector header(m_BP4Deserializer.m_IndexHeaderSize, '\0');
+        m_MDIndexFileManager.ReadFile(
+            header.data(), m_BP4Deserializer.m_IndexHeaderSize, 0, 0);
         bool active = m_BP4Deserializer.ReadActiveFlag(header);
         flag = (active ? 1 : 0);
     }
@@ -520,12 +653,32 @@ bool BP4Reader::CheckWriterActive()
     return m_WriterIsActive;
 }
 
+bool BP4Reader::ProcessNextStepInMemory()
+{
+    if (m_MDFileAlreadyReadSize > m_MDFileProcessedSize)
+    {
+        // Hack: processing metadata for multiple new steps only works
+        // when pretending not to be in streaming mode
+        const bool saveReadStreaming = m_IO.m_ReadStreaming;
+        m_IO.m_ReadStreaming = false;
+        ProcessMetadataForNewSteps(0);
+        m_IO.m_ReadStreaming = saveReadStreaming;
+        return true;
+    }
+    return false;
+}
+
 StepStatus BP4Reader::CheckForNewSteps(Seconds timeoutSeconds)
 {
     /* Do a collective wait for a step within timeout.
        Make sure every reader comes to the same conclusion */
     StepStatus retval = StepStatus::OK;
 
+    if (ProcessNextStepInMemory())
+    {
+        return retval;
+    }
+
     if (timeoutSeconds < Seconds::zero())
     {
         timeoutSeconds = Seconds(999999999); // max 1 billion seconds wait
@@ -545,10 +698,10 @@ StepStatus BP4Reader::CheckForNewSteps(Seconds timeoutSeconds)
     // Hack: processing metadata for multiple new steps only works
     // when pretending not to be in streaming mode
     const bool saveReadStreaming = m_IO.m_ReadStreaming;
+    m_IO.m_ReadStreaming = false;
     size_t newIdxSize = 0;
 
-    m_IO.m_ReadStreaming = false;
-    while (m_WriterIsActive)
+    do
     {
         newIdxSize = UpdateBuffer(timeoutInstant, pollSeconds / 10);
         if (newIdxSize > 0)
@@ -564,12 +717,7 @@ StepStatus BP4Reader::CheckForNewSteps(Seconds timeoutSeconds)
             newIdxSize = UpdateBuffer(timeoutInstant, pollSeconds / 10);
             break;
         }
-        std::this_thread::sleep_for(pollSeconds);
-        if (std::chrono::steady_clock::now() >= timeoutInstant)
-        {
-            break;
-        }
-    }
+    } while (SleepOrQuit(timeoutInstant, pollSeconds));
 
     if (newIdxSize > 0)
     {
diff --git a/source/adios2/engine/bp4/BP4Reader.h b/source/adios2/engine/bp4/BP4Reader.h
index 4136a4fcbb..d5d546c837 100644
--- a/source/adios2/engine/bp4/BP4Reader.h
+++ b/source/adios2/engine/bp4/BP4Reader.h
@@ -61,14 +61,26 @@ class BP4Reader : public Engine
     format::BP4Deserializer m_BP4Deserializer;
     /* transport manager for metadata file */
     transportman::TransportMan m_MDFileManager;
+    /* How many bytes of metadata have we already read in? */
+    size_t m_MDFileAlreadyReadSize = 0;
+    /* How many bytes of metadata have we already processed?
+     * It is <= m_MDFileAlreadyReadSize, at = we need to read more */
     size_t m_MDFileProcessedSize = 0;
+    /* The file position of the first byte that is currently
+     * residing in memory. Needed for skewing positions when
+     * processing metadata index.
+     */
+    size_t m_MDFileAbsolutePos = 0;
+    /* m_MDFileAbsolutePos <= m_MDFileProcessedSize <= m_MDFileAlreadyReadSize
+     */
 
     /* transport manager for managing data file(s) */
     transportman::TransportMan m_DataFileManager;
 
     /* transport manager for managing the metadata index file */
     transportman::TransportMan m_MDIndexFileManager;
-    size_t m_MDIndexFileProcessedSize = 0;
+    /* How many bytes of metadata index have we already read in? */
+    size_t m_MDIndexFileAlreadyReadSize = 0;
 
     /* transport manager for managing the active flag file */
     transportman::TransportMan m_ActiveFlagFileManager;
@@ -82,10 +94,26 @@ class BP4Reader : public Engine
     void Init();
     void InitTransports();
 
+    /* Sleep up to pollSeconds time if we have not reached timeoutInstant.
+     * Return true if slept
+     * return false if sleep was not needed because it was overtime
+     */
+    bool SleepOrQuit(const TimePoint &timeoutInstant,
+                     const Seconds &pollSeconds);
+    /** Open one category of files within timeout.
+     * @return: 0 = OK, 1 = timeout, 2 = error
+     * lasterrmsg contains the error message in case of error
+     */
+    size_t OpenWithTimeout(transportman::TransportMan &tm,
+                           const std::vector &fileNames,
+                           const TimePoint &timeoutInstant,
+                           const Seconds &pollSeconds,
+                           std::string &lasterrmsg /*INOUT*/);
+
     /** Open files within timeout.
      * @return True if files are opened, False in case of timeout
      */
-    void OpenFiles(const TimePoint &timeoutInstant, const Seconds &pollSeconds,
+    void OpenFiles(TimePoint &timeoutInstant, const Seconds &pollSeconds,
                    const Seconds &timeoutSeconds);
     void InitBuffer(const TimePoint &timeoutInstant, const Seconds &pollSeconds,
                     const Seconds &timeoutSeconds);
@@ -108,6 +136,14 @@ class BP4Reader : public Engine
      */
     bool CheckWriterActive();
 
+    /** Check for a step that is already in memory but haven't
+     * been processed yet.
+     *  @return true: if new step has been found and processed, false otherwise
+     *  Used by CheckForNewSteps() to get the next step from memory if there is
+     * one.
+     */
+    bool ProcessNextStepInMemory();
+
     /** Check for new steps withing timeout and only if writer is active.
      *  @return the status flag
      *  Used by BeginStep() to get new steps from file when it reaches the
diff --git a/source/adios2/engine/bp4/BP4Writer.cpp b/source/adios2/engine/bp4/BP4Writer.cpp
index 8e51c2ce80..b47d0aac46 100644
--- a/source/adios2/engine/bp4/BP4Writer.cpp
+++ b/source/adios2/engine/bp4/BP4Writer.cpp
@@ -67,13 +67,13 @@ void BP4Writer::PerformPuts()
 
     for (const std::string &variableName : m_BP4Serializer.m_DeferredVariables)
     {
-        const std::string type = m_IO.InquireVariableType(variableName);
-        if (type == "compound")
+        const DataType type = m_IO.InquireVariableType(variableName);
+        if (type == DataType::Compound)
         {
             // not supported
         }
 #define declare_template_instantiation(T)                                      \
-    else if (type == helper::GetType())                                     \
+    else if (type == helper::GetDataType())                                 \
     {                                                                          \
         Variable &variable = FindVariable(                               \
             variableName, "in call to PerformPuts, EndStep or Close");         \
@@ -84,6 +84,7 @@ void BP4Writer::PerformPuts()
 #undef declare_template_instantiation
     }
     m_BP4Serializer.m_DeferredVariables.clear();
+    m_BP4Serializer.m_DeferredVariablesDataSize = 0;
 }
 
 void BP4Writer::EndStep()
@@ -110,7 +111,7 @@ void BP4Writer::Flush(const int transportIndex)
 {
     TAU_SCOPED_TIMER("BP4Writer::Flush");
     DoFlush(false, transportIndex);
-    m_BP4Serializer.ResetBuffer(m_BP4Serializer.m_Data);
+    m_BP4Serializer.ResetBuffer(m_BP4Serializer.m_Data, false, false);
 
     if (m_BP4Serializer.m_Parameters.CollectiveMetadata)
     {
@@ -122,6 +123,12 @@ void BP4Writer::Flush(const int transportIndex)
 void BP4Writer::Init()
 {
     InitParameters();
+    if (m_BP4Serializer.m_Parameters.NumAggregators <
+        static_cast(m_BP4Serializer.m_SizeMPI))
+    {
+        m_BP4Serializer.m_Aggregator.Init(
+            m_BP4Serializer.m_Parameters.NumAggregators, m_Comm);
+    }
     InitTransports();
     InitBPBuffer();
 }
@@ -203,13 +210,14 @@ void BP4Writer::InitTransports()
 
     /* Create the directories either on target or burst buffer if used */
     m_BP4Serializer.m_Profiler.Start("mkdir");
-    m_FileDataManager.MkDirsBarrier(m_SubStreamNames,
-                                    m_BP4Serializer.m_Parameters.NodeLocal ||
-                                        m_WriteToBB);
+    m_FileDataManager.MkDirsBarrier(
+        m_SubStreamNames, m_IO.m_TransportsParameters,
+        m_BP4Serializer.m_Parameters.NodeLocal || m_WriteToBB);
     if (m_DrainBB)
     {
         /* Create the directories on target anyway by main thread */
         m_FileDataManager.MkDirsBarrier(m_DrainSubStreamNames,
+                                        m_IO.m_TransportsParameters,
                                         m_BP4Serializer.m_Parameters.NodeLocal);
     }
     m_BP4Serializer.m_Profiler.Stop("mkdir");
@@ -469,6 +477,17 @@ void BP4Writer::WriteProfilingJSONFile()
 {
     TAU_SCOPED_TIMER("BP4Writer::WriteProfilingJSONFile");
     auto transportTypes = m_FileDataManager.GetTransportsTypes();
+
+    // find first File type output, where we can write the profile
+    int fileTransportIdx = -1;
+    for (size_t i = 0; i < transportTypes.size(); ++i)
+    {
+        if (transportTypes[i].compare(0, 4, "File") == 0)
+        {
+            fileTransportIdx = static_cast(i);
+        }
+    }
+
     auto transportProfilers = m_FileDataManager.GetTransportsProfilers();
 
     auto transportTypesMD = m_FileMetadataManager.GetTransportsTypes();
@@ -491,19 +510,36 @@ void BP4Writer::WriteProfilingJSONFile()
     if (m_BP4Serializer.m_RankMPI == 0)
     {
         // std::cout << "write profiling file!" << std::endl;
+        std::string profileFileName;
         if (m_DrainBB)
         {
             auto bpTargetNames = m_BP4Serializer.GetBPBaseNames({m_Name});
-            std::string targetProfiler(bpTargetNames[0] + "/profiling.json");
+            if (fileTransportIdx > -1)
+            {
+                profileFileName =
+                    bpTargetNames[fileTransportIdx] + "/profiling.json";
+            }
+            else
+            {
+                profileFileName = bpTargetNames[0] + "_profiling.json";
+            }
             m_FileDrainer.AddOperationWrite(
-                targetProfiler, profilingJSON.size(), profilingJSON.data());
+                profileFileName, profilingJSON.size(), profilingJSON.data());
         }
         else
         {
             transport::FileFStream profilingJSONStream(m_Comm);
             auto bpBaseNames = m_BP4Serializer.GetBPBaseNames({m_BBName});
-            profilingJSONStream.Open(bpBaseNames[0] + "/profiling.json",
-                                     Mode::Write);
+            if (fileTransportIdx > -1)
+            {
+                profileFileName =
+                    bpBaseNames[fileTransportIdx] + "/profiling.json";
+            }
+            else
+            {
+                profileFileName = bpBaseNames[0] + "_profiling.json";
+            }
+            profilingJSONStream.Open(profileFileName, Mode::Write);
             profilingJSONStream.Write(profilingJSON.data(),
                                       profilingJSON.size());
             profilingJSONStream.Close();
@@ -535,12 +571,12 @@ void BP4Writer::UpdateActiveFlag(const bool active)
 {
     const char activeChar = (active ? '\1' : '\0');
     m_FileMetadataIndexManager.WriteFileAt(
-        &activeChar, 1, m_BP4Serializer.m_ActiveFlagPosition, 0);
+        &activeChar, 1, m_BP4Serializer.m_ActiveFlagPosition);
     m_FileMetadataIndexManager.FlushFiles();
     m_FileMetadataIndexManager.SeekToFileEnd();
     if (m_DrainBB)
     {
-        for (int i = 0; i < m_MetadataIndexFileNames.size(); ++i)
+        for (size_t i = 0; i < m_MetadataIndexFileNames.size(); ++i)
         {
             m_FileDrainer.AddOperationWriteAt(
                 m_DrainMetadataIndexFileNames[i],
@@ -574,7 +610,7 @@ void BP4Writer::WriteCollectiveMetadataFile(const bool isFinal)
 
         if (m_DrainBB)
         {
-            for (int i = 0; i < m_MetadataFileNames.size(); ++i)
+            for (size_t i = 0; i < m_MetadataFileNames.size(); ++i)
             {
                 m_FileDrainer.AddOperationCopy(
                     m_MetadataFileNames[i], m_DrainMetadataFileNames[i],
@@ -637,7 +673,7 @@ void BP4Writer::WriteCollectiveMetadataFile(const bool isFinal)
 
         if (m_DrainBB)
         {
-            for (int i = 0; i < m_MetadataIndexFileNames.size(); ++i)
+            for (size_t i = 0; i < m_MetadataIndexFileNames.size(); ++i)
             {
                 m_FileDrainer.AddOperationWrite(
                     m_DrainMetadataIndexFileNames[i],
@@ -647,10 +683,10 @@ void BP4Writer::WriteCollectiveMetadataFile(const bool isFinal)
         }
     }
     /*Clear the local indices buffer at the end of each step*/
-    m_BP4Serializer.ResetBuffer(m_BP4Serializer.m_Metadata, true);
+    m_BP4Serializer.ResetBuffer(m_BP4Serializer.m_Metadata, true, true);
 
     /* clear the metadata index buffer*/
-    m_BP4Serializer.ResetBuffer(m_BP4Serializer.m_MetadataIndex, true);
+    m_BP4Serializer.ResetBuffer(m_BP4Serializer.m_MetadataIndex, true, true);
 
     /* reset the metadata index table*/
     m_BP4Serializer.ResetMetadataIndexTable();
@@ -678,7 +714,7 @@ void BP4Writer::WriteData(const bool isFinal, const int transportIndex)
     m_FileDataManager.FlushFiles(transportIndex);
     if (m_DrainBB)
     {
-        for (int i = 0; i < m_SubStreamNames.size(); ++i)
+        for (size_t i = 0; i < m_SubStreamNames.size(); ++i)
         {
             m_FileDrainer.AddOperationCopy(m_SubStreamNames[i],
                                            m_DrainSubStreamNames[i], dataSize);
@@ -728,7 +764,7 @@ void BP4Writer::AggregateWriteData(const bool isFinal, const int transportIndex)
 
     if (m_DrainBB)
     {
-        for (int i = 0; i < m_SubStreamNames.size(); ++i)
+        for (size_t i = 0; i < m_SubStreamNames.size(); ++i)
         {
             m_FileDrainer.AddOperationCopy(m_SubStreamNames[i],
                                            m_DrainSubStreamNames[i],
@@ -756,6 +792,11 @@ void BP4Writer::AggregateWriteData(const bool isFinal, const int transportIndex)
 ADIOS2_FOREACH_PRIMITVE_STDTYPE_2ARGS(declare_type)
 #undef declare_type
 
+size_t BP4Writer::DebugGetDataBufferSize() const
+{
+    return m_BP4Serializer.DebugGetDataBufferSize();
+}
+
 } // end namespace engine
 } // end namespace core
 } // end namespace adios2
diff --git a/source/adios2/engine/bp4/BP4Writer.h b/source/adios2/engine/bp4/BP4Writer.h
index 097ffd024b..1b04c07d0c 100644
--- a/source/adios2/engine/bp4/BP4Writer.h
+++ b/source/adios2/engine/bp4/BP4Writer.h
@@ -47,6 +47,8 @@ class BP4Writer : public core::Engine
     void EndStep() final;
     void Flush(const int transportIndex = -1) final;
 
+    size_t DebugGetDataBufferSize() const final;
+
 private:
     /** Single object controlling BP buffering */
     format::BP4Serializer m_BP4Serializer;
@@ -115,7 +117,8 @@ class BP4Writer : public core::Engine
 
     template 
     void PutSyncCommon(Variable &variable,
-                       const typename Variable::Info &blockInfo);
+                       const typename Variable::Info &blockInfo,
+                       const bool resize = true);
 
     template 
     void PutDeferredCommon(Variable &variable, const T *data);
diff --git a/source/adios2/engine/bp4/BP4Writer.tcc b/source/adios2/engine/bp4/BP4Writer.tcc
index 94d8791dbd..2edfa7ac68 100644
--- a/source/adios2/engine/bp4/BP4Writer.tcc
+++ b/source/adios2/engine/bp4/BP4Writer.tcc
@@ -63,16 +63,21 @@ void BP4Writer::PutCommon(Variable &variable,
 
 template 
 void BP4Writer::PutSyncCommon(Variable &variable,
-                              const typename Variable::Info &blockInfo)
+                              const typename Variable::Info &blockInfo,
+                              const bool resize)
 {
-    const size_t dataSize =
-        helper::PayloadSize(blockInfo.Data, blockInfo.Count) +
-        m_BP4Serializer.GetBPIndexSizeInData(variable.m_Name, blockInfo.Count);
-
-    const format::BP4Base::ResizeResult resizeResult =
-        m_BP4Serializer.ResizeBuffer(dataSize, "in call to variable " +
-                                                   variable.m_Name + " Put");
+    format::BP4Base::ResizeResult resizeResult =
+        format::BP4Base::ResizeResult::Success;
+    if (resize)
+    {
+        const size_t dataSize =
+            helper::PayloadSize(blockInfo.Data, blockInfo.Count) +
+            m_BP4Serializer.GetBPIndexSizeInData(variable.m_Name,
+                                                 blockInfo.Count);
 
+        resizeResult = m_BP4Serializer.ResizeBuffer(
+            dataSize, "in call to variable " + variable.m_Name + " Put");
+    }
     // if first timestep Write create a new pg index
     if (!m_BP4Serializer.m_MetadataSet.DataPGIsOpen)
     {
@@ -84,7 +89,7 @@ void BP4Writer::PutSyncCommon(Variable &variable,
     if (resizeResult == format::BP4Base::ResizeResult::Flush)
     {
         DoFlush(false);
-        m_BP4Serializer.ResetBuffer(m_BP4Serializer.m_Data);
+        m_BP4Serializer.ResetBuffer(m_BP4Serializer.m_Data, false, false);
 
         // new group index for incoming variable
         m_BP4Serializer.PutProcessGroupIndex(
@@ -133,7 +138,7 @@ void BP4Writer::PerformPutCommon(Variable &variable)
         auto itSpanBlock = variable.m_BlocksSpan.find(b);
         if (itSpanBlock == variable.m_BlocksSpan.end())
         {
-            PutSyncCommon(variable, variable.m_BlocksInfo[b]);
+            PutSyncCommon(variable, variable.m_BlocksInfo[b], false);
         }
         else
         {
diff --git a/source/adios2/engine/dataman/DataManMonitor.cpp b/source/adios2/engine/dataman/DataManMonitor.cpp
new file mode 100644
index 0000000000..86e50c4f9d
--- /dev/null
+++ b/source/adios2/engine/dataman/DataManMonitor.cpp
@@ -0,0 +1,132 @@
+/*
+ * Distributed under the OSI-approved Apache License, Version 2.0.  See
+ * accompanying file Copyright.txt for details.
+ *
+ * DataManMonitor.cpp
+ *
+ *  Created on: Jun 2, 2020
+ *      Author: Jason Wang
+ */
+
+#include "DataManMonitor.h"
+#include 
+
+namespace adios2
+{
+namespace core
+{
+namespace engine
+{
+
+void DataManMonitor::BeginStep(size_t step)
+{
+    if (step == 0)
+    {
+        m_InitialTimer = std::chrono::system_clock::now();
+    }
+    if (m_StepTimers.empty())
+    {
+        m_StepTimers.push(std::chrono::system_clock::now());
+    }
+
+    m_StepBytes.push(0);
+
+    if (m_TotalBytes.empty())
+    {
+        m_TotalBytes.push(0);
+    }
+    else
+    {
+        m_TotalBytes.push(m_TotalBytes.back());
+    }
+
+    ++m_CurrentStep;
+}
+
+void DataManMonitor::EndStep(size_t step)
+{
+    m_StepTimers.push(std::chrono::system_clock::now());
+
+    if (m_StepTimers.size() > m_AverageSteps)
+    {
+        m_StepTimers.pop();
+    }
+    if (m_TotalBytes.size() > m_AverageSteps)
+    {
+        m_TotalBytes.pop();
+    }
+    if (m_StepBytes.size() > m_AverageSteps)
+    {
+        m_StepBytes.pop();
+    }
+
+    m_TotalTime = std::chrono::duration_cast(
+                      (m_StepTimers.back() - m_InitialTimer))
+                      .count();
+    m_AverageTime = std::chrono::duration_cast(
+                        (m_StepTimers.back() - m_StepTimers.front()))
+                        .count();
+    m_TotalRate = static_cast(m_TotalBytes.back()) /
+                  static_cast(m_TotalTime);
+    m_AverageRate =
+        static_cast(m_TotalBytes.back() - m_TotalBytes.front()) /
+        static_cast(m_AverageTime);
+    if (step > 0)
+    {
+        m_DropRate = static_cast((step - m_CurrentStep)) / step;
+    }
+    m_StepsPerSecond = step / m_TotalTime * 1000000;
+
+    if (m_Verbose)
+    {
+        std::lock_guard l(m_PrintMutex);
+        std::cout << "Step " << step << ", Total MBs "
+                  << static_cast(m_TotalBytes.back()) / 1000000.0
+                  << ", Step MBs "
+                  << static_cast(m_StepBytes.back()) / 1000000.0
+                  << ", Total seconds "
+                  << static_cast(m_TotalTime) / 1000000.0 << ", "
+                  << m_StepTimers.size() << " step seconds "
+                  << static_cast(m_AverageTime) / 1000000.0
+                  << ", Total MB/s " << m_TotalRate << ", "
+                  << m_StepTimers.size() << " step average MB/s "
+                  << m_AverageRate << ", Drop rate " << m_DropRate * 100 << "%"
+                  << ", Steps per second " << m_StepsPerSecond << std::endl;
+    }
+}
+
+void DataManMonitor::BeginTransport(size_t step)
+{
+    std::lock_guard l(m_TransportTimersMutex);
+    m_TransportTimers.push({step, std::chrono::system_clock::now()});
+}
+
+void DataManMonitor::EndTransport()
+{
+    std::lock_guard l(m_TransportTimersMutex);
+    if (!m_TransportTimers.empty())
+    {
+        auto latency = std::chrono::duration_cast(
+                           (std::chrono::system_clock::now() -
+                            m_TransportTimers.back().second))
+                           .count();
+        if (m_Verbose)
+        {
+            std::lock_guard l(m_PrintMutex);
+            std::cout << "Step " << m_TransportTimers.back().first
+                      << ", Latency milliseconds "
+                      << static_cast(latency) / 1000.0 << std::endl;
+        }
+        m_TransportTimers.pop();
+    }
+}
+
+void DataManMonitor::AddBytes(size_t bytes)
+{
+    m_TotalBytes.back() += bytes;
+    m_StepBytes.back() += bytes;
+}
+
+} // end namespace engine
+} // end namespace core
+} // end namespace adios2
diff --git a/source/adios2/engine/dataman/DataManMonitor.h b/source/adios2/engine/dataman/DataManMonitor.h
new file mode 100644
index 0000000000..874913c655
--- /dev/null
+++ b/source/adios2/engine/dataman/DataManMonitor.h
@@ -0,0 +1,63 @@
+/*
+ * Distributed under the OSI-approved Apache License, Version 2.0.  See
+ * accompanying file Copyright.txt for details.
+ *
+ * DataManCommon.h
+ *
+ *  Created on: Jun 2, 2020
+ *      Author: Jason Wang
+ */
+
+#ifndef ADIOS2_ENGINE_DATAMAN_DATAMANMONITOR_H_
+#define ADIOS2_ENGINE_DATAMAN_DATAMANMONITOR_H_
+
+#include 
+#include 
+#include 
+
+namespace adios2
+{
+namespace core
+{
+namespace engine
+{
+
+class DataManMonitor
+{
+public:
+    void BeginStep(size_t step);
+    void EndStep(size_t step);
+    void BeginTransport(size_t step);
+    void EndTransport();
+    void AddBytes(size_t bytes);
+
+private:
+    using TimePoint = std::chrono::time_point;
+    std::queue m_StepTimers;
+    TimePoint m_InitialTimer;
+    std::queue m_StepBytes;
+    std::queue m_TotalBytes;
+
+    std::queue> m_TransportTimers;
+    std::mutex m_TransportTimersMutex;
+
+    std::mutex m_PrintMutex;
+
+    size_t m_AverageSteps = 10;
+    int64_t m_CurrentStep = -1;
+
+    double m_TotalTime;
+    double m_AverageTime;
+    double m_TotalRate;
+    double m_AverageRate;
+    double m_DropRate;
+    double m_StepsPerSecond;
+
+    bool m_Verbose = true;
+};
+
+} // end namespace engine
+} // end namespace core
+} // end namespace adios2
+
+#endif /* ADIOS2_ENGINE_DATAMAN_DATAMANMONITOR_H_ */
diff --git a/source/adios2/engine/dataman/DataManReader.cpp b/source/adios2/engine/dataman/DataManReader.cpp
index f8fdf437b7..eeff7dc271 100644
--- a/source/adios2/engine/dataman/DataManReader.cpp
+++ b/source/adios2/engine/dataman/DataManReader.cpp
@@ -21,7 +21,9 @@ namespace engine
 DataManReader::DataManReader(IO &io, const std::string &name,
                              const Mode openMode, helper::Comm comm)
 : Engine("DataManReader", io, name, openMode, std::move(comm)),
-  m_Serializer(m_Comm, helper::IsRowMajor(io.m_HostLanguage))
+  m_Serializer(m_Comm, helper::IsRowMajor(io.m_HostLanguage)),
+  m_RequesterThreadActive(true), m_SubscriberThreadActive(true),
+  m_FinalStep(std::numeric_limits::max())
 {
     m_MpiRank = m_Comm.Rank();
     m_MpiSize = m_Comm.Size();
@@ -31,6 +33,7 @@ DataManReader::DataManReader(IO &io, const std::string &name,
     helper::GetParameter(m_IO.m_Parameters, "Verbose", m_Verbosity);
     helper::GetParameter(m_IO.m_Parameters, "DoubleBuffer", m_DoubleBuffer);
     helper::GetParameter(m_IO.m_Parameters, "TransportMode", m_TransportMode);
+    helper::GetParameter(m_IO.m_Parameters, "Monitor", m_MonitorActive);
 
     m_Requesters.emplace_back();
     m_Requesters[0].OpenRequester(m_Timeout, m_ReceiverBufferSize);
@@ -170,7 +173,7 @@ StepStatus DataManReader::BeginStep(StepMode stepMode,
         if (m_Verbosity >= 5)
         {
             std::cout << "DataManReader::BeginStep() Rank " << m_MpiRank
-                      << "returned EndOfStream due "
+                      << " returned EndOfStream due "
                          "to timeout"
                       << std::endl;
         }
@@ -183,12 +186,12 @@ StepStatus DataManReader::BeginStep(StepMode stepMode,
     {
         if (i.step == m_CurrentStep)
         {
-            if (i.type.empty())
+            if (i.type == DataType::None)
             {
                 throw("unknown data type");
             }
 #define declare_type(T)                                                        \
-    else if (i.type == helper::GetType())                                   \
+    else if (i.type == helper::GetDataType())                               \
     {                                                                          \
         CheckIOVariable(i.name, i.shape, i.start, i.count);                 \
     }
@@ -204,6 +207,11 @@ StepStatus DataManReader::BeginStep(StepMode stepMode,
                   << ", Step " << m_CurrentStep << std::endl;
     }
 
+    if (m_MonitorActive)
+    {
+        m_Monitor.BeginStep(m_CurrentStep);
+    }
+
     return StepStatus::OK;
 }
 
@@ -215,6 +223,11 @@ void DataManReader::EndStep()
 {
     m_Serializer.Erase(m_CurrentStep, true);
     m_CurrentStepMetadata = nullptr;
+
+    if (m_MonitorActive)
+    {
+        m_Monitor.EndStep(m_CurrentStep);
+    }
 }
 
 void DataManReader::Flush(const int transportIndex) {}
@@ -234,7 +247,7 @@ void DataManReader::RequestThread(zmq::ZmqReqRep &requester)
                 {
                     auto jmsg = nlohmann::json::parse(buffer->data());
                     m_FinalStep = jmsg["FinalStep"].get();
-                    continue;
+                    return;
                 }
                 catch (...)
                 {
diff --git a/source/adios2/engine/dataman/DataManReader.h b/source/adios2/engine/dataman/DataManReader.h
index f6836b667c..8314515cf9 100644
--- a/source/adios2/engine/dataman/DataManReader.h
+++ b/source/adios2/engine/dataman/DataManReader.h
@@ -11,8 +11,11 @@
 #ifndef ADIOS2_ENGINE_DATAMAN_DATAMANREADER_H_
 #define ADIOS2_ENGINE_DATAMAN_DATAMANREADER_H_
 
+#include 
+
 #include "adios2/core/Engine.h"
-#include "adios2/toolkit/format/dataman/DataManSerializer.h"
+#include "adios2/engine/dataman/DataManMonitor.h"
+#include "adios2/toolkit/format/dataman/DataManSerializer.tcc"
 #include "adios2/toolkit/zmq/zmqpubsub/ZmqPubSub.h"
 #include "adios2/toolkit/zmq/zmqreqrep/ZmqReqRep.h"
 
@@ -44,6 +47,7 @@ class DataManReader : public Engine
     bool m_DoubleBuffer = true;
     size_t m_ReceiverBufferSize = 128 * 1024 * 1024;
     std::string m_TransportMode = "fast";
+    bool m_MonitorActive = false;
 
     std::vector m_PublisherAddresses;
     std::vector m_ReplierAddresses;
@@ -51,7 +55,7 @@ class DataManReader : public Engine
     int m_MpiSize;
     int64_t m_CurrentStep = -1;
     bool m_InitFailed = false;
-    size_t m_FinalStep = std::numeric_limits::max();
+    std::atomic m_FinalStep;
     format::DmvVecPtr m_CurrentStepMetadata;
 
     format::DataManSerializer m_Serializer;
@@ -59,11 +63,13 @@ class DataManReader : public Engine
     std::vector m_Subscribers;
     std::vector m_Requesters;
 
+    DataManMonitor m_Monitor;
+
     std::vector m_SubscriberThreads;
     std::vector m_RequesterThreads;
 
-    bool m_RequesterThreadActive = true;
-    bool m_SubscriberThreadActive = true;
+    std::atomic m_RequesterThreadActive;
+    std::atomic m_SubscriberThreadActive;
 
     void SubscribeThread(zmq::ZmqPubSub &subscriber);
     void RequestThread(zmq::ZmqReqRep &requester);
diff --git a/source/adios2/engine/dataman/DataManReader.tcc b/source/adios2/engine/dataman/DataManReader.tcc
index 070027acf8..b1b446188b 100644
--- a/source/adios2/engine/dataman/DataManReader.tcc
+++ b/source/adios2/engine/dataman/DataManReader.tcc
@@ -64,6 +64,13 @@ void DataManReader::GetDeferredCommon(Variable &variable, T *data)
             }
         }
     }
+
+    if (m_MonitorActive)
+    {
+        m_Monitor.AddBytes(std::accumulate(variable.m_Count.begin(),
+                                           variable.m_Count.end(), sizeof(T),
+                                           std::multiplies()));
+    }
 }
 
 template 
diff --git a/source/adios2/engine/dataman/DataManWriter.cpp b/source/adios2/engine/dataman/DataManWriter.cpp
index 0840676b87..c115d8da1b 100644
--- a/source/adios2/engine/dataman/DataManWriter.cpp
+++ b/source/adios2/engine/dataman/DataManWriter.cpp
@@ -20,7 +20,8 @@ namespace engine
 DataManWriter::DataManWriter(IO &io, const std::string &name,
                              const Mode openMode, helper::Comm comm)
 : Engine("DataManWriter", io, name, openMode, std::move(comm)),
-  m_Serializer(m_Comm, helper::IsRowMajor(io.m_HostLanguage))
+  m_Serializer(m_Comm, helper::IsRowMajor(io.m_HostLanguage)), m_SentSteps(0),
+  m_ReplyThreadActive(true), m_PublishThreadActive(true)
 {
 
     m_MpiRank = m_Comm.Rank();
@@ -34,6 +35,7 @@ DataManWriter::DataManWriter(IO &io, const std::string &name,
                          m_RendezvousReaderCount);
     helper::GetParameter(m_IO.m_Parameters, "DoubleBuffer", m_DoubleBuffer);
     helper::GetParameter(m_IO.m_Parameters, "TransportMode", m_TransportMode);
+    helper::GetParameter(m_IO.m_Parameters, "Monitor", m_MonitorActive);
 
     if (m_IPAddress.empty())
     {
@@ -85,6 +87,7 @@ DataManWriter::DataManWriter(IO &io, const std::string &name,
 
     if (m_RendezvousReaderCount == 0 || m_TransportMode == "reliable")
     {
+        m_ReplyThreadActive = true;
         m_ReplyThread = std::thread(&DataManWriter::ReplyThread, this);
     }
     else
@@ -95,6 +98,7 @@ DataManWriter::DataManWriter(IO &io, const std::string &name,
 
     if (m_DoubleBuffer && m_TransportMode == "fast")
     {
+        m_PublishThreadActive = true;
         m_PublishThread = std::thread(&DataManWriter::PublishThread, this);
     }
 }
@@ -112,6 +116,16 @@ StepStatus DataManWriter::BeginStep(StepMode mode, const float timeout_sec)
     ++m_CurrentStep;
     m_Serializer.NewWriterBuffer(m_SerializerBufferSize);
 
+    if (m_MonitorActive)
+    {
+        m_Monitor.BeginStep(m_CurrentStep);
+    }
+
+    if (m_Verbosity >= 10)
+    {
+        std::cout << "DataManWriter::BeginStep " << m_CurrentStep << std::endl;
+    }
+
     return StepStatus::OK;
 }
 
@@ -133,6 +147,11 @@ void DataManWriter::EndStep()
         m_SerializerBufferSize = buffer->size();
     }
 
+    if (m_MonitorActive)
+    {
+        m_Monitor.BeginTransport(m_CurrentStep);
+    }
+
     if (m_DoubleBuffer || m_TransportMode == "reliable")
     {
         PushBufferQueue(buffer);
@@ -140,6 +159,20 @@ void DataManWriter::EndStep()
     else
     {
         m_Publisher.Send(buffer);
+        if (m_MonitorActive)
+        {
+            m_Monitor.EndTransport();
+        }
+    }
+
+    if (m_MonitorActive)
+    {
+        m_Monitor.EndStep(m_CurrentStep);
+    }
+
+    if (m_Verbosity >= 10)
+    {
+        std::cout << "DataManWriter::EndStep " << m_CurrentStep << std::endl;
     }
 }
 
@@ -162,7 +195,7 @@ ADIOS2_FOREACH_STDTYPE_1ARG(declare_type)
 void DataManWriter::DoClose(const int transportIndex)
 {
     nlohmann::json endSignal;
-    endSignal["FinalStep"] = m_CurrentStep;
+    endSignal["FinalStep"] = static_cast(m_CurrentStep);
     std::string s = endSignal.dump() + '\0';
     auto cvp = std::make_shared>(s.size());
     std::memcpy(cvp->data(), s.c_str(), s.size());
@@ -176,9 +209,16 @@ void DataManWriter::DoClose(const int transportIndex)
         m_Publisher.Send(cvp);
     }
 
-    m_ReplyThreadActive = false;
     m_PublishThreadActive = false;
 
+    if (m_ReplyThreadActive)
+    {
+        while (m_SentSteps < m_CurrentStep + 2)
+        {
+        }
+        m_ReplyThreadActive = false;
+    }
+
     if (m_ReplyThread.joinable())
     {
         m_ReplyThread.join();
@@ -190,18 +230,10 @@ void DataManWriter::DoClose(const int transportIndex)
     }
 
     m_IsClosed = true;
-}
 
-bool DataManWriter::CheckBufferQueue()
-{
-    std::lock_guard l(m_BufferQueueMutex);
-    if (m_BufferQueue.empty())
+    if (m_Verbosity >= 10)
     {
-        return false;
-    }
-    else
-    {
-        return true;
+        std::cout << "DataManWriter::DoClose " << m_CurrentStep << std::endl;
     }
 }
 
@@ -228,12 +260,16 @@ std::shared_ptr> DataManWriter::PopBufferQueue()
 
 void DataManWriter::PublishThread()
 {
-    while (m_PublishThreadActive || CheckBufferQueue())
+    while (m_PublishThreadActive)
     {
         auto buffer = PopBufferQueue();
         if (buffer != nullptr && buffer->size() > 0)
         {
             m_Publisher.Send(buffer);
+            if (m_MonitorActive)
+            {
+                m_Monitor.EndTransport();
+            }
         }
     }
 }
@@ -241,10 +277,10 @@ void DataManWriter::PublishThread()
 void DataManWriter::ReplyThread()
 {
     int readerCount = 0;
-    while (m_ReplyThreadActive || CheckBufferQueue())
+    while (m_ReplyThreadActive)
     {
         auto request = m_Replier.ReceiveRequest();
-        if (request && request->size() > 0)
+        if (request != nullptr && request->size() > 0)
         {
             std::string r(request->begin(), request->end());
             if (r == "Address")
@@ -262,25 +298,15 @@ void DataManWriter::ReplyThread()
                 auto buffer = PopBufferQueue();
                 while (buffer == nullptr)
                 {
-                    auto buffer = PopBufferQueue();
+                    buffer = PopBufferQueue();
                 }
-                if (buffer != nullptr && buffer->size() > 0)
+                if (buffer->size() > 0)
                 {
                     m_Replier.SendReply(buffer);
-                    if (buffer->size() < 64)
+                    ++m_SentSteps;
+                    if (m_MonitorActive)
                     {
-                        try
-                        {
-                            auto jmsg = nlohmann::json::parse(buffer->data());
-                            auto finalStep = jmsg["FinalStep"].get();
-                            if (finalStep == m_CurrentStep)
-                            {
-                                m_ReplyThreadActive = false;
-                            }
-                        }
-                        catch (...)
-                        {
-                        }
+                        m_Monitor.EndTransport();
                     }
                 }
             }
diff --git a/source/adios2/engine/dataman/DataManWriter.h b/source/adios2/engine/dataman/DataManWriter.h
index 9ad9186bf3..a16f8c3b8f 100644
--- a/source/adios2/engine/dataman/DataManWriter.h
+++ b/source/adios2/engine/dataman/DataManWriter.h
@@ -11,10 +11,12 @@
 #ifndef ADIOS2_ENGINE_DATAMAN_DATAMANWRITER_H_
 #define ADIOS2_ENGINE_DATAMAN_DATAMANWRITER_H_
 
+#include "DataManMonitor.h"
 #include "adios2/core/Engine.h"
 #include "adios2/toolkit/format/dataman/DataManSerializer.tcc"
 #include "adios2/toolkit/zmq/zmqpubsub/ZmqPubSub.h"
 #include "adios2/toolkit/zmq/zmqreqrep/ZmqReqRep.h"
+#include 
 
 namespace adios2
 {
@@ -46,31 +48,34 @@ class DataManWriter : public Engine
     int m_Verbosity = 0;
     bool m_DoubleBuffer = false;
     std::string m_TransportMode = "fast";
+    bool m_MonitorActive = false;
 
     std::string m_AllAddresses;
     std::string m_PublisherAddress;
     std::string m_ReplierAddress;
     int m_MpiRank;
     int m_MpiSize;
+    size_t m_SerializerBufferSize = 1024 * 1024;
     int64_t m_CurrentStep = -1;
-    size_t m_SerializerBufferSize = 128 * 1024 * 1024;
+    std::atomic m_SentSteps;
 
     format::DataManSerializer m_Serializer;
 
     zmq::ZmqPubSub m_Publisher;
     zmq::ZmqReqRep m_Replier;
 
+    DataManMonitor m_Monitor;
+
     std::thread m_ReplyThread;
     std::thread m_PublishThread;
-    bool m_ReplyThreadActive = true;
-    bool m_PublishThreadActive = true;
+    std::atomic m_ReplyThreadActive;
+    bool m_PublishThreadActive;
 
     std::queue>> m_BufferQueue;
     std::mutex m_BufferQueueMutex;
 
     void PushBufferQueue(std::shared_ptr> buffer);
     std::shared_ptr> PopBufferQueue();
-    bool CheckBufferQueue();
 
     void ReplyThread();
     void PublishThread();
diff --git a/source/adios2/engine/dataman/DataManWriter.tcc b/source/adios2/engine/dataman/DataManWriter.tcc
index 26e0520596..ea1f4fc846 100644
--- a/source/adios2/engine/dataman/DataManWriter.tcc
+++ b/source/adios2/engine/dataman/DataManWriter.tcc
@@ -34,8 +34,7 @@ void DataManWriter::PutDeferredCommon(Variable &variable, const T *values)
     variable.SetData(values);
     if (helper::IsRowMajor(m_IO.m_HostLanguage))
     {
-        m_Serializer.PutData(variable, m_Name, CurrentStep(), m_MpiRank, "",
-                             Params());
+        m_Serializer.PutData(variable, m_Name, CurrentStep(), m_MpiRank, "");
     }
     else
     {
@@ -51,7 +50,14 @@ void DataManWriter::PutDeferredCommon(Variable &variable, const T *values)
         std::reverse(memcount.begin(), memcount.end());
         m_Serializer.PutData(variable.m_Data, variable.m_Name, shape, start,
                              count, memstart, memcount, m_Name, CurrentStep(),
-                             m_MpiRank, "", Params());
+                             m_MpiRank, "", variable.m_Operations);
+    }
+
+    if (m_MonitorActive)
+    {
+        m_Monitor.AddBytes(std::accumulate(variable.m_Count.begin(),
+                                           variable.m_Count.end(), sizeof(T),
+                                           std::multiplies()));
     }
 }
 
diff --git a/source/adios2/engine/dataspaces/DataSpacesWriter.tcc b/source/adios2/engine/dataspaces/DataSpacesWriter.tcc
index 90461a181c..c9feaab041 100644
--- a/source/adios2/engine/dataspaces/DataSpacesWriter.tcc
+++ b/source/adios2/engine/dataspaces/DataSpacesWriter.tcc
@@ -87,7 +87,7 @@ void DataSpacesWriter::DoPutSyncCommon(Variable &variable, const T *values)
     }
     gdims_vector.push_back(dims_vec);
     int varType;
-    auto itType = varType_to_ds.find(variable.m_Type);
+    auto itType = varType_to_ds.find(ToString(variable.m_Type));
     if (itType == varType_to_ds.end())
     {
         varType = 2;
diff --git a/source/adios2/engine/hdf5/HDF5ReaderP.cpp b/source/adios2/engine/hdf5/HDF5ReaderP.cpp
index 829a4aa0bd..ee44f0ae98 100644
--- a/source/adios2/engine/hdf5/HDF5ReaderP.cpp
+++ b/source/adios2/engine/hdf5/HDF5ReaderP.cpp
@@ -12,7 +12,7 @@
 #include "HDF5ReaderP.tcc"
 
 #include "adios2/helper/adiosFunctions.h" //CSVToVector
-
+#include "adios2/helper/adiosFunctions.h" //IsHDF5
 #include 
 
 namespace adios2
@@ -27,6 +27,10 @@ HDF5ReaderP::HDF5ReaderP(IO &io, const std::string &name, const Mode openMode,
 : Engine("HDF5Reader", io, name, openMode, std::move(comm))
 {
     m_EndMessage = ", in call to IO HDF5Reader Open " + m_Name + "\n";
+    if (!helper::IsHDF5File(name, m_Comm, {}))
+        throw std::invalid_argument("!ADIOS2 Error: Invalid HDF5 file found" +
+                                    m_EndMessage);
+
     Init();
 }
 
@@ -129,10 +133,10 @@ size_t HDF5ReaderP::ReadDataset(hid_t dataSetId, hid_t h5Type,
         return 0;
     }
 
-    size_t slabsize = 1;
+    size_t slabsize = 1u;
 
-    int ndims = std::max(variable.m_Shape.size(), variable.m_Count.size());
-    if (0 == ndims)
+    size_t ndims = std::max(variable.m_Shape.size(), variable.m_Count.size());
+    if (0u == ndims)
     { // is scalar
         hid_t myclass = H5Tget_class(h5Type);
         if (H5Tget_class(h5Type) == H5T_STRING)
@@ -150,7 +154,7 @@ size_t HDF5ReaderP::ReadDataset(hid_t dataSetId, hid_t h5Type,
         std::vector start(ndims), count(ndims), stride(ndims);
         bool isOrderC = helper::IsRowMajor(m_IO.m_HostLanguage);
 
-        for (int i = 0; i < ndims; i++)
+        for (size_t i = 0u; i < ndims; i++)
         {
             if (isOrderC)
             {
@@ -159,8 +163,8 @@ size_t HDF5ReaderP::ReadDataset(hid_t dataSetId, hid_t h5Type,
             }
             else
             {
-                count[i] = variable.m_Count[ndims - 1 - i];
-                start[i] = variable.m_Start[ndims - 1 - i];
+                count[i] = variable.m_Count[ndims - 1u - i];
+                start[i] = variable.m_Start[ndims - 1u - i];
             }
             slabsize *= count[i];
             stride[i] = 1;
@@ -174,7 +178,7 @@ size_t HDF5ReaderP::ReadDataset(hid_t dataSetId, hid_t h5Type,
         interop::HDF5TypeGuard g_mds(memDataSpace, interop::E_H5_SPACE);
 
         int elementsRead = 1;
-        for (int i = 0; i < ndims; i++)
+        for (size_t i = 0u; i < ndims; i++)
         {
             elementsRead *= count[i];
         }
@@ -207,7 +211,7 @@ void HDF5ReaderP::UseHDFRead(Variable &variable, T *data, hid_t h5Type)
     }
 
     T *values = data;
-    int ts = 0;
+    unsigned int ts = 0;
     // T *values = data;
     size_t variableStart = variable.m_StepsStart;
     /*
@@ -263,7 +267,7 @@ void HDF5ReaderP::UseHDFRead(Variable &variable, T *data, hid_t h5Type)
 
 StepStatus HDF5ReaderP::BeginStep(StepMode mode, const float timeoutSeconds)
 {
-    int ts = m_H5File.GetNumAdiosSteps();
+    const unsigned int ts = m_H5File.GetNumAdiosSteps();
 
     if (m_StreamAt >= ts)
     {
@@ -310,8 +314,8 @@ void HDF5ReaderP::PerformGets()
 #define declare_type(T)                                                        \
     for (std::string variableName : m_DeferredStack)                           \
     {                                                                          \
-        const std::string type = m_IO.InquireVariableType(variableName);       \
-        if (type == helper::GetType())                                      \
+        const DataType type = m_IO.InquireVariableType(variableName);          \
+        if (type == helper::GetDataType())                                  \
         {                                                                      \
             Variable *var = m_IO.InquireVariable(variableName);          \
             if (var != nullptr)                                                \
@@ -372,6 +376,7 @@ void HDF5ReaderP::DoClose(const int transportIndex)
     m_H5File.Close();
 }
 
+size_t HDF5ReaderP::DoSteps() const { return m_H5File.GetAdiosStep(); }
 } // end namespace engine
 } // end namespace core
 } // end namespace adios2
diff --git a/source/adios2/engine/hdf5/HDF5ReaderP.h b/source/adios2/engine/hdf5/HDF5ReaderP.h
index 9208fe6c99..244d5f3c74 100644
--- a/source/adios2/engine/hdf5/HDF5ReaderP.h
+++ b/source/adios2/engine/hdf5/HDF5ReaderP.h
@@ -97,6 +97,8 @@ class HDF5ReaderP : public Engine
     void UseHDFRead(Variable &variable, T *values, hid_t h5Type);
 
     std::vector m_DeferredStack;
+
+    size_t DoSteps() const final;
 };
 
 } // end namespace engine
diff --git a/source/adios2/engine/hdf5/HDF5WriterP.cpp b/source/adios2/engine/hdf5/HDF5WriterP.cpp
index 43d4312d7a..59a5b13db3 100644
--- a/source/adios2/engine/hdf5/HDF5WriterP.cpp
+++ b/source/adios2/engine/hdf5/HDF5WriterP.cpp
@@ -33,16 +33,18 @@ HDF5WriterP::~HDF5WriterP() { DoClose(); }
 StepStatus HDF5WriterP::BeginStep(StepMode mode, const float timeoutSeconds)
 {
     m_IO.m_ReadStreaming = false;
-#ifndef RELAY_DEFINE_TO_HDF5 // RELAY_DEFINE_TO_HDF5 = variables in io are
-                             // created at begin_step
-#else
+
+    // defines variables at this collective call.
+    // this will ensure all vars are defined in hdf5 for all processors
+    // (collective requirement) writing a variable is not a collective call
     m_H5File.CreateVarsFromIO(m_IO);
-#endif
+
     return StepStatus::OK;
 }
 
 void HDF5WriterP::EndStep()
 {
+    m_H5File.CleanUpNullVars(m_IO);
     m_H5File.Advance();
     m_H5File.WriteAttrFromIO(m_IO);
 }
@@ -60,9 +62,17 @@ void HDF5WriterP::Init()
             ", in call to ADIOS Open or HDF5Writer constructor\n");
     }
 
-#ifdef NEVER
-    m_H5File.Init(m_Name, m_Comm, true);
-#else
+    if (m_OpenMode == Mode::Append)
+    {
+        m_H5File.Append(m_Name, m_Comm);
+        m_H5File.ReadAttrToIO(m_IO);
+        m_H5File.ReadAllVariables(m_IO);
+    }
+    else
+        m_H5File.Init(m_Name, m_Comm, true);
+
+    m_H5File.ParseParameters(m_IO); // has to follow m_H5File Init/Append/
+    /*
     // enforce .h5 ending
     std::string suffix = ".h5";
     std::string wrongSuffix = ".bp";
@@ -74,14 +84,26 @@ void HDF5WriterP::Init()
     {
         // is a file with .bp ending
         std::string updatedName = m_Name.substr(0, wpos) + suffix;
-        m_H5File.Init(updatedName, m_Comm, true);
+        if (m_OpenMode == Mode::Append)
+            m_H5File.Append(updatedName, m_Comm);
+        else
+            m_H5File.Init(updatedName, m_Comm, true);
     }
     else
     {
-        m_H5File.Init(m_Name, m_Comm, true);
+        if (m_OpenMode == Mode::Append)
+            m_H5File.Append(m_Name, m_Comm);
+        else
+            m_H5File.Init(m_Name, m_Comm, true);
     }
     m_H5File.ParseParameters(m_IO);
-#endif
+
+    if (m_OpenMode == Mode::Append)
+    {
+        m_H5File.ReadAttrToIO(m_IO);
+        m_H5File.ReadAllVariables(m_IO);
+    }
+    */
 }
 
 #define declare_type(T)                                                        \
@@ -99,34 +121,6 @@ ADIOS2_FOREACH_STDTYPE_1ARG(declare_type)
 template 
 void HDF5WriterP::DoPutSyncCommon(Variable &variable, const T *values)
 {
-
-    bool isOrderC = helper::IsRowMajor(m_IO.m_HostLanguage);
-
-    if (!isOrderC)
-    {
-        int ndims = std::max(variable.m_Shape.size(), variable.m_Count.size());
-
-        if (ndims > 1)
-        {
-            Dims c_shape(ndims), c_offset(ndims), c_count(ndims);
-            for (int i = 0; i < ndims; i++)
-            {
-                c_shape[i] = variable.m_Shape[ndims - i - 1];
-                c_offset[i] = variable.m_Start[ndims - i - 1];
-                c_count[i] = variable.m_Count[ndims - i - 1];
-            }
-
-            Variable dup = Variable(variable.m_Name, c_shape, c_offset,
-                                          c_count, variable.IsConstantDims());
-
-            /*
-             * duplicate var attributes and convert to c order before saving.
-             */
-            dup.SetData(values);
-            m_H5File.Write(dup, values);
-            return;
-        }
-    }
     variable.SetData(values);
     m_H5File.Write(variable, values);
 }
diff --git a/source/adios2/engine/inline/InlineReader.cpp b/source/adios2/engine/inline/InlineReader.cpp
index 4225fb597e..d62edbefd7 100644
--- a/source/adios2/engine/inline/InlineReader.cpp
+++ b/source/adios2/engine/inline/InlineReader.cpp
@@ -38,6 +38,30 @@ InlineReader::InlineReader(IO &io, const std::string &name, const Mode mode,
     }
 }
 
+const InlineWriter *InlineReader::GetWriter() const
+{
+    const auto &engine_map = m_IO.GetEngines();
+    if (engine_map.size() != 2)
+    {
+        throw std::runtime_error("There must be exactly one reader and one "
+                                 "writer for the inline engine.");
+    }
+
+    std::shared_ptr e = engine_map.begin()->second;
+    if (e->OpenMode() == adios2::Mode::Read)
+    {
+        e = engine_map.rbegin()->second;
+    }
+
+    const auto writer = dynamic_cast(e.get());
+    if (!writer)
+    {
+        throw std::runtime_error(
+            "dynamic_cast failed; this is very likely a bug.");
+    }
+    return writer;
+}
+
 StepStatus InlineReader::BeginStep(const StepMode mode,
                                    const float timeoutSeconds)
 {
@@ -48,14 +72,13 @@ StepStatus InlineReader::BeginStep(const StepMode mode,
                                  "reader is already inside a step");
     }
     // Reader should be on step that writer just completed
-    const auto &writer =
-        dynamic_cast(m_IO.GetEngine(m_WriterID));
-    if (writer.IsInsideStep())
+    auto writer = GetWriter();
+    if (writer->IsInsideStep())
     {
         m_InsideStep = false;
         return StepStatus::NotReady;
     }
-    m_CurrentStep = writer.CurrentStep();
+    m_CurrentStep = writer->CurrentStep();
     if (m_CurrentStep == static_cast(-1))
     {
         m_InsideStep = false;
@@ -87,9 +110,8 @@ size_t InlineReader::CurrentStep() const
     // Reader should be on same step as writer
     // added here since it's not really necessary to use beginstep/endstep for
     // this engine's reader so this ensures we do report the correct step
-    const auto &writer =
-        dynamic_cast(m_IO.GetEngine(m_WriterID));
-    return writer.CurrentStep();
+    const auto writer = GetWriter();
+    return writer->CurrentStep();
 }
 
 void InlineReader::EndStep()
@@ -189,15 +211,6 @@ void InlineReader::InitParameters()
                     "integer in the range [0,5], in call to "
                     "Open or Engine constructor\n");
         }
-        else if (key == "writerid")
-        {
-            m_WriterID = value;
-            if (m_Verbosity == 5)
-            {
-                std::cout << "Inline Reader " << m_ReaderRank
-                          << " Init() writerID " << m_WriterID << "\n";
-            }
-        }
     }
 }
 
@@ -223,12 +236,12 @@ void InlineReader::SetDeferredVariablePointers()
     // this will make Variable::Info::Data() work correctly for the user
     for (const auto &varName : m_DeferredVariables)
     {
-        const std::string type = m_IO.InquireVariableType(varName);
-        if (type == "compound")
+        const DataType type = m_IO.InquireVariableType(varName);
+        if (type == DataType::Compound)
         {
         }
 #define declare_type(T)                                                        \
-    else if (type == helper::GetType())                                     \
+    else if (type == helper::GetDataType())                                 \
     {                                                                          \
         Variable &variable =                                                \
             FindVariable(varName, "in call to EndStep");                    \
diff --git a/source/adios2/engine/inline/InlineReader.h b/source/adios2/engine/inline/InlineReader.h
index 9d1ac88dbf..44bf43417e 100644
--- a/source/adios2/engine/inline/InlineReader.h
+++ b/source/adios2/engine/inline/InlineReader.h
@@ -25,6 +25,10 @@ namespace core
 namespace engine
 {
 
+// The inline reader needs to know about the writer, and vice versa.
+// Break cyclic dependency via a forward declaration:
+class InlineWriter;
+
 class InlineReader : public Engine
 {
 public:
@@ -51,6 +55,7 @@ class InlineReader : public Engine
     bool IsInsideStep() const;
 
 private:
+    const InlineWriter *GetWriter() const;
     int m_Verbosity = 0;
     int m_ReaderRank; // my rank in the readers' comm
 
@@ -59,8 +64,6 @@ class InlineReader : public Engine
     bool m_InsideStep = false;
     std::vector m_DeferredVariables;
 
-    std::string m_WriterID;
-
     void Init() final; ///< called from constructor, gets the selected Inline
                        /// transport method from settings
     void InitParameters() final;
diff --git a/source/adios2/engine/inline/InlineReader.tcc b/source/adios2/engine/inline/InlineReader.tcc
index b29d3a7687..bd7e159219 100644
--- a/source/adios2/engine/inline/InlineReader.tcc
+++ b/source/adios2/engine/inline/InlineReader.tcc
@@ -55,8 +55,6 @@ template 
 inline typename Variable::Info *
 InlineReader::GetBlockSyncCommon(Variable &variable)
 {
-    InlineWriter &writer =
-        dynamic_cast(m_IO.GetEngine(m_WriterID));
     if (variable.m_BlockID >= variable.m_BlocksInfo.size())
     {
         throw std::invalid_argument(
@@ -79,8 +77,6 @@ template 
 inline typename Variable::Info *
 InlineReader::GetBlockDeferredCommon(Variable &variable)
 {
-    InlineWriter &writer =
-        dynamic_cast(m_IO.GetEngine(m_WriterID));
     if (variable.m_BlockID >= variable.m_BlocksInfo.size())
     {
         throw std::invalid_argument(
diff --git a/source/adios2/engine/inline/InlineWriter.cpp b/source/adios2/engine/inline/InlineWriter.cpp
index 2af03126a5..e5d47bfb35 100644
--- a/source/adios2/engine/inline/InlineWriter.cpp
+++ b/source/adios2/engine/inline/InlineWriter.cpp
@@ -39,6 +39,30 @@ InlineWriter::InlineWriter(IO &io, const std::string &name, const Mode mode,
     }
 }
 
+const InlineReader *InlineWriter::GetReader() const
+{
+    const auto &engine_map = m_IO.GetEngines();
+    if (engine_map.size() != 2)
+    {
+        throw std::runtime_error("There must be exactly one reader and one "
+                                 "writer for the inline engine.");
+    }
+
+    std::shared_ptr e = engine_map.begin()->second;
+    if (e->OpenMode() == adios2::Mode::Write)
+    {
+        e = engine_map.rbegin()->second;
+    }
+
+    const auto reader = dynamic_cast(e.get());
+    if (!reader)
+    {
+        throw std::runtime_error(
+            "dynamic_cast failed; this is very likely a bug.");
+    }
+    return reader;
+}
+
 StepStatus InlineWriter::BeginStep(StepMode mode, const float timeoutSeconds)
 {
     TAU_SCOPED_TIMER("InlineWriter::BeginStep");
@@ -47,9 +71,9 @@ StepStatus InlineWriter::BeginStep(StepMode mode, const float timeoutSeconds)
         throw std::runtime_error("InlineWriter::BeginStep was called but the "
                                  "writer is already inside a step");
     }
-    const auto &reader =
-        dynamic_cast(m_IO.GetEngine(m_ReaderID));
-    if (reader.IsInsideStep())
+
+    auto reader = GetReader();
+    if (reader->IsInsideStep())
     {
         m_InsideStep = false;
         return StepStatus::NotReady;
@@ -82,13 +106,13 @@ void InlineWriter::ResetVariables()
     for (auto &varPair : availVars)
     {
         const auto &name = varPair.first;
-        const std::string type = m_IO.InquireVariableType(name);
+        const DataType type = m_IO.InquireVariableType(name);
 
-        if (type == "compound")
+        if (type == DataType::Compound)
         {
         }
 #define declare_type(T)                                                        \
-    else if (type == helper::GetType())                                     \
+    else if (type == helper::GetDataType())                                 \
     {                                                                          \
         Variable &variable = FindVariable(name, "in call to BeginStep"); \
         variable.m_BlocksInfo.clear();                                         \
@@ -178,15 +202,6 @@ void InlineWriter::InitParameters()
                     "integer in the range [0,5], in call to "
                     "Open or Engine constructor\n");
         }
-        else if (key == "readerid")
-        {
-            m_ReaderID = value;
-            if (m_Verbosity == 5)
-            {
-                std::cout << "Inline Writer " << m_WriterRank
-                          << " Init() readerID " << m_ReaderID << "\n";
-            }
-        }
     }
 }
 
diff --git a/source/adios2/engine/inline/InlineWriter.h b/source/adios2/engine/inline/InlineWriter.h
index 41566413ce..87aa26d092 100644
--- a/source/adios2/engine/inline/InlineWriter.h
+++ b/source/adios2/engine/inline/InlineWriter.h
@@ -23,6 +23,10 @@ namespace core
 namespace engine
 {
 
+// The inline reader needs to know about the writer, and vice versa.
+// Break cyclic dependency via a forward declaration:
+class InlineReader;
+
 class InlineWriter : public Engine
 {
 
@@ -55,11 +59,10 @@ class InlineWriter : public Engine
     bool m_InsideStep = false;
     bool m_ResetVariables = false; // used when PerformPuts is being used
 
-    std::string m_ReaderID;
-
     void Init() final;
     void InitParameters() final;
     void InitTransports() final;
+    const InlineReader *GetReader() const;
 
 #define declare_type(T)                                                        \
     void DoPutSync(Variable &, const T *) final;                            \
diff --git a/source/adios2/engine/insitumpi/InSituMPIReader.cpp b/source/adios2/engine/insitumpi/InSituMPIReader.cpp
index 20b6f7e192..0b604c5456 100644
--- a/source/adios2/engine/insitumpi/InSituMPIReader.cpp
+++ b/source/adios2/engine/insitumpi/InSituMPIReader.cpp
@@ -265,8 +265,8 @@ StepStatus InSituMPIReader::BeginStep(const StepMode mode,
         if (m_Verbosity == 5)
         {
             std::cout << "InSituMPI Reader " << m_ReaderRank << " found "
-                      << m_IO.GetVariablesDataMap().size() << " variables and "
-                      << m_IO.GetAttributesDataMap().size()
+                      << m_IO.GetVariables().size() << " variables and "
+                      << m_IO.GetAttributes().size()
                       << " attributes in metadata. Is source row major = "
                       << m_BP3Deserializer.m_IsRowMajor << std::endl;
         }
@@ -488,14 +488,14 @@ void InSituMPIReader::AsyncRecvAllVariables()
     for (const auto &variablePair : m_ReadScheduleMap)
     {
         // AsyncRecvVariable(variablePair.first, variablePair.second);
-        const std::string type(m_IO.InquireVariableType(variablePair.first));
+        const DataType type(m_IO.InquireVariableType(variablePair.first));
 
-        if (type == "compound")
+        if (type == DataType::Compound)
         {
             // not supported
         }
 #define declare_template_instantiation(T)                                      \
-    else if (type == helper::GetType())                                     \
+    else if (type == helper::GetDataType())                                 \
     {                                                                          \
         core::Variable *variable =                                          \
             m_IO.InquireVariable(variablePair.first);                       \
diff --git a/source/adios2/engine/insitumpi/InSituMPIWriter.cpp b/source/adios2/engine/insitumpi/InSituMPIWriter.cpp
index ce52a04b17..c846f9d90d 100644
--- a/source/adios2/engine/insitumpi/InSituMPIWriter.cpp
+++ b/source/adios2/engine/insitumpi/InSituMPIWriter.cpp
@@ -281,14 +281,14 @@ void InSituMPIWriter::PerformPuts()
 void InSituMPIWriter::AsyncSendVariable(std::string variableName)
 {
     TAU_SCOPED_TIMER("InSituMPIWriter::AsyncSendVariable");
-    const std::string type(m_IO.InquireVariableType(variableName));
+    const DataType type(m_IO.InquireVariableType(variableName));
 
-    if (type == "compound")
+    if (type == DataType::Compound)
     {
         // not supported
     }
 #define declare_template_instantiation(T)                                      \
-    else if (type == helper::GetType())                                     \
+    else if (type == helper::GetDataType())                                 \
     {                                                                          \
         Variable *variable = m_IO.InquireVariable(variableName);         \
         if (variable == nullptr)                                               \
diff --git a/source/adios2/engine/ssc/SscHelper.cpp b/source/adios2/engine/ssc/SscHelper.cpp
index ccb64092e3..e091fed011 100644
--- a/source/adios2/engine/ssc/SscHelper.cpp
+++ b/source/adios2/engine/ssc/SscHelper.cpp
@@ -24,21 +24,20 @@ namespace engine
 namespace ssc
 {
 
-size_t GetTypeSize(const std::string &type)
+size_t GetTypeSize(DataType type)
 {
-    if (type.empty())
+    if (type == DataType::None)
     {
         throw(std::runtime_error("unknown data type"));
     }
 #define declare_type(T)                                                        \
-    else if (type == helper::GetType()) { return sizeof(T); }
+    else if (type == helper::GetDataType()) { return sizeof(T); }
     ADIOS2_FOREACH_STDTYPE_1ARG(declare_type)
 #undef declare_type
     else { throw(std::runtime_error("unknown data type")); }
 }
 
-size_t TotalDataSize(const Dims &dims, const std::string &type,
-                     const ShapeID &shapeId)
+size_t TotalDataSize(const Dims &dims, DataType type, const ShapeID &shapeId)
 {
     if (shapeId == ShapeID::GlobalArray || shapeId == ShapeID::LocalArray)
     {
@@ -49,11 +48,7 @@ size_t TotalDataSize(const Dims &dims, const std::string &type,
     {
         return GetTypeSize(type);
     }
-    else
-    {
-        throw(std::runtime_error("ShapeID not supported"));
-    }
-    return 0;
+    throw(std::runtime_error("ShapeID not supported"));
 }
 
 size_t TotalDataSize(const BlockVec &bv)
@@ -61,7 +56,7 @@ size_t TotalDataSize(const BlockVec &bv)
     size_t s = 0;
     for (const auto &b : bv)
     {
-        if (b.type == "string")
+        if (b.type == DataType::String)
         {
             s += b.bufferCount;
         }
@@ -95,7 +90,7 @@ RankPosMap CalculateOverlap(BlockVecVec &globalVecVec, const BlockVec &localVec)
                         for (size_t i = 0; i < gBlock.start.size(); ++i)
                         {
                             if (gBlock.start[i] + gBlock.count[i] <=
-                                    lBlock.start[i] or
+                                    lBlock.start[i] ||
                                 lBlock.start[i] + lBlock.count[i] <=
                                     gBlock.start[i])
                             {
@@ -145,17 +140,17 @@ void BlockVecToJson(const BlockVec &input, nlohmann::json &output)
 
 void AttributeMapToJson(IO &input, nlohmann::json &output)
 {
-    const auto &attributeMap = input.GetAttributesDataMap();
+    const auto &attributeMap = input.GetAttributes();
     auto &attributesJson = output["Attributes"];
     for (const auto &attributePair : attributeMap)
     {
         const std::string name(attributePair.first);
-        const std::string type(attributePair.second.first);
-        if (type.empty())
+        const DataType type(attributePair.second->m_Type);
+        if (type == DataType::None)
         {
         }
 #define declare_type(T)                                                        \
-    else if (type == helper::GetType())                                     \
+    else if (type == helper::GetDataType())                                 \
     {                                                                          \
         const auto &attribute = input.InquireAttribute(name);               \
         nlohmann::json attributeJson;                                          \
@@ -218,7 +213,7 @@ void JsonToBlockVecVec(const nlohmann::json &input, BlockVecVec &output)
                 output[i].emplace_back();
                 auto &b = output[i].back();
                 b.name = j["Name"].get();
-                b.type = j["Type"].get();
+                b.type = j["Type"].get();
                 b.shapeId = j["ShapeID"].get();
                 b.start = j["Start"].get();
                 b.count = j["Count"].get();
@@ -281,6 +276,187 @@ bool AreSameDims(const Dims &a, const Dims &b)
     return true;
 }
 
+void MPI_Gatherv64(const void *sendbuf, uint64_t sendcount,
+                   MPI_Datatype sendtype, void *recvbuf,
+                   const uint64_t *recvcounts, const uint64_t *displs,
+                   MPI_Datatype recvtype, int root, MPI_Comm comm,
+                   const int chunksize)
+{
+
+    int mpiSize;
+    int mpiRank;
+    MPI_Comm_size(comm, &mpiSize);
+    MPI_Comm_rank(comm, &mpiRank);
+
+    int recvTypeSize;
+    int sendTypeSize;
+
+    MPI_Type_size(recvtype, &recvTypeSize);
+    MPI_Type_size(sendtype, &sendTypeSize);
+
+    std::vector requests;
+    if (mpiRank == root)
+    {
+        for (int i = 0; i < mpiSize; ++i)
+        {
+            uint64_t recvcount = recvcounts[i];
+            while (recvcount > 0)
+            {
+                requests.emplace_back();
+                if (recvcount > chunksize)
+                {
+                    MPI_Irecv(reinterpret_cast(recvbuf) +
+                                  (displs[i] + recvcounts[i] - recvcount) *
+                                      recvTypeSize,
+                              chunksize, recvtype, i, 0, comm,
+                              &requests.back());
+                    recvcount -= chunksize;
+                }
+                else
+                {
+                    MPI_Irecv(reinterpret_cast(recvbuf) +
+                                  (displs[i] + recvcounts[i] - recvcount) *
+                                      recvTypeSize,
+                              static_cast(recvcount), recvtype, i, 0, comm,
+                              &requests.back());
+                    recvcount = 0;
+                }
+            }
+        }
+    }
+
+    uint64_t sendcountvar = sendcount;
+
+    while (sendcountvar > 0)
+    {
+        requests.emplace_back();
+        if (sendcountvar > chunksize)
+        {
+            MPI_Isend(reinterpret_cast(sendbuf) +
+                          (sendcount - sendcountvar) * sendTypeSize,
+                      chunksize, sendtype, root, 0, comm, &requests.back());
+            sendcountvar -= chunksize;
+        }
+        else
+        {
+            MPI_Isend(reinterpret_cast(sendbuf) +
+                          (sendcount - sendcountvar) * sendTypeSize,
+                      static_cast(sendcountvar), sendtype, root, 0, comm,
+                      &requests.back());
+            sendcountvar = 0;
+        }
+    }
+
+    MPI_Waitall(static_cast(requests.size()), requests.data(),
+                MPI_STATUSES_IGNORE);
+}
+
+void MPI_Gatherv64OneSidedPull(const void *sendbuf, uint64_t sendcount,
+                               MPI_Datatype sendtype, void *recvbuf,
+                               const uint64_t *recvcounts,
+                               const uint64_t *displs, MPI_Datatype recvtype,
+                               int root, MPI_Comm comm, const int chunksize)
+{
+
+    int mpiSize;
+    int mpiRank;
+    MPI_Comm_size(comm, &mpiSize);
+    MPI_Comm_rank(comm, &mpiRank);
+
+    int recvTypeSize;
+    int sendTypeSize;
+
+    MPI_Type_size(recvtype, &recvTypeSize);
+    MPI_Type_size(sendtype, &sendTypeSize);
+
+    MPI_Win win;
+    MPI_Win_create(const_cast(sendbuf), sendcount * sendTypeSize,
+                   sendTypeSize, MPI_INFO_NULL, comm, &win);
+
+    if (mpiRank == root)
+    {
+        for (int i = 0; i < mpiSize; ++i)
+        {
+            uint64_t recvcount = recvcounts[i];
+            while (recvcount > 0)
+            {
+                if (recvcount > chunksize)
+                {
+                    MPI_Get(reinterpret_cast(recvbuf) +
+                                (displs[i] + recvcounts[i] - recvcount) *
+                                    recvTypeSize,
+                            chunksize, recvtype, i, recvcounts[i] - recvcount,
+                            chunksize, recvtype, win);
+                    recvcount -= chunksize;
+                }
+                else
+                {
+                    MPI_Get(reinterpret_cast(recvbuf) +
+                                (displs[i] + recvcounts[i] - recvcount) *
+                                    recvTypeSize,
+                            static_cast(recvcount), recvtype, i,
+                            recvcounts[i] - recvcount,
+                            static_cast(recvcount), recvtype, win);
+                    recvcount = 0;
+                }
+            }
+        }
+    }
+
+    MPI_Win_free(&win);
+}
+
+void MPI_Gatherv64OneSidedPush(const void *sendbuf, uint64_t sendcount,
+                               MPI_Datatype sendtype, void *recvbuf,
+                               const uint64_t *recvcounts,
+                               const uint64_t *displs, MPI_Datatype recvtype,
+                               int root, MPI_Comm comm, const int chunksize)
+{
+
+    int mpiSize;
+    int mpiRank;
+    MPI_Comm_size(comm, &mpiSize);
+    MPI_Comm_rank(comm, &mpiRank);
+
+    int recvTypeSize;
+    int sendTypeSize;
+
+    MPI_Type_size(recvtype, &recvTypeSize);
+    MPI_Type_size(sendtype, &sendTypeSize);
+
+    uint64_t recvsize = displs[mpiSize - 1] + recvcounts[mpiSize - 1];
+
+    MPI_Win win;
+    MPI_Win_create(recvbuf, recvsize * recvTypeSize, recvTypeSize,
+                   MPI_INFO_NULL, comm, &win);
+
+    uint64_t sendcountvar = sendcount;
+
+    while (sendcountvar > 0)
+    {
+        if (sendcountvar > chunksize)
+        {
+            MPI_Put(reinterpret_cast(sendbuf) +
+                        (sendcount - sendcountvar) * sendTypeSize,
+                    chunksize, sendtype, root,
+                    displs[mpiRank] + sendcount - sendcountvar, chunksize,
+                    sendtype, win);
+            sendcountvar -= chunksize;
+        }
+        else
+        {
+            MPI_Put(reinterpret_cast(sendbuf) +
+                        (sendcount - sendcountvar) * sendTypeSize,
+                    static_cast(sendcountvar), sendtype, root,
+                    displs[mpiRank] + sendcount - sendcountvar,
+                    static_cast(sendcountvar), sendtype, win);
+            sendcountvar = 0;
+        }
+    }
+
+    MPI_Win_free(&win);
+}
+
 void PrintDims(const Dims &dims, const std::string &label)
 {
     std::cout << label;
@@ -295,7 +471,7 @@ void PrintBlock(const BlockInfo &b, const std::string &label)
 {
     std::cout << label << std::endl;
     std::cout << b.name << std::endl;
-    std::cout << "    Type : " << b.type << std::endl;
+    std::cout << "    DataType : " << b.type << std::endl;
     PrintDims(b.shape, "    Shape : ");
     PrintDims(b.start, "    Start : ");
     PrintDims(b.count, "    Count : ");
@@ -309,7 +485,7 @@ void PrintBlockVec(const BlockVec &bv, const std::string &label)
     for (const auto &i : bv)
     {
         std::cout << i.name << std::endl;
-        std::cout << "    Type : " << i.type << std::endl;
+        std::cout << "    DataType : " << i.type << std::endl;
         PrintDims(i.shape, "    Shape : ");
         PrintDims(i.start, "    Start : ");
         PrintDims(i.count, "    Count : ");
@@ -328,7 +504,7 @@ void PrintBlockVecVec(const BlockVecVec &bvv, const std::string &label)
         for (const auto &i : bv)
         {
             std::cout << "    " << i.name << std::endl;
-            std::cout << "        Type : " << i.type << std::endl;
+            std::cout << "        DataType : " << i.type << std::endl;
             PrintDims(i.shape, "        Shape : ");
             PrintDims(i.start, "        Start : ");
             PrintDims(i.count, "        Count : ");
diff --git a/source/adios2/engine/ssc/SscHelper.h b/source/adios2/engine/ssc/SscHelper.h
index 8f5679f553..5f7844145d 100644
--- a/source/adios2/engine/ssc/SscHelper.h
+++ b/source/adios2/engine/ssc/SscHelper.h
@@ -14,7 +14,8 @@
 #include "adios2/common/ADIOSTypes.h"
 #include "adios2/core/IO.h"
 #include "nlohmann/json.hpp"
-#include 
+#include 
+#include 
 #include 
 
 namespace adios2
@@ -28,7 +29,7 @@ namespace ssc
 struct BlockInfo
 {
     std::string name;
-    std::string type;
+    DataType type;
     ShapeID shapeId;
     Dims shape;
     Dims start;
@@ -39,7 +40,7 @@ struct BlockInfo
 };
 using BlockVec = std::vector;
 using BlockVecVec = std::vector;
-using RankPosMap = std::map>;
+using RankPosMap = std::unordered_map>;
 using MpiInfo = std::vector>;
 
 void PrintDims(const Dims &dims, const std::string &label = std::string());
@@ -54,8 +55,7 @@ void PrintMpiInfo(const MpiInfo &writersInfo, const MpiInfo &readersInfo);
 
 size_t GetTypeSize(const std::string &type);
 
-size_t TotalDataSize(const Dims &dims, const std::string &type,
-                     const ShapeID &shapeId);
+size_t TotalDataSize(const Dims &dims, DataType type, const ShapeID &shapeId);
 size_t TotalDataSize(const BlockVec &bv);
 
 RankPosMap CalculateOverlap(BlockVecVec &globalPattern,
@@ -71,6 +71,22 @@ void JsonToBlockVecVec(const nlohmann::json &input, BlockVecVec &output);
 void JsonToBlockVecVec(const std::vector &input, BlockVecVec &output);
 void JsonToBlockVecVec(const std::string &input, BlockVecVec &output);
 
+void MPI_Gatherv64OneSidedPush(
+    const void *sendbuf, uint64_t sendcount, MPI_Datatype sendtype,
+    void *recvbuf, const uint64_t *recvcounts, const uint64_t *displs,
+    MPI_Datatype recvtype, int root, MPI_Comm comm,
+    const int chunksize = std::numeric_limits::max());
+void MPI_Gatherv64OneSidedPull(
+    const void *sendbuf, uint64_t sendcount, MPI_Datatype sendtype,
+    void *recvbuf, const uint64_t *recvcounts, const uint64_t *displs,
+    MPI_Datatype recvtype, int root, MPI_Comm comm,
+    const int chunksize = std::numeric_limits::max());
+void MPI_Gatherv64(const void *sendbuf, uint64_t sendcount,
+                   MPI_Datatype sendtype, void *recvbuf,
+                   const uint64_t *recvcounts, const uint64_t *displs,
+                   MPI_Datatype recvtype, int root, MPI_Comm comm,
+                   const int chunksize = std::numeric_limits::max());
+
 bool AreSameDims(const Dims &a, const Dims &b);
 
 } // end namespace ssc
diff --git a/source/adios2/engine/ssc/SscReader.cpp b/source/adios2/engine/ssc/SscReader.cpp
index 79cd144ea4..eb27fd1881 100644
--- a/source/adios2/engine/ssc/SscReader.cpp
+++ b/source/adios2/engine/ssc/SscReader.cpp
@@ -13,6 +13,7 @@
 #include "adios2/helper/adiosCommMPI.h"
 #include "adios2/helper/adiosFunctions.h"
 #include "adios2/helper/adiosJSONcomplex.h"
+#include "adios2/helper/adiosMpiHandshake.h"
 #include "nlohmann/json.hpp"
 
 namespace adios2
@@ -30,114 +31,75 @@ SscReader::SscReader(IO &io, const std::string &name, const Mode mode,
 
     helper::GetParameter(m_IO.m_Parameters, "MpiMode", m_MpiMode);
     helper::GetParameter(m_IO.m_Parameters, "Verbose", m_Verbosity);
-    helper::GetParameter(m_IO.m_Parameters, "MaxFilenameLength",
-                         m_MaxFilenameLength);
-    helper::GetParameter(m_IO.m_Parameters, "RendezvousAppCount",
-                         m_RendezvousAppCount);
-    helper::GetParameter(m_IO.m_Parameters, "MaxStreamsPerApp",
-                         m_MaxStreamsPerApp);
     helper::GetParameter(m_IO.m_Parameters, "OpenTimeoutSecs",
                          m_OpenTimeoutSecs);
 
-    m_Buffer.resize(1);
-
     SyncMpiPattern();
     m_ReaderRank = m_Comm.Rank();
     m_ReaderSize = m_Comm.Size();
     MPI_Comm_rank(m_StreamComm, &m_StreamRank);
     MPI_Comm_size(m_StreamComm, &m_StreamSize);
-
-    m_GlobalWritePattern.resize(m_StreamSize);
 }
 
 SscReader::~SscReader() { TAU_SCOPED_TIMER_FUNC(); }
 
-void SscReader::GetOneSidedPostPush()
+StepStatus SscReader::BeginStep(const StepMode stepMode,
+                                const float timeoutSeconds)
 {
     TAU_SCOPED_TIMER_FUNC();
-    MPI_Win_post(m_MpiAllWritersGroup, 0, m_MpiWin);
-    MPI_Win_wait(m_MpiWin);
-}
 
-void SscReader::GetOneSidedFencePush()
-{
-    TAU_SCOPED_TIMER_FUNC();
-    MPI_Win_fence(0, m_MpiWin);
-    MPI_Win_fence(0, m_MpiWin);
-}
+    ++m_CurrentStep;
 
-void SscReader::GetOneSidedPostPull()
-{
-    TAU_SCOPED_TIMER_FUNC();
-    MPI_Win_start(m_MpiAllWritersGroup, 0, m_MpiWin);
-    for (const auto &i : m_AllReceivingWriterRanks)
-    {
-        MPI_Get(m_Buffer.data() + i.second.first, i.second.second, MPI_CHAR,
-                i.first, 0, i.second.second, MPI_CHAR, m_MpiWin);
-    }
-    MPI_Win_complete(m_MpiWin);
-}
+    m_StepBegun = true;
 
-void SscReader::GetOneSidedFencePull()
-{
-    TAU_SCOPED_TIMER_FUNC();
-    MPI_Win_fence(0, m_MpiWin);
-    for (const auto &i : m_AllReceivingWriterRanks)
+    if (m_Verbosity >= 5)
     {
-        MPI_Get(m_Buffer.data() + i.second.first, i.second.second, MPI_CHAR,
-                i.first, 0, i.second.second, MPI_CHAR, m_MpiWin);
+        std::cout << "SscReader::BeginStep, World Rank " << m_StreamRank
+                  << ", Reader Rank " << m_ReaderRank << ", Step "
+                  << m_CurrentStep << std::endl;
     }
-    MPI_Win_fence(0, m_MpiWin);
-}
 
-void SscReader::GetTwoSided()
-{
-    TAU_SCOPED_TIMER_FUNC();
-    std::vector requests;
-    for (const auto &i : m_AllReceivingWriterRanks)
+    if (m_CurrentStep == 0 || m_WriterDefinitionsLocked == false ||
+        m_ReaderSelectionsLocked == false)
     {
-        requests.emplace_back();
-        MPI_Irecv(m_Buffer.data() + i.second.first, i.second.second, MPI_CHAR,
-                  i.first, 0, m_StreamComm, &requests.back());
-    }
-    MPI_Status statuses[requests.size()];
-    MPI_Waitall(requests.size(), requests.data(), statuses);
-}
-
-StepStatus SscReader::BeginStep(const StepMode stepMode,
-                                const float timeoutSeconds)
-{
-    TAU_SCOPED_TIMER_FUNC();
+        m_AllReceivingWriterRanks.clear();
+        m_ReceivedRanks.clear();
+        m_Buffer.resize(1, 0);
+        m_GlobalWritePattern.clear();
+        m_GlobalWritePattern.resize(m_StreamSize);
+        m_LocalReadPattern.clear();
+        m_GlobalWritePatternJson.clear();
+        bool finalStep = SyncWritePattern();
+        if (finalStep)
+        {
+            return StepStatus::EndOfStream;
+        }
 
-    if (m_InitialStep)
-    {
-        m_InitialStep = false;
-        SyncWritePattern();
         MPI_Win_create(NULL, 0, 1, MPI_INFO_NULL, m_StreamComm, &m_MpiWin);
-        MPI_Win_start(m_MpiAllWritersGroup, 0, m_MpiWin);
     }
     else
     {
-        ++m_CurrentStep;
         if (m_MpiMode == "twosided")
         {
-            GetTwoSided();
+            MPI_Waitall(static_cast(m_MpiRequests.size()),
+                        m_MpiRequests.data(), MPI_STATUS_IGNORE);
+            m_MpiRequests.clear();
         }
         else if (m_MpiMode == "onesidedfencepush")
         {
-            GetOneSidedFencePush();
+            MPI_Win_fence(0, m_MpiWin);
         }
         else if (m_MpiMode == "onesidedpostpush")
         {
-            GetOneSidedPostPush();
+            MPI_Win_wait(m_MpiWin);
         }
         else if (m_MpiMode == "onesidedfencepull")
         {
-            GetOneSidedFencePull();
+            MPI_Win_fence(0, m_MpiWin);
         }
         else if (m_MpiMode == "onesidedpostpull")
         {
-            GetOneSidedPostPull();
+            MPI_Win_complete(m_MpiWin);
         }
     }
 
@@ -149,7 +111,8 @@ StepStatus SscReader::BeginStep(const StepMode stepMode,
                 v.shapeId == ShapeID::LocalValue)
             {
                 std::vector value(v.bufferCount);
-                if (m_CurrentStep == 0)
+                if (m_CurrentStep == 0 || m_WriterDefinitionsLocked == false ||
+                    m_ReaderSelectionsLocked == false)
                 {
                     std::memcpy(value.data(), v.value.data(), v.value.size());
                 }
@@ -158,11 +121,11 @@ StepStatus SscReader::BeginStep(const StepMode stepMode,
                     std::memcpy(value.data(), m_Buffer.data() + v.bufferStart,
                                 v.bufferCount);
                 }
-                if (v.type.empty())
+                if (v.type == DataType::None)
                 {
                     throw(std::runtime_error("unknown data type"));
                 }
-                else if (v.type == "string")
+                else if (v.type == DataType::String)
                 {
                     auto variable = m_IO.InquireVariable(v.name);
                     if (variable)
@@ -176,7 +139,7 @@ StepStatus SscReader::BeginStep(const StepMode stepMode,
                     }
                 }
 #define declare_type(T)                                                        \
-    else if (v.type == helper::GetType())                                   \
+    else if (v.type == helper::GetDataType())                               \
     {                                                                          \
         auto variable = m_IO.InquireVariable(v.name);                       \
         if (variable)                                                          \
@@ -193,13 +156,6 @@ StepStatus SscReader::BeginStep(const StepMode stepMode,
         }
     }
 
-    if (m_Verbosity >= 5)
-    {
-        std::cout << "SscReader::BeginStep, World Rank " << m_StreamRank
-                  << ", Reader Rank " << m_ReaderRank << ", Step "
-                  << m_CurrentStep << std::endl;
-    }
-
     if (m_Buffer[0] == 1)
     {
         return StepStatus::EndOfStream;
@@ -208,88 +164,106 @@ StepStatus SscReader::BeginStep(const StepMode stepMode,
     return StepStatus::OK;
 }
 
-void SscReader::PerformGets() { TAU_SCOPED_TIMER_FUNC(); }
+void SscReader::PerformGets() {}
 
-size_t SscReader::CurrentStep() const
-{
-    TAU_SCOPED_TIMER_FUNC();
-    return m_CurrentStep;
-}
+size_t SscReader::CurrentStep() const { return m_CurrentStep; }
 
 void SscReader::EndStep()
-{
-    TAU_SCOPED_TIMER_FUNC();
-    if (m_CurrentStep == 0)
-    {
-        MPI_Win_complete(m_MpiWin);
-        MPI_Win_free(&m_MpiWin);
-        SyncReadPattern();
-        MPI_Win_create(m_Buffer.data(), m_Buffer.size(), 1, MPI_INFO_NULL,
-                       m_StreamComm, &m_MpiWin);
-    }
-    if (m_Verbosity >= 5)
-    {
-        std::cout << "SscReader::EndStep, World Rank " << m_StreamRank
-                  << ", Reader Rank " << m_ReaderRank << std::endl;
-    }
-}
-
-// PRIVATE
-
-void SscReader::SyncMpiPattern()
 {
     TAU_SCOPED_TIMER_FUNC();
 
     if (m_Verbosity >= 5)
     {
-        std::cout << "SscReader::SyncMpiPattern, World Rank " << m_StreamRank
-                  << ", Reader Rank " << m_ReaderRank << std::endl;
+        std::cout << "SscReader::EndStep, World Rank " << m_StreamRank
+                  << ", Reader Rank " << m_ReaderRank << ", Step "
+                  << m_CurrentStep << std::endl;
     }
 
-    m_MpiHandshake.Handshake(m_Name, 'r', m_OpenTimeoutSecs, m_MaxStreamsPerApp,
-                             m_MaxFilenameLength, m_RendezvousAppCount,
-                             CommAsMPI(m_Comm));
-
-    std::vector allStreamRanks;
-    std::vector allWriterRanks;
-
-    for (const auto &app : m_MpiHandshake.GetWriterMap(m_Name))
+    if (m_WriterDefinitionsLocked &&
+        m_ReaderSelectionsLocked) // fixed IO pattern
     {
-        for (int rank : app.second)
+        if (m_CurrentStep == 0)
+        {
+            MPI_Win_free(&m_MpiWin);
+            SyncReadPattern();
+            MPI_Win_create(m_Buffer.data(), m_Buffer.size(), 1, MPI_INFO_NULL,
+                           m_StreamComm, &m_MpiWin);
+        }
+        if (m_MpiMode == "twosided")
         {
-            allWriterRanks.push_back(rank);
-            allStreamRanks.push_back(rank);
+            for (const auto &i : m_AllReceivingWriterRanks)
+            {
+                m_MpiRequests.emplace_back();
+                MPI_Irecv(m_Buffer.data() + i.second.first,
+                          static_cast(i.second.second), MPI_CHAR, i.first,
+                          0, m_StreamComm, &m_MpiRequests.back());
+            }
+        }
+        else if (m_MpiMode == "onesidedfencepush")
+        {
+            MPI_Win_fence(0, m_MpiWin);
+        }
+        else if (m_MpiMode == "onesidedpostpush")
+        {
+            MPI_Win_post(m_MpiAllWritersGroup, 0, m_MpiWin);
+        }
+        else if (m_MpiMode == "onesidedfencepull")
+        {
+            MPI_Win_fence(0, m_MpiWin);
+            for (const auto &i : m_AllReceivingWriterRanks)
+            {
+                MPI_Get(m_Buffer.data() + i.second.first,
+                        static_cast(i.second.second), MPI_CHAR, i.first, 0,
+                        static_cast(i.second.second), MPI_CHAR, m_MpiWin);
+            }
+        }
+        else if (m_MpiMode == "onesidedpostpull")
+        {
+            MPI_Win_start(m_MpiAllWritersGroup, 0, m_MpiWin);
+            for (const auto &i : m_AllReceivingWriterRanks)
+            {
+                MPI_Get(m_Buffer.data() + i.second.first,
+                        static_cast(i.second.second), MPI_CHAR, i.first, 0,
+                        static_cast(i.second.second), MPI_CHAR, m_MpiWin);
+            }
         }
     }
-
-    for (const auto &app : m_MpiHandshake.GetReaderMap(m_Name))
+    else // flexible IO pattern
     {
-        for (int rank : app.second)
+        MPI_Win_free(&m_MpiWin);
+        if (m_CurrentStep == 0)
         {
-            allStreamRanks.push_back(rank);
+            SyncReadPattern();
         }
     }
 
-    MPI_Group worldGroup;
-    MPI_Comm_group(MPI_COMM_WORLD, &worldGroup);
-    MPI_Group_incl(worldGroup, allWriterRanks.size(), allWriterRanks.data(),
-                   &m_MpiAllWritersGroup);
-
-    MPI_Comm_group(MPI_COMM_WORLD, &worldGroup);
-    std::sort(allStreamRanks.begin(), allStreamRanks.end());
-    MPI_Group allWorkersGroup;
-    MPI_Group_incl(worldGroup, allStreamRanks.size(), allStreamRanks.data(),
-                   &allWorkersGroup);
-    MPI_Comm_create_group(MPI_COMM_WORLD, allWorkersGroup, 0, &m_StreamComm);
+    m_StepBegun = false;
 }
 
-void SscReader::SyncWritePattern()
+// PRIVATE
+
+void SscReader::SyncMpiPattern()
+{
+    TAU_SCOPED_TIMER_FUNC();
+
+    MPI_Group streamGroup;
+    MPI_Group readerGroup;
+    MPI_Comm writerComm;
+    MPI_Comm readerComm;
+
+    helper::HandshakeComm(m_Name, 'r', m_OpenTimeoutSecs, CommAsMPI(m_Comm),
+                          streamGroup, m_MpiAllWritersGroup, readerGroup,
+                          m_StreamComm, writerComm, readerComm);
+}
+
+bool SscReader::SyncWritePattern()
 {
     TAU_SCOPED_TIMER_FUNC();
     if (m_Verbosity >= 5)
     {
         std::cout << "SscReader::SyncWritePattern, World Rank " << m_StreamRank
-                  << ", Reader Rank " << m_ReaderRank << std::endl;
+                  << ", Reader Rank " << m_ReaderRank << ", Step "
+                  << m_CurrentStep << std::endl;
     }
 
     // aggregate global write pattern
@@ -299,29 +273,44 @@ void SscReader::SyncWritePattern()
                   m_StreamComm);
     std::vector localVec(maxLocalSize, '\0');
     std::vector globalVec(maxLocalSize * m_StreamSize);
-    MPI_Allgather(localVec.data(), maxLocalSize, MPI_CHAR, globalVec.data(),
-                  maxLocalSize, MPI_CHAR, m_StreamComm);
+    MPI_Allgather(localVec.data(), static_cast(maxLocalSize), MPI_CHAR,
+                  globalVec.data(), static_cast(maxLocalSize), MPI_CHAR,
+                  m_StreamComm);
 
-    // deserialize global metadata Json
     ssc::LocalJsonToGlobalJson(globalVec, maxLocalSize, m_StreamSize,
                                m_GlobalWritePatternJson);
 
-    // deserialize variables metadata
+    for (int i = 0; i < m_StreamSize; ++i)
+    {
+        if (m_GlobalWritePatternJson[i] == nullptr)
+        {
+            continue;
+        }
+        auto &patternJson = m_GlobalWritePatternJson[i]["FinalStep"];
+        if (patternJson != nullptr)
+        {
+            if (patternJson.get())
+            {
+                return true;
+            }
+        }
+    }
+
     ssc::JsonToBlockVecVec(m_GlobalWritePatternJson, m_GlobalWritePattern);
 
     for (const auto &blockVec : m_GlobalWritePattern)
     {
         for (const auto &b : blockVec)
         {
-            if (b.type.empty())
+            if (b.type == DataType::None)
             {
                 throw(std::runtime_error("unknown data type"));
             }
 #define declare_type(T)                                                        \
-    else if (b.type == helper::GetType())                                   \
+    else if (b.type == helper::GetDataType())                               \
     {                                                                          \
         auto v = m_IO.InquireVariable(b.name);                              \
-        if (not v)                                                             \
+        if (!v)                                                                \
         {                                                                      \
             Dims vStart = b.start;                                             \
             Dims vShape = b.shape;                                             \
@@ -360,6 +349,13 @@ void SscReader::SyncWritePattern()
         {
             continue;
         }
+
+        auto &patternJson = m_GlobalWritePatternJson[i]["Pattern"];
+        if (patternJson != nullptr)
+        {
+            m_WriterDefinitionsLocked = patternJson.get();
+        }
+
         auto &attributesJson = m_GlobalWritePatternJson[i]["Attributes"];
         if (attributesJson == nullptr)
         {
@@ -367,17 +363,16 @@ void SscReader::SyncWritePattern()
         }
         for (const auto &attributeJson : attributesJson)
         {
-            const std::string type(attributeJson["Type"].get());
-            if (type.empty())
+            const DataType type(attributeJson["Type"].get());
+            if (type == DataType::None)
             {
             }
 #define declare_type(T)                                                        \
-    else if (type == helper::GetType())                                     \
+    else if (type == helper::GetDataType())                                 \
     {                                                                          \
-        const auto &attributesDataMap = m_IO.GetAttributesDataMap();           \
-        auto it =                                                              \
-            attributesDataMap.find(attributeJson["Name"].get());  \
-        if (it == attributesDataMap.end())                                     \
+        const auto &attributes = m_IO.GetAttributes();                         \
+        auto it = attributes.find(attributeJson["Name"].get());   \
+        if (it == attributes.end())                                            \
         {                                                                      \
             if (attributeJson["IsSingleValue"].get())                    \
             {                                                                  \
@@ -398,15 +393,23 @@ void SscReader::SyncWritePattern()
 #undef declare_type
         }
     }
+    return false;
 }
 
 void SscReader::SyncReadPattern()
 {
     TAU_SCOPED_TIMER_FUNC();
+
     if (m_Verbosity >= 5)
     {
         std::cout << "SscReader::SyncReadPattern, World Rank " << m_StreamRank
-                  << ", Reader Rank " << m_ReaderRank << std::endl;
+                  << ", Reader Rank " << m_ReaderRank << ", Step "
+                  << m_CurrentStep << std::endl;
+    }
+
+    if (m_ReaderRank == 0)
+    {
+        m_LocalReadPatternJson["Pattern"] = m_ReaderSelectionsLocked;
     }
 
     std::string localStr = m_LocalReadPatternJson.dump();
@@ -419,8 +422,9 @@ void SscReader::SyncReadPattern()
     std::vector localVec(maxLocalSize, '\0');
     std::memcpy(localVec.data(), localStr.c_str(), localStr.size());
     std::vector globalVec(maxLocalSize * m_StreamSize);
-    MPI_Allgather(localVec.data(), maxLocalSize, MPI_CHAR, globalVec.data(),
-                  maxLocalSize, MPI_CHAR, m_StreamComm);
+    MPI_Allgather(localVec.data(), static_cast(maxLocalSize), MPI_CHAR,
+                  globalVec.data(), static_cast(maxLocalSize), MPI_CHAR,
+                  m_StreamComm);
 
     // deserialize global metadata Json
     nlohmann::json globalJson;
@@ -460,7 +464,17 @@ void SscReader::SyncReadPattern()
 
     if (m_Verbosity >= 10)
     {
-        ssc::PrintBlockVec(m_LocalReadPattern, "Local Read Pattern");
+        for (int i = 0; i < m_ReaderSize; ++i)
+        {
+            m_Comm.Barrier();
+            if (i == m_ReaderRank)
+            {
+                ssc::PrintBlockVec(m_LocalReadPattern,
+                                   "\n\nGlobal Read Pattern on Rank " +
+                                       std::to_string(m_ReaderRank));
+            }
+        }
+        m_Comm.Barrier();
     }
 }
 
@@ -471,7 +485,7 @@ void SscReader::CalculatePosition(ssc::BlockVecVec &bvv,
 
     size_t bufferPosition = 0;
 
-    for (size_t rank = 0; rank < bvv.size(); ++rank)
+    for (int rank = 0; rank < bvv.size(); ++rank)
     {
         bool hasOverlap = false;
         for (const auto r : allRanks)
@@ -523,7 +537,16 @@ void SscReader::DoClose(const int transportIndex)
         std::cout << "SscReader::DoClose, World Rank " << m_StreamRank
                   << ", Reader Rank " << m_ReaderRank << std::endl;
     }
-    MPI_Win_free(&m_MpiWin);
+
+    if (!m_StepBegun)
+    {
+        BeginStep();
+    }
+
+    if (m_WriterDefinitionsLocked && m_ReaderSelectionsLocked)
+    {
+        MPI_Win_free(&m_MpiWin);
+    }
 }
 
 } // end namespace engine
diff --git a/source/adios2/engine/ssc/SscReader.h b/source/adios2/engine/ssc/SscReader.h
index 31cc8bb2c2..e5c34adf15 100644
--- a/source/adios2/engine/ssc/SscReader.h
+++ b/source/adios2/engine/ssc/SscReader.h
@@ -17,6 +17,7 @@
 #include "adios2/toolkit/profiling/taustubs/tautimer.hpp"
 #include 
 #include 
+#include 
 
 namespace adios2
 {
@@ -39,8 +40,8 @@ class SscReader : public Engine
     void EndStep() final;
 
 private:
-    size_t m_CurrentStep = 0;
-    bool m_InitialStep = true;
+    int64_t m_CurrentStep = -1;
+    bool m_StepBegun = false;
 
     ssc::BlockVecVec m_GlobalWritePattern;
     ssc::BlockVec m_LocalReadPattern;
@@ -53,22 +54,17 @@ class SscReader : public Engine
     MPI_Group m_MpiAllWritersGroup;
     MPI_Comm m_StreamComm;
     std::string m_MpiMode = "twosided";
+    std::vector m_MpiRequests;
+    std::unordered_set m_ReceivedRanks;
 
     int m_StreamRank;
     int m_StreamSize;
     int m_ReaderRank;
     int m_ReaderSize;
 
-    helper::MpiHandshake m_MpiHandshake;
-
     void SyncMpiPattern();
-    void SyncWritePattern();
+    bool SyncWritePattern();
     void SyncReadPattern();
-    void GetOneSidedFencePush();
-    void GetOneSidedPostPush();
-    void GetOneSidedFencePull();
-    void GetOneSidedPostPull();
-    void GetTwoSided();
 
 #define declare_type(T)                                                        \
     void DoGetSync(Variable &, T *) final;                                  \
@@ -91,9 +87,6 @@ class SscReader : public Engine
                            ssc::RankPosMap &allOverlapRanks);
 
     int m_Verbosity = 0;
-    int m_MaxFilenameLength = 128;
-    int m_MaxStreamsPerApp = 1;
-    int m_RendezvousAppCount = 2;
     int m_OpenTimeoutSecs = 10;
 };
 
diff --git a/source/adios2/engine/ssc/SscReader.tcc b/source/adios2/engine/ssc/SscReader.tcc
index 967eb98f6b..34de0996e6 100644
--- a/source/adios2/engine/ssc/SscReader.tcc
+++ b/source/adios2/engine/ssc/SscReader.tcc
@@ -29,60 +29,76 @@ void SscReader::GetDeferredCommon(Variable &variable,
 {
     TAU_SCOPED_TIMER_FUNC();
     variable.SetData(data);
-    if (m_CurrentStep == 0)
+
+    if (m_WriterDefinitionsLocked && m_ReaderSelectionsLocked)
     {
-        m_LocalReadPattern.emplace_back();
-        auto &b = m_LocalReadPattern.back();
-        b.name = variable.m_Name;
-        b.count = variable.m_Count;
-        b.start = variable.m_Start;
-        b.shape = variable.m_Shape;
-        b.type = "string";
+        if (m_CurrentStep == 0)
+        {
+            m_LocalReadPattern.emplace_back();
+            auto &b = m_LocalReadPattern.back();
+            b.name = variable.m_Name;
+            b.count = variable.m_Count;
+            b.start = variable.m_Start;
+            b.shape = variable.m_Shape;
+            b.type = DataType::String;
 
-        m_LocalReadPatternJson["Variables"].emplace_back();
-        auto &jref = m_LocalReadPatternJson["Variables"].back();
-        jref["Name"] = b.name;
-        jref["Type"] = b.type;
-        jref["ShapeID"] = variable.m_ShapeID;
-        jref["Start"] = b.start;
-        jref["Count"] = b.count;
-        jref["Shape"] = b.shape;
-        jref["BufferStart"] = 0;
-        jref["BufferCount"] = 0;
+            m_LocalReadPatternJson["Variables"].emplace_back();
+            auto &jref = m_LocalReadPatternJson["Variables"].back();
+            jref["Name"] = b.name;
+            jref["Type"] = b.type;
+            jref["ShapeID"] = variable.m_ShapeID;
+            jref["Start"] = b.start;
+            jref["Count"] = b.count;
+            jref["Shape"] = b.shape;
+            jref["BufferStart"] = 0;
+            jref["BufferCount"] = 0;
 
-        ssc::JsonToBlockVecVec(m_GlobalWritePatternJson, m_GlobalWritePattern);
-        m_AllReceivingWriterRanks =
-            ssc::CalculateOverlap(m_GlobalWritePattern, m_LocalReadPattern);
-        CalculatePosition(m_GlobalWritePattern, m_AllReceivingWriterRanks);
-        size_t totalDataSize = 0;
-        for (auto i : m_AllReceivingWriterRanks)
-        {
-            totalDataSize += i.second.second;
+            ssc::JsonToBlockVecVec(m_GlobalWritePatternJson,
+                                   m_GlobalWritePattern);
+            m_AllReceivingWriterRanks =
+                ssc::CalculateOverlap(m_GlobalWritePattern, m_LocalReadPattern);
+            CalculatePosition(m_GlobalWritePattern, m_AllReceivingWriterRanks);
+            size_t totalDataSize = 0;
+            for (auto i : m_AllReceivingWriterRanks)
+            {
+                totalDataSize += i.second.second;
+            }
+            m_Buffer.resize(totalDataSize);
+            for (const auto &i : m_AllReceivingWriterRanks)
+            {
+                MPI_Win_lock(MPI_LOCK_SHARED, i.first, 0, m_MpiWin);
+                MPI_Get(m_Buffer.data() + i.second.first,
+                        static_cast(i.second.second), MPI_CHAR, i.first, 0,
+                        static_cast(i.second.second), MPI_CHAR, m_MpiWin);
+                MPI_Win_unlock(i.first, m_MpiWin);
+            }
         }
-        m_Buffer.resize(totalDataSize);
-        std::vector requests;
+
         for (const auto &i : m_AllReceivingWriterRanks)
         {
-            requests.emplace_back();
-            MPI_Rget(m_Buffer.data() + i.second.first, i.second.second,
-                     MPI_CHAR, i.first, 0, i.second.second, MPI_CHAR, m_MpiWin,
-                     &requests.back());
+            for (const auto &b : m_GlobalWritePattern[i.first])
+            {
+                if (b.name == variable.m_Name)
+                {
+                    std::vector str(b.bufferCount);
+                    std::memcpy(str.data(), m_Buffer.data() + b.bufferStart,
+                                b.bufferCount);
+                    *data = std::string(str.begin(), str.end());
+                }
+            }
         }
-        MPI_Status statuses[requests.size()];
-        MPI_Waitall(requests.size(), requests.data(), statuses);
     }
-
-    for (const auto &i : m_AllReceivingWriterRanks)
+    else
     {
-        const auto &v = m_GlobalWritePattern[i.first];
-        for (const auto &b : v)
+        for (const auto &i : m_AllReceivingWriterRanks)
         {
-            if (b.name == variable.m_Name)
+            const auto &v = m_GlobalWritePattern[i.first];
+            for (const auto &b : v)
             {
-                std::vector str(b.bufferCount);
-                std::memcpy(str.data(), m_Buffer.data() + b.bufferStart,
-                            b.bufferCount);
-                *data = std::string(str.begin(), str.end());
+                if (b.name == variable.m_Name)
+                {
+                    *data = std::string(b.value.begin(), b.value.end());
+                }
             }
         }
     }
@@ -106,7 +122,8 @@ void SscReader::GetDeferredCommon(Variable &variable, T *data)
         std::reverse(vShape.begin(), vShape.end());
     }
 
-    if (m_CurrentStep == 0)
+    if (m_CurrentStep == 0 || m_WriterDefinitionsLocked == false ||
+        m_ReaderSelectionsLocked == false)
     {
         m_LocalReadPattern.emplace_back();
         auto &b = m_LocalReadPattern.back();
@@ -114,7 +131,7 @@ void SscReader::GetDeferredCommon(Variable &variable, T *data)
         b.count = vCount;
         b.start = vStart;
         b.shape = vShape;
-        b.type = helper::GetType();
+        b.type = helper::GetDataType();
 
         for (const auto &d : b.count)
         {
@@ -128,7 +145,7 @@ void SscReader::GetDeferredCommon(Variable &variable, T *data)
         m_LocalReadPatternJson["Variables"].emplace_back();
         auto &jref = m_LocalReadPatternJson["Variables"].back();
         jref["Name"] = variable.m_Name;
-        jref["Type"] = helper::GetType();
+        jref["Type"] = helper::GetDataType();
         jref["ShapeID"] = variable.m_ShapeID;
         jref["Start"] = vStart;
         jref["Count"] = vCount;
@@ -137,25 +154,29 @@ void SscReader::GetDeferredCommon(Variable &variable, T *data)
         jref["BufferCount"] = 0;
 
         ssc::JsonToBlockVecVec(m_GlobalWritePatternJson, m_GlobalWritePattern);
+        size_t oldSize = m_AllReceivingWriterRanks.size();
         m_AllReceivingWriterRanks =
             ssc::CalculateOverlap(m_GlobalWritePattern, m_LocalReadPattern);
         CalculatePosition(m_GlobalWritePattern, m_AllReceivingWriterRanks);
-        size_t totalDataSize = 0;
-        for (auto i : m_AllReceivingWriterRanks)
-        {
-            totalDataSize += i.second.second;
-        }
-        m_Buffer.resize(totalDataSize);
-        std::vector requests;
-        for (const auto &i : m_AllReceivingWriterRanks)
+        size_t newSize = m_AllReceivingWriterRanks.size();
+        if (oldSize != newSize)
         {
-            requests.emplace_back();
-            MPI_Rget(m_Buffer.data() + i.second.first, i.second.second,
-                     MPI_CHAR, i.first, 0, i.second.second, MPI_CHAR, m_MpiWin,
-                     &requests.back());
+            size_t totalDataSize = 0;
+            for (auto i : m_AllReceivingWriterRanks)
+            {
+                totalDataSize += i.second.second;
+            }
+            m_Buffer.resize(totalDataSize);
+            for (const auto &i : m_AllReceivingWriterRanks)
+            {
+                MPI_Win_lock(MPI_LOCK_SHARED, i.first, 0, m_MpiWin);
+                MPI_Get(m_Buffer.data() + i.second.first,
+                        static_cast(i.second.second), MPI_CHAR, i.first, 0,
+                        static_cast(i.second.second), MPI_CHAR, m_MpiWin);
+                MPI_Win_unlock(i.first, m_MpiWin);
+                m_ReceivedRanks.insert(i.first);
+            }
         }
-        MPI_Status statuses[requests.size()];
-        MPI_Waitall(requests.size(), requests.data(), statuses);
     }
 
     for (const auto &i : m_AllReceivingWriterRanks)
@@ -165,6 +186,19 @@ void SscReader::GetDeferredCommon(Variable &variable, T *data)
         {
             if (b.name == variable.m_Name)
             {
+                bool empty = false;
+                for (const auto c : b.count)
+                {
+                    if (c == 0)
+                    {
+                        empty = true;
+                    }
+                }
+                if (empty)
+                {
+                    continue;
+                }
+
                 if (b.shapeId == ShapeID::GlobalArray ||
                     b.shapeId == ShapeID::LocalArray)
                 {
@@ -221,7 +255,9 @@ SscReader::BlocksInfoCommon(const Variable &variable,
                     v.shapeId == ShapeID::LocalValue)
                 {
                     b.IsValue = true;
-                    if (m_CurrentStep == 0)
+                    if (m_CurrentStep == 0 ||
+                        m_WriterDefinitionsLocked == false ||
+                        m_ReaderSelectionsLocked == false)
                     {
                         std::memcpy(&b.Value, v.value.data(), v.value.size());
                     }
diff --git a/source/adios2/engine/ssc/SscWriter.cpp b/source/adios2/engine/ssc/SscWriter.cpp
index 46aba808f6..9d51a2f7b4 100644
--- a/source/adios2/engine/ssc/SscWriter.cpp
+++ b/source/adios2/engine/ssc/SscWriter.cpp
@@ -28,117 +28,62 @@ SscWriter::SscWriter(IO &io, const std::string &name, const Mode mode,
 
     helper::GetParameter(m_IO.m_Parameters, "MpiMode", m_MpiMode);
     helper::GetParameter(m_IO.m_Parameters, "Verbose", m_Verbosity);
-    helper::GetParameter(m_IO.m_Parameters, "MaxFilenameLength",
-                         m_MaxFilenameLength);
-    helper::GetParameter(m_IO.m_Parameters, "RendezvousAppCount",
-                         m_RendezvousAppCount);
-    helper::GetParameter(m_IO.m_Parameters, "MaxStreamsPerApp",
-                         m_MaxStreamsPerApp);
     helper::GetParameter(m_IO.m_Parameters, "OpenTimeoutSecs",
                          m_OpenTimeoutSecs);
 
-    m_Buffer.resize(1);
-
     SyncMpiPattern();
     m_WriterRank = m_Comm.Rank();
     m_WriterSize = m_Comm.Size();
     MPI_Comm_rank(m_StreamComm, &m_StreamRank);
     MPI_Comm_size(m_StreamComm, &m_StreamSize);
-
-    m_GlobalWritePattern.resize(m_StreamSize);
-    m_GlobalReadPattern.resize(m_StreamSize);
 }
 
 StepStatus SscWriter::BeginStep(StepMode mode, const float timeoutSeconds)
 {
     TAU_SCOPED_TIMER_FUNC();
 
-    if (m_InitialStep)
-    {
-        m_InitialStep = false;
-    }
-    else
-    {
-        ++m_CurrentStep;
-    }
+    ++m_CurrentStep;
 
     if (m_Verbosity >= 5)
     {
         std::cout << "SscWriter::BeginStep, World Rank " << m_StreamRank
-                  << ", Writer Rank " << m_WriterRank << ", Step "
+                  << ", Reader Rank " << m_WriterRank << ", Step "
                   << m_CurrentStep << std::endl;
     }
 
-    return StepStatus::OK;
-}
-
-size_t SscWriter::CurrentStep() const
-{
-    TAU_SCOPED_TIMER_FUNC();
-    return m_CurrentStep;
-}
-
-void SscWriter::PerformPuts() { TAU_SCOPED_TIMER_FUNC(); }
-
-void SscWriter::PutOneSidedPostPush()
-{
-    TAU_SCOPED_TIMER_FUNC();
-    MPI_Win_start(m_MpiAllReadersGroup, 0, m_MpiWin);
-    for (const auto &i : m_AllSendingReaderRanks)
+    if (m_CurrentStep == 0 || m_WriterDefinitionsLocked == false ||
+        m_ReaderSelectionsLocked == false)
     {
-        MPI_Put(m_Buffer.data(), m_Buffer.size(), MPI_CHAR, i.first,
-                i.second.first, m_Buffer.size(), MPI_CHAR, m_MpiWin);
+        m_Buffer.resize(1, 0);
+        m_GlobalWritePattern.clear();
+        m_GlobalWritePattern.resize(m_StreamSize);
+        m_GlobalReadPattern.clear();
+        m_GlobalReadPattern.resize(m_StreamSize);
     }
-    MPI_Win_complete(m_MpiWin);
-}
 
-void SscWriter::PutOneSidedFencePush()
-{
-    TAU_SCOPED_TIMER_FUNC();
-    MPI_Win_fence(0, m_MpiWin);
-    for (const auto &i : m_AllSendingReaderRanks)
+    if (m_WriterDefinitionsLocked && m_ReaderSelectionsLocked)
     {
-        MPI_Put(m_Buffer.data(), m_Buffer.size(), MPI_CHAR, i.first,
-                i.second.first, m_Buffer.size(), MPI_CHAR, m_MpiWin);
+        if (m_CurrentStep > 1)
+        {
+            MpiWait();
+        }
     }
-    MPI_Win_fence(0, m_MpiWin);
-}
 
-void SscWriter::PutOneSidedPostPull()
-{
-    TAU_SCOPED_TIMER_FUNC();
-    MPI_Win_post(m_MpiAllReadersGroup, 0, m_MpiWin);
-    MPI_Win_wait(m_MpiWin);
+    return StepStatus::OK;
 }
 
-void SscWriter::PutOneSidedFencePull()
-{
-    TAU_SCOPED_TIMER_FUNC();
-    MPI_Win_fence(0, m_MpiWin);
-    MPI_Win_fence(0, m_MpiWin);
-}
+size_t SscWriter::CurrentStep() const { return m_CurrentStep; }
 
-void SscWriter::PutTwoSided()
-{
-    TAU_SCOPED_TIMER_FUNC();
-    std::vector requests;
-    for (const auto &i : m_AllSendingReaderRanks)
-    {
-        requests.emplace_back();
-        MPI_Isend(m_Buffer.data(), m_Buffer.size(), MPI_CHAR, i.first, 0,
-                  m_StreamComm, &requests.back());
-    }
-    MPI_Status statuses[requests.size()];
-    MPI_Waitall(requests.size(), requests.data(), statuses);
-}
+void SscWriter::PerformPuts() { TAU_SCOPED_TIMER_FUNC(); }
 
 void SscWriter::EndStep()
 {
     TAU_SCOPED_TIMER_FUNC();
+
     if (m_Verbosity >= 5)
     {
         std::cout << "SscWriter::EndStep, World Rank " << m_StreamRank
-                  << ", Writer Rank " << m_WriterRank << ", Step "
+                  << ", Reader Rank " << m_WriterRank << ", Step "
                   << m_CurrentStep << std::endl;
     }
 
@@ -147,33 +92,65 @@ void SscWriter::EndStep()
         SyncWritePattern();
         MPI_Win_create(m_Buffer.data(), m_Buffer.size(), 1, MPI_INFO_NULL,
                        m_StreamComm, &m_MpiWin);
-        PutOneSidedPostPull();
         MPI_Win_free(&m_MpiWin);
         SyncReadPattern();
-        MPI_Win_create(m_Buffer.data(), m_Buffer.size(), 1, MPI_INFO_NULL,
-                       m_StreamComm, &m_MpiWin);
+        if (m_WriterDefinitionsLocked && m_ReaderSelectionsLocked)
+        {
+            MPI_Win_create(m_Buffer.data(), m_Buffer.size(), 1, MPI_INFO_NULL,
+                           m_StreamComm, &m_MpiWin);
+        }
     }
     else
     {
-        if (m_MpiMode == "twosided")
-        {
-            PutTwoSided();
-        }
-        else if (m_MpiMode == "onesidedfencepush")
-        {
-            PutOneSidedFencePush();
-        }
-        else if (m_MpiMode == "onesidedpostpush")
-        {
-            PutOneSidedPostPush();
-        }
-        else if (m_MpiMode == "onesidedfencepull")
+        if (m_WriterDefinitionsLocked && m_ReaderSelectionsLocked)
         {
-            PutOneSidedFencePull();
+            if (m_MpiMode == "twosided")
+            {
+                for (const auto &i : m_AllSendingReaderRanks)
+                {
+                    m_MpiRequests.emplace_back();
+                    MPI_Isend(m_Buffer.data(),
+                              static_cast(m_Buffer.size()), MPI_CHAR,
+                              i.first, 0, m_StreamComm, &m_MpiRequests.back());
+                }
+            }
+            else if (m_MpiMode == "onesidedfencepush")
+            {
+                MPI_Win_fence(0, m_MpiWin);
+                for (const auto &i : m_AllSendingReaderRanks)
+                {
+                    MPI_Put(m_Buffer.data(), static_cast(m_Buffer.size()),
+                            MPI_CHAR, i.first, i.second.first,
+                            static_cast(m_Buffer.size()), MPI_CHAR,
+                            m_MpiWin);
+                }
+            }
+            else if (m_MpiMode == "onesidedpostpush")
+            {
+                MPI_Win_start(m_MpiAllReadersGroup, 0, m_MpiWin);
+                for (const auto &i : m_AllSendingReaderRanks)
+                {
+                    MPI_Put(m_Buffer.data(), static_cast(m_Buffer.size()),
+                            MPI_CHAR, i.first, i.second.first,
+                            static_cast(m_Buffer.size()), MPI_CHAR,
+                            m_MpiWin);
+                }
+            }
+            else if (m_MpiMode == "onesidedfencepull")
+            {
+                MPI_Win_fence(0, m_MpiWin);
+            }
+            else if (m_MpiMode == "onesidedpostpull")
+            {
+                MPI_Win_post(m_MpiAllReadersGroup, 0, m_MpiWin);
+            }
         }
-        else if (m_MpiMode == "onesidedpostpull")
+        else
         {
-            PutOneSidedPostPull();
+            SyncWritePattern();
+            MPI_Win_create(m_Buffer.data(), m_Buffer.size(), 1, MPI_INFO_NULL,
+                           m_StreamComm, &m_MpiWin);
+            MPI_Win_free(&m_MpiWin);
         }
     }
 }
@@ -182,60 +159,54 @@ void SscWriter::Flush(const int transportIndex) { TAU_SCOPED_TIMER_FUNC(); }
 
 // PRIVATE
 
-void SscWriter::SyncMpiPattern()
+void SscWriter::MpiWait()
 {
-    TAU_SCOPED_TIMER_FUNC();
-
-    if (m_Verbosity >= 5)
+    if (m_MpiMode == "twosided")
     {
-        std::cout << "SscWriter::SyncMpiPattern, World Rank " << m_StreamRank
-                  << ", Writer Rank " << m_WriterRank << std::endl;
+        MPI_Waitall(static_cast(m_MpiRequests.size()),
+                    m_MpiRequests.data(), MPI_STATUSES_IGNORE);
+        m_MpiRequests.clear();
     }
-
-    m_MpiHandshake.Handshake(m_Name, 'w', m_OpenTimeoutSecs, m_MaxStreamsPerApp,
-                             m_MaxFilenameLength, m_RendezvousAppCount,
-                             CommAsMPI(m_Comm));
-
-    std::vector allStreamRanks;
-    std::vector allReaderRanks;
-
-    for (const auto &app : m_MpiHandshake.GetWriterMap(m_Name))
+    else if (m_MpiMode == "onesidedfencepush")
     {
-        for (int rank : app.second)
-        {
-            allStreamRanks.push_back(rank);
-        }
+        MPI_Win_fence(0, m_MpiWin);
     }
-
-    for (const auto &app : m_MpiHandshake.GetReaderMap(m_Name))
+    else if (m_MpiMode == "onesidedpostpush")
     {
-        for (int rank : app.second)
-        {
-            allStreamRanks.push_back(rank);
-            allReaderRanks.push_back(rank);
-        }
+        MPI_Win_complete(m_MpiWin);
+    }
+    else if (m_MpiMode == "onesidedfencepull")
+    {
+        MPI_Win_fence(0, m_MpiWin);
+    }
+    else if (m_MpiMode == "onesidedpostpull")
+    {
+        MPI_Win_wait(m_MpiWin);
     }
-    MPI_Group worldGroup;
-    MPI_Group allReadersGroup;
-    MPI_Comm_group(MPI_COMM_WORLD, &worldGroup);
-    MPI_Group_incl(worldGroup, allReaderRanks.size(), allReaderRanks.data(),
-                   &m_MpiAllReadersGroup);
-
-    MPI_Comm_group(MPI_COMM_WORLD, &worldGroup);
-    std::sort(allStreamRanks.begin(), allStreamRanks.end());
-    MPI_Group allWorkersGroup;
-    MPI_Group_incl(worldGroup, allStreamRanks.size(), allStreamRanks.data(),
-                   &allWorkersGroup);
-    MPI_Comm_create_group(MPI_COMM_WORLD, allWorkersGroup, 0, &m_StreamComm);
 }
 
-void SscWriter::SyncWritePattern()
+void SscWriter::SyncMpiPattern()
+{
+    TAU_SCOPED_TIMER_FUNC();
+
+    MPI_Group streamGroup;
+    MPI_Group writerGroup;
+    MPI_Comm writerComm;
+    MPI_Comm readerComm;
+
+    helper::HandshakeComm(m_Name, 'w', m_OpenTimeoutSecs, CommAsMPI(m_Comm),
+                          streamGroup, writerGroup, m_MpiAllReadersGroup,
+                          m_StreamComm, writerComm, readerComm, m_Verbosity);
+}
+
+void SscWriter::SyncWritePattern(bool finalStep)
 {
     TAU_SCOPED_TIMER_FUNC();
     if (m_Verbosity >= 5)
     {
         std::cout << "SscWriter::SyncWritePattern, World Rank " << m_StreamRank
-                  << ", Writer Rank " << m_WriterRank << std::endl;
+                  << ", Writer Rank " << m_WriterRank << ", Step "
+                  << m_CurrentStep << std::endl;
     }
 
     nlohmann::json localRankMetaJ;
@@ -245,6 +216,11 @@ void SscWriter::SyncWritePattern()
     if (m_WriterRank == 0)
     {
         ssc::AttributeMapToJson(m_IO, localRankMetaJ);
+        localRankMetaJ["Pattern"] = m_WriterDefinitionsLocked;
+    }
+    if (finalStep)
+    {
+        localRankMetaJ["FinalStep"] = true;
     }
 
     std::string localStr = localRankMetaJ.dump();
@@ -257,8 +233,9 @@ void SscWriter::SyncWritePattern()
     std::vector localVec(maxLocalSize, '\0');
     std::memcpy(localVec.data(), localStr.data(), localStr.size());
     std::vector globalVec(maxLocalSize * m_StreamSize, '\0');
-    MPI_Allgather(localVec.data(), maxLocalSize, MPI_CHAR, globalVec.data(),
-                  maxLocalSize, MPI_CHAR, m_StreamComm);
+    MPI_Allgather(localVec.data(), static_cast(maxLocalSize), MPI_CHAR,
+                  globalVec.data(), static_cast(maxLocalSize), MPI_CHAR,
+                  m_StreamComm);
 
     // deserialize global metadata Json
     nlohmann::json globalJson;
@@ -267,6 +244,11 @@ void SscWriter::SyncWritePattern()
 
     // deserialize variables metadata
     ssc::JsonToBlockVecVec(globalJson, m_GlobalWritePattern);
+
+    if (m_Verbosity >= 10 && m_WriterRank == 0)
+    {
+        ssc::PrintBlockVecVec(m_GlobalWritePattern, "Global Write Pattern");
+    }
 }
 
 void SscWriter::SyncReadPattern()
@@ -275,7 +257,8 @@ void SscWriter::SyncReadPattern()
     if (m_Verbosity >= 5)
     {
         std::cout << "SscWriter::SyncReadPattern, World Rank " << m_StreamRank
-                  << ", Writer Rank " << m_WriterRank << std::endl;
+                  << ", Writer Rank " << m_WriterRank << ", Step "
+                  << m_CurrentStep << std::endl;
     }
 
     size_t localSize = 0;
@@ -284,8 +267,9 @@ void SscWriter::SyncReadPattern()
                   m_StreamComm);
     std::vector localVec(maxLocalSize, '\0');
     std::vector globalVec(maxLocalSize * m_StreamSize);
-    MPI_Allgather(localVec.data(), maxLocalSize, MPI_CHAR, globalVec.data(),
-                  maxLocalSize, MPI_CHAR, m_StreamComm);
+    MPI_Allgather(localVec.data(), static_cast(maxLocalSize), MPI_CHAR,
+                  globalVec.data(), static_cast(maxLocalSize), MPI_CHAR,
+                  m_StreamComm);
 
     // deserialize global metadata Json
     nlohmann::json globalJson;
@@ -318,11 +302,13 @@ void SscWriter::SyncReadPattern()
     CalculatePosition(m_GlobalWritePattern, m_GlobalReadPattern, m_WriterRank,
                       m_AllSendingReaderRanks);
 
-    if (m_Verbosity >= 10)
+    for (int i = 0; i < m_StreamSize; ++i)
     {
-        ssc::PrintBlockVecVec(m_GlobalWritePattern, "Global Write Pattern");
-        ssc::PrintBlockVec(m_GlobalWritePattern[m_WriterRank],
-                           "Local Write Pattern");
+        auto &patternJson = globalJson[i]["Pattern"];
+        if (patternJson != nullptr)
+        {
+            m_ReaderSelectionsLocked = patternJson.get();
+        }
     }
 }
 
@@ -338,7 +324,7 @@ void SscWriter::CalculatePosition(ssc::BlockVecVec &writerVecVec,
         auto currentReaderOverlapWriterRanks =
             CalculateOverlap(writerVecVec, readerRankMap);
         size_t bufferPosition = 0;
-        for (size_t rank = 0; rank < writerVecVec.size(); ++rank)
+        for (int rank = 0; rank < writerVecVec.size(); ++rank)
         {
             bool hasOverlap = false;
             for (const auto r : currentReaderOverlapWriterRanks)
@@ -380,58 +366,71 @@ ADIOS2_FOREACH_STDTYPE_1ARG(declare_type)
 void SscWriter::DoClose(const int transportIndex)
 {
     TAU_SCOPED_TIMER_FUNC();
+
     if (m_Verbosity >= 5)
     {
         std::cout << "SscWriter::DoClose, World Rank " << m_StreamRank
                   << ", Writer Rank " << m_WriterRank << std::endl;
     }
 
-    m_Buffer[0] = 1;
-
-    if (m_MpiMode == "twosided")
+    if (m_WriterDefinitionsLocked && m_ReaderSelectionsLocked)
     {
-        std::vector requests;
-        for (const auto &i : m_AllSendingReaderRanks)
+        if (m_CurrentStep > 0)
         {
-            requests.emplace_back();
-            MPI_Isend(m_Buffer.data(), 1, MPI_CHAR, i.first, 0, m_StreamComm,
-                      &requests.back());
+            MpiWait();
         }
-        MPI_Status statuses[requests.size()];
-        MPI_Waitall(requests.size(), requests.data(), statuses);
-    }
-    else if (m_MpiMode == "onesidedfencepush")
-    {
-        MPI_Win_fence(0, m_MpiWin);
-        for (const auto &i : m_AllSendingReaderRanks)
+
+        m_Buffer[0] = 1;
+
+        if (m_MpiMode == "twosided")
         {
-            MPI_Put(m_Buffer.data(), 1, MPI_CHAR, i.first, 0, 1, MPI_CHAR,
-                    m_MpiWin);
+            std::vector requests;
+            for (const auto &i : m_AllSendingReaderRanks)
+            {
+                requests.emplace_back();
+                MPI_Isend(m_Buffer.data(), 1, MPI_CHAR, i.first, 0,
+                          m_StreamComm, &requests.back());
+            }
+            MPI_Waitall(static_cast(requests.size()), requests.data(),
+                        MPI_STATUS_IGNORE);
         }
-        MPI_Win_fence(0, m_MpiWin);
-    }
-    else if (m_MpiMode == "onesidedpostpush")
-    {
-        MPI_Win_start(m_MpiAllReadersGroup, 0, m_MpiWin);
-        for (const auto &i : m_AllSendingReaderRanks)
+        else if (m_MpiMode == "onesidedfencepush")
         {
-            MPI_Put(m_Buffer.data(), 1, MPI_CHAR, i.first, 0, 1, MPI_CHAR,
-                    m_MpiWin);
+            MPI_Win_fence(0, m_MpiWin);
+            for (const auto &i : m_AllSendingReaderRanks)
+            {
+                MPI_Put(m_Buffer.data(), 1, MPI_CHAR, i.first, 0, 1, MPI_CHAR,
+                        m_MpiWin);
+            }
+            MPI_Win_fence(0, m_MpiWin);
         }
-        MPI_Win_complete(m_MpiWin);
-    }
-    else if (m_MpiMode == "onesidedfencepull")
-    {
-        MPI_Win_fence(0, m_MpiWin);
-        MPI_Win_fence(0, m_MpiWin);
+        else if (m_MpiMode == "onesidedpostpush")
+        {
+            MPI_Win_start(m_MpiAllReadersGroup, 0, m_MpiWin);
+            for (const auto &i : m_AllSendingReaderRanks)
+            {
+                MPI_Put(m_Buffer.data(), 1, MPI_CHAR, i.first, 0, 1, MPI_CHAR,
+                        m_MpiWin);
+            }
+            MPI_Win_complete(m_MpiWin);
+        }
+        else if (m_MpiMode == "onesidedfencepull")
+        {
+            MPI_Win_fence(0, m_MpiWin);
+            MPI_Win_fence(0, m_MpiWin);
+        }
+        else if (m_MpiMode == "onesidedpostpull")
+        {
+            MPI_Win_post(m_MpiAllReadersGroup, 0, m_MpiWin);
+            MPI_Win_wait(m_MpiWin);
+        }
+
+        MPI_Win_free(&m_MpiWin);
     }
-    else if (m_MpiMode == "onesidedpostpull")
+    else
     {
-        MPI_Win_post(m_MpiAllReadersGroup, 0, m_MpiWin);
-        MPI_Win_wait(m_MpiWin);
+        SyncWritePattern(true);
     }
-
-    MPI_Win_free(&m_MpiWin);
 }
 
 } // end namespace engine
diff --git a/source/adios2/engine/ssc/SscWriter.h b/source/adios2/engine/ssc/SscWriter.h
index dfc66f8ff2..1866f3c0b7 100644
--- a/source/adios2/engine/ssc/SscWriter.h
+++ b/source/adios2/engine/ssc/SscWriter.h
@@ -42,8 +42,7 @@ class SscWriter : public Engine
     void Flush(const int transportIndex = -1) final;
 
 private:
-    size_t m_CurrentStep = 0;
-    bool m_InitialStep = true;
+    int64_t m_CurrentStep = -1;
 
     ssc::BlockVecVec m_GlobalWritePattern;
     ssc::BlockVecVec m_GlobalReadPattern;
@@ -54,22 +53,17 @@ class SscWriter : public Engine
     MPI_Group m_MpiAllReadersGroup;
     MPI_Comm m_StreamComm;
     std::string m_MpiMode = "twosided";
+    std::vector m_MpiRequests;
 
     int m_StreamRank;
     int m_StreamSize;
     int m_WriterRank;
     int m_WriterSize;
 
-    helper::MpiHandshake m_MpiHandshake;
-
     void SyncMpiPattern();
-    void SyncWritePattern();
+    void SyncWritePattern(bool finalStep = false);
     void SyncReadPattern();
-    void PutOneSidedFencePush();
-    void PutOneSidedPostPush();
-    void PutOneSidedFencePull();
-    void PutOneSidedPostPull();
-    void PutTwoSided();
+    void MpiWait();
 
 #define declare_type(T)                                                        \
     void DoPutSync(Variable &, const T *) final;                            \
@@ -87,9 +81,6 @@ class SscWriter : public Engine
                            ssc::RankPosMap &allOverlapRanks);
 
     int m_Verbosity = 0;
-    int m_MaxFilenameLength = 128;
-    int m_MaxStreamsPerApp = 1;
-    int m_RendezvousAppCount = 2;
     int m_OpenTimeoutSecs = 10;
 };
 
diff --git a/source/adios2/engine/ssc/SscWriter.tcc b/source/adios2/engine/ssc/SscWriter.tcc
index 37b08081fa..b31340957e 100644
--- a/source/adios2/engine/ssc/SscWriter.tcc
+++ b/source/adios2/engine/ssc/SscWriter.tcc
@@ -47,12 +47,13 @@ void SscWriter::PutDeferredCommon(Variable &variable,
 
     if (!found)
     {
-        if (m_CurrentStep == 0)
+        if (m_CurrentStep == 0 || m_WriterDefinitionsLocked == false ||
+            m_ReaderSelectionsLocked == false)
         {
             m_GlobalWritePattern[m_StreamRank].emplace_back();
             auto &b = m_GlobalWritePattern[m_StreamRank].back();
             b.name = variable.m_Name;
-            b.type = "string";
+            b.type = DataType::String;
             b.shapeId = variable.m_ShapeID;
             b.shape = variable.m_Shape;
             b.start = variable.m_Start;
@@ -93,8 +94,8 @@ void SscWriter::PutDeferredCommon(Variable &variable, const T *data)
     bool found = false;
     for (const auto &b : m_GlobalWritePattern[m_StreamRank])
     {
-        if (b.name == variable.m_Name and ssc::AreSameDims(vStart, b.start) and
-            ssc::AreSameDims(vCount, b.count) and
+        if (b.name == variable.m_Name && ssc::AreSameDims(vStart, b.start) &&
+            ssc::AreSameDims(vCount, b.count) &&
             ssc::AreSameDims(vShape, b.shape))
         {
             std::memcpy(m_Buffer.data() + b.bufferStart, data, b.bufferCount);
@@ -104,12 +105,13 @@ void SscWriter::PutDeferredCommon(Variable &variable, const T *data)
 
     if (!found)
     {
-        if (m_CurrentStep == 0)
+        if (m_CurrentStep == 0 || m_WriterDefinitionsLocked == false ||
+            m_ReaderSelectionsLocked == false)
         {
             m_GlobalWritePattern[m_StreamRank].emplace_back();
             auto &b = m_GlobalWritePattern[m_StreamRank].back();
             b.name = variable.m_Name;
-            b.type = helper::GetType();
+            b.type = helper::GetDataType();
             b.shapeId = variable.m_ShapeID;
             b.shape = vShape;
             b.start = vStart;
@@ -127,7 +129,7 @@ void SscWriter::PutDeferredCommon(Variable &variable, const T *data)
         }
         else
         {
-            throw std::runtime_error("ssc only accepts fixed IO pattern");
+            throw std::runtime_error("IO pattern changed after locking");
         }
     }
 }
diff --git a/source/adios2/engine/sst/SstReader.cpp b/source/adios2/engine/sst/SstReader.cpp
index ed24fb7311..30a6bd2c8b 100644
--- a/source/adios2/engine/sst/SstReader.cpp
+++ b/source/adios2/engine/sst/SstReader.cpp
@@ -51,16 +51,17 @@ SstReader::SstReader(IO &io, const std::string &name, const Mode mode,
     SstReaderGetParams(m_Input, &m_WriterMarshalMethod);
 
     auto varFFSCallback = [](void *reader, const char *variableName,
-                             const char *type, void *data) {
-        std::string Type(type);
+                             const int type, void *data) {
+        adios2::DataType Type = (adios2::DataType)type;
         class SstReader::SstReader *Reader =
             reinterpret_cast(reader);
-        if (Type == "compound")
+        if (Type == adios2::DataType::Compound)
         {
             return (void *)NULL;
         }
+
 #define declare_type(T)                                                        \
-    else if (Type == helper::GetType())                                     \
+    else if (Type == helper::GetDataType())                                 \
     {                                                                          \
         Variable *variable =                                                \
             &(Reader->m_IO.DefineVariable(variableName));                   \
@@ -76,29 +77,29 @@ SstReader::SstReader(IO &io, const std::string &name, const Mode mode,
     };
 
     auto attrFFSCallback = [](void *reader, const char *attrName,
-                              const char *type, void *data) {
+                              const int type, void *data) {
         class SstReader::SstReader *Reader =
             reinterpret_cast(reader);
+        adios2::DataType Type = (adios2::DataType)type;
         if (attrName == NULL)
         {
             // if attrName is NULL, prepare for attr reinstallation
             Reader->m_IO.RemoveAllAttributes();
             return;
         }
-        std::string Type(type);
         try
         {
-            if (Type == "compound")
+            if (Type == adios2::DataType::Compound)
             {
                 return;
             }
-            else if (Type == helper::GetType())
+            else if (Type == helper::GetDataType())
             {
                 Reader->m_IO.DefineAttribute(attrName,
                                                           *(char **)data);
             }
 #define declare_type(T)                                                        \
-    else if (Type == helper::GetType())                                     \
+    else if (Type == helper::GetDataType())                                 \
     {                                                                          \
         Reader->m_IO.DefineAttribute(attrName, *(T *)data);                 \
     }
@@ -107,8 +108,8 @@ SstReader::SstReader(IO &io, const std::string &name, const Mode mode,
 #undef declare_type
             else
             {
-                std::cout << "Loading attribute matched no type " << Type
-                          << std::endl;
+                std::cout << "Loading attribute matched no type "
+                          << ToString(Type) << std::endl;
             }
         }
         catch (...)
@@ -120,12 +121,12 @@ SstReader::SstReader(IO &io, const std::string &name, const Mode mode,
     };
 
     auto arrayFFSCallback = [](void *reader, const char *variableName,
-                               const char *type, int DimCount, size_t *Shape,
+                               const int type, int DimCount, size_t *Shape,
                                size_t *Start, size_t *Count) {
         std::vector VecShape;
         std::vector VecStart;
         std::vector VecCount;
-        std::string Type(type);
+        adios2::DataType Type = (adios2::DataType)type;
         class SstReader::SstReader *Reader =
             reinterpret_cast(reader);
         /*
@@ -151,12 +152,12 @@ SstReader::SstReader(IO &io, const std::string &name, const Mode mode,
             }
         }
 
-        if (Type == "compound")
+        if (Type == adios2::DataType::Compound)
         {
             return (void *)NULL;
         }
 #define declare_type(T)                                                        \
-    else if (Type == helper::GetType())                                     \
+    else if (Type == helper::GetDataType())                                 \
     {                                                                          \
         Variable *variable = &(Reader->m_IO.DefineVariable(              \
             variableName, VecShape, VecStart, VecCount));                      \
@@ -169,12 +170,12 @@ SstReader::SstReader(IO &io, const std::string &name, const Mode mode,
     };
 
     auto arrayBlocksInfoCallback =
-        [](void *reader, void *variable, const char *type, int WriterRank,
+        [](void *reader, void *variable, const int type, int WriterRank,
            int DimCount, size_t *Shape, size_t *Start, size_t *Count) {
             std::vector VecShape;
             std::vector VecStart;
             std::vector VecCount;
-            std::string Type(type);
+            adios2::DataType Type = (adios2::DataType)type;
             class SstReader::SstReader *Reader =
                 reinterpret_cast(reader);
             size_t currentStep = SstCurrentStep(Reader->m_Input);
@@ -201,12 +202,12 @@ SstReader::SstReader(IO &io, const std::string &name, const Mode mode,
                 }
             }
 
-            if (Type == "compound")
+            if (Type == adios2::DataType::Compound)
             {
                 return;
             }
 #define declare_type(T)                                                        \
-    else if (Type == helper::GetType())                                     \
+    else if (Type == helper::GetDataType())                                 \
     {                                                                          \
         Variable *Var = reinterpret_cast *>(variable);    \
         auto savedShape = Var->m_Shape;                                        \
@@ -406,10 +407,6 @@ void SstReader::Init()
     SstParamParser Parser;
 
     Parser.ParseParams(m_IO, Params);
-
-#define set_params(Param, Type, Typedecl, Default) m_##Param = Params.Param;
-    SST_FOREACH_PARAMETER_TYPE_4ARGS(set_params);
-#undef set_params
 }
 
 #define declare_gets(T)                                                        \
@@ -548,13 +545,13 @@ void SstReader::PerformGets()
 
         for (const std::string &name : m_BP3Deserializer->m_DeferredVariables)
         {
-            const std::string type = m_IO.InquireVariableType(name);
+            const DataType type = m_IO.InquireVariableType(name);
 
-            if (type == "compound")
+            if (type == DataType::Compound)
             {
             }
 #define declare_type(T)                                                        \
-    else if (type == helper::GetType())                                     \
+    else if (type == helper::GetDataType())                                 \
     {                                                                          \
         Variable &variable =                                                \
             FindVariable(name, "in call to PerformGets, EndStep or Close"); \
@@ -579,13 +576,13 @@ void SstReader::PerformGets()
 
         for (const std::string &name : m_BP3Deserializer->m_DeferredVariables)
         {
-            const std::string type = m_IO.InquireVariableType(name);
+            const DataType type = m_IO.InquireVariableType(name);
 
-            if (type == "compound")
+            if (type == DataType::Compound)
             {
             }
 #define declare_type(T)                                                        \
-    else if (type == helper::GetType())                                     \
+    else if (type == helper::GetDataType())                                 \
     {                                                                          \
         Variable &variable =                                                \
             FindVariable(name, "in call to PerformGets, EndStep or Close"); \
diff --git a/source/adios2/engine/sst/SstReader.h b/source/adios2/engine/sst/SstReader.h
index 464ca77ec5..efb0b67b4e 100644
--- a/source/adios2/engine/sst/SstReader.h
+++ b/source/adios2/engine/sst/SstReader.h
@@ -75,10 +75,6 @@ class SstReader : public Engine
     /* --- Used only with BP marshaling --- */
 
     struct _SstParams Params;
-#define declare_locals(Param, Type, Typedecl, Default)                         \
-    Typedecl m_##Param = Default;
-    SST_FOREACH_PARAMETER_TYPE_4ARGS(declare_locals)
-#undef declare_locals
 
 #define declare_type(T)                                                        \
     void DoGetSync(Variable &, T *) final;                                  \
diff --git a/source/adios2/engine/sst/SstReader.tcc b/source/adios2/engine/sst/SstReader.tcc
index 185e47c6e1..ba101728b4 100644
--- a/source/adios2/engine/sst/SstReader.tcc
+++ b/source/adios2/engine/sst/SstReader.tcc
@@ -13,7 +13,7 @@
 
 #include "SstReader.h"
 
-#include "adios2/helper/adiosFunctions.h" //GetType
+#include "adios2/helper/adiosFunctions.h" //GetDataType
 #include "adios2/toolkit/profiling/taustubs/tautimer.hpp"
 
 namespace adios2
diff --git a/source/adios2/engine/sst/SstWriter.cpp b/source/adios2/engine/sst/SstWriter.cpp
index 3ba1ede7f0..3bcb6e6f4f 100644
--- a/source/adios2/engine/sst/SstWriter.cpp
+++ b/source/adios2/engine/sst/SstWriter.cpp
@@ -111,7 +111,7 @@ SstWriter::SstWriter(IO &io, const std::string &name, const Mode mode,
 
     m_Output = SstWriterOpen(name.c_str(), &Params, &m_Comm);
 
-    if (m_MarshalMethod == SstMarshalBP)
+    if (Params.MarshalMethod == SstMarshalBP)
     {
         SstWriterInitMetadataCallback(m_Output, this, AssembleMetadata,
                                       FreeAssembledMetadata);
@@ -131,12 +131,12 @@ StepStatus SstWriter::BeginStep(StepMode mode, const float timeout_sec)
     }
 
     m_BetweenStepPairs = true;
-    if (m_MarshalMethod == SstMarshalFFS)
+    if (Params.MarshalMethod == SstMarshalFFS)
     {
         return (StepStatus)SstFFSWriterBeginStep(m_Output, (int)mode,
                                                  timeout_sec);
     }
-    else if (m_MarshalMethod == SstMarshalBP)
+    else if (Params.MarshalMethod == SstMarshalBP)
     {
         // initialize BP serializer, deleted in
         // SstWriter::EndStep()::lf_FreeBlocks()
@@ -157,24 +157,23 @@ StepStatus SstWriter::BeginStep(StepMode mode, const float timeout_sec)
 void SstWriter::FFSMarshalAttributes()
 {
     TAU_SCOPED_TIMER_FUNC();
-    const auto &attributesDataMap = m_IO.GetAttributesDataMap();
+    const auto &attributes = m_IO.GetAttributes();
 
-    const uint32_t attributesCount =
-        static_cast(attributesDataMap.size());
+    const uint32_t attributesCount = static_cast(attributes.size());
 
     // if there are no new attributes, nothing to do
     if (attributesCount == m_FFSMarshaledAttributesCount)
         return;
 
-    for (const auto &attributePair : attributesDataMap)
+    for (const auto &attributePair : attributes)
     {
         const std::string name(attributePair.first);
-        const std::string type(attributePair.second.first);
+        const DataType type(attributePair.second->m_Type);
 
-        if (type == "unknown")
+        if (type == DataType::None)
         {
         }
-        else if (type == helper::GetType())
+        else if (type == helper::GetDataType())
         {
             core::Attribute &attribute =
                 *m_IO.InquireAttribute(name);
@@ -185,11 +184,11 @@ void SstWriter::FFSMarshalAttributes()
                 //
             }
 
-            SstFFSMarshalAttribute(m_Output, name.c_str(), type.c_str(),
+            SstFFSMarshalAttribute(m_Output, name.c_str(), (int)type,
                                    sizeof(char *), element_count, data_addr);
         }
 #define declare_type(T)                                                        \
-    else if (type == helper::GetType())                                     \
+    else if (type == helper::GetDataType())                                 \
     {                                                                          \
         core::Attribute &attribute = *m_IO.InquireAttribute(name);       \
         int element_count = -1;                                                \
@@ -199,9 +198,8 @@ void SstWriter::FFSMarshalAttributes()
             element_count = attribute.m_Elements;                              \
             data_addr = attribute.m_DataArray.data();                          \
         }                                                                      \
-        SstFFSMarshalAttribute(m_Output, attribute.m_Name.c_str(),             \
-                               type.c_str(), sizeof(T), element_count,         \
-                               data_addr);                                     \
+        SstFFSMarshalAttribute(m_Output, attribute.m_Name.c_str(), (int)type,  \
+                               sizeof(T), element_count, data_addr);           \
     }
 
         ADIOS2_FOREACH_ATTRIBUTE_PRIMITIVE_STDTYPE_1ARG(declare_type)
@@ -223,7 +221,7 @@ void SstWriter::EndStep()
         SstWriterDefinitionLock(m_Output, m_WriterStep);
         m_DefinitionsNotified = true;
     }
-    if (m_MarshalMethod == SstMarshalFFS)
+    if (Params.MarshalMethod == SstMarshalFFS)
     {
         TAU_SCOPED_TIMER("Marshaling Overhead");
         TAU_START("SstMarshalFFS");
@@ -231,7 +229,7 @@ void SstWriter::EndStep()
         TAU_STOP("SstMarshalFFS");
         SstFFSWriterEndStep(m_Output, m_WriterStep);
     }
-    else if (m_MarshalMethod == SstMarshalBP)
+    else if (Params.MarshalMethod == SstMarshalBP)
     {
         // This should finalize BP marshaling at the writer side.  All
         // marshaling methods should result in two blocks, one a block of
@@ -285,9 +283,12 @@ void SstWriter::Init()
 
     Parser.ParseParams(m_IO, Params);
 
-#define set_params(Param, Type, Typedecl, Default) m_##Param = Params.Param;
-    SST_FOREACH_PARAMETER_TYPE_4ARGS(set_params);
-#undef set_params
+    if (Params.verbose < 0 || Params.verbose > 5)
+    {
+        throw std::invalid_argument("ERROR: Method verbose argument must be an "
+                                    "integer in the range [0,5], in call to "
+                                    "Open or Engine constructor\n");
+    }
 }
 
 #define declare_type(T)                                                        \
diff --git a/source/adios2/engine/sst/SstWriter.h b/source/adios2/engine/sst/SstWriter.h
index 0e5086f8b4..4e1099dfa4 100644
--- a/source/adios2/engine/sst/SstWriter.h
+++ b/source/adios2/engine/sst/SstWriter.h
@@ -71,10 +71,6 @@ class SstWriter : public Engine
     bool m_DefinitionsNotified = false;
     size_t m_FFSMarshaledAttributesCount = 0;
     struct _SstParams Params;
-#define declare_locals(Param, Type, Typedecl, Default)                         \
-    Typedecl m_##Param = Default;
-    SST_FOREACH_PARAMETER_TYPE_4ARGS(declare_locals)
-#undef declare_locals
 
     void FFSMarshalAttributes();
     void DoClose(const int transportIndex = -1) final;
diff --git a/source/adios2/engine/sst/SstWriter.tcc b/source/adios2/engine/sst/SstWriter.tcc
index 3f0cdfcbe0..0d4f2bb3eb 100644
--- a/source/adios2/engine/sst/SstWriter.tcc
+++ b/source/adios2/engine/sst/SstWriter.tcc
@@ -13,7 +13,7 @@
 
 #include "SstWriter.h"
 
-#include "adios2/helper/adiosFunctions.h" //GetType
+#include "adios2/helper/adiosFunctions.h" //GetDataType
 #include "adios2/toolkit/profiling/taustubs/tautimer.hpp"
 
 namespace adios2
@@ -36,7 +36,7 @@ void SstWriter::PutSyncCommon(Variable &variable, const T *values)
                                "BeginStep/EndStep pairs");
     }
 
-    if (m_MarshalMethod == SstMarshalFFS)
+    if (Params.MarshalMethod == SstMarshalFFS)
     {
         size_t *Shape = NULL;
         size_t *Start = NULL;
@@ -56,10 +56,10 @@ void SstWriter::PutSyncCommon(Variable &variable, const T *values)
             Count = variable.m_Count.data();
         }
         SstFFSMarshal(m_Output, (void *)&variable, variable.m_Name.c_str(),
-                      variable.m_Type.c_str(), variable.m_ElementSize, DimCount,
+                      (int)variable.m_Type, variable.m_ElementSize, DimCount,
                       Shape, Count, Start, values);
     }
-    else if (m_MarshalMethod == SstMarshalBP)
+    else if (Params.MarshalMethod == SstMarshalBP)
     {
         auto &blockInfo = variable.SetBlockInfo(
             values, m_BP3Serializer->m_MetadataSet.CurrentStep);
diff --git a/source/adios2/engine/table/TableWriter.cpp b/source/adios2/engine/table/TableWriter.cpp
index 3ba159d929..626a252d48 100644
--- a/source/adios2/engine/table/TableWriter.cpp
+++ b/source/adios2/engine/table/TableWriter.cpp
@@ -287,11 +287,11 @@ void TableWriter::PutAggregatorBuffer()
         m_VarInfoMap[v.name].type = v.type;
         m_VarInfoMap[v.name].shape = v.shape;
         size_t elementSize;
-        if (v.type == "")
+        if (v.type == DataType::None)
         {
         }
 #define declare_type(T)                                                        \
-    else if (v.type == helper::GetType()) { elementSize = sizeof(T); }
+    else if (v.type == helper::GetDataType()) { elementSize = sizeof(T); }
         ADIOS2_FOREACH_STDTYPE_1ARG(declare_type)
 #undef declare_type
 
@@ -320,11 +320,11 @@ void TableWriter::PutAggregatorBuffer()
                 }
             }
 
-            if (v.type == "")
+            if (v.type == DataType::None)
             {
             }
 #define declare_type(T)                                                        \
-    else if (v.type == helper::GetType())                                   \
+    else if (v.type == helper::GetDataType())                               \
     {                                                                          \
         helper::NdCopy(                                                     \
             v.buffer->data() + v.position, v.start, v.count, true, true,       \
@@ -376,12 +376,12 @@ void TableWriter::PutSubEngine(bool finalPut)
                 {
                     count[0] = shape[0] - start[0];
                 }
-                const std::string &type = m_VarInfoMap[varPair.first].type;
-                if (type == "")
+                const DataType type = m_VarInfoMap[varPair.first].type;
+                if (type == DataType::None)
                 {
                 }
 #define declare_type(T)                                                        \
-    else if (type == helper::GetType())                                     \
+    else if (type == helper::GetDataType())                                 \
     {                                                                          \
         auto variable = m_SubIO.InquireVariable(varPair.first);             \
         if (not variable)                                                      \
diff --git a/source/adios2/engine/table/TableWriter.h b/source/adios2/engine/table/TableWriter.h
index 91b25aa509..dddfe9a4eb 100644
--- a/source/adios2/engine/table/TableWriter.h
+++ b/source/adios2/engine/table/TableWriter.h
@@ -49,7 +49,7 @@ class TableWriter : public Engine
     struct VarInfo
     {
         Dims shape;
-        std::string type;
+        DataType type;
     };
     int m_Verbosity = 0;
     int m_Timeout = 5;
diff --git a/source/adios2/engine/table/TableWriter.tcc b/source/adios2/engine/table/TableWriter.tcc
index ad0201d507..729a6bf4a0 100644
--- a/source/adios2/engine/table/TableWriter.tcc
+++ b/source/adios2/engine/table/TableWriter.tcc
@@ -89,8 +89,7 @@ void TableWriter::PutDeferredCommon(Variable &variable, const T *data)
     for (auto i : aggregatorIndices)
     {
         auto serializer = m_Serializers[i];
-        serializer->PutData(variable, m_Name, CurrentStep(), m_MpiRank, "",
-                            Params());
+        serializer->PutData(variable, m_Name, CurrentStep(), m_MpiRank, "");
         if (serializer->LocalBufferSize() > m_SerializerBufferSize / 2)
         {
             if (m_MpiSize > 1)
diff --git a/source/adios2/helper/adiosComm.cpp b/source/adios2/helper/adiosComm.cpp
index 01cd58bb4d..42ce4d7fce 100644
--- a/source/adios2/helper/adiosComm.cpp
+++ b/source/adios2/helper/adiosComm.cpp
@@ -70,6 +70,11 @@ Comm Comm::World(const std::string &hint) const
     return Comm(m_Impl->World(hint));
 }
 
+Comm Comm::GroupByShm(const std::string &hint) const
+{
+    return Comm(m_Impl->GroupByShm(hint));
+}
+
 int Comm::Rank() const { return m_Impl->Rank(); }
 
 int Comm::Size() const { return m_Impl->Size(); }
diff --git a/source/adios2/helper/adiosComm.h b/source/adios2/helper/adiosComm.h
index 750f54dcc4..f8e2c7cf9e 100644
--- a/source/adios2/helper/adiosComm.h
+++ b/source/adios2/helper/adiosComm.h
@@ -118,6 +118,13 @@ class Comm
      */
     Comm World(const std::string &hint = std::string()) const;
 
+    /**
+     * @brief Create a communicator for processes that can share memory
+     * @param hint Description of std::runtime_error exception on error.
+     * Useful for grouping processes per compute node
+     */
+    Comm GroupByShm(const std::string &hint = std::string()) const;
+
     int Rank() const;
     int Size() const;
 
@@ -368,6 +375,8 @@ class CommImpl
     virtual std::unique_ptr Split(int color, int key,
                                             const std::string &hint) const = 0;
     virtual std::unique_ptr World(const std::string &hint) const = 0;
+    virtual std::unique_ptr
+    GroupByShm(const std::string &hint) const = 0;
     virtual int Rank() const = 0;
     virtual int Size() const = 0;
     virtual bool IsMPI() const = 0;
diff --git a/source/adios2/helper/adiosComm.inl b/source/adios2/helper/adiosComm.inl
index 981afe8fad..0ff88d892c 100644
--- a/source/adios2/helper/adiosComm.inl
+++ b/source/adios2/helper/adiosComm.inl
@@ -56,6 +56,15 @@ void Comm::GathervArrays(const T *source, size_t sourceCount,
     if (rankDestination == this->Rank())
     {
         displs = GetGathervDisplacements(counts, countsSize);
+        const size_t totalElements =
+            displs[countsSize - 1] + counts[countsSize - 1];
+        if (totalElements > 2147483648)
+        {
+            std::runtime_error(
+                "ERROR: GathervArrays does not support gathering more than "
+                "2^31 elements. Here it was tasked with " +
+                std::to_string(totalElements) + " elements\n");
+        }
     }
     this->Gatherv(source, sourceCount, destination, counts, displs.data(),
                   rankDestination);
@@ -93,7 +102,7 @@ void Comm::GathervVectors(const std::vector &in, std::vector &out,
     }
 
     this->GathervArrays(in.data(), in.size(), counts.data(), counts.size(),
-                        out.data() + position);
+                        out.data() + position, rankDestination);
     position += gatheredSize;
 }
 
diff --git a/source/adios2/helper/adiosCommDummy.cpp b/source/adios2/helper/adiosCommDummy.cpp
index d9c431a096..eddf9827d5 100644
--- a/source/adios2/helper/adiosCommDummy.cpp
+++ b/source/adios2/helper/adiosCommDummy.cpp
@@ -49,6 +49,8 @@ class CommImplDummy : public CommImpl
     std::unique_ptr Split(int color, int key,
                                     const std::string &hint) const override;
     std::unique_ptr World(const std::string &hint) const override;
+    virtual std::unique_ptr
+    GroupByShm(const std::string &hint) const override;
 
     int Rank() const override;
     int Size() const override;
@@ -124,6 +126,11 @@ std::unique_ptr CommImplDummy::World(const std::string &) const
     return std::unique_ptr(new CommImplDummy());
 }
 
+std::unique_ptr CommImplDummy::GroupByShm(const std::string &) const
+{
+    return std::unique_ptr(new CommImplDummy());
+}
+
 int CommImplDummy::Rank() const { return 0; }
 
 int CommImplDummy::Size() const { return 1; }
diff --git a/source/adios2/helper/adiosCommMPI.cpp b/source/adios2/helper/adiosCommMPI.cpp
index 07f33d9abf..eebc4e7251 100644
--- a/source/adios2/helper/adiosCommMPI.cpp
+++ b/source/adios2/helper/adiosCommMPI.cpp
@@ -122,6 +122,8 @@ class CommImplMPI : public CommImpl
     std::unique_ptr Split(int color, int key,
                                     const std::string &hint) const override;
     std::unique_ptr World(const std::string &hint) const override;
+    virtual std::unique_ptr
+    GroupByShm(const std::string &hint) const override;
 
     int Rank() const override;
     int Size() const override;
@@ -224,6 +226,17 @@ std::unique_ptr CommImplMPI::World(const std::string &) const
     return std::unique_ptr(new CommImplMPI(MPI_COMM_WORLD));
 }
 
+std::unique_ptr CommImplMPI::GroupByShm(const std::string &hint) const
+{
+    MPI_Comm nodeComm;
+    MPI_Info info;
+    MPI_Info_create(&info);
+    CheckMPIReturn(MPI_Comm_split_type(m_MPIComm, MPI_COMM_TYPE_SHARED, 0, info,
+                                       &nodeComm),
+                   hint);
+    return std::unique_ptr(new CommImplMPI(nodeComm));
+}
+
 int CommImplMPI::Rank() const
 {
     int rank;
diff --git a/source/adios2/helper/adiosMemory.cpp b/source/adios2/helper/adiosMemory.cpp
index 9154f76e0a..c3a4246a91 100644
--- a/source/adios2/helper/adiosMemory.cpp
+++ b/source/adios2/helper/adiosMemory.cpp
@@ -24,7 +24,7 @@ namespace
 {
 
 void CopyPayloadStride(const char *src, const size_t payloadStride, char *dest,
-                       const bool endianReverse, const std::string destType)
+                       const bool endianReverse, const DataType destType)
 {
 #ifdef ADIOS2_HAVE_ENDIAN_REVERSE
     if (endianReverse)
@@ -33,7 +33,7 @@ void CopyPayloadStride(const char *src, const size_t payloadStride, char *dest,
         {
         }
 #define declare_type(T)                                                        \
-    else if (destType == GetType())                                         \
+    else if (destType == GetDataType())                                     \
     {                                                                          \
         CopyEndianReverse(src, payloadStride, reinterpret_cast(dest)); \
     }
@@ -66,7 +66,7 @@ void ClipRowMajor(char *dest, const Dims &destStart, const Dims &destCount,
                   const Dims &srcStart, const Dims &srcCount,
                   const Dims & /*destMemStart*/, const Dims & /*destMemCount*/,
                   const Dims &srcMemStart, const Dims &srcMemCount,
-                  const bool endianReverse, const std::string destType)
+                  const bool endianReverse, const DataType destType)
 {
     const Dims destStartFinal = DestDimsFinal(destStart, destRowMajor, true);
     const Dims destCountFinal = DestDimsFinal(destCount, destRowMajor, true);
@@ -163,7 +163,7 @@ void ClipColumnMajor(char *dest, const Dims &destStart, const Dims &destCount,
                      const Dims & /*destMemStart*/,
                      const Dims & /*destMemCount*/, const Dims &srcMemStart,
                      const Dims &srcMemCount, const bool endianReverse,
-                     const std::string destType)
+                     const DataType destType)
 {
     const Dims destStartFinal = DestDimsFinal(destStart, destRowMajor, false);
     const Dims destCountFinal = DestDimsFinal(destCount, destRowMajor, false);
@@ -250,7 +250,7 @@ void CopyPayload(char *dest, const Dims &destStart, const Dims &destCount,
                  const Dims &srcCount, const bool srcRowMajor,
                  const Dims &destMemStart, const Dims &destMemCount,
                  const Dims &srcMemStart, const Dims &srcMemCount,
-                 const bool endianReverse, const std::string destType) noexcept
+                 const bool endianReverse, const DataType destType) noexcept
 {
     if (srcStart.size() == 1) // 1D copy memory
     {
diff --git a/source/adios2/helper/adiosMemory.h b/source/adios2/helper/adiosMemory.h
index 7436d0a6c1..9d5e40fbd5 100644
--- a/source/adios2/helper/adiosMemory.h
+++ b/source/adios2/helper/adiosMemory.h
@@ -135,7 +135,7 @@ void CopyPayload(char *dest, const Dims &destStart, const Dims &destCount,
                  const Dims &srcMemStart = Dims(),
                  const Dims &srcMemCount = Dims(),
                  const bool endianReverse = false,
-                 const std::string destType = "") noexcept;
+                 const DataType destType = DataType::None) noexcept;
 
 /**
  * Clips the contiguous memory corresponding to an intersection and puts it in
diff --git a/source/adios2/helper/adiosMemory.inl b/source/adios2/helper/adiosMemory.inl
index 5f3df8bd14..f8d450e70a 100644
--- a/source/adios2/helper/adiosMemory.inl
+++ b/source/adios2/helper/adiosMemory.inl
@@ -331,11 +331,12 @@ void CopyMemoryBlock(T *dest, const Dims &destStart, const Dims &destCount,
     const Dims srcMemStartPayload = PayloadDims(srcMemStart, srcRowMajor);
     const Dims srcMemCountPayload = PayloadDims(srcMemCount, srcRowMajor);
 
-    CopyPayload(
-        reinterpret_cast(dest), destStartPayload, destCountPayload,
-        destRowMajor, reinterpret_cast(src), srcStartPayload,
-        srcCountPayload, srcRowMajor, destMemStartPayload, destMemCountPayload,
-        srcMemStartPayload, srcMemCountPayload, endianReverse, GetType());
+    CopyPayload(reinterpret_cast(dest), destStartPayload,
+                destCountPayload, destRowMajor,
+                reinterpret_cast(src), srcStartPayload,
+                srcCountPayload, srcRowMajor, destMemStartPayload,
+                destMemCountPayload, srcMemStartPayload, srcMemCountPayload,
+                endianReverse, GetDataType());
 }
 
 template 
diff --git a/source/adios2/helper/adiosMpiHandshake.cpp b/source/adios2/helper/adiosMpiHandshake.cpp
index c097d01e7b..44a6b6eb11 100644
--- a/source/adios2/helper/adiosMpiHandshake.cpp
+++ b/source/adios2/helper/adiosMpiHandshake.cpp
@@ -11,328 +11,227 @@
 #include "adiosMpiHandshake.h"
 #include 
 #include 
-#include 
+#include 
+#include 
 #include 
+#include 
 #include 
+#include 
 
 namespace adios2
 {
 namespace helper
 {
 
-std::vector MpiHandshake::m_Buffer;
-std::vector> MpiHandshake::m_SendRequests;
-std::vector> MpiHandshake::m_RecvRequests;
-size_t MpiHandshake::m_MaxStreamsPerApp;
-size_t MpiHandshake::m_MaxFilenameLength;
-size_t MpiHandshake::m_ItemSize;
-std::map MpiHandshake::m_RendezvousAppCounts;
-size_t MpiHandshake::m_StreamID = 0;
-int MpiHandshake::m_WorldSize;
-int MpiHandshake::m_WorldRank;
-int MpiHandshake::m_LocalSize;
-int MpiHandshake::m_LocalRank;
-int MpiHandshake::m_LocalMasterRank;
-std::map>>
-    MpiHandshake::m_WritersMap;
-std::map>>
-    MpiHandshake::m_ReadersMap;
-std::map MpiHandshake::m_AppsSize;
-
-size_t MpiHandshake::PlaceInBuffer(size_t stream, int rank)
+void HandshakeComm(const std::string &filename, const char mode,
+                   const int timeoutSeconds, MPI_Comm localComm,
+                   MPI_Group &streamGroup, MPI_Group &writerGroup,
+                   MPI_Group &readerGroup, MPI_Comm &streamComm,
+                   MPI_Comm &writerComm, MPI_Comm &readerComm, int verbosity)
 {
-    return rank * m_MaxStreamsPerApp * m_ItemSize + stream * m_ItemSize;
+    auto appRankMaps =
+        HandshakeRank(filename, mode, timeoutSeconds, localComm, verbosity);
+    MPI_Group worldGroup;
+    MPI_Comm_group(MPI_COMM_WORLD, &worldGroup);
+    MPI_Group_incl(worldGroup, static_cast(appRankMaps[0].size()),
+                   appRankMaps[0].data(), &streamGroup);
+    MPI_Group_incl(worldGroup, static_cast(appRankMaps[1].size()),
+                   appRankMaps[1].data(), &writerGroup);
+    MPI_Group_incl(worldGroup, static_cast(appRankMaps[2].size()),
+                   appRankMaps[2].data(), &readerGroup);
+#ifdef _WIN32
+    MPI_Comm_create(MPI_COMM_WORLD, streamGroup, &streamComm);
+    MPI_Comm_create(MPI_COMM_WORLD, writerGroup, &writerComm);
+    MPI_Comm_create(MPI_COMM_WORLD, readerGroup, &readerComm);
+#else
+    MPI_Comm_create_group(MPI_COMM_WORLD, streamGroup, 0, &streamComm);
+    MPI_Comm_create_group(MPI_COMM_WORLD, writerGroup, 0, &writerComm);
+    MPI_Comm_create_group(MPI_COMM_WORLD, readerGroup, 0, &readerComm);
+#endif
 }
 
-void MpiHandshake::Test()
+const std::vector>
+HandshakeRank(const std::string &filename, const char mode,
+              const int timeoutSeconds, MPI_Comm localComm, int verbosity)
 {
-    int success;
-    MPI_Status status;
+    std::vector> ret(3);
 
-    for (int rank = 0; rank < m_WorldSize; ++rank)
-    {
-        for (size_t stream = 0; stream < m_MaxStreamsPerApp; ++stream)
-        {
-            MPI_Test(&m_RecvRequests[rank][stream], &success, &status);
-            if (success)
-            {
-                size_t offset = PlaceInBuffer(stream, rank);
-                char mode = m_Buffer[offset];
-                offset += sizeof(char);
-                int appMasterRank;
-                std::memcpy(&appMasterRank, m_Buffer.data() + offset,
-                            sizeof(appMasterRank));
-                offset += sizeof(int);
-                int appSize;
-                std::memcpy(&appSize, m_Buffer.data() + offset,
-                            sizeof(appSize));
-                offset += sizeof(int);
-                std::string filename = m_Buffer.data() + offset;
-                m_AppsSize[appMasterRank] = appSize;
-                if (mode == 'w')
-                {
-                    auto &ranks = m_WritersMap[filename][appMasterRank];
-                    if (std::find(ranks.begin(), ranks.end(), rank) ==
-                        ranks.end())
-                    {
-                        ranks.push_back(rank);
-                    }
-                }
-                else if (mode == 'r')
-                {
-                    auto &ranks = m_ReadersMap[filename][appMasterRank];
-                    if (std::find(ranks.begin(), ranks.end(), rank) ==
-                        ranks.end())
-                    {
-                        ranks.push_back(rank);
-                    }
-                }
-            }
-        }
-    }
-}
+    int localRank;
+    int localSize;
+    int worldRank;
+    int worldSize;
 
-bool MpiHandshake::Check(const std::string &filename)
-{
-    int m_Verbosity = 1;
+    MPI_Comm_rank(localComm, &localRank);
+    MPI_Comm_size(localComm, &localSize);
 
-    Test();
+    MPI_Comm_rank(MPI_COMM_WORLD, &worldRank);
+    MPI_Comm_size(MPI_COMM_WORLD, &worldSize);
 
-    // check if RendezvousAppCount reached
+    std::vector allLocalRanks(localSize);
 
-    if (m_WritersMap[filename].size() + m_ReadersMap[filename].size() !=
-        m_RendezvousAppCounts[filename])
-    {
-        if (m_Verbosity >= 10)
-        {
-            std::cout << "MpiHandshake Rank " << m_WorldRank << " Stream "
-                      << filename << ": " << m_WritersMap[filename].size()
-                      << " writers and " << m_ReadersMap[filename].size()
-                      << " readers found out of "
-                      << m_RendezvousAppCounts[filename]
-                      << " total rendezvous apps" << std::endl;
-        }
-        return false;
-    }
+    MPI_Gather(&worldRank, 1, MPI_INT, allLocalRanks.data(), 1, MPI_INT, 0,
+               localComm);
 
-    // check if all ranks' info is received
-
-    for (const auto &app : m_WritersMap[filename])
+    if (localRank == 0)
     {
-        if (app.second.size() != m_AppsSize[app.first])
+        std::ofstream fs;
+        fs.open(filename + "." + mode);
+        for (auto rank : allLocalRanks)
         {
-            return false;
+            fs << rank << std::endl;
         }
-    }
+        fs.close();
 
-    for (const auto &app : m_ReadersMap[filename])
-    {
-        if (app.second.size() != m_AppsSize[app.first])
+        if (mode == 'r')
         {
-            return false;
-        }
-    }
 
-    return true;
-}
-
-void MpiHandshake::Handshake(const std::string &filename, const char mode,
-                             const int timeoutSeconds,
-                             const size_t maxStreamsPerApp,
-                             const size_t maxFilenameLength,
-                             const size_t rendezvousAppCountForStream,
-                             MPI_Comm localComm)
-{
-
-    // initialize variables
-
-    if (filename.size() > maxFilenameLength)
-    {
-        throw(std::runtime_error("Filename too long"));
-    }
-
-    MPI_Comm_size(MPI_COMM_WORLD, &m_WorldSize);
-    MPI_Comm_rank(MPI_COMM_WORLD, &m_WorldRank);
-    MPI_Comm_size(localComm, &m_LocalSize);
-    MPI_Comm_rank(localComm, &m_LocalRank);
-    m_MaxStreamsPerApp = maxStreamsPerApp;
-    m_MaxFilenameLength = maxFilenameLength;
-    m_RendezvousAppCounts[filename] = rendezvousAppCountForStream;
-
-    m_SendRequests.resize(m_WorldSize);
-    m_RecvRequests.resize(m_WorldSize);
-    for (int rank = 0; rank < m_WorldSize; ++rank)
-    {
-        m_SendRequests[rank].resize(maxStreamsPerApp);
-        m_RecvRequests[rank].resize(maxStreamsPerApp);
-    }
-
-    m_ItemSize = maxFilenameLength + sizeof(char) + sizeof(int) * 2;
-    m_Buffer.resize(m_WorldSize * maxStreamsPerApp * m_ItemSize);
+            for (auto i : allLocalRanks)
+            {
+                ret[0].push_back(i);
+                ret[2].push_back(i);
+            }
 
-    // broadcast local master rank's world rank to use as app ID
+            std::ofstream fsc;
+            fsc.open(filename + ".r.c");
+            fsc << "completed";
+            fsc.close();
 
-    if (m_LocalRank == 0)
-    {
-        m_LocalMasterRank = m_WorldRank;
-    }
-    MPI_Bcast(&m_LocalMasterRank, 1, MPI_INT, 0, localComm);
+            auto startTime = std::chrono::system_clock::now();
+            while (true)
+            {
+                std::ifstream fs;
+                try
+                {
+                    auto nowTime = std::chrono::system_clock::now();
+                    auto duration =
+                        std::chrono::duration_cast(
+                            nowTime - startTime);
+                    if (duration.count() > timeoutSeconds)
+                    {
+                        throw(std::runtime_error(
+                            "Mpi handshake timeout for Stream " + filename));
+                    }
 
-    // start receiving
+                    fs.open(filename + ".w.c");
+                    std::string line;
+                    std::getline(fs, line);
+                    if (line != "completed")
+                    {
+                        continue;
+                    }
+                    fs.close();
+                    remove((filename + ".w.c\0").c_str());
+                    break;
+                }
+                catch (...)
+                {
+                    continue;
+                }
+            }
 
-    for (int rank = 0; rank < m_WorldSize; ++rank)
-    {
-        for (size_t stream = 0; stream < maxStreamsPerApp; ++stream)
-        {
-            MPI_Irecv(m_Buffer.data() + PlaceInBuffer(stream, rank),
-                      static_cast(m_ItemSize), MPI_CHAR, rank, rank,
-                      MPI_COMM_WORLD, &m_RecvRequests[rank][stream]);
+            std::ifstream fs;
+            fs.open(filename + ".w");
+            for (std::string line; std::getline(fs, line);)
+            {
+                ret[0].push_back(std::stoi(line));
+                ret[1].push_back(std::stoi(line));
+            }
+            fs.close();
+            remove((filename + ".w\0").c_str());
         }
-    }
+        else if (mode == 'w')
+        {
+            for (auto i : allLocalRanks)
+            {
+                ret[0].push_back(i);
+                ret[1].push_back(i);
+            }
 
-    // start sending
+            std::ofstream fsc;
+            fsc.open(filename + ".w.c");
+            fsc << "completed";
+            fsc.close();
 
-    size_t offset = 0;
-    std::vector buffer(m_ItemSize);
-    std::memcpy(buffer.data(), &mode, sizeof(char));
-    offset += sizeof(char);
-    std::memcpy(buffer.data() + offset, &m_LocalMasterRank, sizeof(int));
-    offset += sizeof(int);
-    std::memcpy(buffer.data() + offset, &m_LocalSize, sizeof(int));
-    offset += sizeof(int);
-    std::memcpy(buffer.data() + offset, filename.data(), filename.size());
+            while (true)
+            {
+                std::ifstream fs;
+                try
+                {
+                    fs.open(filename + ".r.c");
+                    std::string line;
+                    std::getline(fs, line);
+                    if (line != "completed")
+                    {
+                        continue;
+                    }
+                    fs.close();
+                    remove((filename + ".r.c\0").c_str());
+                    break;
+                }
+                catch (...)
+                {
+                    continue;
+                }
+            }
 
-    for (int rank = 0; rank < m_WorldSize; ++rank)
-    {
-        MPI_Isend(buffer.data(), static_cast(m_ItemSize), MPI_CHAR, rank,
-                  m_WorldRank, MPI_COMM_WORLD,
-                  &m_SendRequests[rank][m_StreamID]);
+            std::ifstream fs;
+            fs.open(filename + ".r");
+            for (std::string line; std::getline(fs, line);)
+            {
+                ret[0].push_back(std::stoi(line));
+                ret[2].push_back(std::stoi(line));
+            }
+            fs.close();
+            remove((filename + ".r\0").c_str());
+        }
     }
 
-    // wait and check if required RendezvousAppCount reached
+    int dims[3];
 
-    auto startTime = std::chrono::system_clock::now();
-    while (!Check(filename))
+    if (localRank == 0)
     {
-        std::this_thread::sleep_for(std::chrono::microseconds(100));
-        auto nowTime = std::chrono::system_clock::now();
-        auto duration = std::chrono::duration_cast(
-            nowTime - startTime);
-        if (duration.count() > timeoutSeconds)
+        for (int i = 0; i < 3; ++i)
         {
-            throw(std::runtime_error("Mpi handshake timeout on Rank" +
-                                     std::to_string(m_WorldRank) +
-                                     " for Stream " + filename));
+            dims[i] = static_cast(ret[i].size());
+            std::sort(ret[i].begin(), ret[i].end());
         }
     }
 
-    // clean up MPI requests
+    MPI_Bcast(dims, 3, MPI_INT, 0, localComm);
 
-    for (auto &rs : m_RecvRequests)
+    if (localRank != 0)
     {
-        for (auto &r : rs)
+        for (int i = 0; i < 3; ++i)
         {
-            MPI_Status status;
-            int success;
-            MPI_Test(&r, &success, &status);
-            if (!success)
-            {
-                MPI_Cancel(&r);
-            }
+            ret[i].resize(dims[i]);
         }
     }
-    m_RecvRequests.clear();
 
-    ++m_StreamID;
-}
-
-const std::map> &
-MpiHandshake::GetWriterMap(const std::string &filename)
-{
-    return m_WritersMap[filename];
-}
-const std::map> &
-MpiHandshake::GetReaderMap(const std::string &filename)
-{
-    return m_ReadersMap[filename];
-}
-
-void MpiHandshake::PrintMaps(const int printRank, const std::string &filename)
-{
-    if (m_WorldRank == printRank)
+    for (int i = 0; i < 3; ++i)
     {
-        std::cout << "Printing MPI handshake map for Stream " << filename
-                  << " from Rank " << printRank << std::endl;
-        std::cout << "    Writers: " << std::endl;
-
-        for (const auto &app : m_WritersMap[filename])
-        {
-            std::cout << "        App Master Rank " << app.first << std::endl;
-            std::cout << "            ";
-            for (const auto &rank : app.second)
-            {
-                std::cout << rank << ", ";
-            }
-            std::cout << std::endl;
-        }
-
-        std::cout << "    Readers: " << std::endl;
-        for (const auto &app : m_ReadersMap[filename])
-        {
-            std::cout << "        App Master Rank " << app.first << std::endl;
-            std::cout << "            ";
-            for (const auto &rank : app.second)
-            {
-                std::cout << rank << ", ";
-            }
-            std::cout << std::endl;
-        }
+        MPI_Bcast(ret[i].data(), static_cast(ret[i].size()), MPI_INT, 0,
+                  localComm);
     }
-}
-void MpiHandshake::PrintMaps()
-{
-    for (int printRank = 0; printRank < m_WorldSize; ++printRank)
+
+    if (verbosity >= 5)
     {
-        MPI_Barrier(MPI_COMM_WORLD);
-        if (m_WorldRank == printRank)
+        std::stringstream output;
+        output << "World Rank " << worldRank << ": " << std::endl;
+        int s = 0;
+        for (const auto &i : ret)
         {
-            std::cout << "For rank " << printRank
-                      << "============================================"
-                      << std::endl;
-            std::cout << "Writers: " << std::endl;
-            for (const auto &stream : m_WritersMap)
+            output << "    " << s << ": ";
+            for (const auto &j : i)
             {
-                std::cout << "    Stream " << stream.first << std::endl;
-                for (const auto &app : stream.second)
-                {
-                    std::cout << "        App Master Rank " << app.first
-                              << std::endl;
-                    std::cout << "            ";
-                    for (const auto &rank : app.second)
-                    {
-                        std::cout << rank << ", ";
-                    }
-                    std::cout << std::endl;
-                }
-            }
-            std::cout << "Readers: " << std::endl;
-            for (const auto &stream : m_ReadersMap)
-            {
-                std::cout << "    Stream " << stream.first << std::endl;
-                for (const auto &app : stream.second)
-                {
-                    std::cout << "        App Master Rank " << app.first
-                              << std::endl;
-                    std::cout << "            ";
-                    for (const auto &rank : app.second)
-                    {
-                        std::cout << rank << ", ";
-                    }
-                    std::cout << std::endl;
-                }
+                output << j << ", ";
             }
+            output << std::endl;
+            ++s;
         }
+        std::cout << output.str();
     }
+
+    return ret;
 }
 
 } // end namespace helper
diff --git a/source/adios2/helper/adiosMpiHandshake.h b/source/adios2/helper/adiosMpiHandshake.h
index 3a128d0f5d..2b89167a01 100644
--- a/source/adios2/helper/adiosMpiHandshake.h
+++ b/source/adios2/helper/adiosMpiHandshake.h
@@ -16,7 +16,6 @@
 #error "Do not include adiosMpiHandshake.h without ADIOS2_HAVE_MPI."
 #endif
 
-#include 
 #include 
 #include 
 #include 
@@ -26,99 +25,39 @@ namespace adios2
 namespace helper
 {
 
-class MpiHandshake
-{
-public:
-    /**
-     * Start the handshake operations and wait until the rendezvous conditions
-     * are reached, or timeout.
-     *
-     * @param filename: name of the staging stream, must be within the length of
-     * maxFilenameLength
-     *
-     * @param mode: 'r' or 'w', read or write
-     *
-     * @param timeoutSeconds: timeout for the handshake, will throw exception
-     * when reaching this timeout
-     *
-     * @param maxStreamsPerApp: the maximum number of streams that all apps
-     * sharing this MPI_COMM_WORLD can possibly open. It is required that this
-     * number is consistent across all ranks. This is used for pre-allocating
-     * the vectors holding MPI requests and must be specified correctly,
-     * otherwise strange errors could occur. This class does not provide any
-     * mechanism to check whether this number being passed is actually correct
-     * or not accross all ranks, because implementing this logic for an
-     * arbitrary communication pattern is overly expensive, if not impossible.
-     *
-     * @param maxFilenameLength: the maximum possible length of filename that
-     * all apps sharing this MPI_COMM_WORLD could possibly define. It is
-     * required that this number is consistent across all ranks. This is used
-     * for pre-allocating the buffer for aggregating the global MPI information.
-     * An exception will be thrown if any filename on any rank is found to be
-     * longer than this.
-     *
-     * @param rendezvousAppCountForStream: the number of apps, including both
-     * writers and readers, that will work on this stream. The function will
-     * block until it receives the MPI handshake information from all these
-     * apps, or until timeoutSeconds is passed.
-     *
-     * @param localComm: local MPI communicator for the app
-     */
-    static void Handshake(const std::string &filename, const char mode,
-                          const int timeoutSeconds,
-                          const size_t maxStreamsPerApp,
-                          const size_t maxFilenameLength,
-                          const size_t rendezvousAppCountForStream,
-                          MPI_Comm localComm);
-
-    /**
-     * Get the writer map of all apps participating the stream.
-     *
-     * @param filename: name of the staging stream
-     *
-     * @return map of all writer apps participating the stream. Key is the world
-     * rank of the master rank of the local communicator of a participating
-     * writer app. Value is a vector of all world ranks of this writer app
-     */
-    static const std::map> &
-    GetWriterMap(const std::string &filename);
-
-    /**
-     * Get the reader map of all apps participating the stream.
-     *
-     * @param filename: name of the staging stream
-     *
-     * @return map of all reader apps participating the stream. Key is the world
-     * rank of the master rank of the local communicator of a participating
-     * reader app. Value is a vector of all world ranks of this reader app
-     */
-    static const std::map> &
-    GetReaderMap(const std::string &filename);
-
-private:
-    static void Test();
-    static bool Check(const std::string &filename);
-    static size_t PlaceInBuffer(const size_t stream, const int rank);
-    static void PrintMaps();
-    static void PrintMaps(const int printRank, const std::string &filename);
+/**
+ * Start the handshake operations and wait until the rendezvous conditions
+ * are reached, or timeout.
+ *
+ * @param filename: name of the staging stream, must be within the length of
+ * maxFilenameLength
+ *
+ * @param mode: 'r' or 'w', read or write
+ *
+ * @param timeoutSeconds: timeout for the handshake, will throw exception
+ * when reaching this timeout
+ *
+ * @param rendezvousAppCountForStream: the number of apps, including both
+ * writers and readers, that will work on this stream. The function will
+ * block until it receives the MPI handshake information from all these
+ * apps, or until timeoutSeconds is passed.
+ *
+ * @param localComm: local MPI communicator for the app
+ *
+ * @return 3 vectors of ranks. [0] is the vector of all writer and reader ranks
+ * for stream *filename*. [1] is the vector of all writer ranks for stream
+ * *filename*. [2] is the vector of all reader ranks for stream *filename*.
+ */
+const std::vector>
+HandshakeRank(const std::string &filename, const char mode,
+              const int timeoutSeconds, MPI_Comm localComm, int verbosity = 0);
 
-    static std::vector m_Buffer;
-    static std::vector> m_SendRequests;
-    static std::vector> m_RecvRequests;
-    static size_t m_MaxStreamsPerApp;
-    static size_t m_MaxFilenameLength;
-    static size_t m_ItemSize;
-    static std::map m_RendezvousAppCounts;
-    static size_t m_StreamID;
-    static int m_WorldSize;
-    static int m_WorldRank;
-    static int m_LocalSize;
-    static int m_LocalRank;
-    static int m_LocalMasterRank;
-    static std::map>> m_WritersMap;
-    static std::map>> m_ReadersMap;
-    static std::map m_AppsSize;
-};
+void HandshakeComm(const std::string &filename, const char mode,
+                   const int timeoutSeconds, MPI_Comm localComm,
+                   MPI_Group &streamGroup, MPI_Group &writerGroup,
+                   MPI_Group &readerGroup, MPI_Comm &streamComm,
+                   MPI_Comm &writerComm, MPI_Comm &readerComm,
+                   int verbosity = 0);
 
 } // end namespace helper
 } // end namespace adios2
diff --git a/source/adios2/helper/adiosType.cpp b/source/adios2/helper/adiosType.cpp
index 64d65c1d04..b334a36255 100644
--- a/source/adios2/helper/adiosType.cpp
+++ b/source/adios2/helper/adiosType.cpp
@@ -20,6 +20,72 @@ namespace adios2
 namespace helper
 {
 
+DataType GetDataTypeFromString(std::string const &type) noexcept
+{
+    // Keep in sync with adios2::ToString(DataType).
+    if (type == "int8_t")
+    {
+        return DataType::Int8;
+    }
+    if (type == "int16_t")
+    {
+        return DataType::Int16;
+    }
+    if (type == "int32_t")
+    {
+        return DataType::Int32;
+    }
+    if (type == "int64_t")
+    {
+        return DataType::Int64;
+    }
+    if (type == "uint8_t")
+    {
+        return DataType::UInt8;
+    }
+    if (type == "uint16_t")
+    {
+        return DataType::UInt16;
+    }
+    if (type == "uint32_t")
+    {
+        return DataType::UInt32;
+    }
+    if (type == "uint64_t")
+    {
+        return DataType::UInt64;
+    }
+    if (type == "float")
+    {
+        return DataType::Float;
+    }
+    if (type == "double")
+    {
+        return DataType::Double;
+    }
+    if (type == "long double")
+    {
+        return DataType::LongDouble;
+    }
+    if (type == "float complex")
+    {
+        return DataType::FloatComplex;
+    }
+    if (type == "double complex")
+    {
+        return DataType::DoubleComplex;
+    }
+    if (type == "string")
+    {
+        return DataType::String;
+    }
+    if (type == "compound")
+    {
+        return DataType::Compound;
+    }
+    return DataType::None;
+}
+
 std::string DimsToCSV(const Dims &dimensions) noexcept
 {
     std::string dimsCSV;
diff --git a/source/adios2/helper/adiosType.h b/source/adios2/helper/adiosType.h
index 8d6a00f950..fe1193c51c 100644
--- a/source/adios2/helper/adiosType.h
+++ b/source/adios2/helper/adiosType.h
@@ -98,23 +98,16 @@ struct SubStreamBoxInfo
 
 /**
  * Gets type from template parameter T
- * @return string with type
+ * @return DataType enumeration value
  */
 template 
-std::string GetType() noexcept;
+DataType GetDataType() noexcept;
 
 /**
- * Check in types set if "type" is one of the aliases for a certain type,
- * (e.g. if type = integer is an accepted alias for "int", returning true)
- * @param type input to be compared with an alias
- * @param aliases set containing aliases to a certain type, typically
- * Support::DatatypesAliases from Support.h
- * @return true: is an alias, false: is not
+ * Gets type from string, inversing ToString(DataType)
+ * @return DataType enumeration value, or DataType::None on failure
  */
-template 
-bool IsTypeAlias(
-    const std::string type,
-    const std::map> &aliases) noexcept;
+DataType GetDataTypeFromString(std::string const &) noexcept;
 
 /**
  * Converts a vector of dimensions to a CSV string
diff --git a/source/adios2/helper/adiosType.inl b/source/adios2/helper/adiosType.inl
index ff1c562e70..50c5e5c67e 100644
--- a/source/adios2/helper/adiosType.inl
+++ b/source/adios2/helper/adiosType.inl
@@ -25,94 +25,75 @@ namespace helper
 {
 
 template <>
-inline std::string GetType() noexcept
+inline DataType GetDataType() noexcept
 {
-    return "string";
+    return DataType::String;
 }
 
 template <>
-inline std::string GetType() noexcept
+inline DataType GetDataType() noexcept
 {
-    return "int8_t";
+    return DataType::Int8;
 }
 template <>
-inline std::string GetType() noexcept
+inline DataType GetDataType() noexcept
 {
-    return "uint8_t";
+    return DataType::UInt8;
 }
 template <>
-inline std::string GetType() noexcept
+inline DataType GetDataType() noexcept
 {
-    return "int16_t";
+    return DataType::Int16;
 }
 template <>
-inline std::string GetType() noexcept
+inline DataType GetDataType() noexcept
 {
-    return "uint16_t";
+    return DataType::UInt16;
 }
 template <>
-inline std::string GetType() noexcept
+inline DataType GetDataType() noexcept
 {
-    return "int32_t";
+    return DataType::Int32;
 }
 template <>
-inline std::string GetType() noexcept
+inline DataType GetDataType() noexcept
 {
-    return "uint32_t";
+    return DataType::UInt32;
 }
 template <>
-inline std::string GetType() noexcept
+inline DataType GetDataType() noexcept
 {
-    return "int64_t";
+    return DataType::Int64;
 }
 template <>
-inline std::string GetType() noexcept
+inline DataType GetDataType() noexcept
 {
-    return "uint64_t";
+    return DataType::UInt64;
 }
 template <>
-inline std::string GetType() noexcept
+inline DataType GetDataType() noexcept
 {
-    return "float";
+    return DataType::Float;
 }
 template <>
-inline std::string GetType() noexcept
+inline DataType GetDataType() noexcept
 {
-    return "double";
+    return DataType::Double;
 }
 template <>
-inline std::string GetType() noexcept
+inline DataType GetDataType() noexcept
 {
-    return "long double";
+    return DataType::LongDouble;
 }
 template <>
-inline std::string GetType>() noexcept
+inline DataType GetDataType>() noexcept
 {
-    return "float complex";
+    return DataType::FloatComplex;
 }
 template <>
-inline std::string GetType>() noexcept
+inline DataType GetDataType>() noexcept
 {
-    return "double complex";
-}
-
-template 
-bool IsTypeAlias(
-    const std::string type,
-    const std::map> &aliases) noexcept
-{
-    if (type == GetType()) // is key itself
-    {
-        return true;
-    }
-
-    bool isAlias = false;
-    if (aliases.at(GetType()).count(type) == 1)
-    {
-        isAlias = true;
-    }
-
-    return isAlias;
+    return DataType::DoubleComplex;
 }
 
 template 
diff --git a/source/adios2/helper/adiosXML.cpp b/source/adios2/helper/adiosXML.cpp
index e969aa6e5b..dbbb3056c9 100644
--- a/source/adios2/helper/adiosXML.cpp
+++ b/source/adios2/helper/adiosXML.cpp
@@ -147,9 +147,10 @@ void ParseConfigXML(
             helper::XMLAttribute("name", io, hint);
 
         // Build the IO object
-        auto itCurrentIO =
-            ios.emplace(ioName->value(), core::IO(adios, ioName->value(), true,
-                                                  adios.m_HostLanguage));
+        auto itCurrentIO = ios.emplace(
+            std::piecewise_construct, std::forward_as_tuple(ioName->value()),
+            std::forward_as_tuple(adios, ioName->value(), true,
+                                  adios.m_HostLanguage));
         core::IO ¤tIO = itCurrentIO.first->second;
 
         // must be unique per io
diff --git a/source/adios2/helper/adiosYAML.cpp b/source/adios2/helper/adiosYAML.cpp
index e12c0a30b1..8cf19179f9 100644
--- a/source/adios2/helper/adiosYAML.cpp
+++ b/source/adios2/helper/adiosYAML.cpp
@@ -123,7 +123,8 @@ void ParseConfigYAML(
     auto lf_IOYAML = [&](const std::string &ioName, const YAML::Node &ioMap) {
         // Build the IO object
         auto itCurrentIO = ios.emplace(
-            ioName, core::IO(adios, ioName, true, adios.m_HostLanguage));
+            std::piecewise_construct, std::forward_as_tuple(ioName),
+            std::forward_as_tuple(adios, ioName, true, adios.m_HostLanguage));
         core::IO ¤tIO = itCurrentIO.first->second;
 
         // Engine parameters
diff --git a/source/adios2/operator/compress/CompressBZIP2.cpp b/source/adios2/operator/compress/CompressBZIP2.cpp
index 0ca7bda055..50860236ab 100644
--- a/source/adios2/operator/compress/CompressBZIP2.cpp
+++ b/source/adios2/operator/compress/CompressBZIP2.cpp
@@ -38,7 +38,7 @@ size_t CompressBZIP2::BufferMaxSize(const size_t sizeIn) const
 }
 
 size_t CompressBZIP2::Compress(const void *dataIn, const Dims &dimensions,
-                               const size_t elementSize, const std::string type,
+                               const size_t elementSize, DataType type,
                                void *bufferOut, const Params ¶meters,
                                Params &info) const
 {
@@ -49,8 +49,8 @@ size_t CompressBZIP2::Compress(const void *dataIn, const Dims &dimensions,
 
     if (!parameters.empty())
     {
-        const std::string hint(" in call to CompressBZIP2 Compress " + type +
-                               "\n");
+        const std::string hint(" in call to CompressBZIP2 Compress " +
+                               ToString(type) + "\n");
         helper::SetParameterValueInt("blockSize100k", parameters, blockSize100k,
                                      hint);
         helper::SetParameterValueInt("verbosity", parameters, verbosity, hint);
diff --git a/source/adios2/operator/compress/CompressBZIP2.h b/source/adios2/operator/compress/CompressBZIP2.h
index aeed0b8417..6332fbef31 100644
--- a/source/adios2/operator/compress/CompressBZIP2.h
+++ b/source/adios2/operator/compress/CompressBZIP2.h
@@ -43,9 +43,8 @@ class CompressBZIP2 : public Operator
      * @return size of compressed buffer in bytes
      */
     size_t Compress(const void *dataIn, const Dims &dimensions,
-                    const size_t elementSize, const std::string type,
-                    void *bufferOut, const Params ¶meters,
-                    Params &info) const final;
+                    const size_t elementSize, DataType type, void *bufferOut,
+                    const Params ¶meters, Params &info) const final;
 
     using Operator::Decompress;
     /**
diff --git a/source/adios2/operator/compress/CompressBlosc.cpp b/source/adios2/operator/compress/CompressBlosc.cpp
index 9b78f2eb80..6104c226b0 100644
--- a/source/adios2/operator/compress/CompressBlosc.cpp
+++ b/source/adios2/operator/compress/CompressBlosc.cpp
@@ -37,7 +37,7 @@ CompressBlosc::CompressBlosc(const Params ¶meters)
 }
 
 size_t CompressBlosc::Compress(const void *dataIn, const Dims &dimensions,
-                               const size_t elementSize, const std::string type,
+                               const size_t elementSize, DataType type,
                                void *bufferOut, const Params ¶meters,
                                Params &info) const
 {
diff --git a/source/adios2/operator/compress/CompressBlosc.h b/source/adios2/operator/compress/CompressBlosc.h
index b468e0df75..7189e9c8ec 100644
--- a/source/adios2/operator/compress/CompressBlosc.h
+++ b/source/adios2/operator/compress/CompressBlosc.h
@@ -44,9 +44,8 @@ class CompressBlosc : public Operator
      * @return size of compressed buffer in bytes
      */
     size_t Compress(const void *dataIn, const Dims &dimensions,
-                    const size_t elementSize, const std::string type,
-                    void *bufferOut, const Params ¶meters,
-                    Params &info) const final;
+                    const size_t elementSize, DataType type, void *bufferOut,
+                    const Params ¶meters, Params &info) const final;
 
     /**
      * Decompression signature for legacy libraries that use void*
diff --git a/source/adios2/operator/compress/CompressMGARD.cpp b/source/adios2/operator/compress/CompressMGARD.cpp
index 942c4b4624..9cae8ecd01 100644
--- a/source/adios2/operator/compress/CompressMGARD.cpp
+++ b/source/adios2/operator/compress/CompressMGARD.cpp
@@ -29,7 +29,7 @@ CompressMGARD::CompressMGARD(const Params ¶meters)
 }
 
 size_t CompressMGARD::Compress(const void *dataIn, const Dims &dimensions,
-                               const size_t elementSize, const std::string type,
+                               const size_t elementSize, DataType type,
                                void *bufferOut, const Params ¶meters,
                                Params &info) const
 {
@@ -43,7 +43,7 @@ size_t CompressMGARD::Compress(const void *dataIn, const Dims &dimensions,
 
     // set type
     int mgardType = -1;
-    if (type == helper::GetType())
+    if (type == helper::GetDataType())
     {
         mgardType = 1;
     }
@@ -93,8 +93,8 @@ size_t CompressMGARD::Compress(const void *dataIn, const Dims &dimensions,
 
     int sizeOut = 0;
     unsigned char *dataOutPtr = mgard_compress(
-        mgardType, const_cast(static_cast(dataIn)),
-        sizeOut, r[0], r[1], r[2], tolerance, s);
+        const_cast(static_cast(dataIn)), sizeOut,
+        r[0], r[1], r[2], tolerance, s);
 
     const size_t sizeOutT = static_cast(sizeOut);
     std::memcpy(bufferOut, dataOutPtr, sizeOutT);
@@ -104,14 +104,14 @@ size_t CompressMGARD::Compress(const void *dataIn, const Dims &dimensions,
 
 size_t CompressMGARD::Decompress(const void *bufferIn, const size_t sizeIn,
                                  void *dataOut, const Dims &dimensions,
-                                 const std::string type,
+                                 DataType type,
                                  const Params & /*parameters*/) const
 {
     int mgardType = -1;
     size_t elementSize = 0;
     double quantizer = 0.0;
 
-    if (type == helper::GetType())
+    if (type == helper::GetDataType())
     {
         mgardType = 1;
         elementSize = 8;
@@ -135,7 +135,6 @@ size_t CompressMGARD::Decompress(const void *bufferIn, const size_t sizeIn,
     }
 
     void *dataPtr = mgard_decompress(
-        mgardType, quantizer,
         reinterpret_cast(const_cast(bufferIn)),
         static_cast(sizeIn), r[0], r[1], r[2], 0.0);
 
diff --git a/source/adios2/operator/compress/CompressMGARD.h b/source/adios2/operator/compress/CompressMGARD.h
index fcc671fc15..9c83175310 100644
--- a/source/adios2/operator/compress/CompressMGARD.h
+++ b/source/adios2/operator/compress/CompressMGARD.h
@@ -41,9 +41,8 @@ class CompressMGARD : public Operator
      * @return size of compressed buffer in bytes
      */
     size_t Compress(const void *dataIn, const Dims &dimensions,
-                    const size_t elementSize, const std::string type,
-                    void *bufferOut, const Params ¶meters,
-                    Params &info) const final;
+                    const size_t elementSize, DataType type, void *bufferOut,
+                    const Params ¶meters, Params &info) const final;
 
     /**
      *
@@ -56,7 +55,7 @@ class CompressMGARD : public Operator
      * @return
      */
     size_t Decompress(const void *bufferIn, const size_t sizeIn, void *dataOut,
-                      const Dims &dimensions, const std::string varType,
+                      const Dims &dimensions, DataType varType,
                       const Params & /*parameters*/) const final;
 };
 
diff --git a/source/adios2/operator/compress/CompressPNG.cpp b/source/adios2/operator/compress/CompressPNG.cpp
index f61806b845..31d1eaaf55 100644
--- a/source/adios2/operator/compress/CompressPNG.cpp
+++ b/source/adios2/operator/compress/CompressPNG.cpp
@@ -48,9 +48,9 @@ CompressPNG::CompressPNG(const Params ¶meters) : Operator("png", parameters)
 }
 
 size_t CompressPNG::Compress(const void *dataIn, const Dims &dimensions,
-                             const size_t elementSize,
-                             const std::string /*type*/, void *bufferOut,
-                             const Params ¶meters, Params &info) const
+                             const size_t elementSize, DataType /*type*/,
+                             void *bufferOut, const Params ¶meters,
+                             Params &info) const
 {
     auto lf_Write = [](png_structp png_ptr, png_bytep data, png_size_t length) {
         DestInfo *pDestInfo =
diff --git a/source/adios2/operator/compress/CompressPNG.h b/source/adios2/operator/compress/CompressPNG.h
index 3a8b14b9a4..bd2d410716 100644
--- a/source/adios2/operator/compress/CompressPNG.h
+++ b/source/adios2/operator/compress/CompressPNG.h
@@ -43,9 +43,8 @@ class CompressPNG : public Operator
      * @return size of compressed buffer in bytes
      */
     size_t Compress(const void *dataIn, const Dims &dimensions,
-                    const size_t elementSize, const std::string type,
-                    void *bufferOut, const Params ¶meters,
-                    Params &info) const final;
+                    const size_t elementSize, DataType type, void *bufferOut,
+                    const Params ¶meters, Params &info) const final;
 
     /**
      * Decompression signature for legacy libraries that use void*
diff --git a/source/adios2/operator/compress/CompressSZ.cpp b/source/adios2/operator/compress/CompressSZ.cpp
index 789bc1e775..e42890bdbf 100644
--- a/source/adios2/operator/compress/CompressSZ.cpp
+++ b/source/adios2/operator/compress/CompressSZ.cpp
@@ -35,7 +35,7 @@ size_t CompressSZ::BufferMaxSize(const size_t sizeIn) const
 }
 
 size_t CompressSZ::Compress(const void *dataIn, const Dims &dimensions,
-                            const size_t elementSize, const std::string varType,
+                            const size_t elementSize, DataType varType,
                             void *bufferOut, const Params ¶meters,
                             Params &info) const
 {
@@ -244,11 +244,11 @@ size_t CompressSZ::Compress(const void *dataIn, const Dims &dimensions,
 
     // Get type info
     int dtype = -1;
-    if (varType == helper::GetType())
+    if (varType == helper::GetDataType())
     {
         dtype = SZ_DOUBLE;
     }
-    else if (varType == helper::GetType())
+    else if (varType == helper::GetDataType())
     {
         dtype = SZ_FLOAT;
     }
@@ -256,7 +256,7 @@ size_t CompressSZ::Compress(const void *dataIn, const Dims &dimensions,
     {
         throw std::invalid_argument("ERROR: ADIOS2 SZ Compression only support "
                                     "double or float, type: " +
-                                    varType + " is unsupported\n");
+                                    ToString(varType) + " is unsupported\n");
     }
 
     // r[0] is the fastest changing dimension and r[4] is the lowest changing
@@ -282,7 +282,7 @@ size_t CompressSZ::Compress(const void *dataIn, const Dims &dimensions,
 
 size_t CompressSZ::Decompress(const void *bufferIn, const size_t sizeIn,
                               void *dataOut, const Dims &dimensions,
-                              const std::string varType,
+                              DataType varType,
                               const Params & /*parameters*/) const
 {
     if (dimensions.size() > 5)
@@ -294,12 +294,12 @@ size_t CompressSZ::Decompress(const void *bufferIn, const size_t sizeIn,
     // Get type info
     int dtype = 0;
     size_t typeSizeBytes = 0;
-    if (varType == helper::GetType())
+    if (varType == helper::GetDataType())
     {
         dtype = SZ_DOUBLE;
         typeSizeBytes = 8;
     }
-    else if (varType == helper::GetType())
+    else if (varType == helper::GetDataType())
     {
         dtype = SZ_FLOAT;
         typeSizeBytes = 4;
diff --git a/source/adios2/operator/compress/CompressSZ.h b/source/adios2/operator/compress/CompressSZ.h
index 9301b57a9c..f8ec7c4eff 100644
--- a/source/adios2/operator/compress/CompressSZ.h
+++ b/source/adios2/operator/compress/CompressSZ.h
@@ -43,9 +43,8 @@ class CompressSZ : public Operator
      * @return size of compressed buffer in bytes
      */
     size_t Compress(const void *dataIn, const Dims &dimensions,
-                    const size_t elementSize, const std::string type,
-                    void *bufferOut, const Params ¶meters,
-                    Params &info) const final;
+                    const size_t elementSize, DataType type, void *bufferOut,
+                    const Params ¶meters, Params &info) const final;
 
     using Operator::Decompress;
 
@@ -59,7 +58,7 @@ class CompressSZ : public Operator
      * @return size of decompressed data in dataOut
      */
     size_t Decompress(const void *bufferIn, const size_t sizeIn, void *dataOut,
-                      const Dims &dimensions, const std::string type,
+                      const Dims &dimensions, DataType type,
                       const Params ¶meters) const final;
 };
 
diff --git a/source/adios2/operator/compress/CompressZFP.cpp b/source/adios2/operator/compress/CompressZFP.cpp
index 17ceeb3961..4fffb23f4f 100644
--- a/source/adios2/operator/compress/CompressZFP.cpp
+++ b/source/adios2/operator/compress/CompressZFP.cpp
@@ -23,7 +23,7 @@ CompressZFP::CompressZFP(const Params ¶meters) : Operator("zfp", parameters)
 }
 
 size_t CompressZFP::DoBufferMaxSize(const void *dataIn, const Dims &dimensions,
-                                    const std::string type,
+                                    DataType type,
                                     const Params ¶meters) const
 {
     zfp_field *field = GetZFPField(dataIn, dimensions, type);
@@ -35,7 +35,7 @@ size_t CompressZFP::DoBufferMaxSize(const void *dataIn, const Dims &dimensions,
 }
 
 size_t CompressZFP::Compress(const void *dataIn, const Dims &dimensions,
-                             const size_t elementSize, const std::string type,
+                             const size_t elementSize, DataType type,
                              void *bufferOut, const Params ¶meters,
                              Params &info) const
 {
@@ -64,8 +64,7 @@ size_t CompressZFP::Compress(const void *dataIn, const Dims &dimensions,
 
 size_t CompressZFP::Decompress(const void *bufferIn, const size_t sizeIn,
                                void *dataOut, const Dims &dimensions,
-                               const std::string type,
-                               const Params ¶meters) const
+                               DataType type, const Params ¶meters) const
 {
     auto lf_GetTypeSize = [](const zfp_type zfpType) -> size_t {
         size_t size = 0;
@@ -109,30 +108,30 @@ size_t CompressZFP::Decompress(const void *bufferIn, const size_t sizeIn,
 }
 
 // PRIVATE
-zfp_type CompressZFP::GetZfpType(const std::string type) const
+zfp_type CompressZFP::GetZfpType(DataType type) const
 {
     zfp_type zfpType = zfp_type_none;
 
-    if (type == helper::GetType())
+    if (type == helper::GetDataType())
     {
         zfpType = zfp_type_double;
     }
-    else if (type == helper::GetType())
+    else if (type == helper::GetDataType())
     {
         zfpType = zfp_type_float;
     }
-    else if (type == helper::GetType())
+    else if (type == helper::GetDataType())
     {
         zfpType = zfp_type_int64;
     }
-    else if (type == helper::GetType())
+    else if (type == helper::GetDataType())
     {
         zfpType = zfp_type_int32;
     }
     else
     {
         throw std::invalid_argument(
-            "ERROR: type " + type +
+            "ERROR: type " + ToString(type) +
             " not supported by zfp, only "
             "signed int32_t, signed int64_t, float, and "
             "double types are acceptable, from class "
@@ -143,16 +142,15 @@ zfp_type CompressZFP::GetZfpType(const std::string type) const
 }
 
 zfp_field *CompressZFP::GetZFPField(const void *data, const Dims &dimensions,
-                                    const std::string type) const
+                                    DataType type) const
 {
     auto lf_CheckField = [](const zfp_field *field,
-                            const std::string zfpFieldFunction,
-                            const std::string type) {
+                            const std::string zfpFieldFunction, DataType type) {
         if (field == nullptr || field == NULL)
         {
             throw std::invalid_argument(
                 "ERROR: " + zfpFieldFunction + " failed for data of type " +
-                type +
+                ToString(type) +
                 ", data pointer might be corrupted, from "
                 "class CompressZfp Transform\n");
         }
@@ -181,7 +179,7 @@ zfp_field *CompressZFP::GetZFPField(const void *data, const Dims &dimensions,
     else
     {
         throw std::invalid_argument(
-            "ERROR: zfp_field* failed for data of type " + type +
+            "ERROR: zfp_field* failed for data of type " + ToString(type) +
             ", only 1D, 2D and 3D dimensions are supported, from "
             "class CompressZfp Transform\n");
     }
@@ -189,8 +187,7 @@ zfp_field *CompressZFP::GetZFPField(const void *data, const Dims &dimensions,
     return field;
 }
 
-zfp_stream *CompressZFP::GetZFPStream(const Dims &dimensions,
-                                      const std::string type,
+zfp_stream *CompressZFP::GetZFPStream(const Dims &dimensions, DataType type,
                                       const Params ¶meters) const
 {
     auto lf_HasKey = [](Params::const_iterator itKey,
diff --git a/source/adios2/operator/compress/CompressZFP.h b/source/adios2/operator/compress/CompressZFP.h
index 20eacbeac5..55d26a9fed 100644
--- a/source/adios2/operator/compress/CompressZFP.h
+++ b/source/adios2/operator/compress/CompressZFP.h
@@ -46,9 +46,8 @@ class CompressZFP : public Operator
      * @return size of compressed buffer in bytes
      */
     size_t Compress(const void *dataIn, const Dims &dimensions,
-                    const size_t elementSize, const std::string type,
-                    void *bufferOut, const Params ¶meters,
-                    Params &info) const final;
+                    const size_t elementSize, DataType type, void *bufferOut,
+                    const Params ¶meters, Params &info) const final;
 
     using Operator::Decompress;
 
@@ -62,17 +61,17 @@ class CompressZFP : public Operator
      * @return size of decompressed data in dataOut
      */
     size_t Decompress(const void *bufferIn, const size_t sizeIn, void *dataOut,
-                      const Dims &dimensions, const std::string type,
+                      const Dims &dimensions, DataType type,
                       const Params ¶meters) const final;
 
 private:
     /**
      * Returns Zfp supported zfp_type based on adios string type
-     * @param type adios type as string, see GetType in
+     * @param type adios type as string, see GetDataType in
      * helper/adiosType.inl
      * @return zfp_type
      */
-    zfp_type GetZfpType(const std::string type) const;
+    zfp_type GetZfpType(DataType type) const;
 
     /**
      * Constructor Zfp zfp_field based on input information around the data
@@ -83,14 +82,13 @@ class CompressZFP : public Operator
      * @return zfp_field*
      */
     zfp_field *GetZFPField(const void *data, const Dims &shape,
-                           const std::string type) const;
+                           DataType type) const;
 
-    zfp_stream *GetZFPStream(const Dims &dimensions, const std::string type,
+    zfp_stream *GetZFPStream(const Dims &dimensions, DataType type,
                              const Params ¶meters) const;
 
     size_t DoBufferMaxSize(const void *dataIn, const Dims &dimensions,
-                           const std::string type,
-                           const Params ¶meters) const final;
+                           DataType type, const Params ¶meters) const final;
 
     /**
      * check status from BZip compression and decompression functions
diff --git a/source/adios2/toolkit/aggregator/mpi/MPIAggregator.cpp b/source/adios2/toolkit/aggregator/mpi/MPIAggregator.cpp
index 8743a6b245..7df033f849 100644
--- a/source/adios2/toolkit/aggregator/mpi/MPIAggregator.cpp
+++ b/source/adios2/toolkit/aggregator/mpi/MPIAggregator.cpp
@@ -96,6 +96,45 @@ void MPIAggregator::InitComm(const size_t subStreams,
     m_SubStreams = subStreams;
 }
 
+void MPIAggregator::InitCommOnePerNode(helper::Comm const &parentComm)
+{
+    m_Comm = parentComm.GroupByShm("creating default aggregator setup at Open");
+    m_Rank = m_Comm.Rank();
+    m_Size = m_Comm.Size();
+
+    if (m_Rank != 0)
+    {
+        m_IsConsumer = false;
+    }
+
+    m_IsActive = true;
+
+    /* Determine number of aggregators (= nodes) */
+
+    /*
+     *  Communicators connecting rank N of each node
+     *  We are only interested in the chain of rank 0s
+     */
+    int color = (m_Rank ? 1 : 0);
+    helper::Comm onePerNodeComm =
+        parentComm.Split(color, 0, "creating default aggregator setup at Open");
+
+    if (!m_Rank)
+    {
+        m_SubStreamIndex = static_cast(onePerNodeComm.Rank());
+        m_SubStreams = static_cast(onePerNodeComm.Size());
+    }
+    m_SubStreams = m_Comm.BroadcastValue(m_SubStreams, 0);
+    m_SubStreamIndex = m_Comm.BroadcastValue(m_SubStreamIndex, 0);
+
+    /* Identify parent rank of aggregator process within each group */
+    if (!m_Rank)
+    {
+        m_ConsumerRank = parentComm.Rank();
+    }
+    m_ConsumerRank = m_Comm.BroadcastValue(m_ConsumerRank, 0);
+}
+
 void MPIAggregator::HandshakeRank(const int rank)
 {
     int message = -1;
diff --git a/source/adios2/toolkit/aggregator/mpi/MPIAggregator.h b/source/adios2/toolkit/aggregator/mpi/MPIAggregator.h
index d15d9751b5..73c4bfb2f3 100644
--- a/source/adios2/toolkit/aggregator/mpi/MPIAggregator.h
+++ b/source/adios2/toolkit/aggregator/mpi/MPIAggregator.h
@@ -98,6 +98,10 @@ class MPIAggregator
      * the last rank) */
     void InitComm(const size_t subStreams, helper::Comm const &parentComm);
 
+    /** A default init function to select one process per node to be aggregator
+     */
+    void InitCommOnePerNode(helper::Comm const &parentComm);
+
     /** handshakes a single rank with the rest of the m_Comm ranks */
     void HandshakeRank(const int rank = 0);
 
diff --git a/source/adios2/toolkit/aggregator/mpi/MPIChain.cpp b/source/adios2/toolkit/aggregator/mpi/MPIChain.cpp
index 4db96961bb..eea9c50f15 100644
--- a/source/adios2/toolkit/aggregator/mpi/MPIChain.cpp
+++ b/source/adios2/toolkit/aggregator/mpi/MPIChain.cpp
@@ -20,8 +20,16 @@ MPIChain::MPIChain() : MPIAggregator() {}
 
 void MPIChain::Init(const size_t subStreams, helper::Comm const &parentComm)
 {
-    InitComm(subStreams, parentComm);
-    HandshakeRank(0);
+    if (subStreams > 0)
+    {
+        InitComm(subStreams, parentComm);
+        HandshakeRank(0);
+    }
+    else
+    {
+        InitCommOnePerNode(parentComm);
+    }
+
     HandshakeLinks();
 
     // add a receiving buffer except for the last rank (only sends)
diff --git a/source/adios2/toolkit/format/bp/BPBase.cpp b/source/adios2/toolkit/format/bp/BPBase.cpp
index 2f92a4cd62..b4788e725c 100644
--- a/source/adios2/toolkit/format/bp/BPBase.cpp
+++ b/source/adios2/toolkit/format/bp/BPBase.cpp
@@ -49,8 +49,6 @@ void BPBase::Init(const Params ¶meters, const std::string hint,
     struct Parameters parsedParameters;
     bool profilePresent = false;
     bool profileValue;
-    bool subStreamsPresent = false;
-    int32_t subStreamsValue;
     for (const auto ¶meter : parameters)
     {
         const std::string key = helper::LowerCase(parameter.first);
@@ -155,24 +153,46 @@ void BPBase::Init(const Params ¶meters, const std::string hint,
             parsedParameters.FlushStepsCount = helper::StringToSizeT(
                 value, " in Parameter key=FlushStepsCount " + hint);
         }
-        else if (key == "substreams")
+        else if (key == "substreams" || key == "numaggregators")
         {
-            int subStreams = static_cast(helper::StringTo(
-                value, " in Parameter key=SubStreams " + hint));
+            int n = static_cast(helper::StringTo(
+                value, " in Parameter key=" + key + " " + hint));
 
-            if (subStreams < 1)
+            if (n < 0)
             {
-                subStreams = 1;
+                n = 0;
             }
-            else if (subStreams > m_SizeMPI)
+            if (n > m_SizeMPI)
             {
-                subStreams = m_SizeMPI;
+                n = m_SizeMPI;
             }
-
-            if (subStreams < m_SizeMPI)
+            parsedParameters.NumAggregators = n;
+        }
+        else if (key == "aggregatorratio")
+        {
+            int ratio = static_cast(helper::StringTo(
+                value, " in Parameter key=AggregatorRatio " + hint));
+            if (ratio > 0)
             {
-                subStreamsPresent = true;
-                subStreamsValue = subStreams;
+                int n = m_SizeMPI / ratio;
+                if ((m_SizeMPI % ratio))
+                {
+                    throw std::invalid_argument(
+                        "ERROR: value for Parameter key=AggregatorRatio=" +
+                        std::to_string(ratio) + " must be " +
+                        "an integer divisor of the number of processes=" +
+                        std::to_string(m_SizeMPI) + " " + hint);
+                }
+
+                if (n < 1)
+                {
+                    n = 1;
+                }
+                else if (n > m_SizeMPI)
+                {
+                    n = m_SizeMPI;
+                }
+                parsedParameters.NumAggregators = n;
             }
         }
         else if (key == "node-local" || key == "nodelocal")
@@ -196,6 +216,11 @@ void BPBase::Init(const Params ¶meters, const std::string hint,
                 static_cast(helper::StringTo(
                     value, " in Parameter key=BurstBufferVerbose " + hint));
         }
+        else if (key == "streamreader")
+        {
+            parsedParameters.StreamReader = helper::StringTo(
+                value, " in Parameter key=StreamReader " + hint);
+        }
     }
     if (!engineType.empty())
     {
@@ -227,10 +252,6 @@ void BPBase::Init(const Params ¶meters, const std::string hint,
             m_Profiler.m_IsActive = profileValue;
         }
         m_Parameters = parsedParameters;
-        if (subStreamsPresent)
-        {
-            m_Aggregator.Init(subStreamsValue, m_Comm);
-        }
     }
     // set timers if active
     if (m_Profiler.m_IsActive)
@@ -505,5 +526,7 @@ std::map> BPBase::SetBPOperations(
 ADIOS2_FOREACH_STDTYPE_1ARG(declare_template_instantiation)
 #undef declare_template_instantiation
 
+size_t BPBase::DebugGetDataBufferSize() const { return m_Data.DebugGetSize(); }
+
 } // end namespace format
 } // end namespace adios2
diff --git a/source/adios2/toolkit/format/bp/BPBase.h b/source/adios2/toolkit/format/bp/BPBase.h
index 99f1e65eee..45e57c08b6 100644
--- a/source/adios2/toolkit/format/bp/BPBase.h
+++ b/source/adios2/toolkit/format/bp/BPBase.h
@@ -210,6 +210,18 @@ class BPBase
         bool BurstBufferDrain = true;
         /** Verbose level for burst buffer draining thread */
         int BurstBufferVerbose = 0;
+
+        /** Stream reader flag: process metadata step-by-step
+         * instead of parsing everything available
+         */
+        bool StreamReader = false;
+
+        /** Number of aggregators.
+         * Must be a value between 1 and number of MPI ranks
+         * 0 as default means that the engine must define the number of
+         * aggregators
+         */
+        unsigned int NumAggregators = 0;
     };
 
     /** Return type of the ResizeBuffer function. */
@@ -294,6 +306,7 @@ class BPBase
      */
     void Init(const Params ¶meters, const std::string hint,
               const std::string engineType = "");
+
     /****************** NEED to check if some are virtual */
 
     /**
@@ -318,6 +331,8 @@ class BPBase
     /** Delete buffer memory manually */
     void DeleteBuffers();
 
+    size_t DebugGetDataBufferSize() const;
+
 protected:
     /** file I/O method type, adios1 legacy, only POSIX and MPI_AGG are used */
     enum IO_METHOD
diff --git a/source/adios2/toolkit/format/bp/BPBase.tcc b/source/adios2/toolkit/format/bp/BPBase.tcc
index 54430fd7be..6ad056796b 100644
--- a/source/adios2/toolkit/format/bp/BPBase.tcc
+++ b/source/adios2/toolkit/format/bp/BPBase.tcc
@@ -301,7 +301,8 @@ inline void BPBase::ParseCharacteristics(const std::vector &buffer,
                         helper::ReadValue(buffer, position,
                                                    isLittleEndian));
                 characteristics.Statistics.SubBlockInfo.SubBlockSize =
-                    helper::ReadValue(buffer, position, isLittleEndian);
+                    static_cast(helper::ReadValue(
+                        buffer, position, isLittleEndian));
 
                 characteristics.Statistics.SubBlockInfo.Div.resize(
                     dimensionsSize);
diff --git a/source/adios2/toolkit/format/bp/BPSerializer.cpp b/source/adios2/toolkit/format/bp/BPSerializer.cpp
index 2ee196b91f..3e1093411f 100644
--- a/source/adios2/toolkit/format/bp/BPSerializer.cpp
+++ b/source/adios2/toolkit/format/bp/BPSerializer.cpp
@@ -336,7 +336,7 @@ void BPSerializer::UpdateOffsetsInMetadata()
             currentPosition, TypeTraits::type_enum, buffer);                \
         break;                                                                 \
     }
-                ADIOS2_FOREACH_ATTRIBUTE_PRIMITIVE_STDTYPE_1ARG(make_case)
+                ADIOS2_FOREACH_PRIMITIVE_STDTYPE_1ARG(make_case)
 #undef make_case
 
             default:
@@ -739,11 +739,11 @@ size_t BPSerializer::GetAttributesSizeInData(core::IO &io) const noexcept
 {
     size_t attributesSizeInData = 0;
 
-    auto &attributes = io.GetAttributesDataMap();
+    auto &attributes = io.GetAttributes();
 
     for (const auto &attribute : attributes)
     {
-        const std::string type = attribute.second.first;
+        const DataType type = attribute.second->m_Type;
 
         // each attribute is only written to output once
         // so filter out the ones already written
@@ -753,11 +753,11 @@ size_t BPSerializer::GetAttributesSizeInData(core::IO &io) const noexcept
             continue;
         }
 
-        if (type == "compound")
+        if (type == DataType::Compound)
         {
         }
 #define declare_type(T)                                                        \
-    else if (type == helper::GetType())                                     \
+    else if (type == helper::GetDataType())                                 \
     {                                                                          \
         const std::string name = attribute.first;                              \
         const core::Attribute &attribute = *io.InquireAttribute(name);   \
@@ -772,7 +772,7 @@ size_t BPSerializer::GetAttributesSizeInData(core::IO &io) const noexcept
 
 void BPSerializer::PutAttributes(core::IO &io)
 {
-    const auto &attributesDataMap = io.GetAttributesDataMap();
+    const auto &attributes = io.GetAttributes();
 
     auto &buffer = m_Data.m_Buffer;
     auto &position = m_Data.m_Position;
@@ -781,8 +781,7 @@ void BPSerializer::PutAttributes(core::IO &io)
     const size_t attributesCountPosition = position;
 
     // count is known ahead of time
-    const uint32_t attributesCount =
-        static_cast(attributesDataMap.size());
+    const uint32_t attributesCount = static_cast(attributes.size());
     helper::CopyToBuffer(buffer, position, &attributesCount);
 
     // will go back
@@ -793,10 +792,10 @@ void BPSerializer::PutAttributes(core::IO &io)
 
     uint32_t memberID = 0;
 
-    for (const auto &attributePair : attributesDataMap)
+    for (const auto &attributePair : attributes)
     {
         const std::string name(attributePair.first);
-        const std::string type(attributePair.second.first);
+        const DataType type(attributePair.second->m_Type);
 
         // each attribute is only written to output once
         // so filter out the ones already written
@@ -806,11 +805,11 @@ void BPSerializer::PutAttributes(core::IO &io)
             continue;
         }
 
-        if (type == "unknown")
+        if (type == DataType::None)
         {
         }
 #define declare_type(T)                                                        \
-    else if (type == helper::GetType())                                     \
+    else if (type == helper::GetDataType())                                 \
     {                                                                          \
         Stats stats;                                                        \
         stats.Offset = absolutePosition + m_PreDataFileLength;                 \
diff --git a/source/adios2/toolkit/format/bp/BPSerializer.tcc b/source/adios2/toolkit/format/bp/BPSerializer.tcc
index 9a33cfbea9..8414fa6eb5 100644
--- a/source/adios2/toolkit/format/bp/BPSerializer.tcc
+++ b/source/adios2/toolkit/format/bp/BPSerializer.tcc
@@ -167,7 +167,7 @@ void BPSerializer::UpdateIndexOffsetsCharacteristics(size_t ¤tPosition,
             currentPosition += 2 * sizeof(T); // block min/max
             if (M > 1)
             {
-                currentPosition += 1 + 4; // method, blockSize
+                currentPosition += 1 + 8; // method (byte), blockSize (uint64_t)
                 currentPosition +=
                     dimensionsSize * sizeof(uint16_t); // N-dim division
                 currentPosition += 2 * M * sizeof(T);  // M * min/max
@@ -210,7 +210,30 @@ void BPSerializer::UpdateIndexOffsetsCharacteristics(size_t ¤tPosition,
                 3 * sizeof(uint64_t) * dimensionsSize + 2; // 2 is for length
             break;
         }
-        // TODO: implement operators
+        case (characteristic_transform_type):
+        {
+            const size_t typeLength =
+                static_cast(helper::ReadValue(
+                    buffer, currentPosition, isLittleEndian));
+            // skip over operator name (transform type) string
+            currentPosition += typeLength;
+
+            // skip over pre-data type (1) and dimensionsSize (1)
+            currentPosition += 2;
+
+            const uint16_t dimensionsLength = helper::ReadValue(
+                buffer, currentPosition, isLittleEndian);
+            // skip over dimensions
+            currentPosition += dimensionsLength;
+
+            const size_t metadataLength =
+                static_cast(helper::ReadValue(
+                    buffer, currentPosition, isLittleEndian));
+            // skip over operator metadata
+            currentPosition += metadataLength;
+
+            break;
+        }
         default:
         {
             throw std::invalid_argument(
diff --git a/source/adios2/toolkit/format/bp/bp3/BP3Deserializer.cpp b/source/adios2/toolkit/format/bp/bp3/BP3Deserializer.cpp
index eaa7c17cab..0698a499d7 100644
--- a/source/adios2/toolkit/format/bp/bp3/BP3Deserializer.cpp
+++ b/source/adios2/toolkit/format/bp/bp3/BP3Deserializer.cpp
@@ -358,13 +358,13 @@ BP3Deserializer::PerformGetsVariablesSubFileInfo(core::IO &io)
     for (auto &subFileInfoPair : m_DeferredVariablesMap)
     {
         const std::string variableName(subFileInfoPair.first);
-        const std::string type(io.InquireVariableType(variableName));
+        const DataType type(io.InquireVariableType(variableName));
 
-        if (type == "compound")
+        if (type == DataType::Compound)
         {
         }
 #define declare_type(T)                                                        \
-    else if (type == helper::GetType())                                     \
+    else if (type == helper::GetDataType())                                 \
     {                                                                          \
         subFileInfoPair.second =                                               \
             GetSubFileInfo(*io.InquireVariable(variableName));              \
@@ -380,13 +380,13 @@ void BP3Deserializer::ClipMemory(const std::string &variableName, core::IO &io,
                                  const Box &blockBox,
                                  const Box &intersectionBox) const
 {
-    const std::string type(io.InquireVariableType(variableName));
+    const DataType type(io.InquireVariableType(variableName));
 
-    if (type == "compound")
+    if (type == DataType::Compound)
     {
     }
 #define declare_type(T)                                                        \
-    else if (type == helper::GetType())                                     \
+    else if (type == helper::GetDataType())                                 \
     {                                                                          \
         core::Variable *variable = io.InquireVariable(variableName);     \
         if (variable != nullptr)                                               \
diff --git a/source/adios2/toolkit/format/bp/bp3/BP3Deserializer.tcc b/source/adios2/toolkit/format/bp/bp3/BP3Deserializer.tcc
index 1daea3db54..f97a0ee98a 100644
--- a/source/adios2/toolkit/format/bp/bp3/BP3Deserializer.tcc
+++ b/source/adios2/toolkit/format/bp/bp3/BP3Deserializer.tcc
@@ -69,7 +69,10 @@ BP3Deserializer::InitVariableBlockInfo(core::Variable &variable,
     }
 
     auto itStep = std::next(indices.begin(), stepsStart);
+    // BlocksInfo() expects absolute step, stepsStart is relative
+    const size_t absStep = itStep->first;
 
+    // Check that we have enough steps from stepsStart
     for (size_t i = 0; i < stepsCount; ++i)
     {
         if (itStep == indices.end())
@@ -88,8 +91,11 @@ BP3Deserializer::InitVariableBlockInfo(core::Variable &variable,
 
     if (variable.m_SelectionType == SelectionType::WriteBlock)
     {
+        // BlocksInfo() expects absolute step, stepsStart is relative
+        // BlocksInfo() adds +1 to match the step starting from 1
+        // but absStep already is the actual step in the map
         const std::vector::Info> blocksInfo =
-            BlocksInfo(variable, stepsStart);
+            BlocksInfo(variable, absStep - 1);
 
         if (variable.m_BlockID >= blocksInfo.size())
         {
@@ -135,7 +141,7 @@ void BP3Deserializer::SetVariableBlockInfo(
         blockOperation.PreShape = bpOpInfo.PreShape;
         blockOperation.PreStart = bpOpInfo.PreStart;
         blockOperation.PreCount = bpOpInfo.PreCount;
-        blockOperation.Info["PreDataType"] = helper::GetType();
+        blockOperation.Info["PreDataType"] = ToString(helper::GetDataType());
         // TODO: need to verify it's a match with PreDataType
         // std::to_string(static_cast(bp3OpInfo.PreDataType));
         blockOperation.Info["Type"] = bpOpInfo.Type;
@@ -300,39 +306,6 @@ void BP3Deserializer::SetVariableBlockInfo(
             return;
         }
 
-        const size_t dimensions = blockCharacteristics.Shape.size();
-        if (dimensions != blockInfo.Shape.size())
-        {
-            throw std::invalid_argument(
-                "ERROR: block Shape (available) and "
-                "selection Shape (requested) number of dimensions, do not "
-                "match "
-                "when reading global array variable " +
-                variableName + ", in call to Get");
-        }
-
-        Dims readInShape = blockCharacteristics.Shape;
-        if (m_ReverseDimensions)
-        {
-            std::reverse(readInShape.begin(), readInShape.end());
-        }
-
-        for (size_t i = 0; i < dimensions; ++i)
-        {
-            if (blockInfo.Start[i] + blockInfo.Count[i] > readInShape[i])
-            {
-                throw std::invalid_argument(
-                    "ERROR: selection Start " +
-                    helper::DimsToString(blockInfo.Start) + " and Count " +
-                    helper::DimsToString(blockInfo.Count) +
-                    " (requested) is out of bounds of (available) "
-                    "Shape " +
-                    helper::DimsToString(readInShape) +
-                    " , when reading global array variable " + variableName +
-                    ", in call to Get");
-            }
-        }
-
         // relative position
         subStreamInfo.Seeks.first =
             sizeof(T) * helper::LinearIndex(subStreamInfo.BlockBox,
@@ -384,6 +357,36 @@ void BP3Deserializer::SetVariableBlockInfo(
 
         if (variable.m_ShapeID == ShapeID::GlobalArray)
         {
+            // Check that dimensions and boundaries are okay
+            Dims readInShape = variable.m_AvailableShapes[step];
+            const size_t dimensions = readInShape.size();
+            if (dimensions != blockInfo.Shape.size())
+            {
+                throw std::invalid_argument(
+                    "ERROR: block Shape (available) and "
+                    "selection Shape (requested) number of dimensions, do not "
+                    "match in step " +
+                    std::to_string(step) +
+                    "when reading global array variable " + variable.m_Name +
+                    ", in call to Get");
+            }
+
+            for (size_t i = 0; i < dimensions; ++i)
+            {
+                if (blockInfo.Start[i] + blockInfo.Count[i] > readInShape[i])
+                {
+                    throw std::invalid_argument(
+                        "ERROR: selection Start " +
+                        helper::DimsToString(blockInfo.Start) + " and Count " +
+                        helper::DimsToString(blockInfo.Count) +
+                        " (requested) is out of bounds of (available) "
+                        "Shape " +
+                        helper::DimsToString(readInShape) +
+                        " , when reading global array variable " +
+                        variable.m_Name + ", in call to Get");
+                }
+            }
+            // Get intersections with each block
             for (const size_t blockOffset : blockOffsets)
             {
                 lf_SetSubStreamInfoGlobalArray(variable.m_Name, selectionBox,
@@ -749,10 +752,8 @@ inline void BP3Deserializer::DefineVariableInEngineIO(
         variable->m_ShapeID = ShapeID::GlobalArray;
         variable->m_SingleValue = true;
     }
-    /* Update variable's starting step, which equals to the min value in
-    the sorted map minus one */
-    variable->m_StepsStart =
-        variable->m_AvailableStepBlockIndexOffsets.begin()->first - 1;
+    /* Update variable's starting step, which is always 0 */
+    variable->m_StepsStart = 0;
 
     // update variable Engine for read streaming functions
     variable->m_Engine = &engine;
@@ -867,22 +868,6 @@ void BP3Deserializer::DefineVariableInEngineIO(const ElementIndexHeader &header,
         const bool isNextStep =
             stepsFound.insert(subsetCharacteristics.Statistics.Step).second;
 
-        // update min max for global values only if new step is found
-        if ((isNextStep &&
-             subsetCharacteristics.EntryShapeID == ShapeID::GlobalValue) ||
-            (subsetCharacteristics.EntryShapeID != ShapeID::GlobalValue))
-        {
-            if (helper::LessThan(blockMin, variable->m_Min))
-            {
-                variable->m_Min = blockMin;
-            }
-
-            if (helper::GreaterThan(blockMax, variable->m_Max))
-            {
-                variable->m_Max = blockMax;
-            }
-        }
-
         if (isNextStep)
         {
             currentStep = subsetCharacteristics.Statistics.Step;
@@ -893,15 +878,6 @@ void BP3Deserializer::DefineVariableInEngineIO(const ElementIndexHeader &header,
                 variable->m_Shape[0] = 1;
                 variable->m_Count[0] = 1;
             }
-            else if (subsetCharacteristics.EntryShapeID == ShapeID::GlobalArray)
-            {
-                if (subsetCharacteristics.Shape !=
-                    variable->m_AvailableShapes.rbegin()->second)
-                {
-                    variable->m_AvailableShapes[currentStep] =
-                        subsetCharacteristics.Shape;
-                }
-            }
         }
         else
         {
@@ -911,6 +887,32 @@ void BP3Deserializer::DefineVariableInEngineIO(const ElementIndexHeader &header,
                 ++variable->m_Count[0];
             }
         }
+        // Shape definition is by the last block now, not the first block
+        if (subsetCharacteristics.EntryShapeID == ShapeID::GlobalArray)
+        {
+            const Dims shape = m_ReverseDimensions
+                                   ? Dims(subsetCharacteristics.Shape.rbegin(),
+                                          subsetCharacteristics.Shape.rend())
+                                   : subsetCharacteristics.Shape;
+            variable->m_Shape = shape;
+            variable->m_AvailableShapes[currentStep] = shape;
+        }
+
+        // update min max for global values only if new step is found
+        if ((isNextStep &&
+             subsetCharacteristics.EntryShapeID == ShapeID::GlobalValue) ||
+            (subsetCharacteristics.EntryShapeID != ShapeID::GlobalValue))
+        {
+            if (helper::LessThan(blockMin, variable->m_Min))
+            {
+                variable->m_Min = blockMin;
+            }
+
+            if (helper::GreaterThan(blockMax, variable->m_Max))
+            {
+                variable->m_Max = blockMax;
+            }
+        }
 
         variable->m_AvailableStepBlockIndexOffsets[currentStep].push_back(
             subsetPosition);
@@ -924,11 +926,8 @@ void BP3Deserializer::DefineVariableInEngineIO(const ElementIndexHeader &header,
         // in metadata
         variable->m_SingleValue = true;
     }
-
-    /* Update variable's starting step, which equals to the min value in the
-     * sorted map minus one */
-    variable->m_StepsStart =
-        variable->m_AvailableStepBlockIndexOffsets.begin()->first - 1;
+    /* Update variable's starting step, which is always 0 */
+    variable->m_StepsStart = 0;
 
     // update variable Engine for read streaming functions
     variable->m_Engine = &engine;
@@ -988,13 +987,14 @@ BP3Deserializer::GetSubFileInfo(const core::Variable &variable) const
 
     const auto &buffer = m_Metadata.m_Buffer;
 
-    const size_t stepStart = variable.m_StepsStart + 1;
-    const size_t stepEnd = stepStart + variable.m_StepsCount; // exclusive
+    auto itStep = variable.m_AvailableStepBlockIndexOffsets.begin();
+    const size_t absStepStart = itStep->first; // absolute step in map
+    const size_t stepEnd = absStepStart + variable.m_StepsCount; // exclusive
 
     const Box selectionBox = helper::StartEndBox(
         variable.m_Start, variable.m_Count, m_ReverseDimensions);
 
-    for (size_t step = stepStart; step < stepEnd; ++step)
+    for (size_t step = absStepStart; step < stepEnd; ++step)
     {
         auto itBlockStarts =
             variable.m_AvailableStepBlockIndexOffsets.find(step);
diff --git a/source/adios2/toolkit/format/bp/bp3/BP3Serializer.cpp b/source/adios2/toolkit/format/bp/bp3/BP3Serializer.cpp
index d20451bd00..8d5f595985 100644
--- a/source/adios2/toolkit/format/bp/bp3/BP3Serializer.cpp
+++ b/source/adios2/toolkit/format/bp/bp3/BP3Serializer.cpp
@@ -16,7 +16,7 @@
 #include 
 #include 
 
-#include "adios2/helper/adiosFunctions.h" //helper::GetType, helper::ReadValue
+#include "adios2/helper/adiosFunctions.h" //helper::GetDataType, helper::ReadValue
 #include "adios2/toolkit/profiling/taustubs/tautimer.hpp"
 
 #ifdef _WIN32
diff --git a/source/adios2/toolkit/format/bp/bp4/BP4Base.h b/source/adios2/toolkit/format/bp/bp4/BP4Base.h
index 054161a765..c5fb29bb4d 100644
--- a/source/adios2/toolkit/format/bp/bp4/BP4Base.h
+++ b/source/adios2/toolkit/format/bp/bp4/BP4Base.h
@@ -56,6 +56,8 @@ class BP4Base : virtual public BPBase
     BufferSTL m_MetadataIndex;
 
     /** Positions of flags in Index Table Header that Reader uses */
+    static constexpr size_t m_IndexHeaderSize = 64;
+    static constexpr size_t m_IndexRecordSize = 64;
     static constexpr size_t m_EndianFlagPosition = 36;
     static constexpr size_t m_BPVersionPosition = 37;
     static constexpr size_t m_ActiveFlagPosition = 38;
diff --git a/source/adios2/toolkit/format/bp/bp4/BP4Deserializer.cpp b/source/adios2/toolkit/format/bp/bp4/BP4Deserializer.cpp
index fac7aa6d01..5ce632c1ef 100644
--- a/source/adios2/toolkit/format/bp/bp4/BP4Deserializer.cpp
+++ b/source/adios2/toolkit/format/bp/bp4/BP4Deserializer.cpp
@@ -48,23 +48,42 @@ size_t BP4Deserializer::ParseMetadata(const BufferSTL &bufferSTL,
     size_t lastposition = 0;
     /* parse the metadata step by step using the pointers saved in the metadata
     index table */
+    auto itStepsString = engine.m_IO.m_Parameters.find(engine.m_Name);
+    std::vector selectedSteps;
+    if (itStepsString != engine.m_IO.m_Parameters.end())
+    {
+        std::string selectedStepsStr = engine.m_IO.m_Parameters[engine.m_Name];
+        std::stringstream ss(selectedStepsStr);
+        std::string item;
+
+        while (std::getline(ss, item, ','))
+        {
+            selectedSteps.push_back(std::stoi(item));
+        }
+    }
     for (size_t i = oldSteps; i < allSteps; i++)
     {
-        ParsePGIndexPerStep(bufferSTL, engine.m_IO.m_HostLanguage, 0, i + 1);
-        ParseVariablesIndexPerStep(bufferSTL, engine, 0, i + 1);
-        ParseAttributesIndexPerStep(bufferSTL, engine, 0, i + 1);
-        lastposition = m_MetadataIndexTable[0][i + 1][3];
+        if (selectedSteps.size() == 0 ||
+            std::find(selectedSteps.begin(), selectedSteps.end(), i) !=
+                selectedSteps.end())
+        {
+            ParsePGIndexPerStep(bufferSTL, engine.m_IO.m_HostLanguage, 0,
+                                i + 1);
+            ParseVariablesIndexPerStep(bufferSTL, engine, 0, i + 1);
+            ParseAttributesIndexPerStep(bufferSTL, engine, 0, i + 1);
+            lastposition = m_MetadataIndexTable[0][i + 1][3];
+        }
     }
     return lastposition;
 }
 
-void BP4Deserializer::ParseMetadataIndex(const BufferSTL &bufferSTL,
+void BP4Deserializer::ParseMetadataIndex(BufferSTL &bufferSTL,
                                          const size_t absoluteStartPos,
-                                         const bool hasHeader)
+                                         const bool hasHeader,
+                                         const bool oneStepOnly)
 {
     const auto &buffer = bufferSTL.m_Buffer;
-    const size_t bufferSize = buffer.size();
-    size_t position = 0;
+    size_t &position = bufferSTL.m_Position;
 
     if (hasHeader)
     {
@@ -113,7 +132,7 @@ void BP4Deserializer::ParseMetadataIndex(const BufferSTL &bufferSTL,
     }
 
     // Read each record now
-    while (position < bufferSize)
+    do
     {
         std::vector ptrs;
         const uint64_t currentStep = helper::ReadValue(
@@ -137,7 +156,7 @@ void BP4Deserializer::ParseMetadataIndex(const BufferSTL &bufferSTL,
         ptrs.push_back(currentTimeStamp);
         m_MetadataIndexTable[mpiRank][currentStep] = ptrs;
         position += 8;
-    }
+    } while (!oneStepOnly && position < buffer.size());
 }
 
 const helper::BlockOperationInfo &BP4Deserializer::InitPostOperatorBlockData(
@@ -589,13 +608,13 @@ BP4Deserializer::PerformGetsVariablesSubFileInfo(core::IO &io)
     for (auto &subFileInfoPair : m_DeferredVariablesMap)
     {
         const std::string variableName(subFileInfoPair.first);
-        const std::string type(io.InquireVariableType(variableName));
+        const DataType type(io.InquireVariableType(variableName));
 
-        if (type == "compound")
+        if (type == DataType::Compound)
         {
         }
 #define declare_type(T)                                                        \
-    else if (type == helper::GetType())                                     \
+    else if (type == helper::GetDataType())                                 \
     {                                                                          \
         subFileInfoPair.second =                                               \
             GetSubFileInfo(*io.InquireVariable(variableName));              \
@@ -611,13 +630,13 @@ void BP4Deserializer::ClipMemory(const std::string &variableName, core::IO &io,
                                  const Box &blockBox,
                                  const Box &intersectionBox) const
 {
-    const std::string type(io.InquireVariableType(variableName));
+    const DataType type(io.InquireVariableType(variableName));
 
-    if (type == "compound")
+    if (type == DataType::Compound)
     {
     }
 #define declare_type(T)                                                        \
-    else if (type == helper::GetType())                                     \
+    else if (type == helper::GetDataType())                                 \
     {                                                                          \
         core::Variable *variable = io.InquireVariable(variableName);     \
         if (variable != nullptr)                                               \
diff --git a/source/adios2/toolkit/format/bp/bp4/BP4Deserializer.h b/source/adios2/toolkit/format/bp/bp4/BP4Deserializer.h
index 0c36e12bbc..149e03c694 100644
--- a/source/adios2/toolkit/format/bp/bp4/BP4Deserializer.h
+++ b/source/adios2/toolkit/format/bp/bp4/BP4Deserializer.h
@@ -46,9 +46,8 @@ class BP4Deserializer : virtual public BP4Base
 
     ~BP4Deserializer() = default;
 
-    void ParseMetadataIndex(const BufferSTL &bufferSTL,
-                            const size_t absoluteStartPos,
-                            const bool hasHeader);
+    void ParseMetadataIndex(BufferSTL &bufferSTL, const size_t absoluteStartPos,
+                            const bool hasHeader, const bool oneStepOnly);
 
     /* Return the position in the buffer where processing ends. The processing
      * is controlled by the number of records in the Index, which may be less
diff --git a/source/adios2/toolkit/format/bp/bp4/BP4Deserializer.tcc b/source/adios2/toolkit/format/bp/bp4/BP4Deserializer.tcc
index 5ff0115c11..c3154bb178 100644
--- a/source/adios2/toolkit/format/bp/bp4/BP4Deserializer.tcc
+++ b/source/adios2/toolkit/format/bp/bp4/BP4Deserializer.tcc
@@ -72,7 +72,10 @@ BP4Deserializer::InitVariableBlockInfo(core::Variable &variable,
     }
 
     auto itStep = std::next(indices.begin(), stepsStart);
+    // BlocksInfo() expects absolute step, stepsStart is relative
+    const size_t absStep = itStep->first;
 
+    // Check that we have enough steps from stepsStart
     for (size_t i = 0; i < stepsCount; ++i)
     {
         if (itStep == indices.end())
@@ -91,8 +94,11 @@ BP4Deserializer::InitVariableBlockInfo(core::Variable &variable,
 
     if (variable.m_SelectionType == SelectionType::WriteBlock)
     {
+        // BlocksInfo() expects absolute step, stepsStart is relative
+        // BlocksInfo() adds +1 to match the step starting from 1
+        // but absStep already is the actual step in the map
         const std::vector::Info> blocksInfo =
-            BlocksInfo(variable, stepsStart);
+            BlocksInfo(variable, absStep - 1);
 
         if (variable.m_BlockID >= blocksInfo.size())
         {
@@ -138,7 +144,7 @@ void BP4Deserializer::SetVariableBlockInfo(
         blockOperation.PreShape = bpOpInfo.PreShape;
         blockOperation.PreStart = bpOpInfo.PreStart;
         blockOperation.PreCount = bpOpInfo.PreCount;
-        blockOperation.Info["PreDataType"] = helper::GetType();
+        blockOperation.Info["PreDataType"] = ToString(helper::GetDataType());
         // TODO: need to verify it's a match with PreDataType
         // std::to_string(static_cast(bp4OpInfo.PreDataType));
         blockOperation.Info["Type"] = bpOpInfo.Type;
@@ -303,39 +309,6 @@ void BP4Deserializer::SetVariableBlockInfo(
             return;
         }
 
-        const size_t dimensions = blockCharacteristics.Shape.size();
-        if (dimensions != blockInfo.Shape.size())
-        {
-            throw std::invalid_argument(
-                "ERROR: block Shape (available) and "
-                "selection Shape (requested) number of dimensions, do not "
-                "match "
-                "when reading global array variable " +
-                variableName + ", in call to Get");
-        }
-
-        Dims readInShape = blockCharacteristics.Shape;
-        if (m_ReverseDimensions)
-        {
-            std::reverse(readInShape.begin(), readInShape.end());
-        }
-
-        for (size_t i = 0; i < dimensions; ++i)
-        {
-            if (blockInfo.Start[i] + blockInfo.Count[i] > readInShape[i])
-            {
-                throw std::invalid_argument(
-                    "ERROR: selection Start " +
-                    helper::DimsToString(blockInfo.Start) + " and Count " +
-                    helper::DimsToString(blockInfo.Count) +
-                    " (requested) is out of bounds of (available) "
-                    "Shape " +
-                    helper::DimsToString(readInShape) +
-                    " , when reading global array variable " + variableName +
-                    ", in call to Get");
-            }
-        }
-
         // relative position
         subStreamInfo.Seeks.first =
             sizeof(T) * helper::LinearIndex(subStreamInfo.BlockBox,
@@ -387,6 +360,37 @@ void BP4Deserializer::SetVariableBlockInfo(
 
         if (variable.m_ShapeID == ShapeID::GlobalArray)
         {
+            // Check that dimensions and boundaries are okay
+            Dims readInShape = variable.m_AvailableShapes[step];
+            const size_t dimensions = readInShape.size();
+            if (dimensions != blockInfo.Shape.size())
+            {
+                throw std::invalid_argument(
+                    "ERROR: variable Shape (available) and "
+                    "selection Shape (requested) number of dimensions, do not "
+                    "match in step " +
+                    std::to_string(step) +
+                    " when reading global array variable " + variable.m_Name +
+                    ", in call to Get");
+            }
+
+            for (size_t i = 0; i < dimensions; ++i)
+            {
+                if (blockInfo.Start[i] + blockInfo.Count[i] > readInShape[i])
+                {
+                    throw std::invalid_argument(
+                        "ERROR: selection Start " +
+                        helper::DimsToString(blockInfo.Start) + " and Count " +
+                        helper::DimsToString(blockInfo.Count) +
+                        " (requested) is out of bounds of (available) "
+                        "Shape " +
+                        helper::DimsToString(readInShape) +
+                        " , when reading global array variable " +
+                        variable.m_Name + " in step " + std::to_string(step) +
+                        ", in call to Get");
+                }
+            }
+            // Get intersections with each block
             for (const size_t blockOffset : blockOffsets)
             {
                 lf_SetSubStreamInfoGlobalArray(variable.m_Name, selectionBox,
@@ -758,10 +762,8 @@ inline void BP4Deserializer::DefineVariableInEngineIOPerStep(
         variable->m_ShapeID = ShapeID::GlobalArray;
         variable->m_SingleValue = true;
     }
-    /* Update variable's starting step, which equals to the min value in the
-     * sorted map minus one */
-    variable->m_StepsStart =
-        variable->m_AvailableStepBlockIndexOffsets.begin()->first - 1;
+    /* Update variable's starting step, which is always 0 */
+    variable->m_StepsStart = 0;
 
     // update variable Engine for read streaming functions
     variable->m_Engine = &engine;
@@ -844,15 +846,15 @@ void BP4Deserializer::DefineVariableInEngineIOPerStep(
             }
             else if (subsetCharacteristics.EntryShapeID == ShapeID::GlobalArray)
             {
-                if (subsetPosition == initialPosition)
-                {
-                    if (subsetCharacteristics.Shape !=
-                        variable->m_AvailableShapes.rbegin()->second)
-                    {
-                        variable->m_AvailableShapes[step] =
-                            subsetCharacteristics.Shape;
-                    }
-                }
+                // Shape definition is by the last block now, not the first
+                // block
+                const Dims shape =
+                    m_ReverseDimensions
+                        ? Dims(subsetCharacteristics.Shape.rbegin(),
+                               subsetCharacteristics.Shape.rend())
+                        : subsetCharacteristics.Shape;
+                variable->m_Shape = shape;
+                variable->m_AvailableShapes[step] = shape;
             }
 
             variable->m_AvailableStepBlockIndexOffsets[step].push_back(
@@ -953,22 +955,6 @@ void BP4Deserializer::DefineVariableInEngineIOPerStep(
         const bool isNextStep =
             stepsFound.insert(subsetCharacteristics.Statistics.Step).second;
 
-        // update min max for global values only if new step is found
-        if ((isNextStep &&
-             subsetCharacteristics.EntryShapeID == ShapeID::GlobalValue) ||
-            (subsetCharacteristics.EntryShapeID != ShapeID::GlobalValue))
-        {
-            if (helper::LessThan(blockMin, variable->m_Min))
-            {
-                variable->m_Min = blockMin;
-            }
-
-            if (helper::GreaterThan(blockMax, variable->m_Max))
-            {
-                variable->m_Max = blockMax;
-            }
-        }
-
         if (isNextStep)
         {
             currentStep = subsetCharacteristics.Statistics.Step;
@@ -979,15 +965,6 @@ void BP4Deserializer::DefineVariableInEngineIOPerStep(
                 variable->m_Shape[0] = 1;
                 variable->m_Count[0] = 1;
             }
-            else if (subsetCharacteristics.EntryShapeID == ShapeID::GlobalArray)
-            {
-                if (subsetCharacteristics.Shape !=
-                    variable->m_AvailableShapes.rbegin()->second)
-                {
-                    variable->m_AvailableShapes[currentStep] =
-                        subsetCharacteristics.Shape;
-                }
-            }
         }
         else
         {
@@ -998,6 +975,33 @@ void BP4Deserializer::DefineVariableInEngineIOPerStep(
             }
         }
 
+        // Shape definition is by the last block now, not the first block
+        if (subsetCharacteristics.EntryShapeID == ShapeID::GlobalArray)
+        {
+            const Dims shape = m_ReverseDimensions
+                                   ? Dims(subsetCharacteristics.Shape.rbegin(),
+                                          subsetCharacteristics.Shape.rend())
+                                   : subsetCharacteristics.Shape;
+            variable->m_Shape = shape;
+            variable->m_AvailableShapes[currentStep] = shape;
+        }
+
+        // update min max for global values only if new step is found
+        if ((isNextStep &&
+             subsetCharacteristics.EntryShapeID == ShapeID::GlobalValue) ||
+            (subsetCharacteristics.EntryShapeID != ShapeID::GlobalValue))
+        {
+            if (helper::LessThan(blockMin, variable->m_Min))
+            {
+                variable->m_Min = blockMin;
+            }
+
+            if (helper::GreaterThan(blockMax, variable->m_Max))
+            {
+                variable->m_Max = blockMax;
+            }
+        }
+
         variable->m_AvailableStepBlockIndexOffsets[currentStep].push_back(
             subsetPosition);
         position = subsetPosition + subsetCharacteristics.EntryLength + 5;
@@ -1010,10 +1014,8 @@ void BP4Deserializer::DefineVariableInEngineIOPerStep(
         // in metadata
         variable->m_SingleValue = true;
     }
-    /* Update variable's starting step, which equals to the min value in the
-     * sorted map minus one */
-    variable->m_StepsStart =
-        variable->m_AvailableStepBlockIndexOffsets.begin()->first - 1;
+    /* Update variable's starting step, which is always 0 */
+    variable->m_StepsStart = 0;
 
     // update variable Engine for read streaming functions
     variable->m_Engine = &engine;
diff --git a/source/adios2/toolkit/format/bp/bp4/BP4Serializer.cpp b/source/adios2/toolkit/format/bp/bp4/BP4Serializer.cpp
index 4a3903d37a..ff3caaeafc 100644
--- a/source/adios2/toolkit/format/bp/bp4/BP4Serializer.cpp
+++ b/source/adios2/toolkit/format/bp/bp4/BP4Serializer.cpp
@@ -19,7 +19,7 @@
 #include 
 #include 
 
-#include "adios2/helper/adiosFunctions.h" //helper::GetType, helper::ReadValue
+#include "adios2/helper/adiosFunctions.h" //helper::GetDataType, helper::ReadValue
 
 #ifdef _WIN32
 #pragma warning(disable : 4503) // Windows complains about SubFileInfoMap levels
@@ -280,35 +280,6 @@ size_t BP4Serializer::CloseStream(core::IO &io, const bool addMetadata)
     return dataEndsAt;
 }
 
-size_t BP4Serializer::CloseStream(core::IO &io, size_t &metadataStart,
-                                  size_t &metadataCount, const bool addMetadata)
-{
-
-    m_Profiler.Start("buffering");
-    if (m_MetadataSet.DataPGIsOpen)
-    {
-        SerializeDataBuffer(io);
-    }
-    size_t dataEndsAt = m_Data.m_Position;
-    metadataStart = m_Data.m_Position;
-    SerializeMetadataInData(false, addMetadata);
-
-    metadataCount = m_Data.m_Position - metadataStart;
-
-    if (m_Profiler.m_IsActive)
-    {
-        m_Profiler.m_Bytes.at("buffering") += m_Data.m_Position;
-    }
-    m_Profiler.Stop("buffering");
-    return dataEndsAt;
-}
-
-void BP4Serializer::ResetIndices()
-{
-    m_MetadataSet.AttributesIndices.clear();
-    m_MetadataSet.VarsIndices.clear();
-}
-
 /* Reset the local metadata indices */
 void BP4Serializer::ResetAllIndices()
 {
@@ -401,13 +372,24 @@ void BP4Serializer::SerializeDataBuffer(core::IO &io) noexcept
     if (attributesSizeInData)
     {
         attributesSizeInData += 12; // count + length + end ID
-        m_Data.Resize(position + attributesSizeInData + 4,
-                      "when writing Attributes in rank=0\n");
+        // Take care not to shrink the buffer size in this.
+        // Otherwise, growing the buffer again later on is expensive
+        // - not because of reallocation, but because of (re)initialization
+        // with zeros by BufferSTL::Resize.
+        const size_t minSize = position + attributesSizeInData + 4;
+        if (m_Data.m_Buffer.size() < minSize)
+        {
+            m_Data.Resize(minSize, "when writing Attributes in rank=0\n");
+        }
         PutAttributes(io);
     }
     else
     {
-        m_Data.Resize(position + 12 + 4, "for empty Attributes\n");
+        const size_t minSize = position + 12 + 4;
+        if (m_Data.m_Buffer.size() < minSize)
+        {
+            m_Data.Resize(minSize, "for empty Attributes\n");
+        }
         // Attribute index header for zero attributes: 0, 0LL
         // Resize() already takes care of this
         position += 12;
@@ -532,316 +514,6 @@ void BP4Serializer::SerializeMetadataInData(const bool updateAbsolutePosition,
     }
 }
 
-void BP4Serializer::AggregateIndex(const SerialElementIndex &index,
-                                   const size_t count, helper::Comm const &comm,
-                                   BufferSTL &bufferSTL)
-{
-    auto &buffer = bufferSTL.m_Buffer;
-    auto &position = bufferSTL.m_Position;
-    int rank = comm.Rank();
-
-    size_t countPosition = position;
-    const size_t totalCount = comm.ReduceValues(count);
-
-    if (rank == 0)
-    {
-        // Write count
-        position += 16;
-        bufferSTL.Resize(position, " in call to AggregateIndex BP4 metadata");
-        const uint64_t totalCountU64 = static_cast(totalCount);
-        helper::CopyToBuffer(buffer, countPosition, &totalCountU64);
-    }
-
-    // write contents
-    comm.GathervVectors(index.Buffer, buffer, position);
-
-    // get total length and write it after count and before index
-    if (rank == 0)
-    {
-        const uint64_t totalLengthU64 =
-            static_cast(position - countPosition - 8);
-        helper::CopyToBuffer(buffer, countPosition, &totalLengthU64);
-    }
-}
-
-void BP4Serializer::AggregateMergeIndex(
-    const std::unordered_map &indices,
-    helper::Comm const &comm, BufferSTL &bufferSTL, const bool isRankConstant)
-{
-    // first serialize index
-    std::vector serializedIndices = SerializeIndices(indices, comm);
-    // gather in rank 0
-    std::vector gatheredSerialIndices;
-    size_t gatheredSerialIndicesPosition = 0;
-
-    comm.GathervVectors(serializedIndices, gatheredSerialIndices,
-                        gatheredSerialIndicesPosition);
-
-    // deallocate local serialized Indices
-    std::vector().swap(serializedIndices);
-
-    // deserialize in [name][rank] order
-    const std::unordered_map>
-        nameRankIndices = DeserializeIndicesPerRankThreads(
-            gatheredSerialIndices, comm, isRankConstant);
-
-    // deallocate gathered serial indices (full in rank 0 only)
-    std::vector().swap(gatheredSerialIndices);
-
-    int rank = comm.Rank();
-
-    if (rank == 0)
-    {
-        // to write count and length
-        auto &buffer = bufferSTL.m_Buffer;
-        auto &position = bufferSTL.m_Position;
-        size_t countPosition = position;
-
-        // Write count
-        position += 12;
-        bufferSTL.Resize(position + gatheredSerialIndicesPosition +
-                             m_MetadataSet.MiniFooterSize,
-                         ", in call to AggregateMergeIndex BP4 metadata");
-        const uint32_t totalCountU32 =
-            static_cast(nameRankIndices.size());
-        helper::CopyToBuffer(buffer, countPosition, &totalCountU32);
-
-        // MergeSerializeIndices(nameRankIndices, comm, bufferSTL);
-        /* merge and serialize all the indeices at each step */
-        MergeSerializeIndicesPerStep(nameRankIndices, comm, bufferSTL);
-
-        // Write length
-        const uint64_t totalLengthU64 =
-            static_cast(position - countPosition - 8);
-        helper::CopyToBuffer(buffer, countPosition, &totalLengthU64);
-    }
-}
-
-std::vector BP4Serializer::SerializeIndices(
-    const std::unordered_map &indices,
-    helper::Comm const &comm) const noexcept
-{
-    // pre-allocate
-    size_t serializedIndicesSize = 0;
-    for (const auto &indexPair : indices)
-    {
-        const SerialElementIndex &index = indexPair.second;
-        if (!index.Valid)
-        {
-            continue; // if the variable is not put (not valid) at current step,
-                      // skip
-        }
-        serializedIndicesSize += 4 + index.Buffer.size();
-    }
-
-    std::vector serializedIndices;
-    serializedIndices.reserve(serializedIndicesSize);
-
-    int rank = comm.Rank();
-
-    for (const auto &indexPair : indices)
-    {
-        const SerialElementIndex &index = indexPair.second;
-
-        if (!index.Valid)
-        {
-            continue; // if the variable is not put (not valid) at current step,
-                      // skip
-        }
-
-        // add rank at the beginning
-        const uint32_t rankSource = static_cast(rank);
-        helper::InsertToBuffer(serializedIndices, &rankSource);
-
-        // insert buffer
-        helper::InsertToBuffer(serializedIndices, index.Buffer.data(),
-                               index.Buffer.size());
-    }
-
-    return serializedIndices;
-}
-
-std::unordered_map>
-BP4Serializer::DeserializeIndicesPerRankSingleThread(
-    const std::vector &serialized, helper::Comm const &comm,
-    const bool isRankConstant) const noexcept
-{
-    std::unordered_map>
-        deserialized;
-
-    const size_t serializedSize = serialized.size();
-    int rank = comm.Rank();
-
-    if (rank != 0 || serializedSize < 8)
-    {
-        return deserialized;
-    }
-
-    size_t serializedPosition = 0;
-
-    while (serializedPosition < serializedSize - 4)
-    {
-        const int rankSource = static_cast(
-            helper::ReadValue(serialized, serializedPosition));
-
-        if (serializedPosition <= serializedSize)
-        {
-            size_t localPosition = serializedPosition;
-
-            ElementIndexHeader header =
-                ReadElementIndexHeader(serialized, localPosition);
-
-            if (!isRankConstant || deserialized.count(header.Name) != 1)
-            {
-
-                std::vector *deserializedIndexes;
-
-                auto search = deserialized.find(header.Name);
-                if (search == deserialized.end())
-                {
-                    // variable does not exist, we need to add it
-                    deserializedIndexes =
-                        &(deserialized
-                              .emplace(std::piecewise_construct,
-                                       std::forward_as_tuple(header.Name),
-                                       std::forward_as_tuple(
-                                           m_SizeMPI, SerialElementIndex(
-                                                          header.MemberID, 0)))
-                              .first->second);
-                }
-                else
-                {
-                    deserializedIndexes = &(search->second);
-                }
-
-                const size_t bufferSize =
-                    static_cast(header.Length) + 4;
-                SerialElementIndex &index = deserializedIndexes->at(rankSource);
-                helper::InsertToBuffer(
-                    index.Buffer, &serialized[serializedPosition], bufferSize);
-            }
-        }
-
-        const size_t bufferSize = static_cast(
-            helper::ReadValue(serialized, serializedPosition));
-        serializedPosition += bufferSize;
-    }
-
-    return deserialized;
-}
-
-std::unordered_map>
-BP4Serializer::DeserializeIndicesPerRankThreads(
-    const std::vector &serialized, helper::Comm const &comm,
-    const bool isRankConstant) const noexcept
-{
-    if (m_Parameters.Threads == 1)
-    {
-        return BP4Serializer::DeserializeIndicesPerRankSingleThread(
-            serialized, comm, isRankConstant);
-    }
-
-    std::unordered_map>
-        deserialized;
-
-    auto lf_Deserialize = [&](const int rankSource,
-                              const size_t serializedPosition,
-                              const bool isRankConstant) {
-        size_t localPosition = serializedPosition;
-        ElementIndexHeader header =
-            ReadElementIndexHeader(serialized, localPosition);
-
-        if (isRankConstant)
-        {
-            if (deserialized.count(header.Name) == 1)
-            {
-                return;
-            }
-        }
-
-        std::vector *deserializedIndexes;
-        // mutex portion
-        {
-            std::lock_guard lock(m_Mutex);
-            deserializedIndexes =
-                &(deserialized
-                      .emplace(std::piecewise_construct,
-                               std::forward_as_tuple(header.Name),
-                               std::forward_as_tuple(
-                                   m_SizeMPI,
-                                   SerialElementIndex(header.MemberID, 0)))
-                      .first->second);
-        }
-
-        const size_t bufferSize = static_cast(header.Length) + 4;
-        SerialElementIndex &index = deserializedIndexes->at(rankSource);
-        helper::InsertToBuffer(index.Buffer, &serialized[serializedPosition],
-                               bufferSize);
-    };
-
-    // BODY OF FUNCTION starts here
-    const size_t serializedSize = serialized.size();
-    int rank = comm.Rank();
-
-    if (rank != 0 || serializedSize < 8)
-    {
-        return deserialized;
-    }
-
-    size_t serializedPosition = 0;
-
-    std::vector> asyncs(m_Parameters.Threads);
-    std::vector asyncPositions(m_Parameters.Threads);
-    std::vector asyncRankSources(m_Parameters.Threads);
-
-    bool launched = false;
-
-    while (serializedPosition < serializedSize)
-    {
-        // extract rank and index buffer size
-        for (unsigned int t = 0; t < m_Parameters.Threads; ++t)
-        {
-            if (serializedPosition >= serializedSize)
-            {
-                break;
-            }
-
-            const int rankSource = static_cast(
-                helper::ReadValue(serialized, serializedPosition));
-            asyncRankSources[t] = rankSource;
-            asyncPositions[t] = serializedPosition;
-
-            const size_t bufferSize = static_cast(
-                helper::ReadValue(serialized, serializedPosition));
-            serializedPosition += bufferSize;
-
-            if (launched)
-            {
-                asyncs[t].get();
-            }
-
-            if (serializedPosition <= serializedSize)
-            {
-                asyncs[t] = std::async(std::launch::async, lf_Deserialize,
-                                       asyncRankSources[t], asyncPositions[t],
-                                       isRankConstant);
-            }
-        }
-
-        launched = true;
-    }
-
-    for (auto &async : asyncs)
-    {
-        if (async.valid())
-        {
-            async.wait();
-        }
-    }
-
-    return deserialized;
-}
-
 void BP4Serializer::AggregateCollectiveMetadataIndices(helper::Comm const &comm,
                                                        BufferSTL &outBufferSTL)
 {
@@ -1297,12 +969,10 @@ void BP4Serializer::AggregateCollectiveMetadataIndices(helper::Comm const &comm,
 
                 const uint64_t variablesIndexStart = position;
                 ptrs.push_back(variablesIndexStart);
-                if (!varIndicesInfo.empty())
+                const auto itvars = varIndicesInfo.find(t);
+                if (itvars != varIndicesInfo.end())
                 {
-
-                    std::unordered_map>>
-                        perStepVarIndicesInfo = varIndicesInfo.at(t);
+                    const auto perStepVarIndicesInfo = itvars->second;
                     size_t perStepVarCountPosition = position;
                     const uint32_t perStepVarCountU32 =
                         static_cast(perStepVarIndicesInfo.size());
@@ -1330,7 +1000,7 @@ void BP4Serializer::AggregateCollectiveMetadataIndices(helper::Comm const &comm,
                                       serialized.begin() + start + length,
                                       buffer.begin() + position);
                             position += length - headerSize;
-                            setsCount++;
+                            setsCount += header.CharacteristicsSetsCount;
                         }
                         const uint32_t entryLength = static_cast(
                             position - entryLengthPosition - 4);
@@ -1365,11 +1035,10 @@ void BP4Serializer::AggregateCollectiveMetadataIndices(helper::Comm const &comm,
                 const uint64_t attributesIndexStart = position;
                 ptrs.push_back(attributesIndexStart);
 
-                if (!attrIndicesInfo.empty())
+                const auto itattrs = attrIndicesInfo.find(t);
+                if (itattrs != attrIndicesInfo.end())
                 {
-                    std::unordered_map>>
-                        perStepAttrIndicesInfo = attrIndicesInfo.at(t);
+                    const auto perStepAttrIndicesInfo = itattrs->second;
                     size_t perStepAttrCountPosition = position;
                     const uint32_t perStepAttrCountU32 =
                         static_cast(perStepAttrIndicesInfo.size());
@@ -1466,316 +1135,6 @@ void BP4Serializer::AggregateCollectiveMetadataIndices(helper::Comm const &comm,
     }
 }
 
-/* Merge and serialize all the indices at each step */
-void BP4Serializer::MergeSerializeIndicesPerStep(
-    const std::unordered_map>
-        &nameRankIndices,
-    helper::Comm const &comm, BufferSTL &bufferSTL)
-{
-    auto lf_GetCharacteristics = [&](const std::vector &buffer,
-                                     size_t &position, const uint8_t dataType,
-                                     uint8_t &count, uint32_t &length,
-                                     uint32_t &timeStep)
-
-    {
-        const DataTypes dataTypeEnum = static_cast(dataType);
-
-        switch (dataTypeEnum)
-        {
-
-#define make_case(T)                                                           \
-    case (TypeTraits::type_enum):                                           \
-    {                                                                          \
-        const auto characteristics = ReadElementIndexCharacteristics(       \
-            buffer, position, TypeTraits::type_enum, true);                 \
-        count = characteristics.EntryCount;                                    \
-        length = characteristics.EntryLength;                                  \
-        timeStep = characteristics.Statistics.Step;                            \
-        break;                                                                 \
-    }
-            ADIOS2_FOREACH_STDTYPE_1ARG(make_case)
-#undef make_case
-
-        case (type_string_array):
-        {
-            const auto characteristics =
-                ReadElementIndexCharacteristics(
-                    buffer, position, type_string_array, true);
-            count = characteristics.EntryCount;
-            length = characteristics.EntryLength;
-            timeStep = characteristics.Statistics.Step;
-            break;
-        }
-
-        default:
-            // TODO: complex, long double
-            throw std::invalid_argument(
-                "ERROR: type " + std::to_string(dataType) +
-                " not supported in BP4 Metadata Merge\n");
-
-        } // end switch
-    };
-
-    auto lf_MergeRankSerial =
-        [&](const std::vector &indices,
-            BufferSTL &bufferSTL) {
-            auto &bufferOut = bufferSTL.m_Buffer;
-            auto &positionOut = bufferSTL.m_Position;
-
-            // extract header
-            ElementIndexHeader header;
-            // index non-empty buffer
-            size_t firstRank = 0;
-            // index positions per rank
-            std::vector positions(indices.size(), 0);
-            // merge index length
-            size_t headerSize = 0;
-
-            for (size_t r = 0; r < indices.size(); ++r)
-            {
-                const auto &buffer = indices[r].Buffer;
-                if (buffer.empty())
-                {
-                    continue;
-                }
-                size_t &position = positions[r];
-
-                header = ReadElementIndexHeader(buffer, position);
-                firstRank = r;
-
-                headerSize = position;
-                break;
-            }
-
-            if (header.DataType == std::numeric_limits::max() - 1)
-            {
-                throw std::runtime_error(
-                    "ERROR: invalid data type for variable " + header.Name +
-                    "when writing metadata index\n");
-            }
-
-            // move all positions to headerSize
-            for (size_t r = 0; r < indices.size(); ++r)
-            {
-                const auto &buffer = indices[r].Buffer;
-                if (buffer.empty())
-                {
-                    continue;
-                }
-                positions[r] = headerSize;
-            }
-
-            uint64_t setsCount = 0;
-            // unsigned int currentTimeStep = 1;
-
-            const size_t entryLengthPosition = positionOut;
-            positionOut += headerSize;
-
-            for (size_t r = firstRank; r < indices.size(); ++r)
-            {
-                const auto &buffer = indices[r].Buffer;
-                if (buffer.empty())
-                {
-                    continue;
-                }
-                auto &position = positions[r];
-                if (position >= buffer.size())
-                {
-                    continue;
-                }
-
-                uint8_t count = 0;
-                uint32_t length = 0;
-                uint32_t timeStep = 1;
-
-                while (true)
-                {
-                    if (position >= buffer.size())
-                    {
-                        break;
-                    }
-                    size_t localPosition = position;
-                    lf_GetCharacteristics(buffer, localPosition,
-                                          header.DataType, count, length,
-                                          timeStep);
-
-                    ++setsCount;
-
-                    helper::CopyToBuffer(bufferOut, positionOut,
-                                         &buffer[position], length + 5);
-
-                    position += length + 5;
-                }
-            }
-
-            const uint32_t entryLength =
-                static_cast(positionOut - entryLengthPosition - 4);
-
-            size_t backPosition = entryLengthPosition;
-            helper::CopyToBuffer(bufferOut, backPosition, &entryLength);
-            helper::CopyToBuffer(bufferOut, backPosition,
-                                 &indices[firstRank].Buffer[4],
-                                 headerSize - 8 - 4);
-            helper::CopyToBuffer(bufferOut, backPosition, &setsCount);
-        };
-
-    auto lf_MergeRank = [&](const std::vector &indices,
-                            BufferSTL &bufferSTL) {
-        ElementIndexHeader header;
-        size_t firstRank = 0;
-        // index positions per rank
-        std::vector positions(indices.size(), 0);
-        // merge index length
-        size_t headerSize = 0;
-
-        for (size_t r = 0; r < indices.size(); ++r)
-        {
-            const auto &buffer = indices[r].Buffer;
-            if (buffer.empty())
-            {
-                continue;
-            }
-            size_t &position = positions[r];
-
-            header = ReadElementIndexHeader(buffer, position);
-            firstRank = r;
-
-            headerSize = position;
-            break;
-        }
-
-        if (header.DataType == std::numeric_limits::max() - 1)
-        {
-            throw std::runtime_error("ERROR: invalid data type for variable " +
-                                     header.Name +
-                                     "when writing collective metadata\n");
-        }
-
-        // move all positions to headerSize
-        for (size_t r = 0; r < indices.size(); ++r)
-        {
-            const auto &buffer = indices[r].Buffer;
-            if (buffer.empty())
-            {
-                continue;
-            }
-            positions[r] = headerSize;
-        }
-
-        uint64_t setsCount = 0;
-        unsigned int currentTimeStep = 1;
-        std::vector sorted;
-
-        for (size_t r = firstRank; r < indices.size(); ++r)
-        {
-            const auto &buffer = indices[r].Buffer;
-            if (buffer.empty())
-            {
-                continue;
-            }
-
-            auto &position = positions[r];
-            if (position >= buffer.size())
-            {
-                continue;
-            }
-
-            uint8_t count = 0;
-            uint32_t length = 0;
-            uint32_t timeStep = static_cast(currentTimeStep);
-
-            size_t localPosition = position;
-            lf_GetCharacteristics(buffer, localPosition, header.DataType, count,
-                                  length, timeStep);
-            ++setsCount;
-
-            // here copy to sorted buffer
-            helper::InsertToBuffer(sorted, &buffer[position], length + 5);
-            position += length + 5;
-        }
-
-        const uint32_t entryLength =
-            static_cast(headerSize + sorted.size() - 4);
-        // Copy header to metadata buffer, need mutex here
-        {
-            std::lock_guard lock(m_Mutex);
-            auto &buffer = bufferSTL.m_Buffer;
-            auto &position = bufferSTL.m_Position;
-
-            helper::CopyToBuffer(buffer, position, &entryLength);
-            helper::CopyToBuffer(buffer, position,
-                                 &indices[firstRank].Buffer[4],
-                                 headerSize - 8 - 4);
-            helper::CopyToBuffer(buffer, position, &setsCount);
-            helper::CopyToBuffer(buffer, position, sorted.data(),
-                                 sorted.size());
-        }
-    };
-
-    auto lf_MergeRankRange =
-        [&](const std::unordered_map<
-                std::string, std::vector> &nameRankIndices,
-            const std::vector &names, const size_t start,
-            const size_t end, BufferSTL &bufferSTL)
-
-    {
-        for (auto i = start; i < end; ++i)
-        {
-            auto itIndex = nameRankIndices.find(names[i]);
-            lf_MergeRank(itIndex->second, bufferSTL);
-        }
-    };
-
-    // BODY OF FUNCTION STARTS HERE
-    if (m_Parameters.Threads == 1) // enforcing serial version for now
-    {
-        for (const auto &rankIndices : nameRankIndices)
-        {
-            lf_MergeRankSerial(rankIndices.second, bufferSTL);
-        }
-        return;
-    }
-
-    // TODO need to debug this part, if threaded per variable
-    const size_t elements = nameRankIndices.size();
-    const size_t stride =
-        elements / m_Parameters.Threads; // elements per thread
-    const size_t last =
-        stride + elements % m_Parameters.Threads; // remainder to last
-
-    std::vector threads;
-    threads.reserve(m_Parameters.Threads);
-
-    // copy names in order to use threads
-    std::vector names;
-    names.reserve(nameRankIndices.size());
-
-    for (const auto &nameRankIndexPair : nameRankIndices)
-    {
-        names.push_back(nameRankIndexPair.first);
-    }
-
-    for (unsigned int t = 0; t < m_Parameters.Threads; ++t)
-    {
-        const size_t start = stride * t;
-        size_t end = start + stride;
-
-        if (t == m_Parameters.Threads - 1)
-        {
-            end = start + last;
-        }
-
-        threads.push_back(
-            std::thread(lf_MergeRankRange, std::ref(nameRankIndices),
-                        std::ref(names), start, end, std::ref(bufferSTL)));
-    }
-
-    for (auto &thread : threads)
-    {
-        thread.join();
-    }
-}
-
 #define declare_template_instantiation(T)                                      \
     void BP4Serializer::DoPutAttributeInData(                                  \
         const core::Attribute &attribute, Stats &stats) noexcept         \
diff --git a/source/adios2/toolkit/format/bp/bp4/BP4Serializer.h b/source/adios2/toolkit/format/bp/bp4/BP4Serializer.h
index de0a1f2df5..bdc29045bd 100644
--- a/source/adios2/toolkit/format/bp/bp4/BP4Serializer.h
+++ b/source/adios2/toolkit/format/bp/bp4/BP4Serializer.h
@@ -118,10 +118,6 @@ class BP4Serializer : public BP4Base, public BPSerializer
      * footer into it (in case someone wants to write only the data portion)
      */
     size_t CloseStream(core::IO &io, const bool addMetadata = true);
-    size_t CloseStream(core::IO &io, size_t &metadataStart,
-                       size_t &metadataCount, const bool addMetadata = true);
-
-    void ResetIndices();
 
     /* Reset all metadata indices at the end of each step */
     void ResetAllIndices();
@@ -256,78 +252,8 @@ class BP4Serializer : public BP4Base, public BPSerializer
      */
     void SerializeDataBuffer(core::IO &io) noexcept final;
 
-    /**
-     * Used for PG index, aggregates without merging
-     * @param index
-     * @param count
-     * @param comm
-     * @param bufferSTL
-     */
-    void AggregateIndex(const SerialElementIndex &index, const size_t count,
-                        helper::Comm const &comm, BufferSTL &bufferSTL);
-
-    /**
-     * Collective operation to aggregate and merge (sort) indices (variables and
-     * attributes)
-     * @param indices has containing indices per unique variable/attribute name
-     * @param comm communicator domain, allows reusing the function in
-     * aggregation
-     * @param bufferSTL buffer where merged indices will be placed (metadata or
-     * data footer in aggregation)
-     * @param isRankConstant true: use for attributes as values are constant
-     * across all ranks, false: default used for variables as values  can vary
-     * across ranks
-     */
-    void AggregateMergeIndex(
-        const std::unordered_map &indices,
-        helper::Comm const &comm, BufferSTL &bufferSTL,
-        const bool isRankConstant = false);
-
     void AggregateCollectiveMetadataIndices(helper::Comm const &comm,
                                             BufferSTL &bufferSTL);
-
-    /**
-     * Returns a serialized buffer with all indices with format:
-     * Rank (4 bytes), Buffer
-     * @param indices input of all indices to be serialized
-     * @return buffer with serialized indices
-     */
-    std::vector SerializeIndices(
-        const std::unordered_map &indices,
-        helper::Comm const &comm) const noexcept;
-
-    /**
-     * In rank=0, deserialize gathered indices
-     * @param serializedIndices input gathered indices
-     * @param comm establishes MPI domain
-     * @param true: constant across ranks, no need to merge (attributes), false:
-     * variable across ranks, can merge (variables)
-     * @return hash[name][rank] = bp index buffer
-     */
-    std::unordered_map>
-    DeserializeIndicesPerRankThreads(const std::vector &serializedIndices,
-                                     helper::Comm const &comm,
-                                     const bool isRankConstant) const noexcept;
-
-    /** private function called by DeserializeIndicesPerRankThreads
-     * in case of a single thread
-     */
-    std::unordered_map>
-    DeserializeIndicesPerRankSingleThread(const std::vector &serialized,
-                                          helper::Comm const &comm,
-                                          const bool isRankConstant) const
-        noexcept;
-
-    /**
-     * Only merge indices of each time step and write to
-     * m_HeapBuffer.m_Metadata, clear indices of current time step at the end of
-     * each step
-     * @param nameRankIndices
-     */
-    void MergeSerializeIndicesPerStep(
-        const std::unordered_map>
-            &nameRankIndices,
-        helper::Comm const &comm, BufferSTL &bufferSTL);
 };
 
 #define declare_template_instantiation(T)                                      \
diff --git a/source/adios2/toolkit/format/bp/bp4/BP4Serializer.tcc b/source/adios2/toolkit/format/bp/bp4/BP4Serializer.tcc
index c60924c2bd..b3b4296033 100644
--- a/source/adios2/toolkit/format/bp/bp4/BP4Serializer.tcc
+++ b/source/adios2/toolkit/format/bp/bp4/BP4Serializer.tcc
@@ -620,8 +620,9 @@ void BP4Serializer::PutBoundsRecord(const bool singleValue,
                 uint8_t method =
                     static_cast(stats.SubBlockInfo.DivisionMethod);
                 helper::InsertToBuffer(buffer, &method);
-                helper::InsertToBuffer(buffer,
-                                       &stats.SubBlockInfo.SubBlockSize);
+                uint64_t subBlockSize =
+                    static_cast(stats.SubBlockInfo.SubBlockSize);
+                helper::InsertToBuffer(buffer, &subBlockSize);
 
                 const uint16_t N =
                     static_cast(stats.SubBlockInfo.Div.size());
@@ -673,8 +674,9 @@ void BP4Serializer::PutBoundsRecord(const bool singleValue,
                 uint8_t method =
                     static_cast(stats.SubBlockInfo.DivisionMethod);
                 helper::CopyToBuffer(buffer, position, &method);
-                helper::CopyToBuffer(buffer, position,
-                                     &stats.SubBlockInfo.SubBlockSize);
+                uint64_t subBlockSize =
+                    static_cast(stats.SubBlockInfo.SubBlockSize);
+                helper::CopyToBuffer(buffer, position, &subBlockSize);
 
                 const uint16_t N =
                     static_cast(stats.SubBlockInfo.Div.size());
diff --git a/source/adios2/toolkit/format/bp/bpOperation/compress/BPMGARD.cpp b/source/adios2/toolkit/format/bp/bpOperation/compress/BPMGARD.cpp
index 01253abf31..4c8e748fe1 100644
--- a/source/adios2/toolkit/format/bp/bpOperation/compress/BPMGARD.cpp
+++ b/source/adios2/toolkit/format/bp/bpOperation/compress/BPMGARD.cpp
@@ -11,6 +11,7 @@
 #include "BPMGARD.h"
 
 #include "adios2/helper/adiosFunctions.h"
+#include "adios2/helper/adiosType.h"
 
 #ifdef ADIOS2_HAVE_MGARD
 #include "adios2/operator/compress/CompressMGARD.h"
@@ -70,7 +71,8 @@ void BPMGARD::GetData(const char *input,
     core::compress::CompressMGARD op((Params()));
     op.Decompress(input, blockOperationInfo.PayloadSize, dataOutput,
                   blockOperationInfo.PreCount,
-                  blockOperationInfo.Info.at("PreDataType"),
+                  helper::GetDataTypeFromString(
+                      blockOperationInfo.Info.at("PreDataType")),
                   blockOperationInfo.Info);
 
 #else
diff --git a/source/adios2/toolkit/format/bp/bpOperation/compress/BPSZ.cpp b/source/adios2/toolkit/format/bp/bpOperation/compress/BPSZ.cpp
index 0d87ddfbff..e1d9b107aa 100644
--- a/source/adios2/toolkit/format/bp/bpOperation/compress/BPSZ.cpp
+++ b/source/adios2/toolkit/format/bp/bpOperation/compress/BPSZ.cpp
@@ -11,6 +11,7 @@
 #include "BPSZ.h"
 
 #include "adios2/helper/adiosFunctions.h"
+#include "adios2/helper/adiosType.h"
 
 #ifdef ADIOS2_HAVE_SZ
 #include "adios2/operator/compress/CompressSZ.h"
@@ -69,7 +70,8 @@ void BPSZ::GetData(const char *input,
     core::compress::CompressSZ op((Params()));
     op.Decompress(input, blockOperationInfo.PayloadSize, dataOutput,
                   blockOperationInfo.PreCount,
-                  blockOperationInfo.Info.at("PreDataType"),
+                  helper::GetDataTypeFromString(
+                      blockOperationInfo.Info.at("PreDataType")),
                   blockOperationInfo.Info);
 
 #else
diff --git a/source/adios2/toolkit/format/bp/bpOperation/compress/BPZFP.cpp b/source/adios2/toolkit/format/bp/bpOperation/compress/BPZFP.cpp
index 980b94c92e..0d8059008b 100644
--- a/source/adios2/toolkit/format/bp/bpOperation/compress/BPZFP.cpp
+++ b/source/adios2/toolkit/format/bp/bpOperation/compress/BPZFP.cpp
@@ -12,6 +12,7 @@
 #include "BPZFP.tcc"
 
 #include "adios2/helper/adiosFunctions.h"
+#include "adios2/helper/adiosType.h"
 
 #ifdef ADIOS2_HAVE_ZFP
 #include "adios2/operator/compress/CompressZFP.h"
@@ -90,7 +91,8 @@ void BPZFP::GetData(const char *input,
     core::compress::CompressZFP op((Params()));
     op.Decompress(input, blockOperationInfo.PayloadSize, dataOutput,
                   blockOperationInfo.PreCount,
-                  blockOperationInfo.Info.at("PreDataType"),
+                  helper::GetDataTypeFromString(
+                      blockOperationInfo.Info.at("PreDataType")),
                   blockOperationInfo.Info);
 #else
     throw std::runtime_error(
diff --git a/source/adios2/toolkit/format/buffer/heap/BufferSTL.cpp b/source/adios2/toolkit/format/buffer/heap/BufferSTL.cpp
index a08a9fc422..3d862e9469 100644
--- a/source/adios2/toolkit/format/buffer/heap/BufferSTL.cpp
+++ b/source/adios2/toolkit/format/buffer/heap/BufferSTL.cpp
@@ -10,6 +10,8 @@
 
 #include "BufferSTL.h"
 #include "BufferSTL.tcc"
+#include 
+#include 
 
 namespace adios2
 {
@@ -51,7 +53,24 @@ void BufferSTL::Reset(const bool resetAbsolutePosition,
     }
     if (zeroInitialize)
     {
-        m_Buffer.assign(m_Buffer.size(), '\0');
+        std::fill(m_Buffer.begin(), m_Buffer.end(), 0);
+    }
+    else
+    {
+        // just zero out the first and last 1kb
+        const size_t bufsize = m_Buffer.size();
+        size_t s = (bufsize < 1024 ? bufsize : 1024);
+        std::fill_n(m_Buffer.begin(), s, 0);
+        if (bufsize > 1024)
+        {
+            size_t pos = bufsize - 1024;
+            if (pos < 1024)
+            {
+                pos = 1024;
+            }
+            s = bufsize - pos;
+            std::fill_n(next(m_Buffer.begin(), pos), s, 0);
+        }
     }
 }
 
@@ -68,5 +87,7 @@ ADIOS2_FOREACH_PRIMITIVE_STDTYPE_1ARG(declare_template_instantiation)
 
 void BufferSTL::Delete() { std::vector().swap(m_Buffer); }
 
+size_t BufferSTL::DebugGetSize() const { return m_Buffer.size(); };
+
 } // end namespace format
 } // end namespace adios2
diff --git a/source/adios2/toolkit/format/buffer/heap/BufferSTL.h b/source/adios2/toolkit/format/buffer/heap/BufferSTL.h
index 9287701a63..f9b1fe2ffe 100644
--- a/source/adios2/toolkit/format/buffer/heap/BufferSTL.h
+++ b/source/adios2/toolkit/format/buffer/heap/BufferSTL.h
@@ -42,6 +42,8 @@ class BufferSTL : public Buffer
     size_t Align() const noexcept;
 
     void Delete();
+
+    size_t DebugGetSize() const;
 };
 
 #define declare_template_instantiation(T)                                      \
diff --git a/source/adios2/toolkit/format/dataman/DataManSerializer.cpp b/source/adios2/toolkit/format/dataman/DataManSerializer.cpp
index 697d080ae8..ba659b5274 100644
--- a/source/adios2/toolkit/format/dataman/DataManSerializer.cpp
+++ b/source/adios2/toolkit/format/dataman/DataManSerializer.cpp
@@ -8,7 +8,6 @@
  *      Author: Jason Wang
  */
 
-#include "DataManSerializer.h"
 #include "DataManSerializer.tcc"
 
 #include 
@@ -22,8 +21,7 @@ namespace format
 DataManSerializer::DataManSerializer(helper::Comm const &comm,
                                      const bool isRowMajor)
 : m_Comm(comm), m_IsRowMajor(isRowMajor),
-  m_IsLittleEndian(helper::IsLittleEndian()),
-  m_DeferredRequestsToSend(std::make_shared())
+  m_IsLittleEndian(helper::IsLittleEndian())
 {
     m_MpiRank = m_Comm.Rank();
     m_MpiSize = m_Comm.Size();
@@ -64,241 +62,16 @@ VecPtr DataManSerializer::GetLocalPack()
     return m_LocalBuffer;
 }
 
-void DataManSerializer::AggregateMetadata()
-{
-    TAU_SCOPED_TIMER_FUNC();
-
-    m_ProtectedStepsMutex.lock();
-    for (const auto &idMap : m_ProtectedStepsToAggregate)
-    {
-        m_MetadataJson["P"][std::to_string(idMap.first)] = idMap.second;
-    }
-    m_ProtectedStepsMutex.unlock();
-
-    auto localJsonPack = SerializeJson(m_MetadataJson);
-    unsigned int size = localJsonPack->size();
-    unsigned int maxSize;
-    m_Comm.Allreduce(&size, &maxSize, 1, helper::Comm::Op::Max);
-    maxSize += sizeof(uint64_t);
-    localJsonPack->resize(maxSize, '\0');
-    *(reinterpret_cast(localJsonPack->data() +
-                                   localJsonPack->size()) -
-      1) = size;
-
-    std::vector globalJsonStr(m_MpiSize * maxSize);
-    m_Comm.Allgather(localJsonPack->data(), maxSize, globalJsonStr.data(),
-                     maxSize);
-
-    nlohmann::json aggMetadata;
-
-    for (int i = 0; i < m_MpiSize; ++i)
-    {
-        size_t deserializeSize =
-            *(reinterpret_cast(globalJsonStr.data() +
-                                           (i + 1) * maxSize) -
-              1);
-        nlohmann::json metaj = DeserializeJson(
-            globalJsonStr.data() + i * maxSize, deserializeSize);
-        for (auto stepMapIt = metaj.begin(); stepMapIt != metaj.end();
-             ++stepMapIt)
-        {
-            if (stepMapIt.key() == "P")
-            {
-                std::lock_guard l(m_ProtectedStepsMutex);
-                for (auto appidMapIt = stepMapIt.value().begin();
-                     appidMapIt != stepMapIt.value().end(); ++appidMapIt)
-                {
-                    auto stepVecToAdd = appidMapIt->get>();
-                    auto &stepVecExisted =
-                        m_ProtectedStepsAggregated[stoull(appidMapIt.key())];
-                    for (const auto &protectedStep : stepVecToAdd)
-                    {
-                        auto it =
-                            std::find(stepVecExisted.begin(),
-                                      stepVecExisted.end(), protectedStep);
-                        if (it == stepVecExisted.end())
-                        {
-                            stepVecExisted.push_back(protectedStep);
-                        }
-                    }
-                    std::sort(stepVecExisted.begin(), stepVecExisted.end());
-                }
-                if (m_Verbosity >= 5)
-                {
-                    std::cout << "Rank ";
-                    std::cout << m_MpiRank;
-                    std::cout << " All protected steps aggregated before "
-                                 "reducing are: ";
-                    for (const auto &stepVecPair : m_ProtectedStepsAggregated)
-                    {
-                        for (const auto &step : stepVecPair.second)
-                        {
-                            std::cout << step << ", ";
-                        }
-                    }
-                    std::cout << std::endl;
-                }
-                for (auto &stepVecPair : m_ProtectedStepsAggregated)
-                {
-                    while (stepVecPair.second.size() > 3)
-                    {
-                        stepVecPair.second.erase(stepVecPair.second.begin());
-                    }
-                }
-                if (m_Verbosity >= 5)
-                {
-                    std::cout << "Rank ";
-                    std::cout << m_MpiRank;
-                    std::cout << " All protected steps aggregated after "
-                                 "reducing are: ";
-                    for (const auto &stepVecPair : m_ProtectedStepsAggregated)
-                    {
-                        for (const auto &step : stepVecPair.second)
-                        {
-                            std::cout << step << ", ";
-                        }
-                    }
-                    std::cout << std::endl;
-                }
-            }
-            else
-            {
-                for (auto rankMapIt = stepMapIt.value().begin();
-                     rankMapIt != stepMapIt.value().end(); ++rankMapIt)
-                {
-                    aggMetadata[stepMapIt.key()][rankMapIt.key()] =
-                        rankMapIt.value();
-                }
-            }
-        }
-    }
-
-    m_AggregatedMetadataJsonMutex.lock();
-    for (auto stepMapIt = aggMetadata.begin(); stepMapIt != aggMetadata.end();
-         ++stepMapIt)
-    {
-        m_AggregatedMetadataJson[stepMapIt.key()] = stepMapIt.value();
-    }
-    m_AggregatedMetadataJsonMutex.unlock();
-}
-
-VecPtr DataManSerializer::GetAggregatedMetadataPack(const int64_t stepRequested,
-                                                    int64_t &stepProvided,
-                                                    const int64_t appID)
-{
-
-    TAU_SCOPED_TIMER_FUNC();
-
-    std::lock_guard l(m_AggregatedMetadataJsonMutex);
-
-    VecPtr ret = nullptr;
-
-    stepProvided = -1;
-
-    if (stepRequested == -1) // getting the earliest step
-    {
-        int64_t min = std::numeric_limits::max();
-        for (auto stepMapIt = m_AggregatedMetadataJson.begin();
-             stepMapIt != m_AggregatedMetadataJson.end(); ++stepMapIt)
-        {
-            int64_t step = stoll(stepMapIt.key());
-            if (min > step)
-            {
-                min = step;
-            }
-        }
-        if (min < std::numeric_limits::max())
-        {
-            nlohmann::json retJ;
-            retJ[std::to_string(min)] =
-                m_AggregatedMetadataJson[std::to_string(min)];
-            ret = SerializeJson(retJ);
-            stepProvided = min;
-        }
-    }
-    else if (stepRequested == -2) // getting the latest step
-    {
-        int64_t max = std::numeric_limits::min();
-        for (auto stepMapIt = m_AggregatedMetadataJson.begin();
-             stepMapIt != m_AggregatedMetadataJson.end(); ++stepMapIt)
-        {
-            int64_t step = stoll(stepMapIt.key());
-            if (max < step)
-            {
-                max = step;
-            }
-        }
-        if (max >= 0)
-        {
-            nlohmann::json retJ;
-            retJ[std::to_string(max)] =
-                m_AggregatedMetadataJson[std::to_string(max)];
-            ret = SerializeJson(retJ);
-            stepProvided = max;
-        }
-    }
-    else if (stepRequested == -3) // getting static variables
-    {
-        ret = SerializeJson(m_StaticDataJson);
-    }
-    else if (stepRequested == -4) // getting all steps
-    {
-        ret = SerializeJson(m_AggregatedMetadataJson);
-    }
-    else
-    {
-        auto it = m_AggregatedMetadataJson.find(std::to_string(stepRequested));
-        if (it != m_AggregatedMetadataJson.end())
-        {
-            nlohmann::json retJ;
-            retJ[std::to_string(stepRequested)] = *it;
-            ret = SerializeJson(retJ);
-            stepProvided = stepRequested;
-        }
-    }
-
-    return ret;
-}
-
-void DataManSerializer::PutAggregatedMetadata(VecPtr input,
-                                              helper::Comm const &comm)
-{
-    TAU_SCOPED_TIMER_FUNC();
-    if (input == nullptr)
-    {
-        Log(1,
-            "DataManSerializer::PutAggregatedMetadata received nullptr input",
-            true, true);
-        return;
-    }
-
-    comm.BroadcastVector(*input);
-
-    if (input->size() > 0)
-    {
-        nlohmann::json metaJ = DeserializeJson(input->data(), input->size());
-        JsonToVarMap(metaJ, nullptr);
-
-        if (m_Verbosity >= 100)
-        {
-            std::cout << "DataManSerializer::PutAggregatedMetadata: "
-                      << std::endl;
-            std::cout << metaJ.dump(4) << std::endl;
-        }
-    }
-}
-
 bool DataManSerializer::IsCompressionAvailable(const std::string &method,
-                                               const std::string &type,
-                                               const Dims &count)
+                                               DataType type, const Dims &count)
 {
     TAU_SCOPED_TIMER_FUNC();
     if (method == "zfp")
     {
-        if (type == helper::GetType() ||
-            type == helper::GetType() ||
-            type == helper::GetType() ||
-            type == helper::GetType())
+        if (type == helper::GetDataType() ||
+            type == helper::GetDataType() ||
+            type == helper::GetDataType() ||
+            type == helper::GetDataType())
         {
             if (count.size() <= 3)
             {
@@ -308,8 +81,8 @@ bool DataManSerializer::IsCompressionAvailable(const std::string &method,
     }
     else if (method == "sz")
     {
-        if (type == helper::GetType() ||
-            type == helper::GetType())
+        if (type == helper::GetDataType() ||
+            type == helper::GetDataType())
         {
             if (count.size() <= 5)
             {
@@ -319,10 +92,10 @@ bool DataManSerializer::IsCompressionAvailable(const std::string &method,
     }
     else if (method == "bzip2")
     {
-        if (type == helper::GetType() ||
-            type == helper::GetType() ||
-            type == helper::GetType() ||
-            type == helper::GetType())
+        if (type == helper::GetDataType() ||
+            type == helper::GetDataType() ||
+            type == helper::GetDataType() ||
+            type == helper::GetDataType())
         {
             return true;
         }
@@ -333,17 +106,17 @@ bool DataManSerializer::IsCompressionAvailable(const std::string &method,
 void DataManSerializer::PutAttributes(core::IO &io)
 {
     TAU_SCOPED_TIMER_FUNC();
-    const auto &attributesDataMap = io.GetAttributesDataMap();
+    const auto &attributes = io.GetAttributes();
     bool attributePut = false;
-    for (const auto &attributePair : attributesDataMap)
+    for (const auto &attributePair : attributes)
     {
         const std::string name(attributePair.first);
-        const std::string type(attributePair.second.first);
-        if (type == "unknown")
+        const DataType type(attributePair.second->m_Type);
+        if (type == DataType::None)
         {
         }
 #define declare_type(T)                                                        \
-    else if (type == helper::GetType())                                     \
+    else if (type == helper::GetDataType())                                 \
     {                                                                          \
         core::Attribute &attribute = *io.InquireAttribute(name);         \
         PutAttribute(attribute);                                               \
@@ -376,16 +149,17 @@ void DataManSerializer::GetAttributes(core::IO &io)
     std::lock_guard lStaticDataJson(m_StaticDataJsonMutex);
     for (const auto &staticVar : m_StaticDataJson["S"])
     {
-        const std::string type(staticVar["Y"].get());
-        if (type == "")
+        const DataType type(
+            helper::GetDataTypeFromString(staticVar["Y"].get()));
+        if (type == DataType::None)
         {
         }
 #define declare_type(T)                                                        \
-    else if (type == helper::GetType())                                     \
+    else if (type == helper::GetDataType())                                 \
     {                                                                          \
-        const auto &attributesDataMap = io.GetAttributesDataMap();             \
-        auto it = attributesDataMap.find(staticVar["N"].get());   \
-        if (it == attributesDataMap.end())                                     \
+        const auto &attributes = io.GetAttributes();                           \
+        auto it = attributes.find(staticVar["N"].get());          \
+        if (it == attributes.end())                                            \
         {                                                                      \
             if (staticVar["V"].get())                                    \
             {                                                                  \
@@ -460,7 +234,8 @@ void DataManSerializer::JsonToVarMap(nlohmann::json &metaJ, VecPtr pack)
                     var.start = varBlock["O"].get();
                     var.count = varBlock["C"].get();
                     var.size = varBlock["I"].get();
-                    var.type = varBlock["Y"].get();
+                    var.type = helper::GetDataTypeFromString(
+                        varBlock["Y"].get());
                     var.rank = stoi(rankMapIt.key());
                 }
                 catch (std::exception &e)
@@ -531,13 +306,10 @@ void DataManSerializer::JsonToVarMap(nlohmann::json &metaJ, VecPtr pack)
                     var.compression = it->get();
                 }
 
-                for (auto i = varBlock.begin(); i != varBlock.end(); ++i)
+                it = varBlock.find("ZP");
+                if (it != varBlock.end())
                 {
-                    auto pos = i.key().find(":");
-                    if (pos != std::string::npos)
-                    {
-                        var.params[i.key().substr(pos + 1)] = i.value();
-                    }
+                    var.params = it->get();
                 }
 
                 if (m_DataManVarMap[var.step] == nullptr)
@@ -611,11 +383,11 @@ void DataManSerializer::Erase(const size_t step, const bool allPreviousSteps)
         }
         for (auto it : its)
         {
-            m_DataManVarMap.erase(it);
             Log(5,
-                "DataManSerializer::Erase() erased step " +
+                "DataManSerializer::Erase() erasing step " +
                     std::to_string(it->first),
                 true, true);
+            m_DataManVarMap.erase(it);
         }
         if (m_AggregatedMetadataJson != nullptr)
         {
@@ -636,13 +408,14 @@ void DataManSerializer::Erase(const size_t step, const bool allPreviousSteps)
     }
     else
     {
+        Log(5,
+            "DataManSerializer::Erase() erasing step " + std::to_string(step),
+            true, true);
         m_DataManVarMap.erase(step);
         if (m_AggregatedMetadataJson != nullptr)
         {
             m_AggregatedMetadataJson.erase(std::to_string(step));
         }
-        Log(5, "DataManSerializer::Erase() erased step " + std::to_string(step),
-            true, true);
     }
 }
 
@@ -653,298 +426,6 @@ DmvVecPtrMap DataManSerializer::GetFullMetadataMap()
     return m_DataManVarMap;
 }
 
-DmvVecPtr DataManSerializer::GetStepMetadata(const size_t step)
-{
-    TAU_SCOPED_TIMER_FUNC();
-    std::lock_guard l(m_DataManVarMapMutex);
-    auto it = m_DataManVarMap.find(step);
-    if (it != m_DataManVarMap.end())
-    {
-        return it->second;
-    }
-    return nullptr;
-}
-
-int DataManSerializer::PutDeferredRequest(const std::string &variable,
-                                          const size_t step, const Dims &start,
-                                          const Dims &count, void *data)
-{
-
-    TAU_SCOPED_TIMER_FUNC();
-    DmvVecPtr varVec;
-
-    m_DataManVarMapMutex.lock();
-    auto stepVecIt = m_DataManVarMap.find(step);
-    if (stepVecIt == m_DataManVarMap.end())
-    {
-        // aggregated metadata does not have this step
-        std::cout << "aggregated metadata does not have Step " << step
-                  << std::endl;
-        return -1;
-    }
-    else
-    {
-        varVec = stepVecIt->second;
-    }
-    m_DataManVarMapMutex.unlock();
-
-    std::unordered_map jmap;
-
-    for (const auto &var : *varVec)
-    {
-        if (var.name == variable)
-        {
-            if (var.start.size() != start.size() ||
-                var.count.size() != count.size() ||
-                start.size() != count.size())
-            {
-                throw("DataManSerializer::PutDeferredRequest() requested "
-                      "start, count and shape do not match");
-                continue;
-            }
-            bool toContinue = false;
-            for (size_t i = 0; i < start.size(); ++i)
-            {
-                if (start[i] >= var.start[i] + var.count[i] ||
-                    start[i] + count[i] <= var.start[i])
-                {
-                    toContinue = true;
-                }
-            }
-            if (toContinue)
-            {
-                continue;
-            }
-
-            jmap[var.address].emplace_back();
-            nlohmann::json &j = jmap[var.address].back();
-            j["N"] = variable;
-            j["O"] = var.start;
-            j["C"] = var.count;
-            j["T"] = step;
-        }
-    }
-
-    for (const auto &i : jmap)
-    {
-        auto charVec = (*m_DeferredRequestsToSend)[i.first];
-        if (charVec == nullptr)
-        {
-            charVec = std::make_shared>();
-        }
-        nlohmann::json jsonSer;
-        if (charVec->size() > 0)
-        {
-            jsonSer = DeserializeJson(charVec->data(), charVec->size());
-        }
-        for (auto j = i.second.begin(); j != i.second.end(); ++j)
-        {
-            jsonSer.push_back(*j);
-        }
-        (*m_DeferredRequestsToSend)[i.first] = SerializeJson(jsonSer);
-    }
-
-    return 0;
-}
-
-DeferredRequestMapPtr DataManSerializer::GetDeferredRequest()
-{
-    TAU_SCOPED_TIMER_FUNC();
-    auto t = m_DeferredRequestsToSend;
-    m_DeferredRequestsToSend = std::make_shared();
-    return t;
-}
-
-VecPtr DataManSerializer::GenerateReply(
-    const std::vector &request, size_t &step,
-    const std::unordered_map &compressionParams)
-{
-    TAU_SCOPED_TIMER_FUNC();
-    auto replyMetaJ = std::make_shared();
-    auto replyLocalBuffer =
-        std::make_shared>(sizeof(uint64_t) * 2);
-
-    nlohmann::json metaj;
-    try
-    {
-        metaj = DeserializeJson(request.data(), request.size());
-    }
-    catch (std::exception &e)
-    {
-        Log(1,
-            "DataManSerializer::GenerateReply() received staging request "
-            "but failed to deserialize due to " +
-                std::string(e.what()),
-            true, true);
-        step = -1;
-        return replyLocalBuffer;
-    }
-
-    for (const auto &req : metaj)
-    {
-        std::string variable = req["N"].get();
-        Dims start = req["O"].get();
-        Dims count = req["C"].get();
-        step = req["T"].get();
-
-        DmvVecPtr varVec;
-
-        {
-            std::lock_guard l(m_DataManVarMapMutex);
-            auto itVarVec = m_DataManVarMap.find(step);
-            if (itVarVec == m_DataManVarMap.end())
-            {
-                Log(1,
-                    "DataManSerializer::GenerateReply() received staging "
-                    "request but DataManVarMap does not have Step " +
-                        std::to_string(step),
-                    true, true);
-                if (m_Verbosity >= 1)
-                {
-                    std::string msg =
-                        "DataManSerializer::GenerateReply() current steps are ";
-                    for (auto s : m_DataManVarMap)
-                    {
-                        msg += std::to_string(s.first) + ", ";
-                    }
-                    Log(1, msg, true, true);
-                }
-                return replyLocalBuffer;
-            }
-            else
-            {
-                varVec = itVarVec->second;
-                if (varVec == nullptr)
-                {
-                    Log(1,
-                        "DataManSerializer::GenerateReply() received "
-                        "staging request but DataManVarMap contains a "
-                        "nullptr for Step " +
-                            std::to_string(step),
-                        true, true);
-                    return replyLocalBuffer;
-                }
-            }
-        }
-        for (const auto &var : *varVec)
-        {
-            if (var.name == variable)
-            {
-                Params compressionParamsVar;
-                auto compressionParamsIter = compressionParams.find(var.name);
-                if (compressionParamsIter != compressionParams.end())
-                {
-                    compressionParamsVar = compressionParamsIter->second;
-                }
-                Dims ovlpStart, ovlpCount;
-                bool ovlp = CalculateOverlap(var.start, var.count, start, count,
-                                             ovlpStart, ovlpCount);
-                if (ovlp)
-                {
-                    std::vector tmpBuffer;
-                    if (var.type == "compound")
-                    {
-                        throw("Compound type is not supported yet.");
-                    }
-#define declare_type(T)                                                        \
-    else if (var.type == helper::GetType())                                 \
-    {                                                                          \
-        tmpBuffer.reserve(std::accumulate(ovlpCount.begin(), ovlpCount.end(),  \
-                                          sizeof(T),                           \
-                                          std::multiplies()));         \
-        GetData(reinterpret_cast(tmpBuffer.data()), variable, ovlpStart,  \
-                ovlpCount, step);                                              \
-        PutData(reinterpret_cast(tmpBuffer.data()), variable, var.shape,  \
-                ovlpStart, ovlpCount, ovlpStart, ovlpCount, var.doid, step,    \
-                var.rank, var.address, compressionParamsVar, replyLocalBuffer, \
-                replyMetaJ);                                                   \
-    }
-                    ADIOS2_FOREACH_STDTYPE_1ARG(declare_type)
-#undef declare_type
-
-                    auto metapack = SerializeJson(*replyMetaJ);
-                    size_t metasize = metapack->size();
-                    (reinterpret_cast(
-                        replyLocalBuffer->data()))[0] =
-                        replyLocalBuffer->size();
-                    (reinterpret_cast(
-                        replyLocalBuffer->data()))[1] = metasize;
-                    replyLocalBuffer->resize(replyLocalBuffer->size() +
-                                             metasize);
-                    std::memcpy(replyLocalBuffer->data() +
-                                    replyLocalBuffer->size() - metasize,
-                                metapack->data(), metasize);
-                }
-            }
-        }
-    }
-    if (m_Verbosity >= 1)
-    {
-        if (replyLocalBuffer->size() <= 16)
-        {
-            std::cout << "DataManSerializer::GenerateReply returns a buffer "
-                         "with size "
-                      << replyLocalBuffer->size()
-                      << ", which means no data is contained in the buffer. "
-                         "This will cause the deserializer to unpack incorrect "
-                         "data for Step "
-                      << step << "." << std::endl;
-        }
-    }
-    return replyLocalBuffer;
-}
-
-bool DataManSerializer::CalculateOverlap(const Dims &inStart,
-                                         const Dims &inCount,
-                                         const Dims &outStart,
-                                         const Dims &outCount, Dims &ovlpStart,
-                                         Dims &ovlpCount)
-{
-    TAU_SCOPED_TIMER_FUNC();
-
-    if (inStart.size() != inCount.size() ||
-        outStart.size() != outCount.size() || inStart.size() != outStart.size())
-    {
-        return false;
-    }
-    if (ovlpStart.size() != inStart.size())
-    {
-        ovlpStart.resize(inStart.size());
-    }
-    if (ovlpCount.size() != inStart.size())
-    {
-        ovlpCount.resize(inStart.size());
-    }
-    for (size_t i = 0; i < inStart.size(); ++i)
-    {
-        if (inStart[i] + inCount[i] <= outStart[i])
-        {
-            return false;
-        }
-        if (outStart[i] + outCount[i] <= inStart[i])
-        {
-            return false;
-        }
-        if (inStart[i] < outStart[i])
-        {
-            ovlpStart[i] = outStart[i];
-        }
-        else
-        {
-            ovlpStart[i] = inStart[i];
-        }
-        if (inStart[i] + inCount[i] < outStart[i] + outCount[i])
-        {
-            ovlpCount[i] = inStart[i] + inCount[i] - ovlpStart[i];
-        }
-        else
-        {
-            ovlpCount[i] = outStart[i] + outCount[i] - ovlpStart[i];
-        }
-    }
-    return true;
-}
-
 size_t DataManSerializer::LocalBufferSize() { return m_LocalBuffer->size(); }
 
 VecPtr DataManSerializer::SerializeJson(const nlohmann::json &message)
@@ -1003,15 +484,15 @@ nlohmann::json DataManSerializer::DeserializeJson(const char *start,
     nlohmann::json message;
     if (m_UseJsonSerialization == "msgpack")
     {
-        message = nlohmann::json::from_msgpack(start, size);
+        message = nlohmann::json::from_msgpack(start, start + size);
     }
     else if (m_UseJsonSerialization == "cbor")
     {
-        message = nlohmann::json::from_cbor(start, size);
+        message = nlohmann::json::from_cbor(start, start + size);
     }
     else if (m_UseJsonSerialization == "ubjson")
     {
-        message = nlohmann::json::from_ubjson(start, size);
+        message = nlohmann::json::from_ubjson(start, start + size);
     }
     else if (m_UseJsonSerialization == "string")
     {
@@ -1120,7 +601,6 @@ DmvVecPtr DataManSerializer::GetEarliestLatestStep(
             return nullptr;
         }
     }
-    return nullptr;
 }
 
 void DataManSerializer::Log(const int level, const std::string &message,
diff --git a/source/adios2/toolkit/format/dataman/DataManSerializer.h b/source/adios2/toolkit/format/dataman/DataManSerializer.h
index 4dcbd6e05a..2e6b183ba8 100644
--- a/source/adios2/toolkit/format/dataman/DataManSerializer.h
+++ b/source/adios2/toolkit/format/dataman/DataManSerializer.h
@@ -61,7 +61,7 @@ struct DataManVar
     Dims start;
     std::string name;
     std::string doid;
-    std::string type;
+    DataType type;
     std::vector min;
     std::vector max;
     std::vector value;
@@ -103,36 +103,21 @@ class DataManSerializer
                  const Dims &varCount, const Dims &varMemStart,
                  const Dims &varMemCount, const std::string &doid,
                  const size_t step, const int rank, const std::string &address,
-                 const Params ¶ms, VecPtr localBuffer = nullptr,
-                 JsonPtr metadataJson = nullptr);
+                 const std::vector &ops,
+                 VecPtr localBuffer = nullptr, JsonPtr metadataJson = nullptr);
 
     // another wrapper for PutData which accepts adios2::core::Variable
     template 
     void PutData(const core::Variable &variable, const std::string &doid,
                  const size_t step, const int rank, const std::string &address,
-                 const Params ¶ms, VecPtr localBuffer = nullptr,
-                 JsonPtr metadataJson = nullptr);
+                 VecPtr localBuffer = nullptr, JsonPtr metadataJson = nullptr);
 
     // attach attributes to local pack
     void AttachAttributesToLocalPack();
 
-    // aggregate metadata across all writer ranks and put it into map
-    void AggregateMetadata();
-
-    // get aggregated metadata pack for sending from staging writer to staging
-    // reader
-    VecPtr GetAggregatedMetadataPack(const int64_t stepRequested,
-                                     int64_t &stepProvided,
-                                     const int64_t appID);
-
     // put local metadata and data buffer together and return the merged buffer
     VecPtr GetLocalPack();
 
-    // generate reply on staging writer based on the request from reader
-    VecPtr GenerateReply(
-        const std::vector &request, size_t &step,
-        const std::unordered_map &compressionParams);
-
     // ************ deserializer functions
 
     // put binary pack for deserialization
@@ -154,18 +139,6 @@ class DataManSerializer
     // deserializer
     DmvVecPtrMap GetFullMetadataMap();
 
-    DmvVecPtr GetStepMetadata(const size_t step);
-
-    void PutAggregatedMetadata(VecPtr input, helper::Comm const &comm);
-
-    int PutDeferredRequest(const std::string &variable, const size_t step,
-                           const Dims &start, const Dims &count, void *data);
-    DeferredRequestMapPtr GetDeferredRequest();
-
-    bool CalculateOverlap(const Dims &inStart, const Dims &inCount,
-                          const Dims &outStart, const Dims &outCount,
-                          Dims &ovlpStart, Dims &ovlpCount);
-
     void SetDestination(const std::string &dest);
 
     std::string GetDestination();
@@ -193,8 +166,8 @@ class DataManSerializer
     template 
     void PutAttribute(const core::Attribute &attribute);
 
-    bool IsCompressionAvailable(const std::string &method,
-                                const std::string &type, const Dims &count);
+    bool IsCompressionAvailable(const std::string &method, DataType type,
+                                const Dims &count);
 
     void JsonToVarMap(nlohmann::json &metaJ, VecPtr pack);
 
@@ -228,10 +201,6 @@ class DataManSerializer
     DmvVecPtrMap m_DataManVarMap;
     std::mutex m_DataManVarMapMutex;
 
-    std::unordered_map> m_ProtectedStepsToAggregate;
-    std::unordered_map> m_ProtectedStepsAggregated;
-    std::mutex m_ProtectedStepsMutex;
-
     // used to count buffers that have been put into deserializer, asynchronous
     // engines such as dataman use this to tell if a certain step has received
     // all blocks from all writers
@@ -248,17 +217,13 @@ class DataManSerializer
     std::mutex m_StaticDataJsonMutex;
     bool m_StaticDataFinished = false;
 
-    // for generating deferred requests, only accessed from reader main thread,
-    // does not need mutex
-    DeferredRequestMapPtr m_DeferredRequestsToSend;
-
     // string, msgpack, cbor, ubjson
     std::string m_UseJsonSerialization = "string";
 
     std::string m_Destination;
     bool m_IsRowMajor;
     bool m_IsLittleEndian;
-    bool m_ContiguousMajor;
+    bool m_ContiguousMajor = true;
     bool m_EnableStat = true;
     int m_MpiRank;
     int m_MpiSize;
diff --git a/source/adios2/toolkit/format/dataman/DataManSerializer.tcc b/source/adios2/toolkit/format/dataman/DataManSerializer.tcc
index 8f1c75f234..00355293f8 100644
--- a/source/adios2/toolkit/format/dataman/DataManSerializer.tcc
+++ b/source/adios2/toolkit/format/dataman/DataManSerializer.tcc
@@ -80,25 +80,23 @@ template 
 void DataManSerializer::PutData(const core::Variable &variable,
                                 const std::string &doid, const size_t step,
                                 const int rank, const std::string &address,
-                                const Params ¶ms, VecPtr localBuffer,
-                                JsonPtr metadataJson)
+                                VecPtr localBuffer, JsonPtr metadataJson)
 {
     TAU_SCOPED_TIMER_FUNC();
     PutData(variable.GetData(), variable.m_Name, variable.m_Shape,
             variable.m_Start, variable.m_Count, variable.m_MemoryStart,
-            variable.m_MemoryCount, doid, step, rank, address, params,
-            localBuffer, metadataJson);
+            variable.m_MemoryCount, doid, step, rank, address,
+            variable.m_Operations, localBuffer, metadataJson);
 }
 
 template 
-void DataManSerializer::PutData(const T *inputData, const std::string &varName,
-                                const Dims &varShape, const Dims &varStart,
-                                const Dims &varCount, const Dims &varMemStart,
-                                const Dims &varMemCount,
-                                const std::string &doid, const size_t step,
-                                const int rank, const std::string &address,
-                                const Params ¶ms, VecPtr localBuffer,
-                                JsonPtr metadataJson)
+void DataManSerializer::PutData(
+    const T *inputData, const std::string &varName, const Dims &varShape,
+    const Dims &varStart, const Dims &varCount, const Dims &varMemStart,
+    const Dims &varMemCount, const std::string &doid, const size_t step,
+    const int rank, const std::string &address,
+    const std::vector &ops, VecPtr localBuffer,
+    JsonPtr metadataJson)
 {
     TAU_SCOPED_TIMER_FUNC();
     Log(1,
@@ -117,7 +115,7 @@ void DataManSerializer::PutData(const T *inputData, const std::string &varName,
     metaj["O"] = varStart;
     metaj["C"] = varCount;
     metaj["S"] = varShape;
-    metaj["Y"] = helper::GetType();
+    metaj["Y"] = ToString(helper::GetDataType());
     metaj["P"] = localBuffer->size();
 
     if (not address.empty())
@@ -139,64 +137,72 @@ void DataManSerializer::PutData(const T *inputData, const std::string &varName,
         metaj["E"] = m_IsLittleEndian;
     }
 
-    size_t datasize = 0;
-    bool compressed = false;
-    if (not params.empty())
+    for (const auto &op : ops)
     {
-        const auto i = params.find("CompressionMethod");
-        if (i != params.end())
+        const auto opName = op.Op->m_Type;
+        if (opName == "zfp" or opName == "bzip2" or opName == "sz")
         {
-            std::string compressionMethod = i->second;
-            std::transform(compressionMethod.begin(), compressionMethod.end(),
-                           compressionMethod.begin(), ::tolower);
-            if (compressionMethod == "zfp")
+            /*
+            m_CompressionParams[variable.m_Name]["CompressionMethod"] =
+                opName;
+            for (const auto &p : op.Parameters)
             {
-                if (IsCompressionAvailable(compressionMethod,
-                                           helper::GetType(), varCount))
-                {
-                    compressed =
-                        PutZfp(metaj, datasize, inputData, varCount, params);
-                    if (compressed)
-                    {
-                        metaj["Z"] = "zfp";
-                    }
-                }
+                m_CompressionParams[variable.m_Name]
+                    [opName + ":" + p.first] = p.second;
             }
-            else if (compressionMethod == "sz")
+            break;
+            */
+        }
+    }
+
+    size_t datasize = 0;
+    bool compressed = false;
+    std::string compressionMethod;
+    if (not ops.empty())
+    {
+        compressionMethod = ops[0].Op->m_Type;
+        std::transform(compressionMethod.begin(), compressionMethod.end(),
+                       compressionMethod.begin(), ::tolower);
+        if (compressionMethod == "zfp")
+        {
+            if (IsCompressionAvailable(compressionMethod,
+                                       helper::GetDataType(), varCount))
             {
-                if (IsCompressionAvailable(compressionMethod,
-                                           helper::GetType(), varCount))
-                {
-                    compressed =
-                        PutSz(metaj, datasize, inputData, varCount, params);
-                    if (compressed)
-                    {
-                        metaj["Z"] = "sz";
-                    }
-                }
+                compressed = PutZfp(metaj, datasize, inputData, varCount,
+                                       ops[0].Parameters);
             }
-            else if (compressionMethod == "bzip2")
+        }
+        else if (compressionMethod == "sz")
+        {
+            if (IsCompressionAvailable(compressionMethod,
+                                       helper::GetDataType(), varCount))
             {
-                if (IsCompressionAvailable(compressionMethod,
-                                           helper::GetType(), varCount))
-                {
-                    compressed = PutBZip2(metaj, datasize, inputData,
-                                             varCount, params);
-                    if (compressed)
-                    {
-                        metaj["Z"] = "bzip2";
-                    }
-                }
+                compressed = PutSz(metaj, datasize, inputData, varCount,
+                                      ops[0].Parameters);
             }
-            else
+        }
+        else if (compressionMethod == "bzip2")
+        {
+            if (IsCompressionAvailable(compressionMethod,
+                                       helper::GetDataType(), varCount))
             {
-                throw(std::invalid_argument("Compression method " + i->second +
-                                            " not supported."));
+                compressed = PutBZip2(metaj, datasize, inputData, varCount,
+                                         ops[0].Parameters);
             }
         }
+        else
+        {
+            throw(std::invalid_argument("Compression method " +
+                                        compressionMethod + " not supported."));
+        }
     }
 
-    if (compressed == false)
+    if (compressed)
+    {
+        metaj["Z"] = compressionMethod;
+        metaj["ZP"] = ops[0].Parameters;
+    }
+    else
     {
         datasize = std::accumulate(varCount.begin(), varCount.end(), sizeof(T),
                                    std::multiplies());
@@ -238,155 +244,6 @@ void DataManSerializer::PutData(const T *inputData, const std::string &varName,
         true, true);
 }
 
-template 
-bool DataManSerializer::PutZfp(nlohmann::json &metaj, size_t &datasize,
-                               const T *inputData, const Dims &varCount,
-                               const Params ¶ms)
-{
-    TAU_SCOPED_TIMER_FUNC();
-#ifdef ADIOS2_HAVE_ZFP
-    Params p;
-    for (const auto &i : params)
-    {
-        std::string prefix = i.first.substr(0, 4);
-        if (prefix == "zfp:" || prefix == "Zfp:" || prefix == "ZFP:")
-        {
-            std::string key = i.first.substr(4);
-            metaj[i.first] = i.second;
-            p[key] = i.second;
-        }
-    }
-    core::compress::CompressZFP compressor(p);
-    m_CompressBuffer.reserve(std::accumulate(varCount.begin(), varCount.end(),
-                                             sizeof(T),
-                                             std::multiplies()));
-    try
-    {
-        Params info;
-        datasize = compressor.Compress(inputData, varCount, sizeof(T),
-                                       helper::GetType(),
-                                       m_CompressBuffer.data(), p, info);
-        return true;
-    }
-    catch (std::exception &e)
-    {
-        std::cout << "Got exception " << e.what()
-                  << " from ZFP. Turned off compression." << std::endl;
-    }
-#else
-    throw(std::invalid_argument(
-        "ZFP compression used but ZFP library is not linked to ADIOS2"));
-#endif
-    return false;
-}
-
-template 
-bool DataManSerializer::PutSz(nlohmann::json &metaj, size_t &datasize,
-                              const T *inputData, const Dims &varCount,
-                              const Params ¶ms)
-{
-    TAU_SCOPED_TIMER_FUNC();
-#ifdef ADIOS2_HAVE_SZ
-    Params p;
-    for (const auto &i : params)
-    {
-        std::string prefix = i.first.substr(0, 3);
-        if (prefix == "sz:" || prefix == "Sz:" || prefix == "SZ:")
-        {
-            std::string key = i.first.substr(3);
-            metaj[i.first] = i.second;
-            p[key] = i.second;
-        }
-    }
-    m_CompressBuffer.reserve(std::accumulate(varCount.begin(), varCount.end(),
-                                             sizeof(T),
-                                             std::multiplies()));
-    core::compress::CompressSZ compressor(p);
-    try
-    {
-        Params info;
-        datasize = compressor.Compress(inputData, varCount, sizeof(T),
-                                       helper::GetType(),
-                                       m_CompressBuffer.data(), p, info);
-        return true;
-    }
-    catch (std::exception &e)
-    {
-        std::cout << "Got exception " << e.what()
-                  << " from SZ. Turned off compression." << std::endl;
-    }
-#else
-    throw(std::invalid_argument(
-        "SZ compression used but SZ library is not linked to ADIOS2"));
-#endif
-    return false;
-}
-
-template 
-bool DataManSerializer::PutBZip2(nlohmann::json &metaj, size_t &datasize,
-                                 const T *inputData, const Dims &varCount,
-                                 const Params ¶ms)
-{
-    TAU_SCOPED_TIMER_FUNC();
-#ifdef ADIOS2_HAVE_BZIP2
-    Params p;
-    for (const auto &i : params)
-    {
-        std::string prefix = i.first.substr(0, 6);
-        if (prefix == "bzip2:" || prefix == "Bzip2:" || prefix == "BZip2:" ||
-            prefix == "BZIP2:")
-        {
-            std::string key = i.first.substr(6);
-            metaj[i.first] = i.second;
-            p[key] = i.second;
-        }
-    }
-    m_CompressBuffer.reserve(std::accumulate(varCount.begin(), varCount.end(),
-                                             sizeof(T),
-                                             std::multiplies()));
-    core::compress::CompressBZIP2 compressor(p);
-    try
-    {
-        Params info;
-        datasize = compressor.Compress(inputData, varCount, sizeof(T),
-                                       helper::GetType(),
-                                       m_CompressBuffer.data(), p, info);
-        return true;
-    }
-    catch (std::exception &e)
-    {
-        std::cout << "Got exception " << e.what()
-                  << " from BZip2. Turned off compression." << std::endl;
-    }
-#else
-    throw(std::invalid_argument(
-        "BZip2 compression used but BZip2 library is not linked to ADIOS2"));
-#endif
-    return false;
-}
-
-template 
-void DataManSerializer::PutAttribute(const core::Attribute &attribute)
-{
-    TAU_SCOPED_TIMER_FUNC();
-    nlohmann::json staticVar;
-    staticVar["N"] = attribute.m_Name;
-    staticVar["Y"] = attribute.m_Type;
-    staticVar["V"] = attribute.m_IsSingleValue;
-    if (attribute.m_IsSingleValue)
-    {
-        staticVar["G"] = attribute.m_DataSingleValue;
-    }
-    else
-    {
-        staticVar["G"] = attribute.m_DataArray;
-    }
-
-    m_StaticDataJsonMutex.lock();
-    m_StaticDataJson["S"].emplace_back(std::move(staticVar));
-    m_StaticDataJsonMutex.unlock();
-}
-
 template 
 int DataManSerializer::GetData(T *outputData, const std::string &varName,
                                const Dims &varStart, const Dims &varCount,
@@ -559,13 +416,128 @@ int DataManSerializer::GetData(T *outputData, const std::string &varName,
             }
             if (j.shape.empty() or (j.shape.size() == 1 and j.shape[0] == 1))
             {
-                *outputData = *reinterpret_cast(input_data);
+                std::memcpy(outputData, input_data, sizeof(T));
             }
         }
     }
     return 0;
 }
 
+template 
+bool DataManSerializer::PutZfp(nlohmann::json &metaj, size_t &datasize,
+                               const T *inputData, const Dims &varCount,
+                               const Params ¶ms)
+{
+    TAU_SCOPED_TIMER_FUNC();
+#ifdef ADIOS2_HAVE_ZFP
+    core::compress::CompressZFP compressor(params);
+    m_CompressBuffer.reserve(std::accumulate(varCount.begin(), varCount.end(),
+                                             sizeof(T),
+                                             std::multiplies()));
+    try
+    {
+        Params info;
+        datasize = compressor.Compress(inputData, varCount, sizeof(T),
+                                       helper::GetDataType(),
+                                       m_CompressBuffer.data(), params, info);
+        return true;
+    }
+    catch (std::exception &e)
+    {
+        std::cout << "Got exception " << e.what()
+                  << " from ZFP. Turned off compression." << std::endl;
+    }
+#else
+    throw(std::invalid_argument(
+        "ZFP compression used but ZFP library is not linked to ADIOS2"));
+#endif
+    return false;
+}
+
+template 
+bool DataManSerializer::PutSz(nlohmann::json &metaj, size_t &datasize,
+                              const T *inputData, const Dims &varCount,
+                              const Params ¶ms)
+{
+    TAU_SCOPED_TIMER_FUNC();
+#ifdef ADIOS2_HAVE_SZ
+    m_CompressBuffer.reserve(std::accumulate(varCount.begin(), varCount.end(),
+                                             sizeof(T),
+                                             std::multiplies()));
+    core::compress::CompressSZ compressor(params);
+    try
+    {
+        Params info;
+        datasize = compressor.Compress(inputData, varCount, sizeof(T),
+                                       helper::GetDataType(),
+                                       m_CompressBuffer.data(), params, info);
+        return true;
+    }
+    catch (std::exception &e)
+    {
+        std::cout << "Got exception " << e.what()
+                  << " from SZ. Turned off compression." << std::endl;
+    }
+#else
+    throw(std::invalid_argument(
+        "SZ compression used but SZ library is not linked to ADIOS2"));
+#endif
+    return false;
+}
+
+template 
+bool DataManSerializer::PutBZip2(nlohmann::json &metaj, size_t &datasize,
+                                 const T *inputData, const Dims &varCount,
+                                 const Params ¶ms)
+{
+    TAU_SCOPED_TIMER_FUNC();
+#ifdef ADIOS2_HAVE_BZIP2
+    m_CompressBuffer.reserve(std::accumulate(varCount.begin(), varCount.end(),
+                                             sizeof(T),
+                                             std::multiplies()));
+    core::compress::CompressBZIP2 compressor(params);
+    try
+    {
+        Params info;
+        datasize = compressor.Compress(inputData, varCount, sizeof(T),
+                                       helper::GetDataType(),
+                                       m_CompressBuffer.data(), params, info);
+        return true;
+    }
+    catch (std::exception &e)
+    {
+        std::cout << "Got exception " << e.what()
+                  << " from BZip2. Turned off compression." << std::endl;
+    }
+#else
+    throw(std::invalid_argument(
+        "BZip2 compression used but BZip2 library is not linked to ADIOS2"));
+#endif
+    return false;
+}
+
+template 
+void DataManSerializer::PutAttribute(const core::Attribute &attribute)
+{
+    TAU_SCOPED_TIMER_FUNC();
+    nlohmann::json staticVar;
+    staticVar["N"] = attribute.m_Name;
+    staticVar["Y"] = ToString(attribute.m_Type);
+    staticVar["V"] = attribute.m_IsSingleValue;
+    if (attribute.m_IsSingleValue)
+    {
+        staticVar["G"] = attribute.m_DataSingleValue;
+    }
+    else
+    {
+        staticVar["G"] = attribute.m_DataArray;
+    }
+
+    m_StaticDataJsonMutex.lock();
+    m_StaticDataJson["S"].emplace_back(std::move(staticVar));
+    m_StaticDataJsonMutex.unlock();
+}
+
 } // namespace format
 } // namespace adios2
 
diff --git a/source/adios2/toolkit/interop/hdf5/HDF5Common.cpp b/source/adios2/toolkit/interop/hdf5/HDF5Common.cpp
index 8c04512760..65d8989570 100644
--- a/source/adios2/toolkit/interop/hdf5/HDF5Common.cpp
+++ b/source/adios2/toolkit/interop/hdf5/HDF5Common.cpp
@@ -47,6 +47,7 @@ const std::string HDF5Common::PREFIX_STAT = "ADIOS_STAT_";
 const std::string HDF5Common::PARAMETER_COLLECTIVE = "H5CollectiveMPIO";
 const std::string HDF5Common::PARAMETER_CHUNK_FLAG = "H5ChunkDim";
 const std::string HDF5Common::PARAMETER_CHUNK_VARS = "H5ChunkVars";
+const std::string HDF5Common::PARAMETER_HAS_IDLE_WRITER_RANK = "IdleH5Writer";
 
 /*
    //need to know ndim before defining this.
@@ -90,12 +91,21 @@ void HDF5Common::ParseParameters(core::IO &io)
 {
     if (m_MPI)
     {
+        m_MPI->set_dxpl_mpio(m_PropertyTxfID,
+                             H5FD_MPIO_INDEPENDENT); // explicit
         auto itKey = io.m_Parameters.find(PARAMETER_COLLECTIVE);
         if (itKey != io.m_Parameters.end())
         {
             if (itKey->second == "yes" || itKey->second == "true")
                 m_MPI->set_dxpl_mpio(m_PropertyTxfID, H5FD_MPIO_COLLECTIVE);
         }
+
+        itKey = io.m_Parameters.find(PARAMETER_HAS_IDLE_WRITER_RANK);
+        if (itKey != io.m_Parameters.end())
+        {
+            if (itKey->second == "yes" || itKey->second == "true")
+                m_IdleWriterOn = true;
+        }
     }
 
     m_ChunkVarNames.clear();
@@ -133,6 +143,54 @@ void HDF5Common::ParseParameters(core::IO &io)
                 m_ChunkVarNames.insert(token);
         }
     }
+
+    m_OrderByC = helper::IsRowMajor(io.m_HostLanguage);
+}
+
+void HDF5Common::Append(const std::string &name, helper::Comm const &comm)
+{
+    m_PropertyListId = H5Pcreate(H5P_FILE_ACCESS);
+
+    if (MPI_API const *mpi = GetHDF5Common_MPI_API())
+    {
+        if (mpi && mpi->init(comm, m_PropertyListId, &m_CommRank, &m_CommSize))
+        {
+            m_MPI = mpi;
+        }
+    }
+
+    m_FileId = H5Fopen(name.c_str(), H5F_ACC_RDWR, m_PropertyListId);
+    H5Pclose(m_PropertyListId);
+
+    std::string ts0;
+    StaticGetAdiosStepString(ts0, 0);
+
+    if (m_FileId >= 0)
+    {
+        if (H5Lexists(m_FileId, ts0.c_str(), H5P_DEFAULT) != 0)
+        {
+            m_IsGeneratedByAdios = true;
+        }
+        if (!m_IsGeneratedByAdios)
+            throw std::ios_base::failure(
+                "HDF5Engine Append error. Likely no such file." + name);
+
+        GetNumAdiosSteps(); // read how many steps exists in this file
+
+        if (0 == m_NumAdiosSteps)
+            throw std::ios_base::failure(
+                "HDF5Engine Append error. No valid steps found in " + name);
+        if (1 == m_NumAdiosSteps)
+            m_GroupId = H5Gopen(m_FileId, ts0.c_str(), H5P_DEFAULT);
+        else
+            SetAdiosStep(m_NumAdiosSteps - 1);
+
+        m_WriteMode = true;
+        Advance();
+    }
+    else
+        throw std::ios_base::failure(
+            "HDF5Engine Append error. Likely no such file." + name);
 }
 
 void HDF5Common::Init(const std::string &name, helper::Comm const &comm,
@@ -204,9 +262,13 @@ void HDF5Common::WriteAdiosSteps()
     }
 
     hid_t s = H5Screate(H5S_SCALAR);
-    hid_t attr =
-        H5Acreate(m_FileId, ATTRNAME_NUM_STEPS.c_str(),
-                  /*"NumSteps",*/ H5T_NATIVE_UINT, s, H5P_DEFAULT, H5P_DEFAULT);
+    hid_t attr = H5Aexists(m_FileId, ATTRNAME_NUM_STEPS.c_str());
+    if (0 == attr)
+        attr = H5Acreate(m_FileId, ATTRNAME_NUM_STEPS.c_str(), H5T_NATIVE_UINT,
+                         s, H5P_DEFAULT, H5P_DEFAULT);
+    else
+        attr = H5Aopen(m_FileId, ATTRNAME_NUM_STEPS.c_str(), H5P_DEFAULT);
+
     unsigned int totalAdiosSteps = m_CurrentAdiosStep + 1;
 
     if (m_GroupId < 0)
@@ -220,6 +282,8 @@ void HDF5Common::WriteAdiosSteps()
     H5Aclose(attr);
 }
 
+unsigned int HDF5Common::GetAdiosStep() const { return m_NumAdiosSteps; }
+
 unsigned int HDF5Common::GetNumAdiosSteps()
 {
     if (m_WriteMode)
@@ -642,6 +706,46 @@ void HDF5Common::SetAdiosStep(int step)
     m_CurrentAdiosStep = step;
 }
 
+//
+// This function is intend to pair with CreateVarsFromIO().
+//
+// Because of the collective call requirement, we creata
+// all variables through CreateVarFromIO() at BeginStep().
+// At EndStep(), this function is called to remove unwritten variables
+// to comply with ADIOS custom that define all vars, use a few per step
+// and only see these few at the step.
+//
+// note that this works with 1 rank currently.
+// need to find a general way in parallel HDF5 to
+// detect whether a dataset called H5Dwrite  or not
+//
+void HDF5Common::CleanUpNullVars(core::IO &io)
+{
+    if (!m_WriteMode)
+        return;
+
+    if (m_CommSize != 1) // because the H5D_storage_size > 0 after H5Dcreate
+                         // when there are multiple processors
+        return;
+
+    const core::VarMap &variables = io.GetVariables();
+    for (const auto &vpair : variables)
+    {
+        const std::string &varName = vpair.first;
+        const DataType varType = vpair.second->m_Type;
+#define declare_template_instantiation(T)                                      \
+    if (varType == helper::GetDataType())                                   \
+    {                                                                          \
+        core::Variable *v = io.InquireVariable(varName);                 \
+        if (!v)                                                                \
+            return;                                                            \
+        RemoveEmptyDataset(varName);                                           \
+    }
+        ADIOS2_FOREACH_STDTYPE_1ARG(declare_template_instantiation)
+#undef declare_template_instantiation
+    }
+}
+
 void HDF5Common::Advance()
 {
     if (m_WriteMode)
@@ -781,14 +885,23 @@ void HDF5Common::CreateDataset(const std::string &varName, hid_t h5Type,
             varCreateProperty = m_ChunkPID;
     }
 
-    hid_t dsetID = H5Dcreate(topId, list.back().c_str(), h5Type, filespaceID,
-                             H5P_DEFAULT, varCreateProperty, H5P_DEFAULT);
-
-    if (list.back().compare(varName) != 0)
+    hid_t dsetID = -1;
+    if (H5Lexists(topId, list.back().c_str(), H5P_DEFAULT) == 0)
     {
-        StoreADIOSName(varName, dsetID); // only stores when not the same
+        dsetID = H5Dcreate(topId, list.back().c_str(), h5Type, filespaceID,
+                           H5P_DEFAULT, varCreateProperty, H5P_DEFAULT);
+        if (list.back().compare(varName) != 0)
+        {
+            StoreADIOSName(varName, dsetID); // only stores when not the same
+        }
     }
+    else
+        dsetID = H5Dopen(topId, list.back().c_str(), H5P_DEFAULT);
+
     datasetChain.push_back(dsetID);
+
+    hid_t dspace = H5Dget_space(dsetID);
+    const int ndims = H5Sget_simple_extent_ndims(dspace);
     // return dsetID;
 }
 
@@ -857,9 +970,17 @@ bool HDF5Common::OpenDataset(const std::string &varName,
 
     if (list.size() == 1)
     {
-        hid_t dsetID = H5Dopen(m_GroupId, list[0].c_str(), H5P_DEFAULT);
-        datasetChain.push_back(dsetID);
-        return true;
+        if (H5Lexists(m_GroupId, list[0].c_str(), H5P_DEFAULT) == 0)
+        {
+            datasetChain.push_back(-1);
+            return false;
+        }
+        else
+        {
+            hid_t dsetID = H5Dopen(m_GroupId, list[0].c_str(), H5P_DEFAULT);
+            datasetChain.push_back(dsetID);
+            return true;
+        }
     }
 
     hid_t topId = m_GroupId;
@@ -887,6 +1008,74 @@ bool HDF5Common::OpenDataset(const std::string &varName,
     return true;
 }
 
+//
+// We use H5Dget_storage_size to see whether H5Dwrite has been called.
+// and looks like when there are multiple processors, H5Dcreate causes
+// storage allcoation already. So we limit this function when there is only
+// one rank.
+//
+void HDF5Common::RemoveEmptyDataset(const std::string &varName)
+{
+    if (m_CommSize > 1)
+        return;
+
+    std::vector list;
+    char delimiter = '/';
+    int delimiterLength = 1;
+    std::string s = std::string(varName);
+    size_t pos = 0;
+    std::string token;
+    while ((pos = s.find(delimiter)) != std::string::npos)
+    {
+        if (pos > 0)
+        { // "///a/b/c" == "a/b/c"
+            token = s.substr(0, pos);
+            list.push_back(token);
+        }
+        s.erase(0, pos + delimiterLength);
+    }
+    list.push_back(s);
+
+    if (list.size() == 1)
+    {
+        if (H5Lexists(m_GroupId, list[0].c_str(), H5P_DEFAULT) != 0)
+        {
+            hid_t dsetID = H5Dopen(m_GroupId, list[0].c_str(), H5P_DEFAULT);
+            HDF5TypeGuard d(dsetID, E_H5_DATASET);
+
+            H5D_space_status_t status;
+            herr_t s1 = H5Dget_space_status(dsetID, &status);
+
+            if (0 == H5Dget_storage_size(dsetID)) /*nothing is written */
+                H5Ldelete(m_GroupId, list[0].c_str(), H5P_DEFAULT);
+        }
+        return;
+    }
+
+    hid_t topId = m_GroupId;
+    std::vector datasetChain;
+
+    for (int i = 0; i < list.size() - 1; i++)
+    {
+        if (H5Lexists(topId, list[i].c_str(), H5P_DEFAULT) == 0)
+            break;
+        else
+            topId = H5Gopen(topId, list[i].c_str(), H5P_DEFAULT);
+
+        datasetChain.push_back(topId);
+    }
+    hid_t dsetID = H5Dopen(topId, list.back().c_str(), H5P_DEFAULT);
+    datasetChain.push_back(dsetID);
+
+    HDF5DatasetGuard g(datasetChain);
+
+    if (H5Lexists(topId, list.back().c_str(), H5P_DEFAULT) != 0)
+    {
+        if (0 == H5Dget_storage_size(dsetID)) // nothing is written
+            H5Ldelete(topId, list.back().c_str(), H5P_DEFAULT);
+    }
+}
+
 // trim from right
 inline std::string &rtrim(std::string &s, const char *t = " \t\n\r\f\v")
 {
@@ -1188,14 +1377,21 @@ void HDF5Common::LocateAttrParent(const std::string &attrName,
 //
 void HDF5Common::CreateVarsFromIO(core::IO &io)
 {
-    CheckWriteGroup();
-    const core::DataMap &variables = io.GetVariablesDataMap();
+    if (!m_WriteMode)
+        return;
+
+    CheckWriteGroup(); // making sure all processors are creating new step
+
+    if (!m_IdleWriterOn)
+        return;
+
+    const core::VarMap &variables = io.GetVariables();
     for (const auto &vpair : variables)
     {
         const std::string &varName = vpair.first;
-        const std::string &varType = vpair.second.first;
+        const DataType varType = vpair.second->m_Type;
 #define declare_template_instantiation(T)                                      \
-    if (varType == helper::GetType())                                       \
+    if (varType == helper::GetDataType())                                   \
     {                                                                          \
         core::Variable *v = io.InquireVariable(varName);                 \
         if (!v)                                                                \
@@ -1229,7 +1425,7 @@ void HDF5Common::WriteAttrFromIO(core::IO &io)
     {
         std::string attrName = apair.first;
         Params temp = apair.second;
-        std::string attrType = temp["Type"];
+        DataType attrType = helper::GetDataTypeFromString(temp["Type"]);
 
         hid_t parentID = m_FileId;
 #ifdef NO_ATTR_VAR_ASSOC
@@ -1254,11 +1450,11 @@ void HDF5Common::WriteAttrFromIO(core::IO &io)
             continue;
         }
 
-        if (attrType == "compound")
+        if (attrType == DataType::Compound)
         {
             // not supported
         }
-        else if (attrType == helper::GetType())
+        else if (attrType == helper::GetDataType())
         {
             // WriteStringAttr(io, attrName, parentID);
             core::Attribute *adiosAttr =
@@ -1269,7 +1465,7 @@ void HDF5Common::WriteAttrFromIO(core::IO &io)
 // note no std::complext attr types
 //
 #define declare_template_instantiation(T)                                      \
-    else if (attrType == helper::GetType())                                 \
+    else if (attrType == helper::GetDataType())                             \
     {                                                                          \
         core::Attribute *adiosAttr = io.InquireAttribute(attrName);      \
         WriteNonStringAttr(io, adiosAttr, parentID, list.back().c_str());      \
@@ -1277,8 +1473,6 @@ void HDF5Common::WriteAttrFromIO(core::IO &io)
         ADIOS2_FOREACH_ATTRIBUTE_STDTYPE_1ARG(declare_template_instantiation)
 #undef declare_template_instantiation
     }
-
-    // std::string attrType = attributesInfo[attrName]["Type"];
 }
 
 //
diff --git a/source/adios2/toolkit/interop/hdf5/HDF5Common.h b/source/adios2/toolkit/interop/hdf5/HDF5Common.h
index 39bdb44398..e657be6fc4 100644
--- a/source/adios2/toolkit/interop/hdf5/HDF5Common.h
+++ b/source/adios2/toolkit/interop/hdf5/HDF5Common.h
@@ -124,9 +124,11 @@ class HDF5Common
     static const std::string PARAMETER_COLLECTIVE;
     static const std::string PARAMETER_CHUNK_FLAG;
     static const std::string PARAMETER_CHUNK_VARS;
+    static const std::string PARAMETER_HAS_IDLE_WRITER_RANK;
 
     void ParseParameters(core::IO &io);
     void Init(const std::string &name, helper::Comm const &comm, bool toWrite);
+    void Append(const std::string &name, helper::Comm const &comm);
 
     template 
     void Write(core::Variable &variable, const T *values);
@@ -143,7 +145,7 @@ class HDF5Common
     void CreateDataset(const std::string &varName, hid_t h5Type,
                        hid_t filespaceID, std::vector &chain);
     bool OpenDataset(const std::string &varName, std::vector &chain);
-
+    void RemoveEmptyDataset(const std::string &varName);
     void StoreADIOSName(const std::string adiosName, hid_t dsetID);
     void ReadADIOSName(hid_t dsetID, std::string &adiosName);
 
@@ -162,6 +164,7 @@ class HDF5Common
      * required by HDF5
      */
     void CreateVarsFromIO(core::IO &io);
+    void CleanUpNullVars(core::IO &io);
 
     void WriteAttrFromIO(core::IO &io);
     void ReadAttrToIO(core::IO &io);
@@ -172,6 +175,7 @@ class HDF5Common
     void SetAdiosStep(int ts);
 
     unsigned int GetNumAdiosSteps();
+    unsigned int GetAdiosStep() const;
     void WriteAdiosSteps();
 
     void ReadVariables(unsigned int ts, core::IO &io);
@@ -255,6 +259,11 @@ class HDF5Common
     hid_t m_ChunkPID;
     int m_ChunkDim;
     std::set m_ChunkVarNames;
+    bool m_OrderByC = true; // C or fortran
+
+    // Some write rank can be idle. This causes conflict with HDF5 collective
+    // requirement in functions Guard this by load vars in beginStep
+    bool m_IdleWriterOn = false;
 };
 
 // Explicit declaration of the public template methods
diff --git a/source/adios2/toolkit/interop/hdf5/HDF5Common.tcc b/source/adios2/toolkit/interop/hdf5/HDF5Common.tcc
index 2207681b7e..4275aa541e 100644
--- a/source/adios2/toolkit/interop/hdf5/HDF5Common.tcc
+++ b/source/adios2/toolkit/interop/hdf5/HDF5Common.tcc
@@ -95,6 +95,18 @@ void HDF5Common::GetHDF5SpaceSpec(const core::Variable &variable,
             offset.push_back(0);
         }
     }
+
+    if (dimSize <= 1)
+        return;
+    if (m_OrderByC)
+        return;
+
+    for (int i = 0; i < dimSize / 2; ++i)
+    {
+        std::swap(dimsf[i], dimsf[dimSize - 1 - i]);
+        std::swap(count[i], count[dimSize - 1 - i]);
+        std::swap(offset[i], offset[dimSize - 1 - i]);
+    }
 }
 
 template 
diff --git a/source/adios2/toolkit/query/JsonWorker.cpp b/source/adios2/toolkit/query/JsonWorker.cpp
index d2657382a0..10c4d0caec 100644
--- a/source/adios2/toolkit/query/JsonWorker.cpp
+++ b/source/adios2/toolkit/query/JsonWorker.cpp
@@ -70,7 +70,6 @@ void LoadVarQuery(QueryVar *q, nlohmann::json &varO)
         adios2::query::JsonUtil::ConstructTree(q->m_RangeTree, opO);
     }
 } // LoadVarQuery
-
 }
 }
 }
@@ -94,8 +93,8 @@ void JsonWorker::ParseJson()
             throw std::ios_base::failure("No var name specified!!");
         auto varName = (varO)["name"];
         adios2::core::IO &currIO = m_SourceReader->m_IO;
-        const std::string varType = currIO.InquireVariableType(varName);
-        if (varType.size() == 0)
+        const DataType varType = currIO.InquireVariableType(varName);
+        if (varType == DataType::None)
         {
             std::cerr << "No such variable: " << varName << std::endl;
             return nullptr;
@@ -167,6 +166,5 @@ void JsonWorker::ParseJson()
     m_Query = result;
     return;
 } // parse
-
 }
 }
diff --git a/source/adios2/toolkit/query/Query.cpp b/source/adios2/toolkit/query/Query.cpp
index 39a3b4584d..fb23d45994 100644
--- a/source/adios2/toolkit/query/Query.cpp
+++ b/source/adios2/toolkit/query/Query.cpp
@@ -186,14 +186,14 @@ void QueryVar::BlockIndexEvaluate(adios2::core::IO &io,
                                   adios2::core::Engine &reader,
                                   std::vector> &touchedBlocks)
 {
-    const std::string varType = io.InquireVariableType(m_VarName);
+    const DataType varType = io.InquireVariableType(m_VarName);
 
     // Variable var = io.InquireVariable(m_VarName);
     // BlockIndex idx(io, reader);
 
     // var already exists when loading query. skipping validity checking
 #define declare_type(T)                                                        \
-    if (varType == adios2::helper::GetType())                               \
+    if (varType == adios2::helper::GetDataType())                           \
     {                                                                          \
         core::Variable *var = io.InquireVariable(m_VarName);             \
         BlockIndex idx(*var, io, reader);                                   \
diff --git a/source/adios2/toolkit/query/Query.tcc b/source/adios2/toolkit/query/Query.tcc
index e00cd505b0..7ed046a420 100644
--- a/source/adios2/toolkit/query/Query.tcc
+++ b/source/adios2/toolkit/query/Query.tcc
@@ -69,6 +69,5 @@ bool RangeTree::CheckInterval(T &min, T &max) const
     // anything else are false
     return false;
 }
-
 }
 }
diff --git a/source/adios2/toolkit/query/Worker.cpp b/source/adios2/toolkit/query/Worker.cpp
index ccf387d2c4..defcb42da6 100644
--- a/source/adios2/toolkit/query/Worker.cpp
+++ b/source/adios2/toolkit/query/Worker.cpp
@@ -20,14 +20,14 @@ Worker::~Worker()
 QueryVar *Worker::GetBasicVarQuery(adios2::core::IO ¤tIO,
                                    const std::string &variableName)
 {
-    const std::string varType = currentIO.InquireVariableType(variableName);
-    if (varType.size() == 0)
+    const DataType varType = currentIO.InquireVariableType(variableName);
+    if (varType == DataType::None)
     {
         std::cerr << "No such variable: " << variableName << std::endl;
         return nullptr;
     }
 #define declare_type(T)                                                        \
-    if (varType == helper::GetType())                                       \
+    if (varType == helper::GetDataType())                                   \
     {                                                                          \
         core::Variable *var = currentIO.InquireVariable(variableName);   \
         if (var)                                                               \
diff --git a/source/adios2/toolkit/query/XmlWorker.cpp b/source/adios2/toolkit/query/XmlWorker.cpp
index 67ee2aa4b0..451de605a9 100644
--- a/source/adios2/toolkit/query/XmlWorker.cpp
+++ b/source/adios2/toolkit/query/XmlWorker.cpp
@@ -136,14 +136,14 @@ QueryVar *XmlWorker::ParseVarNode(const pugi::xml_node &node,
         adios2::helper::XMLAttribute("name", node, "in query")->value());
 
     // const std::string varType = currentIO.VariableType(variableName);
-    const std::string varType = currentIO.InquireVariableType(variableName);
-    if (varType.size() == 0)
+    const DataType varType = currentIO.InquireVariableType(variableName);
+    if (varType == DataType::None)
     {
         std::cerr << "No such variable: " << variableName << std::endl;
         return nullptr;
     }
 #define declare_type(T)                                                        \
-    if (varType == helper::GetType())                                       \
+    if (varType == helper::GetDataType())                                   \
     {                                                                          \
         core::Variable *var = currentIO.InquireVariable(variableName);   \
         if (var)                                                               \
diff --git a/source/adios2/toolkit/sst/CMakeLists.txt b/source/adios2/toolkit/sst/CMakeLists.txt
index 80474dd962..6f0551be5b 100644
--- a/source/adios2/toolkit/sst/CMakeLists.txt
+++ b/source/adios2/toolkit/sst/CMakeLists.txt
@@ -27,12 +27,6 @@ if(ADIOS2_SST_HAVE_LIBFABRIC)
   endif()
 endif()
 
-if(ADIOS2_SST_HAVE_NVStream)
-  target_sources(sst PRIVATE dp/nvstream_dp.c dp/nvswrapper.cpp)
-  target_link_libraries(sst PRIVATE NVStream::NVStream ${Boost_LIBRARIES})
-  set(CMAKE_REQUIRED_INCLUDES ${NVSTREAM_INCLUDE_DIRS})
-endif()
-
 if(ADIOS2_HAVE_ZFP)
   target_sources(sst PRIVATE cp/ffs_zfp.c)
   target_link_libraries(sst PRIVATE zfp::zfp)
diff --git a/source/adios2/toolkit/sst/cp/cp_common.c b/source/adios2/toolkit/sst/cp/cp_common.c
index 7fe5be1f1a..d8090e6c7d 100644
--- a/source/adios2/toolkit/sst/cp/cp_common.c
+++ b/source/adios2/toolkit/sst/cp/cp_common.c
@@ -39,8 +39,9 @@ void CP_validateParams(SstStream Stream, SstParams Params, int Writer)
                 "Invalid QueueLimit parameter value (%d) for SST Stream %s\n",
                 Params->QueueLimit, Stream->Filename);
     }
-    Stream->QueueFullPolicy = Params->QueueFullPolicy;
-    Stream->RegistrationMethod = Params->RegistrationMethod;
+    Stream->QueueFullPolicy = (SstQueueFullPolicy)Params->QueueFullPolicy;
+    Stream->RegistrationMethod =
+        (SstRegistrationMethod)Params->RegistrationMethod;
     if (Params->DataTransport != NULL)
     {
         int i;
@@ -50,6 +51,8 @@ void CP_validateParams(SstStream Stream, SstParams Params, int Writer)
             SelectedTransport[i] = tolower(Params->DataTransport[i]);
         }
         SelectedTransport[i] = 0;
+        /* free old */
+        free(Params->DataTransport);
 
         /* canonicalize SelectedTransport */
         if ((strcmp(SelectedTransport, "wan") == 0) ||
@@ -114,10 +117,12 @@ void CP_validateParams(SstStream Stream, SstParams Params, int Writer)
         {
             Stream->ConnectionUsleepMultiplier = tmp;
         }
-        CP_verbose(Stream, "USING %d as usleep multiplier before connections\n",
+        CP_verbose(Stream, PerStepVerbose,
+                   "USING %d as usleep multiplier before connections\n",
                    Stream->ConnectionUsleepMultiplier);
     }
-    CP_verbose(Stream, "Sst set to use %s as a Control Transport\n",
+    CP_verbose(Stream, PerStepVerbose,
+               "Sst set to use %s as a Control Transport\n",
                Params->ControlTransport);
     if (Params->ControlModule != NULL)
     {
@@ -150,6 +155,14 @@ void CP_validateParams(SstStream Stream, SstParams Params, int Writer)
     {
         Params->ControlModule = strdup("select");
     }
+    if (Params->verbose > Stream->CPVerbosityLevel)
+    {
+        Stream->CPVerbosityLevel = Params->verbose;
+    }
+    else if (Params->verbose < Stream->CPVerbosityLevel)
+    {
+        Params->verbose = Stream->CPVerbosityLevel;
+    }
 }
 
 static char *SstRegStr[] = {"File", "Screen", "Cloud"};
@@ -162,7 +175,7 @@ static char *SstPreloadModeStr[] = {"Off", "On", "Auto"};
 extern void CP_dumpParams(SstStream Stream, struct _SstParams *Params,
                           int ReaderSide)
 {
-    if (!Stream->CPVerbose)
+    if (Stream->CPVerbosityLevel < SummaryVerbose)
         return;
 
     fprintf(stderr, "Param -   RegistrationMethod=%s\n",
@@ -358,11 +371,11 @@ static FMField MetaDataPlusDPInfoList[] = {
 static FMField FFSFormatBlockList[] = {
     {"FormatServerRep", "char[FormatServerRepLen]", 1,
      FMOffset(struct FFSFormatBlock *, FormatServerRep)},
-    {"FormatServerRepLen", "integer", sizeof(int),
+    {"FormatServerRepLen", "integer", sizeof(size_t),
      FMOffset(struct FFSFormatBlock *, FormatServerRepLen)},
     {"FormatIDRep", "char[FormatIDRepLen]", 1,
      FMOffset(struct FFSFormatBlock *, FormatIDRep)},
-    {"FormatIDRepLen", "integer", sizeof(int),
+    {"FormatIDRepLen", "integer", sizeof(size_t),
      FMOffset(struct FFSFormatBlock *, FormatIDRepLen)},
     {"Next", "*FFSFormatBlock", sizeof(struct FFSFormatBlock),
      FMOffset(struct FFSFormatBlock *, Next)},
@@ -840,40 +853,29 @@ static void initAtomList()
     CM_ENET_CONN_TIMEOUT = attr_atom_from_string("CM_ENET_CONN_TIMEOUT");
 }
 
-static void AddCustomStruct(CP_GlobalInfo CPInfo, FMStructDescList Struct)
+static void AddCustomStruct(CP_StructList *List, FMStructDescList Struct)
 {
-    CPInfo->CustomStructCount++;
-    CPInfo->CustomStructList =
-        realloc(CPInfo->CustomStructList,
-                sizeof(FMStructDescList) * CPInfo->CustomStructCount);
-    CPInfo->CustomStructList[CPInfo->CustomStructCount - 1] = Struct;
+    List->CustomStructCount++;
+    List->CustomStructList =
+        realloc(List->CustomStructList,
+                sizeof(FMStructDescList) * List->CustomStructCount);
+    List->CustomStructList[List->CustomStructCount - 1] = Struct;
 }
 
-static void FreeCustomStructs(CP_GlobalInfo CPInfo)
+static void FreeCustomStructs(CP_StructList *List)
 {
-    for (int i = 0; i < CPInfo->CustomStructCount; i++)
+    for (int i = 0; i < List->CustomStructCount; i++)
     {
-        FMfree_struct_list(CPInfo->CustomStructList[i]);
+        FMfree_struct_list(List->CustomStructList[i]);
     }
-    free(CPInfo->CustomStructList);
+    free(List->CustomStructList);
 }
 
-static void doFormatRegistration(CP_GlobalInfo CPInfo, CP_DP_Interface DPInfo)
+static void doCMFormatRegistration(CP_GlobalCMInfo CPInfo,
+                                   CP_DP_Interface DPInfo)
 {
-    FMStructDescList PerRankReaderStructs, FullReaderRegisterStructs,
-        CombinedReaderStructs;
-    FMStructDescList PerRankWriterStructs, FullWriterResponseStructs,
-        CombinedWriterStructs;
-    FMStructDescList CombinedMetadataStructs, CombinedTimestepMetadataStructs;
-    FMFormat f;
-
-    PerRankReaderStructs = combineCpDpFormats(
-        CP_DP_PairStructs, CP_ReaderInitStructs, DPInfo->ReaderContactFormats);
-    f = FMregister_data_format(CPInfo->fm_c, PerRankReaderStructs);
-    CPInfo->PerRankReaderInfoFormat =
-        FFSTypeHandle_by_index(CPInfo->ffs_c, FMformat_index(f));
-    FFSset_fixed_target(CPInfo->ffs_c, PerRankReaderStructs);
-    AddCustomStruct(CPInfo, PerRankReaderStructs);
+    FMStructDescList FullReaderRegisterStructs, FullWriterResponseStructs,
+        CombinedTimestepMetadataStructs;
 
     FullReaderRegisterStructs =
         combineCpDpFormats(CP_ReaderRegisterStructs, CP_ReaderInitStructs,
@@ -882,7 +884,66 @@ static void doFormatRegistration(CP_GlobalInfo CPInfo, CP_DP_Interface DPInfo)
         CMregister_format(CPInfo->cm, FullReaderRegisterStructs);
     CMregister_handler(CPInfo->ReaderRegisterFormat, CP_ReaderRegisterHandler,
                        NULL);
-    AddCustomStruct(CPInfo, FullReaderRegisterStructs);
+    AddCustomStruct(&CPInfo->CustomStructs, FullReaderRegisterStructs);
+
+    FullWriterResponseStructs =
+        combineCpDpFormats(CP_WriterResponseStructs, CP_WriterInitStructs,
+                           DPInfo->WriterContactFormats);
+    CPInfo->WriterResponseFormat =
+        CMregister_format(CPInfo->cm, FullWriterResponseStructs);
+    CMregister_handler(CPInfo->WriterResponseFormat, CP_WriterResponseHandler,
+                       NULL);
+    AddCustomStruct(&CPInfo->CustomStructs, FullWriterResponseStructs);
+
+    CombinedTimestepMetadataStructs = combineCpDpFormats(
+        TimestepMetadataStructs, NULL, DPInfo->TimestepInfoFormats);
+    CPInfo->DeliverTimestepMetadataFormat =
+        CMregister_format(CPInfo->cm, CombinedTimestepMetadataStructs);
+    CMregister_handler(CPInfo->DeliverTimestepMetadataFormat,
+                       CP_TimestepMetadataHandler, NULL);
+    AddCustomStruct(&CPInfo->CustomStructs, CombinedTimestepMetadataStructs);
+
+    CPInfo->PeerSetupFormat = CMregister_format(CPInfo->cm, PeerSetupStructs);
+    CMregister_handler(CPInfo->PeerSetupFormat, CP_PeerSetupHandler, NULL);
+
+    CPInfo->ReaderActivateFormat =
+        CMregister_format(CPInfo->cm, ReaderActivateStructs);
+    CMregister_handler(CPInfo->ReaderActivateFormat, CP_ReaderActivateHandler,
+                       NULL);
+    CPInfo->ReleaseTimestepFormat =
+        CMregister_format(CPInfo->cm, ReleaseTimestepStructs);
+    CMregister_handler(CPInfo->ReleaseTimestepFormat, CP_ReleaseTimestepHandler,
+                       NULL);
+    CPInfo->LockReaderDefinitionsFormat =
+        CMregister_format(CPInfo->cm, LockReaderDefinitionsStructs);
+    CMregister_handler(CPInfo->LockReaderDefinitionsFormat,
+                       CP_LockReaderDefinitionsHandler, NULL);
+    CPInfo->CommPatternLockedFormat =
+        CMregister_format(CPInfo->cm, CommPatternLockedStructs);
+    CMregister_handler(CPInfo->CommPatternLockedFormat,
+                       CP_CommPatternLockedHandler, NULL);
+    CPInfo->WriterCloseFormat =
+        CMregister_format(CPInfo->cm, WriterCloseStructs);
+    CMregister_handler(CPInfo->WriterCloseFormat, CP_WriterCloseHandler, NULL);
+    CPInfo->ReaderCloseFormat =
+        CMregister_format(CPInfo->cm, ReaderCloseStructs);
+    CMregister_handler(CPInfo->ReaderCloseFormat, CP_ReaderCloseHandler, NULL);
+}
+
+static void doFFSFormatRegistration(CP_Info CPInfo, CP_DP_Interface DPInfo)
+{
+    FMStructDescList PerRankReaderStructs, CombinedReaderStructs;
+    FMStructDescList PerRankWriterStructs, CombinedWriterStructs;
+    FMStructDescList CombinedMetadataStructs;
+    FMFormat f;
+
+    PerRankReaderStructs = combineCpDpFormats(
+        CP_DP_PairStructs, CP_ReaderInitStructs, DPInfo->ReaderContactFormats);
+    f = FMregister_data_format(CPInfo->fm_c, PerRankReaderStructs);
+    CPInfo->PerRankReaderInfoFormat =
+        FFSTypeHandle_by_index(CPInfo->ffs_c, FMformat_index(f));
+    FFSset_fixed_target(CPInfo->ffs_c, PerRankReaderStructs);
+    AddCustomStruct(&CPInfo->CustomStructs, PerRankReaderStructs);
 
     CombinedReaderStructs =
         combineCpDpFormats(CP_DP_ReaderArrayStructs, CP_ReaderInitStructs,
@@ -891,7 +952,7 @@ static void doFormatRegistration(CP_GlobalInfo CPInfo, CP_DP_Interface DPInfo)
     CPInfo->CombinedReaderInfoFormat =
         FFSTypeHandle_by_index(CPInfo->ffs_c, FMformat_index(f));
     FFSset_fixed_target(CPInfo->ffs_c, CombinedReaderStructs);
-    AddCustomStruct(CPInfo, CombinedReaderStructs);
+    AddCustomStruct(&CPInfo->CustomStructs, CombinedReaderStructs);
 
     PerRankWriterStructs =
         combineCpDpFormats(CP_DP_WriterPairStructs, CP_WriterInitStructs,
@@ -900,16 +961,7 @@ static void doFormatRegistration(CP_GlobalInfo CPInfo, CP_DP_Interface DPInfo)
     CPInfo->PerRankWriterInfoFormat =
         FFSTypeHandle_by_index(CPInfo->ffs_c, FMformat_index(f));
     FFSset_fixed_target(CPInfo->ffs_c, PerRankWriterStructs);
-    AddCustomStruct(CPInfo, PerRankWriterStructs);
-
-    FullWriterResponseStructs =
-        combineCpDpFormats(CP_WriterResponseStructs, CP_WriterInitStructs,
-                           DPInfo->WriterContactFormats);
-    CPInfo->WriterResponseFormat =
-        CMregister_format(CPInfo->cm, FullWriterResponseStructs);
-    CMregister_handler(CPInfo->WriterResponseFormat, CP_WriterResponseHandler,
-                       NULL);
-    AddCustomStruct(CPInfo, FullWriterResponseStructs);
+    AddCustomStruct(&CPInfo->CustomStructs, PerRankWriterStructs);
 
     CombinedWriterStructs =
         combineCpDpFormats(CP_DP_WriterArrayStructs, CP_WriterInitStructs,
@@ -918,7 +970,7 @@ static void doFormatRegistration(CP_GlobalInfo CPInfo, CP_DP_Interface DPInfo)
     CPInfo->CombinedWriterInfoFormat =
         FFSTypeHandle_by_index(CPInfo->ffs_c, FMformat_index(f));
     FFSset_fixed_target(CPInfo->ffs_c, CombinedWriterStructs);
-    AddCustomStruct(CPInfo, CombinedWriterStructs);
+    AddCustomStruct(&CPInfo->CustomStructs, CombinedWriterStructs);
 
     CombinedMetadataStructs = combineCpDpFormats(
         MetaDataPlusDPInfoStructs, NULL, DPInfo->TimestepInfoFormats);
@@ -926,15 +978,7 @@ static void doFormatRegistration(CP_GlobalInfo CPInfo, CP_DP_Interface DPInfo)
     CPInfo->PerRankMetadataFormat =
         FFSTypeHandle_by_index(CPInfo->ffs_c, FMformat_index(f));
     FFSset_fixed_target(CPInfo->ffs_c, CombinedMetadataStructs);
-    AddCustomStruct(CPInfo, CombinedMetadataStructs);
-
-    CombinedTimestepMetadataStructs = combineCpDpFormats(
-        TimestepMetadataStructs, NULL, DPInfo->TimestepInfoFormats);
-    CPInfo->DeliverTimestepMetadataFormat =
-        CMregister_format(CPInfo->cm, CombinedTimestepMetadataStructs);
-    CMregister_handler(CPInfo->DeliverTimestepMetadataFormat,
-                       CP_TimestepMetadataHandler, NULL);
-    AddCustomStruct(CPInfo, CombinedTimestepMetadataStructs);
+    AddCustomStruct(&CPInfo->CustomStructs, CombinedMetadataStructs);
 
     CombinedMetadataStructs = combineCpDpFormats(
         TimestepMetadataDistributionStructs, NULL, DPInfo->TimestepInfoFormats);
@@ -942,7 +986,7 @@ static void doFormatRegistration(CP_GlobalInfo CPInfo, CP_DP_Interface DPInfo)
     CPInfo->TimestepDistributionFormat =
         FFSTypeHandle_by_index(CPInfo->ffs_c, FMformat_index(f));
     FFSset_fixed_target(CPInfo->ffs_c, CombinedMetadataStructs);
-    AddCustomStruct(CPInfo, CombinedMetadataStructs);
+    AddCustomStruct(&CPInfo->CustomStructs, CombinedMetadataStructs);
 
     CombinedMetadataStructs = combineCpDpFormats(
         ReturnMetadataInfoStructs, NULL, DPInfo->TimestepInfoFormats);
@@ -950,45 +994,111 @@ static void doFormatRegistration(CP_GlobalInfo CPInfo, CP_DP_Interface DPInfo)
     CPInfo->ReturnMetadataInfoFormat =
         FFSTypeHandle_by_index(CPInfo->ffs_c, FMformat_index(f));
     FFSset_fixed_target(CPInfo->ffs_c, CombinedMetadataStructs);
-    AddCustomStruct(CPInfo, CombinedMetadataStructs);
+    AddCustomStruct(&CPInfo->CustomStructs, CombinedMetadataStructs);
+}
 
-    CPInfo->PeerSetupFormat = CMregister_format(CPInfo->cm, PeerSetupStructs);
-    CMregister_handler(CPInfo->PeerSetupFormat, CP_PeerSetupHandler, NULL);
+static pthread_mutex_t StateMutex = PTHREAD_MUTEX_INITIALIZER;
+static CP_GlobalCMInfo SharedCMInfo = NULL;
+static int SharedCMInfoRefCount = 0;
 
-    CPInfo->ReaderActivateFormat =
-        CMregister_format(CPInfo->cm, ReaderActivateStructs);
-    CMregister_handler(CPInfo->ReaderActivateFormat, CP_ReaderActivateHandler,
-                       NULL);
-    CPInfo->ReleaseTimestepFormat =
-        CMregister_format(CPInfo->cm, ReleaseTimestepStructs);
-    CMregister_handler(CPInfo->ReleaseTimestepFormat, CP_ReleaseTimestepHandler,
-                       NULL);
-    CPInfo->LockReaderDefinitionsFormat =
-        CMregister_format(CPInfo->cm, LockReaderDefinitionsStructs);
-    CMregister_handler(CPInfo->LockReaderDefinitionsFormat,
-                       CP_LockReaderDefinitionsHandler, NULL);
-    CPInfo->CommPatternLockedFormat =
-        CMregister_format(CPInfo->cm, CommPatternLockedStructs);
-    CMregister_handler(CPInfo->CommPatternLockedFormat,
-                       CP_CommPatternLockedHandler, NULL);
-    CPInfo->WriterCloseFormat =
-        CMregister_format(CPInfo->cm, WriterCloseStructs);
-    CMregister_handler(CPInfo->WriterCloseFormat, CP_WriterCloseHandler, NULL);
-    CPInfo->ReaderCloseFormat =
-        CMregister_format(CPInfo->cm, ReaderCloseStructs);
-    CMregister_handler(CPInfo->ReaderCloseFormat, CP_ReaderCloseHandler, NULL);
+extern void AddToLastCallFreeList(void *Block)
+{
+    SharedCMInfo->LastCallFreeList =
+        realloc(SharedCMInfo->LastCallFreeList,
+                sizeof(void *) * (SharedCMInfo->LastCallFreeCount + 1));
+    SharedCMInfo->LastCallFreeList[SharedCMInfo->LastCallFreeCount] = Block;
+    SharedCMInfo->LastCallFreeCount++;
 }
 
-static CP_GlobalInfo CPInfo = NULL;
-static int CPInfoRefCount = 0;
+static void ReadableSizeString(size_t SizeInBytes, char *Output, size_t size)
+{
+    int i = 0;
+    size_t LastSizeInBytes = SizeInBytes;
+    char *byteUnits[] = {"bytes", "kB", "MB", "GB", "TB",
+                         "PB",    "EB", "ZB", "YB"};
 
-extern void AddToLastCallFreeList(void *Block)
+    while (SizeInBytes > 1024)
+    {
+        LastSizeInBytes = SizeInBytes;
+        SizeInBytes = SizeInBytes / 1024;
+        i++;
+    }
+
+    if ((SizeInBytes < 100) && (i != 0))
+    {
+        snprintf(Output, size, "%.1f %s", ((double)LastSizeInBytes) / 1024,
+                 byteUnits[i]);
+    }
+    else
+    {
+        snprintf(Output, size, "%ld %s", SizeInBytes, byteUnits[i]);
+    }
+};
+
+extern void DoStreamSummary(SstStream Stream)
 {
-    CPInfo->LastCallFreeList =
-        realloc(CPInfo->LastCallFreeList,
-                sizeof(void *) * (CPInfo->LastCallFreeCount + 1));
-    CPInfo->LastCallFreeList[CPInfo->LastCallFreeCount] = Block;
-    CPInfo->LastCallFreeCount++;
+    SstStats AllStats = NULL;
+
+    if (Stream->Rank == 0)
+        AllStats = malloc(sizeof(struct _SstStats) * Stream->CohortSize);
+
+    SMPI_Gather(&Stream->Stats, sizeof(struct _SstStats), SMPI_CHAR, AllStats,
+                sizeof(struct _SstStats), SMPI_CHAR, 0, Stream->mpiComm);
+
+    if (Stream->Rank != 0)
+    {
+        return;
+    }
+
+    for (int i = 1; i < Stream->CohortSize; i++)
+    {
+        AllStats[0].MetadataBytesReceived += AllStats[i].MetadataBytesReceived;
+        AllStats[0].DataBytesReceived += AllStats[i].DataBytesReceived;
+        AllStats[0].PreloadBytesReceived += AllStats[i].PreloadBytesReceived;
+        AllStats[0].RunningFanIn += AllStats[i].RunningFanIn;
+    }
+    AllStats[0].RunningFanIn /= Stream->CohortSize;
+
+    CP_verbose(Stream, SummaryVerbose, "\nStream \"%s\" (%p) summary info:\n",
+               Stream->Filename, (void *)Stream);
+    CP_verbose(Stream, SummaryVerbose, "\tDuration (secs) = %g\n",
+               Stream->Stats.StreamValidTimeSecs);
+    if (Stream->Role == WriterRole)
+    {
+        CP_verbose(Stream, SummaryVerbose, "\tTimesteps Created = %zu\n",
+                   Stream->Stats.TimestepsCreated);
+        CP_verbose(Stream, SummaryVerbose, "\tTimesteps Delivered = %zu\n",
+                   Stream->Stats.TimestepsDelivered);
+    }
+    else if (Stream->Role == ReaderRole)
+    {
+        char OutputString[256];
+        CP_verbose(Stream, SummaryVerbose,
+                   "\tTimestep Metadata Received = %zu\n",
+                   Stream->Stats.TimestepMetadataReceived);
+        CP_verbose(Stream, SummaryVerbose, "\tTimesteps Consumed = %zu\n",
+                   Stream->Stats.TimestepsConsumed);
+        ReadableSizeString(AllStats[0].MetadataBytesReceived, OutputString,
+                           sizeof(OutputString));
+        CP_verbose(Stream, SummaryVerbose,
+                   "\tMetadataBytesReceived = %zu (%s)\n",
+                   AllStats[0].MetadataBytesReceived, OutputString);
+        ReadableSizeString(AllStats[0].DataBytesReceived, OutputString,
+                           sizeof(OutputString));
+        CP_verbose(Stream, SummaryVerbose, "\tDataBytesReceived = %zu (%s)\n",
+                   AllStats[0].DataBytesReceived, OutputString);
+        ReadableSizeString(AllStats[0].PreloadBytesReceived, OutputString,
+                           sizeof(OutputString));
+        CP_verbose(Stream, SummaryVerbose,
+                   "\tPreloadBytesReceived = %zu (%s)\n",
+                   AllStats[0].PreloadBytesReceived, OutputString);
+        CP_verbose(Stream, SummaryVerbose, "\tPreloadTimestepsReceived = %zu\n",
+                   Stream->Stats.PreloadTimestepsReceived);
+        CP_verbose(Stream, SummaryVerbose, "\tAverageReadRankFanIn = %.1f\n",
+                   AllStats[0].RunningFanIn);
+    }
+    CP_verbose(Stream, SummaryVerbose, "\n");
+    free(AllStats);
 }
 
 extern void SstStreamDestroy(SstStream Stream)
@@ -999,8 +1109,8 @@ extern void SstStreamDestroy(SstStream Stream)
      */
     struct _SstStream StackStream;
     pthread_mutex_lock(&Stream->DataLock);
-    CP_verbose(Stream, "Destroying stream %p, name %s\n", Stream,
-               Stream->Filename);
+    CP_verbose(Stream, PerStepVerbose, "Destroying stream %p, name %s\n",
+               Stream, Stream->Filename);
     StackStream = *Stream;
     Stream->Status = Destroyed;
     struct _TimestepMetadataList *Next = Stream->Timesteps;
@@ -1102,6 +1212,8 @@ extern void SstStreamDestroy(SstStream Stream)
             Stream->ConnectionsToWriter = NULL;
         }
         free(Stream->Peers);
+        if (Stream->RanksRead)
+            free(Stream->RanksRead);
     }
     else if (Stream->ConfigParams->MarshalMethod == SstMarshalFFS)
     {
@@ -1109,7 +1221,9 @@ extern void SstStreamDestroy(SstStream Stream)
     }
     if (Stream->ConfigParams->DataTransport)
         free(Stream->ConfigParams->DataTransport);
-    if (Stream->ConfigParams->DataTransport)
+    if (Stream->ConfigParams->WANDataTransport)
+        free(Stream->ConfigParams->WANDataTransport);
+    if (Stream->ConfigParams->ControlTransport)
         free(Stream->ConfigParams->ControlTransport);
     if (Stream->ConfigParams->NetworkInterface)
         free(Stream->ConfigParams->NetworkInterface);
@@ -1136,34 +1250,40 @@ extern void SstStreamDestroy(SstStream Stream)
         free(Stream->ParamsBlock);
         Stream->ParamsBlock = NULL;
     }
+    if (Stream->CPInfo->ffs_c)
+        free_FFSContext(Stream->CPInfo->ffs_c);
+    if (Stream->CPInfo->fm_c)
+        free_FMcontext(Stream->CPInfo->fm_c);
+    FreeCustomStructs(&Stream->CPInfo->CustomStructs);
+    free(Stream->CPInfo);
+
     pthread_mutex_unlock(&Stream->DataLock);
     //   Stream is free'd in LastCall
 
-    CPInfoRefCount--;
-    if (CPInfoRefCount == 0)
+    pthread_mutex_lock(&StateMutex);
+    SharedCMInfoRefCount--;
+    if (SharedCMInfoRefCount == 0)
     {
         CP_verbose(
-            Stream,
+            Stream, PerStepVerbose,
             "Reference count now zero, Destroying process SST info cache\n");
-        CManager_close(CPInfo->cm);
-        if (CPInfo->ffs_c)
-            free_FFSContext(CPInfo->ffs_c);
-        if (CPInfo->fm_c)
-            free_FMcontext(CPInfo->fm_c);
-        FreeCustomStructs(CPInfo);
-        CP_verbose(Stream, "Freeing LastCallList\n");
-        for (int i = 0; i < CPInfo->LastCallFreeCount; i++)
+        CManager_close(SharedCMInfo->cm);
+        FreeCustomStructs(&SharedCMInfo->CustomStructs);
+        CP_verbose(Stream, PerStepVerbose, "Freeing LastCallList\n");
+        for (int i = 0; i < SharedCMInfo->LastCallFreeCount; i++)
         {
-            free(CPInfo->LastCallFreeList[i]);
+            free(SharedCMInfo->LastCallFreeList[i]);
         }
-        free(CPInfo->LastCallFreeList);
-        free(CPInfo);
-        CPInfo = NULL;
+        free(SharedCMInfo->LastCallFreeList);
+        free(SharedCMInfo);
+        SharedCMInfo = NULL;
         if (CP_SstParamsList)
             free_FMfield_list(CP_SstParamsList);
         CP_SstParamsList = NULL;
     }
-    CP_verbose(&StackStream, "SstStreamDestroy successful, returning\n");
+    pthread_mutex_unlock(&StateMutex);
+    CP_verbose(&StackStream, PerStepVerbose,
+               "SstStreamDestroy successful, returning\n");
 }
 
 extern char *CP_GetContactString(SstStream Stream, attr_list DPAttrs)
@@ -1181,7 +1301,10 @@ extern char *CP_GetContactString(SstStream Stream, attr_list DPAttrs)
         set_string_attr(ListenList, IP_INTERFACE_ATOM,
                         strdup(Stream->ConfigParams->NetworkInterface));
     }
-    ContactList = CMget_specific_contact_list(Stream->CPInfo->cm, ListenList);
+    ContactList =
+        CMget_specific_contact_list(Stream->CPInfo->SharedCM->cm, ListenList);
+    ContactList =
+        CMderef_and_copy_list(Stream->CPInfo->SharedCM->cm, ContactList);
     if (strcmp(Stream->ConfigParams->ControlTransport, "enet") == 0)
     {
         set_int_attr(ContactList, CM_ENET_CONN_TIMEOUT, 60000); /* 60 seconds */
@@ -1196,92 +1319,116 @@ extern char *CP_GetContactString(SstStream Stream, attr_list DPAttrs)
     return ret;
 }
 
-extern CP_GlobalInfo CP_getCPInfo(CP_DP_Interface DPInfo, char *ControlModule)
+static void CP_versionError(CMConnection conn, char *formatName)
 {
+    fprintf(stderr,
+            " * An invalid message of type \"%s\" has been received on an "
+            "incoming connection.\n",
+            formatName);
+    fprintf(stderr, " * In ADIOS2/SST this likely means a version mismatch "
+                    "between stream participants.\n");
+    fprintf(stderr, " * Please ensure that all writers and readers are built "
+                    "with the same version of ADIOS2.\n");
+}
 
-    if (CPInfo)
-    {
-        CPInfoRefCount++;
-        return CPInfo;
-    }
+extern CP_Info CP_getCPInfo(CP_DP_Interface DPInfo, char *ControlModule)
+{
+    CP_Info StreamCP;
 
-    initAtomList();
+    pthread_mutex_lock(&StateMutex);
+    if (!SharedCMInfo)
+    {
 
-    CPInfo = malloc(sizeof(*CPInfo));
-    memset(CPInfo, 0, sizeof(*CPInfo));
+        initAtomList();
 
-    CPInfo->cm = CManager_create_control(ControlModule);
-    if (CMfork_comm_thread(CPInfo->cm) == 0)
-    {
-        fprintf(stderr, "ADIOS2 SST Engine failed to fork a communication "
-                        "thread.\nThis is a fatal condition, please check "
-                        "resources or system settings.\nDying now.\n");
-        exit(1);
-    }
+        SharedCMInfo = malloc(sizeof(*SharedCMInfo));
+        memset(SharedCMInfo, 0, sizeof(*SharedCMInfo));
 
-    if (globalNetinfoCallback)
-    {
-        IPDiagString = CMget_ip_config_diagnostics(CPInfo->cm);
-    }
+        SharedCMInfo->cm = CManager_create_control(ControlModule);
+        if (CMfork_comm_thread(SharedCMInfo->cm) == 0)
+        {
+            fprintf(stderr, "ADIOS2 SST Engine failed to fork a communication "
+                            "thread.\nThis is a fatal condition, please check "
+                            "resources or system settings.\nDying now.\n");
+            exit(1);
+        }
 
-    CMlisten(CPInfo->cm);
+        if (globalNetinfoCallback)
+        {
+            IPDiagString = CMget_ip_config_diagnostics(SharedCMInfo->cm);
+        }
 
-    CPInfo->fm_c = create_local_FMcontext();
-    CPInfo->ffs_c = create_FFSContext_FM(CPInfo->fm_c);
+        CMlisten(SharedCMInfo->cm);
+        CMregister_invalid_message_handler(SharedCMInfo->cm, CP_versionError);
 
-    if (!CP_SstParamsList)
-    {
-        int i = 0;
-        /* need to pre-process the CP_SstParamsList to fix typedecl values */
-        CP_SstParamsList = copy_field_list(CP_SstParamsList_RAW);
-        while (CP_SstParamsList[i].field_name)
+        if (!CP_SstParamsList)
         {
-            if ((strcmp(CP_SstParamsList[i].field_type, "int") == 0) ||
-                (strcmp(CP_SstParamsList[i].field_type, "size_t") == 0))
+            int i = 0;
+            /* need to pre-process the CP_SstParamsList to fix typedecl values
+             */
+            CP_SstParamsList = copy_field_list(CP_SstParamsList_RAW);
+            while (CP_SstParamsList[i].field_name)
             {
-                free((void *)CP_SstParamsList[i].field_type);
-                CP_SstParamsList[i].field_type = strdup("integer");
-            }
-            else if ((strcmp(CP_SstParamsList[i].field_type, "char*") == 0) ||
-                     (strcmp(CP_SstParamsList[i].field_type, "char *") == 0))
-            {
-                free((void *)CP_SstParamsList[i].field_type);
-                CP_SstParamsList[i].field_type = strdup("string");
+                if ((strcmp(CP_SstParamsList[i].field_type, "int") == 0) ||
+                    (strcmp(CP_SstParamsList[i].field_type, "size_t") == 0))
+                {
+                    free((void *)CP_SstParamsList[i].field_type);
+                    CP_SstParamsList[i].field_type = strdup("integer");
+                }
+                else if ((strcmp(CP_SstParamsList[i].field_type, "char*") ==
+                          0) ||
+                         (strcmp(CP_SstParamsList[i].field_type, "char *") ==
+                          0))
+                {
+                    free((void *)CP_SstParamsList[i].field_type);
+                    CP_SstParamsList[i].field_type = strdup("string");
+                }
+                i++;
             }
-            i++;
         }
-    }
-    for (int i = 0; i < sizeof(CP_DP_WriterArrayStructs) /
-                            sizeof(CP_DP_WriterArrayStructs[0]);
-         i++)
-    {
-        if (CP_DP_WriterArrayStructs[i].format_name &&
-            (strcmp(CP_DP_WriterArrayStructs[i].format_name, "SstParams") == 0))
+        for (int i = 0; i < sizeof(CP_DP_WriterArrayStructs) /
+                                sizeof(CP_DP_WriterArrayStructs[0]);
+             i++)
         {
-            CP_DP_WriterArrayStructs[i].field_list = CP_SstParamsList;
+            if (CP_DP_WriterArrayStructs[i].format_name &&
+                (strcmp(CP_DP_WriterArrayStructs[i].format_name, "SstParams") ==
+                 0))
+            {
+                CP_DP_WriterArrayStructs[i].field_list = CP_SstParamsList;
+            }
         }
-    }
 
-    for (int i = 0; i < sizeof(CP_WriterResponseStructs) /
-                            sizeof(CP_WriterResponseStructs[0]);
-         i++)
-    {
-        if (CP_WriterResponseStructs[i].format_name &&
-            (strcmp(CP_WriterResponseStructs[i].format_name, "SstParams") == 0))
+        for (int i = 0; i < sizeof(CP_WriterResponseStructs) /
+                                sizeof(CP_WriterResponseStructs[0]);
+             i++)
         {
-            CP_WriterResponseStructs[i].field_list = CP_SstParamsList;
+            if (CP_WriterResponseStructs[i].format_name &&
+                (strcmp(CP_WriterResponseStructs[i].format_name, "SstParams") ==
+                 0))
+            {
+                CP_WriterResponseStructs[i].field_list = CP_SstParamsList;
+            }
         }
+        doCMFormatRegistration(SharedCMInfo, DPInfo);
     }
+    SharedCMInfoRefCount++;
+    pthread_mutex_unlock(&StateMutex);
 
-    doFormatRegistration(CPInfo, DPInfo);
+    StreamCP = calloc(1, sizeof(*StreamCP));
+    StreamCP->SharedCM = SharedCMInfo;
+    StreamCP->fm_c = create_local_FMcontext();
+    StreamCP->ffs_c = create_FFSContext_FM(StreamCP->fm_c);
 
-    CPInfoRefCount++;
-    return CPInfo;
+    doFFSFormatRegistration(StreamCP, DPInfo);
+
+    return StreamCP;
 }
 
 SstStream CP_newStream()
 {
     SstStream Stream = malloc(sizeof(*Stream));
+    char *CPEnvValue = NULL;
+    char *DPEnvValue = NULL;
     memset(Stream, 0, sizeof(*Stream));
     pthread_mutex_init(&Stream->DataLock, NULL);
     pthread_cond_init(&Stream->DataCondition, NULL);
@@ -1290,21 +1437,28 @@ SstStream CP_newStream()
     Stream->LastReleasedTimestep = -1;
     Stream->DiscardPriorTimestep =
         -1; // Timesteps prior to this discarded/released upon arrival
-    Stream->CPVerbose = 0;
-    Stream->DPVerbose = 0;
-    if (getenv("SstVerbose"))
+    Stream->CPVerbosityLevel = CriticalVerbose;
+    Stream->DPVerbosityLevel = CriticalVerbose;
+    if ((CPEnvValue = getenv("SstVerbose")))
+    {
+        DPEnvValue = CPEnvValue;
+    }
+    else
+    {
+        CPEnvValue = getenv("SstCPVerbose");
+    }
+    if (CPEnvValue)
     {
-        Stream->CPVerbose = 1;
-        Stream->DPVerbose = 1;
+        sscanf(CPEnvValue, "%d", &Stream->CPVerbosityLevel);
     }
-    if (getenv("SstCPVerbose"))
+    if (DPEnvValue)
     {
-        Stream->CPVerbose = 1;
+        sscanf(DPEnvValue, "%d", &Stream->DPVerbosityLevel);
     }
     return Stream;
 }
 
-static void DP_verbose(SstStream Stream, char *Format, ...);
+static void DP_verbose(SstStream Stream, int Level, char *Format, ...);
 static CManager CP_getCManager(SstStream Stream);
 static int CP_sendToPeer(SstStream Stream, CP_PeerCohort cohort, int rank,
                          CMFormat Format, void *data);
@@ -1405,44 +1559,61 @@ extern void getPeerArrays(int MySize, int MyRank, int PeerSize,
     }
 }
 
-extern void SstSetStatsSave(SstStream Stream, SstStats Stats)
+static void DP_verbose(SstStream s, int Level, char *Format, ...)
 {
-    Stats->OpenTimeSecs = Stream->OpenTimeSecs;
-    Stream->Stats = Stats;
-}
-
-static void DP_verbose(SstStream s, char *Format, ...)
-{
-    if (s->DPVerbose)
+    if (s->DPVerbosityLevel >= Level)
     {
         va_list Args;
         va_start(Args, Format);
+        char *Role = "Writer";
         if (s->Role == ReaderRole)
         {
-            fprintf(stderr, "DP Reader %d (%p): ", s->Rank, s);
+            Role = "Reader";
         }
-        else
+        switch (s->CPVerbosityLevel)
         {
-            fprintf(stderr, "DP Writer %d (%p): ", s->Rank, s);
+        case TraceVerbose:
+        case PerRankVerbose:
+        case CriticalVerbose:
+            fprintf(stderr, "DP %s %d (%p): ", Role, s->Rank, s);
+            break;
+        case PerStepVerbose:
+            fprintf(stderr, "DP %s (%p): ", Role, s);
+            break;
+        case SummaryVerbose:
+        default:
+            break;
         }
         vfprintf(stderr, Format, Args);
         va_end(Args);
     }
 }
 
-extern void CP_verbose(SstStream s, char *Format, ...)
+extern void CP_verbose(SstStream s, enum VerbosityLevel Level, char *Format,
+                       ...)
 {
-    if (s->CPVerbose)
+    if (s->CPVerbosityLevel >= (int)Level)
     {
         va_list Args;
         va_start(Args, Format);
+        char *Role = "Writer";
         if (s->Role == ReaderRole)
         {
-            fprintf(stderr, "Reader %d (%p): ", s->Rank, s);
+            Role = "Reader";
         }
-        else
+        switch (s->CPVerbosityLevel)
         {
-            fprintf(stderr, "Writer %d (%p): ", s->Rank, s);
+        case TraceVerbose:
+        case PerRankVerbose:
+        case CriticalVerbose:
+            fprintf(stderr, "%s %d (%p): ", Role, s->Rank, s);
+            break;
+        case PerStepVerbose:
+            fprintf(stderr, "%s (%p): ", Role, s);
+            break;
+        case SummaryVerbose:
+        default:
+            break;
         }
         vfprintf(stderr, Format, Args);
         va_end(Args);
@@ -1465,7 +1636,10 @@ extern void CP_error(SstStream s, char *Format, ...)
     va_end(Args);
 }
 
-static CManager CP_getCManager(SstStream Stream) { return Stream->CPInfo->cm; }
+static CManager CP_getCManager(SstStream Stream)
+{
+    return Stream->CPInfo->SharedCM->cm;
+}
 
 static SMPI_Comm CP_getMPIComm(SstStream Stream) { return Stream->mpiComm; }
 
@@ -1479,7 +1653,8 @@ static int CP_sendToPeer(SstStream s, CP_PeerCohort Cohort, int Rank,
     CP_PeerConnection *Peers = (CP_PeerConnection *)Cohort;
     if (Peers[Rank].CMconn == NULL)
     {
-        Peers[Rank].CMconn = CMget_conn(s->CPInfo->cm, Peers[Rank].ContactList);
+        Peers[Rank].CMconn =
+            CMget_conn(s->CPInfo->SharedCM->cm, Peers[Rank].ContactList);
         if (!Peers[Rank].CMconn)
         {
             CP_error(s,
@@ -1490,7 +1665,7 @@ static int CP_sendToPeer(SstStream s, CP_PeerCohort Cohort, int Rank,
         if (s->Role == ReaderRole)
         {
             CP_verbose(
-                s,
+                s, TraceVerbose,
                 "Registering reader close handler for peer %d CONNECTION %p\n",
                 Rank, Peers[Rank].CMconn);
             CMconn_register_close_handler(Peers[Rank].CMconn,
@@ -1502,7 +1677,7 @@ static int CP_sendToPeer(SstStream s, CP_PeerCohort Cohort, int Rank,
             {
                 if (Peers == s->Readers[i]->Connections)
                 {
-                    CP_verbose(s,
+                    CP_verbose(s, TraceVerbose,
                                "Registering writer close handler for peer %d, "
                                "CONNECTION %p\n",
                                Rank, Peers[Rank].CMconn);
@@ -1516,7 +1691,7 @@ static int CP_sendToPeer(SstStream s, CP_PeerCohort Cohort, int Rank,
     }
     if (CMwrite(Peers[Rank].CMconn, Format, Data) != 1)
     {
-        CP_verbose(s,
+        CP_verbose(s, CriticalVerbose,
                    "Message failed to send to peer %d CONNECTION %p in "
                    "CP_sendToPeer()\n",
                    Rank, Peers[Rank].CMconn);
diff --git a/source/adios2/toolkit/sst/cp/cp_internal.h b/source/adios2/toolkit/sst/cp/cp_internal.h
index c69b99324e..d8d2e9d9ab 100644
--- a/source/adios2/toolkit/sst/cp/cp_internal.h
+++ b/source/adios2/toolkit/sst/cp/cp_internal.h
@@ -3,21 +3,18 @@
 
 #define SSTMAGICV0 "#ADIOS2-SST v0\n"
 
-typedef struct _CP_GlobalInfo
+typedef struct StructList
+{
+    int CustomStructCount;
+    FMStructDescList *CustomStructList;
+} CP_StructList;
+
+typedef struct _CP_GlobalCMInfo
 {
     /* exchange info */
     CManager cm;
-    FFSContext ffs_c;
-    FMContext fm_c;
-    FFSTypeHandle PerRankReaderInfoFormat;
-    FFSTypeHandle CombinedReaderInfoFormat;
     CMFormat ReaderRegisterFormat;
-    FFSTypeHandle PerRankWriterInfoFormat;
-    FFSTypeHandle CombinedWriterInfoFormat;
     CMFormat WriterResponseFormat;
-    FFSTypeHandle PerRankMetadataFormat;
-    FFSTypeHandle TimestepDistributionFormat;
-    FFSTypeHandle ReturnMetadataInfoFormat;
     CMFormat DeliverTimestepMetadataFormat;
     CMFormat PeerSetupFormat;
     CMFormat ReaderActivateFormat;
@@ -26,11 +23,25 @@ typedef struct _CP_GlobalInfo
     CMFormat CommPatternLockedFormat;
     CMFormat WriterCloseFormat;
     CMFormat ReaderCloseFormat;
-    int CustomStructCount;
-    FMStructDescList *CustomStructList;
     int LastCallFreeCount;
     void **LastCallFreeList;
-} * CP_GlobalInfo;
+    struct StructList CustomStructs;
+} * CP_GlobalCMInfo;
+
+typedef struct _CP_Info
+{
+    CP_GlobalCMInfo SharedCM;
+    FFSContext ffs_c;
+    FMContext fm_c;
+    FFSTypeHandle PerRankReaderInfoFormat;
+    FFSTypeHandle CombinedReaderInfoFormat;
+    FFSTypeHandle PerRankWriterInfoFormat;
+    FFSTypeHandle CombinedWriterInfoFormat;
+    FFSTypeHandle PerRankMetadataFormat;
+    FFSTypeHandle TimestepDistributionFormat;
+    FFSTypeHandle ReturnMetadataInfoFormat;
+    struct StructList CustomStructs;
+} * CP_Info;
 
 struct _ReaderRegisterMsg;
 
@@ -105,6 +116,7 @@ typedef struct _CPTimestepEntry
     long Timestep;
     struct _SstData Data;
     struct _TimestepMetadataMsg *Msg;
+    int MetaDataSendCount;
     int ReferenceCount;
     int Expired;
     int PreciousTimestep;
@@ -121,7 +133,7 @@ typedef struct FFSFormatBlock *FFSFormatList;
 
 struct _SstStream
 {
-    CP_GlobalInfo CPInfo;
+    CP_Info CPInfo;
 
     SMPI_Comm mpiComm;
     enum StreamRole Role;
@@ -131,11 +143,12 @@ struct _SstStream
     SstRegistrationMethod RegistrationMethod;
 
     /* state */
-    int CPVerbose;
-    int DPVerbose;
+    int CPVerbosityLevel;
+    int DPVerbosityLevel;
     double OpenTimeSecs;
     struct timeval ValidStartTime;
-    SstStats Stats;
+    struct _SstStats Stats;
+    char *RanksRead;
 
     /* MPI info */
     int Rank;
@@ -247,9 +260,9 @@ struct _CP_DP_PairInfo
 struct FFSFormatBlock
 {
     char *FormatServerRep;
-    int FormatServerRepLen;
+    size_t FormatServerRepLen;
     char *FormatIDRep;
-    int FormatIDRepLen;
+    size_t FormatIDRepLen;
     struct FFSFormatBlock *Next;
 };
 
@@ -462,7 +475,7 @@ typedef struct _MetadataPlusDPInfo *MetadataPlusDPInfo;
 extern atom_t CM_TRANSPORT_ATOM;
 
 void CP_validateParams(SstStream stream, SstParams Params, int Writer);
-extern CP_GlobalInfo CP_getCPInfo(CP_DP_Interface DPInfo, char *ControlModule);
+extern CP_Info CP_getCPInfo(CP_DP_Interface DPInfo, char *ControlModule);
 extern char *CP_GetContactString(SstStream s, attr_list DPAttrs);
 extern SstStream CP_newStream();
 extern void SstInternalProvideTimestep(
@@ -513,7 +526,23 @@ extern void FFSFreeMarshalData(SstStream Stream);
 extern void getPeerArrays(int MySize, int MyRank, int PeerSize,
                           int **forwardArray, int **reverseArray);
 extern void AddToLastCallFreeList(void *Block);
-extern void CP_verbose(SstStream Stream, char *Format, ...);
+
+enum VerbosityLevel
+{
+    NoVerbose = 0,       // Generally no output (but not absolutely quiet?)
+    CriticalVerbose = 1, // Informational output for failures only
+    SummaryVerbose =
+        2, // One-time summary output containing general info (transports used,
+           // timestep count, stream duration, etc.)
+    PerStepVerbose = 3, // One-per-step info, generally from rank 0 (metadata
+                        // read, Begin/EndStep verbosity, etc.)
+    PerRankVerbose = 4, // Per-step info from each rank (for those things that
+                        // might be different per rank).
+    TraceVerbose = 5,   // All debugging available
+};
+
+extern void CP_verbose(SstStream Stream, enum VerbosityLevel Level,
+                       char *Format, ...);
 extern void CP_error(SstStream Stream, char *Format, ...);
 extern struct _CP_Services Svcs;
 extern void CP_dumpParams(SstStream Stream, struct _SstParams *Params,
@@ -524,3 +553,4 @@ typedef void (*CPNetworkInfoFunc)(int dataID, const char *net_string,
 extern char *IPDiagString;
 extern CPNetworkInfoFunc globalNetinfoCallback;
 extern void SSTSetNetworkCallback(CPNetworkInfoFunc callback);
+extern void DoStreamSummary(SstStream Stream);
diff --git a/source/adios2/toolkit/sst/cp/cp_reader.c b/source/adios2/toolkit/sst/cp/cp_reader.c
index 53869154b8..3d0cd95436 100644
--- a/source/adios2/toolkit/sst/cp/cp_reader.c
+++ b/source/adios2/toolkit/sst/cp/cp_reader.c
@@ -98,7 +98,7 @@ static char *readContactInfoFile(const char *Name, SstStream Stream,
     int64_t WaitWarningRemaining = 5 * 1000 * 1000;
     long SleepInterval = 100000;
     snprintf(FileName, len, "%s" SST_POSTFIX, Name);
-    CP_verbose(Stream,
+    CP_verbose(Stream, PerRankVerbose,
                "Looking for writer contact in file %s, with timeout %d secs\n",
                FileName, Timeout);
 redo:
@@ -206,14 +206,11 @@ static char *readContactInfo(const char *Name, SstStream Stream, int Timeout)
     {
     case SstRegisterFile:
         return readContactInfoFile(Name, Stream, Timeout);
-        break;
     case SstRegisterScreen:
         return readContactInfoScreen(Name, Stream);
-        break;
     case SstRegisterCloud:
         /* not yet */
         return NULL;
-        break;
     }
     return NULL;
 }
@@ -227,7 +224,7 @@ extern void ReaderConnCloseHandler(CManager cm, CMConnection ClosedConn,
     SstStream Stream = (SstStream)client_data;
     int FailedPeerRank = -1;
     STREAM_MUTEX_LOCK(Stream);
-    CP_verbose(Stream, "Reader-side close handler invoked\n");
+    CP_verbose(Stream, PerRankVerbose, "Reader-side close handler invoked\n");
     if ((Stream->Status == Destroyed) || (!Stream->ConnectionsToWriter))
     {
         STREAM_MUTEX_UNLOCK(Stream);
@@ -246,10 +243,11 @@ extern void ReaderConnCloseHandler(CManager cm, CMConnection ClosedConn,
         if ((Stream->WriterConfigParams->CPCommPattern == SstCPCommMin) &&
             (Stream->Rank != 0))
         {
-            CP_verbose(Stream, "Reader-side Rank received a "
-                               "connection-close event during normal "
-                               "operations, but might be part of shutdown  "
-                               "Don't change stream status.\n");
+            CP_verbose(Stream, PerRankVerbose,
+                       "Reader-side Rank received a "
+                       "connection-close event during normal "
+                       "operations, but might be part of shutdown  "
+                       "Don't change stream status.\n");
             /* if this happens and *is* a failure, we'll get the status from
              * rank 0 later */
         }
@@ -259,9 +257,10 @@ extern void ReaderConnCloseHandler(CManager cm, CMConnection ClosedConn,
              * tag our reader instance as failed, IFF this came from someone we
              * should have gotten a CLOSE from. I.E. a reverse peer
              */
-            CP_verbose(Stream, "Reader-side Rank received a "
-                               "connection-close event during normal "
-                               "operations, peer likely failed\n");
+            CP_verbose(Stream, PerRankVerbose,
+                       "Reader-side Rank received a "
+                       "connection-close event during normal "
+                       "operations, peer likely failed\n");
             if (FailedPeerRank == Stream->FailureContactRank)
             {
                 Stream->Status = PeerFailed;
@@ -269,7 +268,7 @@ extern void ReaderConnCloseHandler(CManager cm, CMConnection ClosedConn,
             }
         }
         CP_verbose(
-            Stream,
+            Stream, PerRankVerbose,
             "The close was for connection to writer peer %d, notifying DP\n",
             FailedPeerRank);
         STREAM_MUTEX_UNLOCK(Stream);
@@ -282,9 +281,10 @@ extern void ReaderConnCloseHandler(CManager cm, CMConnection ClosedConn,
     {
         /* ignore this.  We expect a close after the connection is marked closed
          */
-        CP_verbose(Stream, "Reader-side Rank received a "
-                           "connection-close event after close, "
-                           "not unexpected\n");
+        CP_verbose(Stream, PerRankVerbose,
+                   "Reader-side Rank received a "
+                   "connection-close event after close, "
+                   "not unexpected\n");
         STREAM_MUTEX_UNLOCK(Stream);
         // Don't notify DP, because this is part of normal shutdown and we don't
         // want to kill pending reads
@@ -292,7 +292,7 @@ extern void ReaderConnCloseHandler(CManager cm, CMConnection ClosedConn,
     else if (Stream->Status == PeerFailed)
     {
         CP_verbose(
-            Stream,
+            Stream, PerRankVerbose,
             "Reader-side Rank received a "
             "connection-close event after PeerFailed, already notified DP \n");
         // Don't notify DP, because we already have */
@@ -300,8 +300,9 @@ extern void ReaderConnCloseHandler(CManager cm, CMConnection ClosedConn,
     }
     else
     {
-        fprintf(stderr, "Got an unexpected connection close event\n");
-        CP_verbose(Stream,
+        CP_verbose(Stream, CriticalVerbose,
+                   "Got an unexpected connection close event\n");
+        CP_verbose(Stream, PerStepVerbose,
                    "Reader-side Rank received a "
                    "connection-close event in unexpected "
                    "status %s\n",
@@ -347,7 +348,7 @@ static int HasAllPeers(SstStream Stream)
     int i, StillWaiting = 0;
     if (!Stream->ConnectionsToWriter)
     {
-        CP_verbose(Stream,
+        CP_verbose(Stream, PerRankVerbose,
                    "(PID %lx, TID %lx) Waiting for first Peer notification\n",
                    (long)gettid(), (long)getpid());
         return 0;
@@ -362,13 +363,14 @@ static int HasAllPeers(SstStream Stream)
     }
     if (StillWaiting == 0)
     {
-        CP_verbose(Stream, "Rank %d has all forward peer connections\n",
-                   Stream->Rank);
+        CP_verbose(Stream, PerRankVerbose,
+                   "Rank %d has all forward peer connections\n", Stream->Rank);
         return 1;
     }
     else
     {
-        CP_verbose(Stream, "Rank %d waiting for %d forward peer connections\n",
+        CP_verbose(Stream, PerRankVerbose,
+                   "Rank %d waiting for %d forward peer connections\n",
                    Stream->Rank, StillWaiting);
         return 0;
     }
@@ -407,12 +409,12 @@ attr_list ContactWriter(SstStream Stream, char *Filename, SstParams Params,
                 (globalNetinfoCallback)(2, CMContactString, NULL);
             }
             WriterRank0Contact = attr_list_from_string(CMContactString);
-            conn = CMget_conn(Stream->CPInfo->cm, WriterRank0Contact);
+            conn = CMget_conn(Stream->CPInfo->SharedCM->cm, WriterRank0Contact);
             free_attr_list(WriterRank0Contact);
         }
         if (conn)
         {
-            DataSize = strlen(CMContactString);
+            DataSize = strlen(CMContactString) + 1;
             *conn_p = conn;
         }
         else
@@ -473,7 +475,8 @@ SstStream SstReaderOpen(const char *Name, SstParams Params, SMPI_Comm comm)
     CP_validateParams(Stream, Params, 0 /* reader */);
     Stream->ConfigParams = Params;
 
-    Stream->DP_Interface = SelectDP(&Svcs, Stream, Stream->ConfigParams);
+    Stream->DP_Interface =
+        SelectDP(&Svcs, Stream, Stream->ConfigParams, Stream->Rank);
 
     Stream->CPInfo =
         CP_getCPInfo(Stream->DP_Interface, Stream->ConfigParams->ControlModule);
@@ -495,7 +498,8 @@ SstStream SstReaderOpen(const char *Name, SstParams Params, SMPI_Comm comm)
     }
 
     Stream->DP_Stream = Stream->DP_Interface->initReader(
-        &Svcs, Stream, &dpInfo, Stream->ConfigParams, WriterContactAttributes);
+        &Svcs, Stream, &dpInfo, Stream->ConfigParams, WriterContactAttributes,
+        &Stream->Stats);
 
     free_attr_list(WriterContactAttributes);
 
@@ -508,16 +512,19 @@ SstStream SstReaderOpen(const char *Name, SstParams Params, SMPI_Comm comm)
         struct _ReaderRegisterMsg ReaderRegister;
 
         memset(&ReaderRegister, 0, sizeof(ReaderRegister));
+        memset(&WriterData, 0, sizeof(WriterData));
+        WriterData.WriterCohortSize = -1;
         ReaderRegister.WriterFile = WriterFileID;
         ReaderRegister.WriterResponseCondition =
-            CMCondition_get(Stream->CPInfo->cm, rank0_to_rank0_conn);
+            CMCondition_get(Stream->CPInfo->SharedCM->cm, rank0_to_rank0_conn);
         ReaderRegister.ReaderCohortSize = Stream->CohortSize;
         switch (Stream->ConfigParams->SpeculativePreloadMode)
         {
         case SpecPreloadOff:
         case SpecPreloadOn:
             ReaderRegister.SpecPreload =
-                Stream->ConfigParams->SpeculativePreloadMode;
+                (SpeculativePreloadMode)
+                    Stream->ConfigParams->SpeculativePreloadMode;
             break;
         case SpecPreloadAuto:
             ReaderRegister.SpecPreload = SpecPreloadOff;
@@ -542,15 +549,16 @@ SstStream SstReaderOpen(const char *Name, SstParams Params, SMPI_Comm comm)
         free(pointers);
 
         /* the response value is set in the handler */
-        struct _WriterResponseMsg *response = NULL;
-        CMCondition_set_client_data(Stream->CPInfo->cm,
+        volatile struct _WriterResponseMsg *response = NULL;
+        CMCondition_set_client_data(Stream->CPInfo->SharedCM->cm,
                                     ReaderRegister.WriterResponseCondition,
                                     &response);
 
-        if (CMwrite(rank0_to_rank0_conn, Stream->CPInfo->ReaderRegisterFormat,
+        if (CMwrite(rank0_to_rank0_conn,
+                    Stream->CPInfo->SharedCM->ReaderRegisterFormat,
                     &ReaderRegister) != 1)
         {
-            CP_verbose(Stream,
+            CP_verbose(Stream, CriticalVerbose,
                        "Message failed to send to writer in SstReaderOpen\n");
         }
         free(ReaderRegister.CP_ReaderInfo);
@@ -558,12 +566,12 @@ SstStream SstReaderOpen(const char *Name, SstParams Params, SMPI_Comm comm)
 
         /* wait for "go" from writer */
         CP_verbose(
-            Stream,
+            Stream, PerRankVerbose,
             "Waiting for writer response message in SstReadOpen(\"%s\")\n",
             Filename, ReaderRegister.WriterResponseCondition);
-        CMCondition_wait(Stream->CPInfo->cm,
+        CMCondition_wait(Stream->CPInfo->SharedCM->cm,
                          ReaderRegister.WriterResponseCondition);
-        CP_verbose(Stream,
+        CP_verbose(Stream, PerRankVerbose,
                    "finished wait writer response message in read_open\n");
 
         if (response)
@@ -596,11 +604,11 @@ SstStream SstReaderOpen(const char *Name, SstParams Params, SMPI_Comm comm)
 
     if (Stream->Rank == 0)
     {
-        CP_verbose(Stream,
+        CP_verbose(Stream, SummaryVerbose,
                    "Opening Reader Stream.\nWriter stream params are:\n");
         CP_dumpParams(Stream, ReturnData->WriterConfigParams,
                       0 /* writer side */);
-        CP_verbose(Stream, "Reader stream params are:\n");
+        CP_verbose(Stream, SummaryVerbose, "Reader stream params are:\n");
         CP_dumpParams(Stream, Stream->ConfigParams, 1 /* reader side */);
     }
 
@@ -611,23 +619,29 @@ SstStream SstReaderOpen(const char *Name, SstParams Params, SMPI_Comm comm)
 
     Stream->WriterCohortSize = ReturnData->WriterCohortSize;
     Stream->WriterConfigParams = ReturnData->WriterConfigParams;
-    if (Stream->WriterConfigParams->MarshalMethod == SstMarshalFFS)
+    if ((Stream->WriterConfigParams->MarshalMethod == SstMarshalFFS) &&
+        (Stream->Rank == 0))
     {
-        CP_verbose(Stream, "Writer is doing FFS-based marshalling\n");
+        CP_verbose(Stream, SummaryVerbose,
+                   "Writer is doing FFS-based marshalling\n");
     }
-    if (Stream->WriterConfigParams->MarshalMethod == SstMarshalBP)
+    if ((Stream->WriterConfigParams->MarshalMethod == SstMarshalBP) &&
+        (Stream->Rank == 0))
     {
-        CP_verbose(Stream, "Writer is doing BP-based marshalling\n");
+        CP_verbose(Stream, SummaryVerbose,
+                   "Writer is doing BP-based marshalling\n");
     }
-    if (Stream->WriterConfigParams->CPCommPattern == SstCPCommMin)
+    if ((Stream->WriterConfigParams->CPCommPattern == SstCPCommMin) &&
+        (Stream->Rank == 0))
     {
         CP_verbose(
-            Stream,
+            Stream, SummaryVerbose,
             "Writer is using Minimum Connection Communication pattern (min)\n");
     }
-    if (Stream->WriterConfigParams->CPCommPattern == SstCPCommPeer)
+    if ((Stream->WriterConfigParams->CPCommPattern == SstCPCommPeer) &&
+        (Stream->Rank == 0))
     {
-        CP_verbose(Stream,
+        CP_verbose(Stream, SummaryVerbose,
                    "Writer is using Peer-based Communication pattern (peer)\n");
     }
     STREAM_MUTEX_LOCK(Stream);
@@ -699,11 +713,13 @@ SstStream SstReaderOpen(const char *Name, SstParams Params, SMPI_Comm comm)
     Stream->DP_Interface->provideWriterDataToReader(
         &Svcs, Stream->DP_Stream, ReturnData->WriterCohortSize,
         Stream->ConnectionsToWriter, ReturnData->DP_WriterInfo);
-    CP_verbose(Stream, "Sending Reader Activate messages to writer\n");
+    CP_verbose(Stream, PerRankVerbose,
+               "Sending Reader Activate messages to writer\n");
     memset(&Msg, 0, sizeof(Msg));
-    sendOneToEachWriterRank(Stream, Stream->CPInfo->ReaderActivateFormat, &Msg,
-                            &Msg.WSR_Stream);
-    CP_verbose(Stream,
+    sendOneToEachWriterRank(Stream,
+                            Stream->CPInfo->SharedCM->ReaderActivateFormat,
+                            &Msg, &Msg.WSR_Stream);
+    CP_verbose(Stream, PerStepVerbose,
                "Finish opening Stream \"%s\", starting with Step number %d\n",
                Filename, ReturnData->StartingStepNumber);
 
@@ -717,7 +733,8 @@ SstStream SstReaderOpen(const char *Name, SstParams Params, SMPI_Comm comm)
 extern void SstReaderGetParams(SstStream Stream,
                                SstMarshalMethod *WriterMarshalMethod)
 {
-    *WriterMarshalMethod = Stream->WriterConfigParams->MarshalMethod;
+    *WriterMarshalMethod =
+        (SstMarshalMethod)Stream->WriterConfigParams->MarshalMethod;
 }
 
 /*
@@ -733,16 +750,18 @@ extern void CP_PeerSetupHandler(CManager cm, CMConnection conn, void *Msg_v,
     struct _PeerSetupMsg *Msg = (struct _PeerSetupMsg *)Msg_v;
     Stream = (SstStream)Msg->RS_Stream;
     STREAM_MUTEX_LOCK(Stream);
-    CP_verbose(Stream, "Received peer setup from rank %d, conn %p\n",
-               Msg->WriterRank, conn);
+    CP_verbose(Stream, TraceVerbose,
+               "Received peer setup from rank %d, conn %p\n", Msg->WriterRank,
+               conn);
     if (!Stream->ConnectionsToWriter)
     {
-        CP_verbose(Stream, "Allocating connections to writer\n");
+        CP_verbose(Stream, TraceVerbose, "Allocating connections to writer\n");
         Stream->ConnectionsToWriter =
             calloc(sizeof(CP_PeerConnection), Msg->WriterCohortSize);
     }
-    CP_verbose(Stream, "Received peer setup from rank %d, conn %p\n",
-               Msg->WriterRank, conn);
+    CP_verbose(Stream, TraceVerbose,
+               "Received peer setup from rank %d, conn %p\n", Msg->WriterRank,
+               conn);
     if (Msg->WriterRank != -1)
     {
         Stream->ConnectionsToWriter[Msg->WriterRank].CMconn = conn;
@@ -770,23 +789,19 @@ void queueTimestepMetadataMsgAndNotify(SstStream Stream,
          * send each writer rank a release for this timestep (actually goes to
          * WSR Streams)
          */
-        CP_verbose(Stream,
-                   "Sending ReleaseTimestep message for PRIOR DISCARD "
-                   "timestep %d, one to each writer\n",
-                   tsm->Timestep);
         if (tsm->Metadata != NULL)
         {
-            CP_verbose(Stream,
+            CP_verbose(Stream, PerStepVerbose,
                        "Sending ReleaseTimestep message for PRIOR DISCARD "
                        "timestep %d, one to each writer\n",
                        tsm->Timestep);
-            sendOneToEachWriterRank(Stream,
-                                    Stream->CPInfo->ReleaseTimestepFormat, &Msg,
-                                    &Msg.WSR_Stream);
+            sendOneToEachWriterRank(
+                Stream, Stream->CPInfo->SharedCM->ReleaseTimestepFormat, &Msg,
+                &Msg.WSR_Stream);
         }
         else
         {
-            CP_verbose(Stream,
+            CP_verbose(Stream, PerStepVerbose,
                        "Received discard notice for timestep %d, "
                        "ignoring in PRIOR DISCARD\n",
                        tsm->Timestep);
@@ -809,7 +824,13 @@ void queueTimestepMetadataMsgAndNotify(SstStream Stream,
     {
         Stream->Timesteps = New;
     }
-    CP_verbose(Stream,
+    Stream->Stats.TimestepMetadataReceived++;
+    if (tsm->Metadata)
+    {
+        Stream->Stats.MetadataBytesReceived +=
+            (tsm->Metadata->DataSize + tsm->AttributeData->DataSize);
+    }
+    CP_verbose(Stream, PerRankVerbose,
                "Received a Timestep metadata message for timestep %d, "
                "signaling condition\n",
                tsm->Timestep);
@@ -825,7 +846,7 @@ void queueTimestepMetadataMsgAndNotify(SstStream Stream,
          * we want to release timesteps that are older than it, NOT
          * INCLUDING ANY TIMESTEP IN CURRENT USE.
          */
-        CP_verbose(Stream,
+        CP_verbose(Stream, TraceVerbose,
                    "Got a new timestep in AlwaysProvideLatestTimestep mode, "
                    "discard older than %d\n",
                    tsm->Timestep);
@@ -850,7 +871,7 @@ void CP_TimestepMetadataHandler(CManager cm, CMConnection conn, void *Msg_v,
         if (Msg->Metadata == NULL)
         {
             CP_verbose(
-                Stream,
+                Stream, PerRankVerbose,
                 "Received a message that timestep %d has been discarded\n",
                 Msg->Timestep);
 
@@ -869,7 +890,7 @@ void CP_TimestepMetadataHandler(CManager cm, CMConnection conn, void *Msg_v,
         else
         {
             CP_verbose(
-                Stream,
+                Stream, PerStepVerbose,
                 "Received an incoming metadata message for timestep %d\n",
                 Msg->Timestep);
         }
@@ -943,7 +964,7 @@ extern void CP_WriterCloseHandler(CManager cm, CMConnection conn, void *Msg_v,
     SstStream Stream = (SstStream)Msg->RS_Stream;
 
     STREAM_MUTEX_LOCK(Stream);
-    CP_verbose(Stream,
+    CP_verbose(Stream, PerStepVerbose,
                "Received a writer close message. "
                "Timestep %d was the final timestep.\n",
                Msg->FinalTimestep);
@@ -969,7 +990,7 @@ extern void CP_CommPatternLockedHandler(CManager cm, CMConnection conn,
 
     STREAM_MUTEX_LOCK(Stream);
     CP_verbose(
-        Stream,
+        Stream, PerStepVerbose,
         "Received a CommPatternLocked message, beginning with Timestep %d.\n",
         Msg->Timestep);
 
@@ -986,7 +1007,7 @@ static long MaxQueuedMetadata(SstStream Stream)
     Next = Stream->Timesteps;
     if (Next == NULL)
     {
-        CP_verbose(Stream, "MaxQueued Timestep returning -1\n");
+        CP_verbose(Stream, TraceVerbose, "MaxQueued Timestep returning -1\n");
         return -1;
     }
     while (Next)
@@ -997,7 +1018,8 @@ static long MaxQueuedMetadata(SstStream Stream)
         }
         Next = Next->Next;
     }
-    CP_verbose(Stream, "MaxQueued Timestep returning %ld\n", MaxTimestep);
+    CP_verbose(Stream, TraceVerbose, "MaxQueued Timestep returning %ld\n",
+               MaxTimestep);
     return MaxTimestep;
 }
 
@@ -1009,7 +1031,7 @@ static long NextQueuedMetadata(SstStream Stream)
     Next = Stream->Timesteps;
     if (Next == NULL)
     {
-        CP_verbose(Stream, "NextQueued Timestep returning -1\n");
+        CP_verbose(Stream, TraceVerbose, "NextQueued Timestep returning -1\n");
         return -1;
     }
     while (Next)
@@ -1020,7 +1042,8 @@ static long NextQueuedMetadata(SstStream Stream)
         }
         Next = Next->Next;
     }
-    CP_verbose(Stream, "NextQueued Timestep returning %ld\n", MinTimestep);
+    CP_verbose(Stream, TraceVerbose, "NextQueued Timestep returning %ld\n",
+               MinTimestep);
     return MinTimestep;
 }
 
@@ -1047,12 +1070,13 @@ static void waitForMetadataWithTimeout(SstStream Stream, float timeout_secs)
     gettimeofday(&start, NULL);
     Next = Stream->Timesteps;
     CP_verbose(
-        Stream,
+        Stream, PerRankVerbose,
         "Wait for metadata with timeout %g secs starting at time %ld.%06ld \n",
         timeout_secs, start.tv_sec, start.tv_usec);
     if (Next)
     {
-        CP_verbose(Stream, "Returning from wait with timeout, NO TIMEOUT\n");
+        CP_verbose(Stream, PerRankVerbose,
+                   "Returning from wait with timeout, NO TIMEOUT\n");
     }
     end.tv_sec = start.tv_sec + timeout_int_sec;
     end.tv_usec = start.tv_usec + timeout_int_usec;
@@ -1070,13 +1094,13 @@ static void waitForMetadataWithTimeout(SstStream Stream, float timeout_secs)
     if (timeout_secs == 0.0)
     {
         CP_verbose(
-            Stream,
+            Stream, PerRankVerbose,
             "Returning from wait With no data after zero timeout poll\n");
         return;
     }
 
     TimeoutTask =
-        CMadd_delayed_task(Stream->CPInfo->cm, timeout_int_sec,
+        CMadd_delayed_task(Stream->CPInfo->SharedCM->cm, timeout_int_sec,
                            timeout_int_usec, triggerDataCondition, Stream);
     while (1)
     {
@@ -1084,22 +1108,25 @@ static void waitForMetadataWithTimeout(SstStream Stream, float timeout_secs)
         if (Next)
         {
             CMremove_task(TimeoutTask);
-            CP_verbose(Stream,
+            CP_verbose(Stream, PerRankVerbose,
                        "Returning from wait with timeout, NO TIMEOUT\n");
             return;
         }
         if (Stream->Status != Established)
         {
-            CP_verbose(Stream, "Returning from wait with timeout, STREAM NO "
-                               "LONGER ESTABLISHED\n");
+            CP_verbose(Stream, PerRankVerbose,
+                       "Returning from wait with timeout, STREAM NO "
+                       "LONGER ESTABLISHED\n");
             return;
         }
         gettimeofday(&now, NULL);
-        CP_verbose(Stream, "timercmp, now is %ld.%06ld    end is %ld.%06ld \n",
+        CP_verbose(Stream, TraceVerbose,
+                   "timercmp, now is %ld.%06ld    end is %ld.%06ld \n",
                    now.tv_sec, now.tv_usec, end.tv_sec, end.tv_usec);
         if (timercmp(&now, &end, >))
         {
-            CP_verbose(Stream, "Returning from wait after timing out\n");
+            CP_verbose(Stream, PerRankVerbose,
+                       "Returning from wait after timing out\n");
             return;
         }
         /* wait until we get the timestep metadata or something else changes */
@@ -1112,7 +1139,8 @@ static void releasePriorTimesteps(SstStream Stream, long Latest)
 {
     struct _TimestepMetadataList *Next, *Last;
     STREAM_ASSERT_LOCKED(Stream);
-    CP_verbose(Stream, "Releasing any timestep earlier than %d\n", Latest);
+    CP_verbose(Stream, PerRankVerbose,
+               "Releasing any timestep earlier than %d\n", Latest);
     Next = Stream->Timesteps;
     Last = NULL;
     while (Next)
@@ -1141,7 +1169,7 @@ static void releasePriorTimesteps(SstStream Stream, long Latest)
              * to WSR
              * Streams)
              */
-            CP_verbose(Stream,
+            CP_verbose(Stream, PerRankVerbose,
                        "Sending ReleaseTimestep message for RELEASE "
                        "PRIOR timestep %d, one to each writer\n",
                        This->MetadataMsg->Timestep);
@@ -1155,13 +1183,13 @@ static void releasePriorTimesteps(SstStream Stream, long Latest)
                 Last->Next = Next;
             }
             STREAM_MUTEX_UNLOCK(Stream);
-            sendOneToEachWriterRank(Stream,
-                                    Stream->CPInfo->ReleaseTimestepFormat, &Msg,
-                                    &Msg.WSR_Stream);
+            sendOneToEachWriterRank(
+                Stream, Stream->CPInfo->SharedCM->ReleaseTimestepFormat, &Msg,
+                &Msg.WSR_Stream);
             if (This->MetadataMsg == NULL)
                 printf("READER RETURN_BUFFER, metadatamsg == %p, line %d\n",
                        This->MetadataMsg, __LINE__);
-            CMreturn_buffer(Stream->CPInfo->cm, This->MetadataMsg);
+            CMreturn_buffer(Stream->CPInfo->SharedCM->cm, This->MetadataMsg);
             STREAM_MUTEX_LOCK(Stream);
             free(This);
         }
@@ -1187,7 +1215,7 @@ static void FreeTimestep(SstStream Stream, long Timestep)
         if (List->MetadataMsg == NULL)
             printf("READER RETURN_BUFFER, List->MEtadataMsg == %p, line %d\n",
                    List->MetadataMsg, __LINE__);
-        CMreturn_buffer(Stream->CPInfo->cm, List->MetadataMsg);
+        CMreturn_buffer(Stream->CPInfo->SharedCM->cm, List->MetadataMsg);
 
         free(List);
     }
@@ -1204,7 +1232,8 @@ static void FreeTimestep(SstStream Stream, long Timestep)
                     printf("READER RETURN_BUFFER, List->MEtadataMsg == %p, "
                            "line %d\n",
                            List->MetadataMsg, __LINE__);
-                CMreturn_buffer(Stream->CPInfo->cm, List->MetadataMsg);
+                CMreturn_buffer(Stream->CPInfo->SharedCM->cm,
+                                List->MetadataMsg);
 
                 free(List);
                 break;
@@ -1218,15 +1247,16 @@ static void FreeTimestep(SstStream Stream, long Timestep)
 static TSMetadataList waitForNextMetadata(SstStream Stream, long LastTimestep)
 {
     TSMetadataList FoundTS = NULL;
-    CP_verbose(Stream, "Wait for next metadata after last timestep %d\n",
-               LastTimestep);
+    CP_verbose(Stream, PerRankVerbose,
+               "Wait for next metadata after last timestep %d\n", LastTimestep);
     while (1)
     {
         struct _TimestepMetadataList *Next;
         Next = Stream->Timesteps;
         while (Next)
         {
-            CP_verbose(Stream, "Examining metadata for Timestep %d\n",
+            CP_verbose(Stream, TraceVerbose,
+                       "Examining metadata for Timestep %d\n",
                        Next->MetadataMsg->Timestep);
             if (((Next->MetadataMsg->Metadata == NULL) ||
                  (Next->MetadataMsg->Timestep <
@@ -1241,7 +1271,7 @@ static TSMetadataList waitForNextMetadata(SstStream Stream, long LastTimestep)
                  * time to install the 'precious' info that it carried
                  * (Attributes and formats) and then discard it.
                  */
-                CP_verbose(Stream,
+                CP_verbose(Stream, PerRankVerbose,
                            "SstAdvanceStep installing precious "
                            "metadata for discarded TS %d\n",
                            Next->MetadataMsg->Timestep);
@@ -1270,7 +1300,8 @@ static TSMetadataList waitForNextMetadata(SstStream Stream, long LastTimestep)
         }
         if (FoundTS)
         {
-            CP_verbose(Stream, "Returning metadata for Timestep %d\n",
+            CP_verbose(Stream, PerRankVerbose,
+                       "Returning metadata for Timestep %d\n",
                        FoundTS->MetadataMsg->Timestep);
             Stream->CurrentWorkingTimestep = FoundTS->MetadataMsg->Timestep;
             return FoundTS;
@@ -1280,25 +1311,25 @@ static TSMetadataList waitForNextMetadata(SstStream Stream, long LastTimestep)
             ((Stream->FinalTimestep != INT_MAX) &&
              (Stream->FinalTimestep >= LastTimestep)))
         {
-            CP_verbose(Stream,
+            CP_verbose(Stream, TraceVerbose,
                        "Stream Final Timestep is %d, last timestep was %d\n",
                        Stream->FinalTimestep, LastTimestep);
             if (Stream->Status == NotOpen)
             {
-                CP_verbose(Stream,
+                CP_verbose(Stream, PerRankVerbose,
                            "Wait for next metadata returning NULL because "
                            "channel was never fully established\n");
             }
             else if (Stream->Status == PeerFailed)
             {
-                CP_verbose(Stream,
+                CP_verbose(Stream, PerRankVerbose,
                            "Wait for next metadata returning NULL because "
                            "the connection failed before final timestep "
                            "notification\n");
             }
             else
             {
-                CP_verbose(Stream,
+                CP_verbose(Stream, PerStepVerbose,
                            "Wait for next metadata returning NULL, status %d ",
                            Stream->Status);
             }
@@ -1306,12 +1337,12 @@ static TSMetadataList waitForNextMetadata(SstStream Stream, long LastTimestep)
             Stream->CurrentWorkingTimestep = -1;
             return NULL;
         }
-        CP_verbose(Stream,
+        CP_verbose(Stream, PerRankVerbose,
                    "Waiting for metadata for a Timestep later than TS %d\n",
                    LastTimestep);
-        CP_verbose(Stream, "(PID %lx, TID %lx) Stream status is %s\n",
-                   (long)getpid(), (long)gettid(),
-                   SSTStreamStatusStr[Stream->Status]);
+        CP_verbose(Stream, TraceVerbose,
+                   "(PID %lx, TID %lx) Stream status is %s\n", (long)getpid(),
+                   (long)gettid(), SSTStreamStatusStr[Stream->Status]);
         /* wait until we get the timestep metadata or something else changes */
         STREAM_CONDITION_WAIT(Stream);
     }
@@ -1327,53 +1358,121 @@ extern SstFullMetadata SstGetCurMetadata(SstStream Stream)
     return Stream->CurrentMetadata;
 }
 
+static void AddToReadStats(SstStream Stream, int Rank, long Timestep,
+                           size_t Length)
+{
+    if (!Stream->RanksRead)
+        Stream->RanksRead = calloc(1, Stream->WriterCohortSize);
+    Stream->RanksRead[Rank] = 1;
+    Stream->Stats.BytesRead += Length;
+}
+
+#ifndef min
+#define min(a, b) (((a) < (b)) ? (a) : (b))
+#endif
+
+static void ReleaseTSReadStats(SstStream Stream, long Timestep)
+{
+    int ThisFanIn = 0;
+    if (Stream->RanksRead)
+    {
+        for (int i = 0; i < Stream->WriterCohortSize; i++)
+        {
+            if (Stream->RanksRead[i])
+                ThisFanIn++;
+        }
+        memset(Stream->RanksRead, 0, Stream->WriterCohortSize);
+    }
+    if (Stream->Stats.TimestepsConsumed == 1)
+    {
+        Stream->Stats.RunningFanIn = ThisFanIn;
+    }
+    else
+    {
+        Stream->Stats.RunningFanIn =
+            Stream->Stats.RunningFanIn +
+            ((double)ThisFanIn - Stream->Stats.RunningFanIn) /
+                min(Stream->Stats.TimestepsConsumed, 100);
+    }
+}
+
 //  SstReadRemotememory is only called by the main
 //  program thread.
 extern void *SstReadRemoteMemory(SstStream Stream, int Rank, long Timestep,
                                  size_t Offset, size_t Length, void *Buffer,
                                  void *DP_TimestepInfo)
 {
-    if (Stream->Stats)
-        Stream->Stats->BytesTransferred += Length;
+    if (Stream->ConfigParams->ReaderShortCircuitReads)
+        return NULL;
+    Stream->Stats.BytesTransferred += Length;
+    AddToReadStats(Stream, Rank, Timestep, Length);
     return Stream->DP_Interface->readRemoteMemory(
         &Svcs, Stream->DP_Stream, Rank, Timestep, Offset, Length, Buffer,
         DP_TimestepInfo);
 }
 
-static void sendOneToEachWriterRank(SstStream s, CMFormat f, void *Msg,
+static void sendOneToEachWriterRank(SstStream Stream, CMFormat f, void *Msg,
                                     void **WS_StreamPtr)
 {
-    if (s->WriterConfigParams->CPCommPattern == SstCPCommPeer)
+    if (Stream->WriterConfigParams->CPCommPattern == SstCPCommPeer)
     {
         int i = 0;
-        while (s->Peers[i] != -1)
+        while (Stream->Peers[i] != -1)
         {
-            int peer = s->Peers[i];
-            CMConnection conn = s->ConnectionsToWriter[peer].CMconn;
+            int peer = Stream->Peers[i];
+            CMConnection conn = Stream->ConnectionsToWriter[peer].CMconn;
             /* add the writer Stream identifier to each outgoing
              * message */
-            *WS_StreamPtr = s->ConnectionsToWriter[peer].RemoteStreamID;
+            *WS_StreamPtr = Stream->ConnectionsToWriter[peer].RemoteStreamID;
             if (CMwrite(conn, f, Msg) != 1)
             {
-                CP_verbose(s, "Message failed to send to writer %d (%p)\n",
-                           peer, *WS_StreamPtr);
+                switch (Stream->Status)
+                {
+                case NotOpen:
+                case Opening:
+                case Established:
+                    CP_verbose(Stream, CriticalVerbose,
+                               "Message failed to send to writer %d (%p)\n",
+                               peer, *WS_StreamPtr);
+                    break;
+                case PeerClosed:
+                case PeerFailed:
+                case Closed:
+                case Destroyed:
+                    // Don't warn on send failures for closing/closed clients
+                    break;
+                }
             }
             i++;
         }
     }
     else
     {
-        if (s->Rank == 0)
+        if (Stream->Rank == 0)
         {
             int peer = 0;
-            CMConnection conn = s->ConnectionsToWriter[peer].CMconn;
+            CMConnection conn = Stream->ConnectionsToWriter[peer].CMconn;
             /* add the writer Stream identifier to each outgoing
              * message */
-            *WS_StreamPtr = s->ConnectionsToWriter[peer].RemoteStreamID;
+            *WS_StreamPtr = Stream->ConnectionsToWriter[peer].RemoteStreamID;
             if (CMwrite(conn, f, Msg) != 1)
             {
-                CP_verbose(s, "Message failed to send to writer %d (%p)\n",
-                           peer, *WS_StreamPtr);
+                switch (Stream->Status)
+                {
+                case NotOpen:
+                case Opening:
+                case Established:
+                    CP_verbose(Stream, CriticalVerbose,
+                               "Message failed to send to writer %d (%p)\n",
+                               peer, *WS_StreamPtr);
+                    break;
+                case PeerClosed:
+                case PeerFailed:
+                case Closed:
+                case Destroyed:
+                    // Don't warn on send failures for closing/closed clients
+                    break;
+                }
             }
         }
     }
@@ -1388,8 +1487,9 @@ extern void SstReaderDefinitionLock(SstStream Stream, long EffectiveTimestep)
     memset(&Msg, 0, sizeof(Msg));
     Msg.Timestep = EffectiveTimestep;
 
-    sendOneToEachWriterRank(Stream, Stream->CPInfo->LockReaderDefinitionsFormat,
-                            &Msg, &Msg.WSR_Stream);
+    sendOneToEachWriterRank(
+        Stream, Stream->CPInfo->SharedCM->LockReaderDefinitionsFormat, &Msg,
+        &Msg.WSR_Stream);
 }
 
 //  SstReleaseStep is only called by the main program thread.  It
@@ -1407,6 +1507,7 @@ extern void SstReleaseStep(SstStream Stream)
         (Stream->DP_Interface->RSReleaseTimestep)(&Svcs, Stream->DP_Stream,
                                                   Timestep);
     }
+    ReleaseTSReadStats(Stream, Timestep);
     STREAM_MUTEX_UNLOCK(Stream);
 
     if ((Stream->WriterConfigParams->CPCommPattern == SstCPCommPeer) ||
@@ -1427,11 +1528,12 @@ extern void SstReleaseStep(SstStream Stream)
      * Streams)
      */
     CP_verbose(
-        Stream,
+        Stream, PerRankVerbose,
         "Sending ReleaseTimestep message for timestep %d, one to each writer\n",
         Timestep);
-    sendOneToEachWriterRank(Stream, Stream->CPInfo->ReleaseTimestepFormat, &Msg,
-                            &Msg.WSR_Stream);
+    sendOneToEachWriterRank(Stream,
+                            Stream->CPInfo->SharedCM->ReleaseTimestepFormat,
+                            &Msg, &Msg.WSR_Stream);
 
     if (Stream->WriterConfigParams->MarshalMethod == SstMarshalFFS)
     {
@@ -1440,24 +1542,19 @@ extern void SstReleaseStep(SstStream Stream)
     TAU_STOP_FUNC();
 }
 
-static void NotifyDPArrivedMetadata(SstStream Stream)
+static void NotifyDPArrivedMetadata(SstStream Stream,
+                                    struct _TimestepMetadataMsg *MetadataMsg)
 {
-    struct _TimestepMetadataList *TS;
-    TS = Stream->Timesteps;
-    while (TS)
+    if ((MetadataMsg->Metadata != NULL) &&
+        (MetadataMsg->Timestep > Stream->LastDPNotifiedTimestep))
     {
-        if ((TS->MetadataMsg->Metadata != NULL) &&
-            (TS->MetadataMsg->Timestep > Stream->LastDPNotifiedTimestep))
+        if (Stream->DP_Interface->timestepArrived)
         {
-            if (Stream->DP_Interface->timestepArrived)
-            {
-                Stream->DP_Interface->timestepArrived(
-                    &Svcs, Stream->DP_Stream, TS->MetadataMsg->Timestep,
-                    TS->MetadataMsg->PreloadMode);
-            }
-            Stream->LastDPNotifiedTimestep = TS->MetadataMsg->Timestep;
+            Stream->DP_Interface->timestepArrived(&Svcs, Stream->DP_Stream,
+                                                  MetadataMsg->Timestep,
+                                                  MetadataMsg->PreloadMode);
         }
-        TS = TS->Next;
+        Stream->LastDPNotifiedTimestep = MetadataMsg->Timestep;
     }
 }
 
@@ -1488,7 +1585,7 @@ static SstStatusValue SstAdvanceStepPeer(SstStream Stream, SstStepMode mode,
         if (Stream->Rank == 0)
         {
             global_info = malloc(sizeof(my_info) * Stream->CohortSize);
-            CP_verbose(Stream,
+            CP_verbose(Stream, PerRankVerbose,
                        "In special case of advancestep, mode is %d, "
                        "Timeout Sec is %g, flt_max is %g\n",
                        mode, timeout_sec, FLT_MAX);
@@ -1559,7 +1656,7 @@ static SstStatusValue SstAdvanceStepPeer(SstStream Stream, SstStepMode mode,
                 if (mode == SstLatestAvailable)
                 {
                     // latest available
-                    CP_verbose(Stream,
+                    CP_verbose(Stream, PerRankVerbose,
                                "Returning Biggest timestep available "
                                "%ld because LatestAvailable "
                                "specified\n",
@@ -1569,7 +1666,7 @@ static SstStatusValue SstAdvanceStepPeer(SstStream Stream, SstStepMode mode,
                 else
                 {
                     // next available (take the oldest that everyone has)
-                    CP_verbose(Stream,
+                    CP_verbose(Stream, PerRankVerbose,
                                "Returning Smallest timestep available "
                                "%ld because NextAvailable specified\n",
                                Smallest);
@@ -1598,7 +1695,7 @@ static SstStatusValue SstAdvanceStepPeer(SstStream Stream, SstStepMode mode,
         {
             /* there was a peerClosed setting on rank0, we'll close */
             Stream->Status = PeerClosed;
-            CP_verbose(Stream,
+            CP_verbose(Stream, PerStepVerbose,
                        "SstAdvanceStep returning EndOfStream at timestep %d\n",
                        Stream->ReaderTimestep);
             return SstEndOfStream;
@@ -1607,7 +1704,7 @@ static SstStatusValue SstAdvanceStepPeer(SstStream Stream, SstStepMode mode,
         {
             /* there was a peerFailed setting on rank0, we'll fail */
             Stream->Status = PeerFailed;
-            CP_verbose(Stream,
+            CP_verbose(Stream, PerStepVerbose,
                        "SstAdvanceStep returning EndOfStream at timestep %d\n",
                        Stream->ReaderTimestep);
             STREAM_MUTEX_UNLOCK(Stream);
@@ -1618,7 +1715,8 @@ static SstStatusValue SstAdvanceStepPeer(SstStream Stream, SstStepMode mode,
         }
         if (NextTimestep == -1)
         {
-            CP_verbose(Stream, "AdvancestepPeer timing out on no data\n");
+            CP_verbose(Stream, PerStepVerbose,
+                       "AdvancestepPeer timing out on no data\n");
             return SstTimeout;
         }
         if (mode == SstLatestAvailable)
@@ -1629,7 +1727,7 @@ static SstStatusValue SstAdvanceStepPeer(SstStream Stream, SstStepMode mode,
             /* Side note: It is possible that someone could get a "prior"
              * timestep after this point.  It has to be released upon
              * arrival */
-            CP_verbose(Stream,
+            CP_verbose(Stream, PerStepVerbose,
                        "timed or Latest timestep, determined NextTimestep %d\n",
                        NextTimestep);
             Stream->DiscardPriorTimestep = NextTimestep;
@@ -1641,9 +1739,10 @@ static SstStatusValue SstAdvanceStepPeer(SstStream Stream, SstStepMode mode,
 
     TAU_STOP("Waiting on metadata per rank per timestep");
 
-    NotifyDPArrivedMetadata(Stream);
     if (Entry)
     {
+        NotifyDPArrivedMetadata(Stream, Entry->MetadataMsg);
+
         if (Stream->WriterConfigParams->MarshalMethod == SstMarshalFFS)
         {
             TAU_START("FFS marshaling case");
@@ -1672,20 +1771,21 @@ static SstStatusValue SstAdvanceStepPeer(SstStream Stream, SstStepMode mode,
         Stream->CurrentWorkingTimestep = Entry->MetadataMsg->Timestep;
         Stream->CurrentMetadata = Mdata;
 
-        CP_verbose(Stream, "SstAdvanceStep returning Success on timestep %d\n",
+        CP_verbose(Stream, PerStepVerbose,
+                   "SstAdvanceStep returning Success on timestep %d\n",
                    Entry->MetadataMsg->Timestep);
         return SstSuccess;
     }
     if (Stream->Status == PeerClosed)
     {
-        CP_verbose(Stream,
+        CP_verbose(Stream, PerStepVerbose,
                    "SstAdvanceStepPeer returning EndOfStream at timestep %d\n",
                    Stream->ReaderTimestep);
         return SstEndOfStream;
     }
     else
     {
-        CP_verbose(Stream,
+        CP_verbose(Stream, PerStepVerbose,
                    "SstAdvanceStep returning FatalError at timestep %d\n",
                    Stream->ReaderTimestep);
         return SstFatalError;
@@ -1754,7 +1854,7 @@ static SstStatusValue SstAdvanceStepMin(SstStream Stream, SstStepMode mode,
                 if (mode == SstLatestAvailable)
                 {
                     // latest available
-                    CP_verbose(Stream,
+                    CP_verbose(Stream, PerStepVerbose,
                                "Returning latest timestep available "
                                "%ld because LatestAvailable "
                                "specified\n",
@@ -1765,7 +1865,7 @@ static SstStatusValue SstAdvanceStepMin(SstStream Stream, SstStepMode mode,
                 {
                     // next available (take the oldest that everyone has)
                     NextTimestep = NextQueuedMetadata(Stream);
-                    CP_verbose(Stream,
+                    CP_verbose(Stream, PerStepVerbose,
                                "Returning Smallest timestep available "
                                "%ld because NextAvailable specified\n",
                                NextTimestep);
@@ -1773,23 +1873,24 @@ static SstStatusValue SstAdvanceStepMin(SstStream Stream, SstStepMode mode,
             }
             if (Stream->Status == PeerFailed)
             {
-                CP_verbose(Stream,
+                CP_verbose(Stream, PerStepVerbose,
                            "SstAdvanceStepMin returning FatalError because of "
-                           "conn failure at timestep %d\n",
+                           "connection failure at timestep %d\n",
                            Stream->ReaderTimestep);
                 return_value = SstFatalError;
             }
             else if ((NextTimestep == -1) && (Stream->Status == PeerClosed))
             {
                 CP_verbose(
-                    Stream,
+                    Stream, PerStepVerbose,
                     "SstAdvanceStepMin returning EndOfStream at timestep %d\n",
                     Stream->ReaderTimestep);
                 return_value = SstEndOfStream;
             }
             else if (NextTimestep == -1)
             {
-                CP_verbose(Stream, "AdvancestepMin timing out on no data\n");
+                CP_verbose(Stream, PerStepVerbose,
+                           "AdvancestepMin timing out on no data\n");
                 return_value = SstTimeout;
             }
             else if (mode == SstLatestAvailable)
@@ -1801,7 +1902,7 @@ static SstStatusValue SstAdvanceStepMin(SstStream Stream, SstStepMode mode,
                  * timestep after this point.  It has to be released upon
                  * arrival */
                 CP_verbose(
-                    Stream,
+                    Stream, PerStepVerbose,
                     "timed or Latest timestep, determined NextTimestep %d\n",
                     NextTimestep);
                 Stream->DiscardPriorTimestep = NextTimestep;
@@ -1810,7 +1911,7 @@ static SstStatusValue SstAdvanceStepMin(SstStream Stream, SstStepMode mode,
         }
         if (Stream->Status == PeerFailed)
         {
-            CP_verbose(Stream,
+            CP_verbose(Stream, PerStepVerbose,
                        "SstAdvanceStepMin returning FatalError because of "
                        "conn failure at timestep %d\n",
                        Stream->ReaderTimestep);
@@ -1824,7 +1925,8 @@ static SstStatusValue SstAdvanceStepMin(SstStream Stream, SstStepMode mode,
         {
             msg.TSmsg = RootEntry->MetadataMsg;
             msg.ReturnValue = return_value;
-            CP_verbose(Stream, "Setting TSmsg to Rootentry value\n");
+            CP_verbose(Stream, TraceVerbose,
+                       "Setting TSmsg to Rootentry value\n");
         }
         else
         {
@@ -1832,7 +1934,7 @@ static SstStatusValue SstAdvanceStepMin(SstStream Stream, SstStepMode mode,
             {
                 if (Stream->Status == PeerClosed)
                 {
-                    CP_verbose(Stream,
+                    CP_verbose(Stream, PerStepVerbose,
                                "SstAdvanceStepMin rank 0 returning "
                                "EndOfStream at timestep %d\n",
                                Stream->ReaderTimestep);
@@ -1840,13 +1942,13 @@ static SstStatusValue SstAdvanceStepMin(SstStream Stream, SstStepMode mode,
                 }
                 else
                 {
-                    CP_verbose(Stream,
+                    CP_verbose(Stream, PerStepVerbose,
                                "SstAdvanceStepMin rank 0 returning "
                                "FatalError at timestep %d\n",
                                Stream->ReaderTimestep);
                     msg.ReturnValue = SstFatalError;
                 }
-                CP_verbose(Stream, "Setting TSmsg to NULL\n");
+                CP_verbose(Stream, TraceVerbose, "Setting TSmsg to NULL\n");
                 msg.TSmsg = NULL;
             }
             else
@@ -1868,9 +1970,7 @@ static SstStatusValue SstAdvanceStepMin(SstStream Stream, SstStepMode mode,
             &free_block);
         STREAM_MUTEX_LOCK(Stream);
     }
-    ret = ReturnData->ReturnValue;
-
-    NotifyDPArrivedMetadata(Stream);
+    ret = (SstStatusValue)ReturnData->ReturnValue;
 
     if (ReturnData->ReturnValue != SstSuccess)
     {
@@ -1878,16 +1978,18 @@ static SstStatusValue SstAdvanceStepMin(SstStream Stream, SstStepMode mode,
             (ReturnData->TSmsg))
         {
             CP_verbose(
-                Stream,
+                Stream, PerRankVerbose,
                 "SstAdvanceStep installing precious metadata before exiting\n");
             FFSMarshalInstallPreciousMetadata(Stream, ReturnData->TSmsg);
         }
 
         free(free_block);
-        CP_verbose(Stream, "SstAdvanceStep returning FAILURE\n");
+        CP_verbose(Stream, PerStepVerbose,
+                   "SstAdvanceStep returning FAILURE\n");
         return ret;
     }
     MetadataMsg = ReturnData->TSmsg;
+
     if (ReturnData->CommPatternLockedTimestep != -1)
     {
         Stream->CommPatternLockedTimestep =
@@ -1903,15 +2005,16 @@ static SstStatusValue SstAdvanceStepMin(SstStream Stream, SstStepMode mode,
     }
     if (MetadataMsg)
     {
+        NotifyDPArrivedMetadata(Stream, MetadataMsg);
+
+        Stream->ReaderTimestep = MetadataMsg->Timestep;
         if (Stream->WriterConfigParams->MarshalMethod == SstMarshalFFS)
         {
-            CP_verbose(
-                Stream,
-                "Calling install precious metadata from metadata block %p\n",
-                MetadataMsg);
+            CP_verbose(Stream, TraceVerbose,
+                       "Calling install  metadata from metadata block %p\n",
+                       MetadataMsg);
             FFSMarshalInstallMetadata(Stream, MetadataMsg);
         }
-        Stream->ReaderTimestep = MetadataMsg->Timestep;
         SstFullMetadata Mdata = malloc(sizeof(struct _SstFullMetadata));
         memset(Mdata, 0, sizeof(struct _SstFullMetadata));
         Mdata->WriterCohortSize = MetadataMsg->CohortSize;
@@ -1934,11 +2037,12 @@ static SstStatusValue SstAdvanceStepMin(SstStream Stream, SstStepMode mode,
         Mdata->FreeBlock = free_block;
         Stream->CurrentMetadata = Mdata;
 
-        CP_verbose(Stream, "SstAdvanceStep returning Success on timestep %d\n",
+        CP_verbose(Stream, PerStepVerbose,
+                   "SstAdvanceStep returning Success on timestep %d\n",
                    MetadataMsg->Timestep);
         return SstSuccess;
     }
-    CP_verbose(Stream, "SstAdvanceStep final return\n");
+    CP_verbose(Stream, TraceVerbose, "SstAdvanceStep final return\n");
     return ret;
 }
 
@@ -1975,6 +2079,10 @@ extern SstStatusValue SstAdvanceStep(SstStream Stream, const float timeout_sec)
     {
         result = SstAdvanceStepMin(Stream, mode, timeout_sec);
     }
+    if (result == SstSuccess)
+    {
+        Stream->Stats.TimestepsConsumed++;
+    }
     STREAM_MUTEX_UNLOCK(Stream);
     return result;
 }
@@ -1993,12 +2101,16 @@ extern void SstReaderClose(SstStream Stream)
     gettimeofday(&CloseTime, NULL);
     timersub(&CloseTime, &Stream->ValidStartTime, &Diff);
     memset(&Msg, 0, sizeof(Msg));
-    sendOneToEachWriterRank(Stream, Stream->CPInfo->ReaderCloseFormat, &Msg,
-                            &Msg.WSR_Stream);
-    if (Stream->Stats)
-        Stream->Stats->ValidTimeSecs = (double)Diff.tv_usec / 1e6 + Diff.tv_sec;
+    sendOneToEachWriterRank(Stream, Stream->CPInfo->SharedCM->ReaderCloseFormat,
+                            &Msg, &Msg.WSR_Stream);
+    Stream->Stats.StreamValidTimeSecs =
+        (double)Diff.tv_usec / 1e6 + Diff.tv_sec;
 
-    CMusleep(Stream->CPInfo->cm, 100000);
+    if (Stream->CPVerbosityLevel >= (int)SummaryVerbose)
+    {
+        DoStreamSummary(Stream);
+    }
+    CMusleep(Stream->CPInfo->SharedCM->cm, 100000);
     if (Stream->CurrentMetadata != NULL)
     {
         if (Stream->CurrentMetadata->FreeBlock)
@@ -2014,6 +2126,8 @@ extern void SstReaderClose(SstStream Stream)
 //  needs no locking
 extern SstStatusValue SstWaitForCompletion(SstStream Stream, void *handle)
 {
+    if (Stream->ConfigParams->ReaderShortCircuitReads)
+        return SstSuccess;
     if (Stream->DP_Interface->waitForCompletion(&Svcs, handle) != 1)
     {
         return SstFatalError;
diff --git a/source/adios2/toolkit/sst/cp/cp_writer.c b/source/adios2/toolkit/sst/cp/cp_writer.c
index 007b0ba22e..b819fb29c5 100644
--- a/source/adios2/toolkit/sst/cp/cp_writer.c
+++ b/source/adios2/toolkit/sst/cp/cp_writer.c
@@ -20,7 +20,6 @@
 #include "cp_internal.h"
 
 #define gettid() pthread_self()
-extern void CP_verbose(SstStream Stream, char *Format, ...);
 
 static void CP_PeerFailCloseWSReader(WS_ReaderInfo CP_WSR_Stream,
                                      enum StreamStatus NewState);
@@ -280,7 +279,11 @@ static void RemoveQueueEntries(SstStream Stream)
             }
 
             Stream->QueuedTimestepCount--;
-            CP_verbose(Stream,
+            if (ItemToFree->MetaDataSendCount)
+            {
+                Stream->Stats.TimestepsDelivered++;
+            }
+            CP_verbose(Stream, PerRankVerbose,
                        "Remove queue Entries removing Timestep %ld (exp %d, "
                        "Prec %d, Ref %d), Count now %d\n",
                        ItemToFree->Timestep, ItemToFree->Expired,
@@ -318,20 +321,6 @@ static void RemoveQueueEntries(SstStream Stream)
     }
 }
 
-static void ReleaseAndDiscardRemainingTimesteps(SstStream Stream)
-{
-    CPTimestepList List = Stream->QueuedTimesteps;
-
-    while (List)
-    {
-        List->Expired = 1;
-        List->PreciousTimestep = 0;
-        List->ReferenceCount = 0;
-        List = List->Next;
-    }
-    RemoveQueueEntries(Stream);
-}
-
 /*
 Queue maintenance:    (ASSUME LOCKED)
         calculate smallest entry for CurrentTimestep in a reader.  Update that
@@ -352,7 +341,7 @@ static void QueueMaintenance(SstStream Stream)
     CPTimestepList List;
     for (int i = 0; i < Stream->ReaderCount; i++)
     {
-        CP_verbose(Stream,
+        CP_verbose(Stream, TraceVerbose,
                    "Reader %d status %s has last released %ld, last sent %ld\n",
                    i, SSTStreamStatusStr[Stream->Readers[i]->ReaderStatus],
                    Stream->Readers[i]->LastReleasedTimestep,
@@ -372,21 +361,22 @@ static void QueueMaintenance(SstStream Stream)
     if (SmallestLastReleasedTimestep != LONG_MAX)
     {
         CP_verbose(
-            Stream,
+            Stream, TraceVerbose,
             "QueueMaintenance, smallest last released = %ld, count = %d\n",
             SmallestLastReleasedTimestep, Stream->QueuedTimestepCount);
     }
     else
     {
         CP_verbose(
-            Stream,
+            Stream, TraceVerbose,
             "QueueMaintenance, smallest last released = LONG_MAX, count = %d\n",
             Stream->QueuedTimestepCount);
     }
     if (SomeReaderIsOpening)
     {
-        CP_verbose(Stream, "Some Reader is in status \"Opening\", abandon "
-                           "queue maintenance until it's fully open");
+        CP_verbose(Stream, TraceVerbose,
+                   "Some Reader is in status \"Opening\", abandon "
+                   "queue maintenance until it's fully open");
         return;
     }
     /* Count precious */
@@ -412,7 +402,7 @@ static void QueueMaintenance(SstStream Stream)
             {
                 if (List->Expired == 0)
                 {
-                    CP_verbose(Stream,
+                    CP_verbose(Stream, PerRankVerbose,
                                "Writer tagging timestep %ld as expired\n",
                                List->Timestep);
                 }
@@ -429,9 +419,9 @@ static void QueueMaintenance(SstStream Stream)
         }
         List = List->Next;
     }
-    CP_verbose(Stream, "Removing dead entries\n");
+    CP_verbose(Stream, PerRankVerbose, "Removing dead entries\n");
     RemoveQueueEntries(Stream);
-    CP_verbose(Stream, "QueueMaintenance complete\n");
+    CP_verbose(Stream, PerRankVerbose, "QueueMaintenance complete\n");
 }
 
 /*
@@ -452,7 +442,7 @@ extern void WriterConnCloseHandler(CManager cm, CMConnection closed_conn,
     STREAM_MUTEX_LOCK(ParentWriterStream);
     if (ParentWriterStream->Status == Destroyed)
     {
-        CP_verbose(ParentWriterStream,
+        CP_verbose(ParentWriterStream, PerRankVerbose,
                    "Writer-side Rank received a "
                    "connection-close event on destroyed stream %p, ignored\n");
         STREAM_MUTEX_UNLOCK(ParentWriterStream);
@@ -465,9 +455,10 @@ extern void WriterConnCloseHandler(CManager cm, CMConnection closed_conn,
          * If any instance is failed, we should remove all, but that requires a
          * global operation, so prep.
          */
-        CP_verbose(ParentWriterStream, "Writer-side Rank received a "
-                                       "connection-close event during normal "
-                                       "operations, peer likely failed\n");
+        CP_verbose(ParentWriterStream, PerStepVerbose,
+                   "Writer-side Rank received a "
+                   "connection-close event during normal "
+                   "operations, peer likely failed\n");
         CP_PeerFailCloseWSReader(WSreader, PeerFailed);
     }
     else if (WSreader->ReaderStatus == Opening)
@@ -475,7 +466,7 @@ extern void WriterConnCloseHandler(CManager cm, CMConnection closed_conn,
         /* ignore this.  We expect a close after the connection is marked closed
          */
         CP_verbose(
-            ParentWriterStream,
+            ParentWriterStream, PerStepVerbose,
             "Writer-side Rank received a "
             "connection-close event in state opening, handling failure\n");
         /* main thread will be waiting for this */
@@ -486,14 +477,16 @@ extern void WriterConnCloseHandler(CManager cm, CMConnection closed_conn,
     {
         /* ignore this.  We expect a close after the connection is marked closed
          */
-        CP_verbose(ParentWriterStream, "Writer-side Rank received a "
-                                       "connection-close event after close, "
-                                       "not unexpected\n");
+        CP_verbose(ParentWriterStream, TraceVerbose,
+                   "Writer-side Rank received a "
+                   "connection-close event after close, "
+                   "not unexpected\n");
     }
     else
     {
-        fprintf(stderr, "Got an unexpected connection close event\n");
-        CP_verbose(ParentWriterStream,
+        CP_verbose(ParentWriterStream, CriticalVerbose,
+                   "Got an unexpected connection close event\n");
+        CP_verbose(ParentWriterStream, PerRankVerbose,
                    "Writer-side Rank received a "
                    "connection-close event in unexpected "
                    "state %s\n",
@@ -517,14 +510,13 @@ static void SendPeerSetupMsg(WS_ReaderInfo reader, int reversePeer, int myRank)
     setup.WriterRank = myRank;
     setup.WriterCohortSize = Stream->CohortSize;
     STREAM_ASSERT_UNLOCKED(Stream);
-    if (CMwrite(conn, Stream->CPInfo->PeerSetupFormat, &setup) != 1)
+    if (CMwrite(conn, Stream->CPInfo->SharedCM->PeerSetupFormat, &setup) != 1)
     {
-        CP_verbose(Stream,
-                   "Message failed to send to reader in sendPeerSetup in "
-                   "reader open\n");
-        printf("FAILED TO Send Peer setup message to rank %d remote stream "
-               "(%p) \n",
-               reversePeer, setup.RS_Stream);
+        CP_verbose(
+            Stream, CriticalVerbose,
+            "Message failed to send to reader peer rank %d in sendPeerSetup in "
+            "reader open\n",
+            reversePeer);
     }
 }
 
@@ -606,7 +598,7 @@ static int initWSReader(WS_ReaderInfo reader, int ReaderSize,
             if (!reader->Connections[peer].CMconn)
             {
                 reader->Connections[peer].CMconn =
-                    CMget_conn(reader->ParentStream->CPInfo->cm,
+                    CMget_conn(reader->ParentStream->CPInfo->SharedCM->cm,
                                reader->Connections[peer].ContactList);
             }
 
@@ -623,7 +615,7 @@ static int initWSReader(WS_ReaderInfo reader, int ReaderSize,
             }
 
             CP_verbose(
-                Stream,
+                Stream, TraceVerbose,
                 "Registering a close handler for connection %p, to peer %d\n",
                 reader->Connections[peer].CMconn, peer);
             CMconn_register_close_handler(reader->Connections[peer].CMconn,
@@ -632,13 +624,13 @@ static int initWSReader(WS_ReaderInfo reader, int ReaderSize,
             if (i == 0)
             {
                 /* failure awareness for reader rank */
-                CP_verbose(reader->ParentStream,
+                CP_verbose(reader->ParentStream, TraceVerbose,
                            "Sending peer setup to rank %d\n", peer);
                 SendPeerSetupMsg(reader, peer, reader->ParentStream->Rank);
             }
             else
             {
-                CP_verbose(reader->ParentStream,
+                CP_verbose(reader->ParentStream, TraceVerbose,
                            "Sending peer setup to rank %d\n", peer);
                 /* failure awareness for reader rank */
                 SendPeerSetupMsg(reader, peer, -1);
@@ -660,7 +652,7 @@ static int initWSReader(WS_ReaderInfo reader, int ReaderSize,
                 usleep(WriterRank *
                        reader->ParentStream->ConnectionUsleepMultiplier);
             reader->Connections[peer].CMconn =
-                CMget_conn(reader->ParentStream->CPInfo->cm,
+                CMget_conn(reader->ParentStream->CPInfo->SharedCM->cm,
                            reader->Connections[peer].ContactList);
 
             if (!reader->Connections[peer].CMconn)
@@ -679,8 +671,8 @@ static int initWSReader(WS_ReaderInfo reader, int ReaderSize,
                                           WriterConnCloseHandler,
                                           (void *)reader);
             /* failure awareness for reader rank */
-            CP_verbose(reader->ParentStream, "Sending peer setup to rank %d\n",
-                       peer);
+            CP_verbose(reader->ParentStream, TraceVerbose,
+                       "Sending peer setup to rank %d\n", peer);
             SendPeerSetupMsg(reader, peer, reader->ParentStream->Rank);
             i++;
         }
@@ -694,7 +686,7 @@ static int initWSReader(WS_ReaderInfo reader, int ReaderSize,
             if (!reader->Connections[0].CMconn)
             {
                 reader->Connections[0].CMconn =
-                    CMget_conn(reader->ParentStream->CPInfo->cm,
+                    CMget_conn(reader->ParentStream->CPInfo->SharedCM->cm,
                                reader->Connections[0].ContactList);
             }
             if (!reader->Connections[0].CMconn)
@@ -726,7 +718,7 @@ static long earliestAvailableTimestepNumber(SstStream Stream,
     STREAM_MUTEX_LOCK(Stream);
     while (List)
     {
-        CP_verbose(Stream,
+        CP_verbose(Stream, TraceVerbose,
                    "Earliest available : Writer-side Timestep %ld "
                    "now has reference count %d, expired %d, precious %d\n",
                    List->Timestep, List->ReferenceCount, List->Expired,
@@ -750,7 +742,7 @@ static void UntagPreciousTimesteps(SstStream Stream)
     {
         if (List->PreciousTimestep)
         {
-            CP_verbose(Stream,
+            CP_verbose(Stream, TraceVerbose,
                        "Precious Timestep %d untagged, reference count is %d\n",
                        List->Timestep, List->ReferenceCount);
             List->PreciousTimestep = 0;
@@ -770,7 +762,7 @@ static void SubRefTimestep(SstStream Stream, long Timestep, int SetLast)
         if (List->Timestep == Timestep)
         {
             List->ReferenceCount--;
-            CP_verbose(Stream,
+            CP_verbose(Stream, TraceVerbose,
                        "SubRef : Writer-side Timestep %ld "
                        "now has reference count %d, expired %d, precious %d\n",
                        List->Timestep, List->ReferenceCount, List->Expired,
@@ -790,7 +782,8 @@ WS_ReaderInfo WriterParticipateInReaderOpen(SstStream Stream)
     long MyStartingTimestep, GlobalStartingTimestep;
     WS_ReaderInfo CP_WSR_Stream = malloc(sizeof(*CP_WSR_Stream));
 
-    CP_verbose(Stream, "Beginning writer-side reader open protocol\n");
+    CP_verbose(Stream, PerRankVerbose,
+               "Beginning writer-side reader open protocol\n");
     if (Stream->Rank == 0)
     {
         STREAM_MUTEX_LOCK(Stream);
@@ -811,7 +804,7 @@ WS_ReaderInfo WriterParticipateInReaderOpen(SstStream Stream)
             &free_block);
         WriterResponseCondition = Req->Msg->WriterResponseCondition;
         conn = Req->Conn;
-        CMreturn_buffer(Stream->CPInfo->cm, Req->Msg);
+        CMreturn_buffer(Stream->CPInfo->SharedCM->cm, Req->Msg);
         free(Req);
     }
     else
@@ -874,7 +867,8 @@ WS_ReaderInfo WriterParticipateInReaderOpen(SstStream Stream)
 
         CP_WSR_Stream->PreloadMode = SstPreloadSpeculative;
         CP_WSR_Stream->PreloadModeActiveTimestep = 0;
-        CP_verbose(Stream, "Setting SpeculativePreload ON for new reader\n");
+        CP_verbose(Stream, PerStepVerbose,
+                   "Setting SpeculativePreload ON for new reader\n");
     }
 
     int MySuccess = initWSReader(CP_WSR_Stream, ReturnData->ReaderCohortSize,
@@ -919,7 +913,7 @@ WS_ReaderInfo WriterParticipateInReaderOpen(SstStream Stream)
     SMPI_Allreduce(&MyStartingTimestep, &GlobalStartingTimestep, 1, SMPI_LONG,
                    SMPI_MAX, Stream->mpiComm);
 
-    CP_verbose(Stream,
+    CP_verbose(Stream, TraceVerbose,
                "My oldest timestep was %ld, global oldest timestep was %ld\n",
                MyStartingTimestep, GlobalStartingTimestep);
 
@@ -948,10 +942,11 @@ WS_ReaderInfo WriterParticipateInReaderOpen(SstStream Stream)
             response.DP_WriterInfo[i] = pointers[i]->DP_Info;
         }
         STREAM_ASSERT_UNLOCKED(Stream);
-        if (CMwrite(conn, Stream->CPInfo->WriterResponseFormat, &response) != 1)
+        if (CMwrite(conn, Stream->CPInfo->SharedCM->WriterResponseFormat,
+                    &response) != 1)
         {
             CP_verbose(
-                Stream,
+                Stream, CriticalVerbose,
                 "Message failed to send to reader in participate in "
                 "reader open!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n");
         }
@@ -964,7 +959,7 @@ WS_ReaderInfo WriterParticipateInReaderOpen(SstStream Stream)
     if (pointers)
         free(pointers);
     Stream->NewReaderPresent = 1;
-    CP_verbose(Stream,
+    CP_verbose(Stream, PerStepVerbose,
                "Finish writer-side reader open protocol for reader %p, "
                "reader ready response pending\n",
                CP_WSR_Stream);
@@ -987,7 +982,8 @@ void sendOneToWSRCohort(WS_ReaderInfo CP_WSR_Stream, CMFormat f, void *Msg,
             /* add the reader-rank-specific Stream identifier to each outgoing
              * message */
             *RS_StreamPtr = CP_WSR_Stream->Connections[peer].RemoteStreamID;
-            CP_verbose(Stream, "Sending a message to reader %d (%p)\n", peer,
+            CP_verbose(Stream, TraceVerbose,
+                       "Sending a message to reader %d (%p)\n", peer,
                        *RS_StreamPtr);
 
             if (conn)
@@ -998,7 +994,7 @@ void sendOneToWSRCohort(WS_ReaderInfo CP_WSR_Stream, CMFormat f, void *Msg,
                 STREAM_MUTEX_LOCK(Stream);
                 if (res != 1)
                 {
-                    CP_verbose(Stream,
+                    CP_verbose(Stream, PerStepVerbose,
                                "Message failed to send to reader %d (%p)\n",
                                peer, *RS_StreamPtr);
                     CP_PeerFailCloseWSReader(CP_WSR_Stream, PeerFailed);
@@ -1017,7 +1013,8 @@ void sendOneToWSRCohort(WS_ReaderInfo CP_WSR_Stream, CMFormat f, void *Msg,
             /* add the reader-rank-specific Stream identifier to each outgoing
              * message */
             *RS_StreamPtr = CP_WSR_Stream->Connections[peer].RemoteStreamID;
-            CP_verbose(Stream, "Sending a message to reader %d (%p)\n", peer,
+            CP_verbose(Stream, TraceVerbose,
+                       "Sending a message to reader %d (%p)\n", peer,
                        *RS_StreamPtr);
 
             if (conn)
@@ -1028,7 +1025,7 @@ void sendOneToWSRCohort(WS_ReaderInfo CP_WSR_Stream, CMFormat f, void *Msg,
                 STREAM_MUTEX_LOCK(Stream);
                 if (res != 1)
                 {
-                    CP_verbose(Stream,
+                    CP_verbose(Stream, PerStepVerbose,
                                "Message failed to send to reader %d (%p)\n",
                                peer, *RS_StreamPtr);
                     CP_PeerFailCloseWSReader(CP_WSR_Stream, PeerFailed);
@@ -1063,7 +1060,8 @@ static void DerefSentTimestep(SstStream Stream, WS_ReaderInfo Reader,
                               long Timestep)
 {
     struct _SentTimestepRec *List = Reader->SentTimestepList, *Last = NULL;
-    CP_verbose(Stream, "Reader sent timestep list %p, trying to release %ld\n",
+    CP_verbose(Stream, PerRankVerbose,
+               "Reader sent timestep list %p, trying to release %ld\n",
                Reader->SentTimestepList, Timestep);
 
     while (List)
@@ -1072,7 +1070,7 @@ static void DerefSentTimestep(SstStream Stream, WS_ReaderInfo Reader,
         int Freed = 0;
         struct _SentTimestepRec *Next = List->Next;
         CP_verbose(
-            Stream,
+            Stream, TraceVerbose,
             "Reader considering sent timestep %ld,trying to release %ld\n",
             List->Timestep, Timestep);
         if (List->Timestep == Timestep)
@@ -1112,16 +1110,17 @@ static void DerefAllSentTimesteps(SstStream Stream, WS_ReaderInfo Reader)
     CPTimestepList List = Stream->QueuedTimesteps;
 
     STREAM_ASSERT_LOCKED(Stream);
-    CP_verbose(Stream, "Dereferencing all timesteps sent to reader %p\n",
-               Reader);
+    CP_verbose(Stream, PerRankVerbose,
+               "Dereferencing all timesteps sent to reader %p\n", Reader);
     while (List)
     {
         CPTimestepList Next = List->Next;
-        CP_verbose(Stream, "Checking on timestep %d\n", List->Timestep);
+        CP_verbose(Stream, TraceVerbose, "Checking on timestep %d\n",
+                   List->Timestep);
         DerefSentTimestep(Stream, Reader, List->Timestep);
         List = Next;
     }
-    CP_verbose(Stream, "DONE DEREFERENCING\n");
+    CP_verbose(Stream, PerRankVerbose, "DONE DEREFERENCING\n");
 }
 
 static void SendTimestepEntryToSingleReader(SstStream Stream,
@@ -1135,12 +1134,13 @@ static void SendTimestepEntryToSingleReader(SstStream Stream,
         CP_WSR_Stream->LastSentTimestep = Entry->Timestep;
         if (rank != -1)
         {
-            CP_verbose(Stream, "Sent timestep %ld to reader cohort %d\n",
+            CP_verbose(Stream, PerRankVerbose,
+                       "Sent timestep %ld to reader cohort %d\n",
                        Entry->Timestep, rank);
         }
         Entry->ReferenceCount++;
-
-        CP_verbose(Stream,
+        Entry->MetaDataSendCount++;
+        CP_verbose(Stream, PerRankVerbose,
                    "ADDING timestep %ld to sent list for reader cohort %d, "
                    "READER %p, reference count is now %d\n",
                    Entry->Timestep, rank, CP_WSR_Stream, Entry->ReferenceCount);
@@ -1151,7 +1151,7 @@ static void SendTimestepEntryToSingleReader(SstStream Stream,
             (CP_WSR_Stream->PreloadMode != SstPreloadNone))
         {
             PMode = CP_WSR_Stream->PreloadMode;
-            CP_verbose(Stream,
+            CP_verbose(Stream, PerStepVerbose,
                        "PRELOADMODE for timestep %ld non-default for reader , "
                        "active at timestep %ld, mode %d\n",
                        Entry->Timestep,
@@ -1167,9 +1167,10 @@ static void SendTimestepEntryToSingleReader(SstStream Stream,
         Entry->Msg->PreloadMode = PMode;
         STREAM_MUTEX_LOCK(Stream);
         if (CP_WSR_Stream->ReaderStatus == Established)
-            sendOneToWSRCohort(CP_WSR_Stream,
-                               Stream->CPInfo->DeliverTimestepMetadataFormat,
-                               Entry->Msg, &Entry->Msg->RS_Stream);
+            sendOneToWSRCohort(
+                CP_WSR_Stream,
+                Stream->CPInfo->SharedCM->DeliverTimestepMetadataFormat,
+                Entry->Msg, &Entry->Msg->RS_Stream);
     }
 }
 
@@ -1189,7 +1190,7 @@ static void waitForReaderResponseAndSendQueued(WS_ReaderInfo Reader)
     STREAM_MUTEX_LOCK(Stream);
     while (Reader->ReaderStatus == Opening)
     {
-        CP_verbose(Stream,
+        CP_verbose(Stream, PerRankVerbose,
                    "(PID %lx, TID %lx) Waiting for Reader ready on WSR %p.\n",
                    (long)getpid(), (long)gettid(), Reader);
         STREAM_CONDITION_WAIT(Stream);
@@ -1197,7 +1198,8 @@ static void waitForReaderResponseAndSendQueued(WS_ReaderInfo Reader)
 
     if (Reader->ReaderStatus != Established)
     {
-        CP_verbose(Stream, "Reader WSR %p, Failed during startup.\n", Reader);
+        CP_verbose(Stream, CriticalVerbose,
+                   "Reader WSR %p, Failed during startup.\n", Reader);
         STREAM_MUTEX_UNLOCK(Stream);
     }
     /* LOCK */
@@ -1217,7 +1219,7 @@ static void waitForReaderResponseAndSendQueued(WS_ReaderInfo Reader)
 
     /* send any queued metadata necessary */
     Reader->OldestUnreleasedTimestep = Reader->StartingTimestep;
-    CP_verbose(Stream,
+    CP_verbose(Stream, PerStepVerbose,
                "Reader ready on WSR %p, Stream established, Starting %d "
                "LastProvided %d.\n",
                Reader, Reader->StartingTimestep, Stream->LastProvidedTimestep);
@@ -1228,7 +1230,7 @@ static void waitForReaderResponseAndSendQueued(WS_ReaderInfo Reader)
         while (List)
         {
             CP_verbose(
-                Stream,
+                Stream, TraceVerbose,
                 "In send queued, trying to send TS %ld, examining TS %ld\n", TS,
                 List->Timestep);
             if (Reader->ReaderStatus != Established)
@@ -1241,7 +1243,7 @@ static void waitForReaderResponseAndSendQueued(WS_ReaderInfo Reader)
                 FFSFormatList SavedFormats = List->Msg->Formats;
                 if (List->Expired && !List->PreciousTimestep)
                 {
-                    CP_verbose(Stream,
+                    CP_verbose(Stream, TraceVerbose,
                                "Reader send queued skipping  TS %d, expired "
                                "and not precious\n",
                                List->Timestep, TS);
@@ -1254,7 +1256,7 @@ static void waitForReaderResponseAndSendQueued(WS_ReaderInfo Reader)
                     /* For first Msg, send all previous formats */
                     List->Msg->Formats = Stream->PreviousFormats;
                 }
-                CP_verbose(Stream,
+                CP_verbose(Stream, PerStepVerbose,
                            "Sending Queued TimestepMetadata for timestep %d, "
                            "reference count = %d\n",
                            TS, List->ReferenceCount);
@@ -1291,11 +1293,13 @@ SstStream SstWriterOpen(const char *Name, SstParams Params, SMPI_Comm comm)
     //    printf("WRITER main program thread PID is %lx, TID %lx in writer
     //    open\n",
     //           (long)getpid(), (long)gettid());
-    Stream->DP_Interface = SelectDP(&Svcs, Stream, Stream->ConfigParams);
+    Stream->DP_Interface =
+        SelectDP(&Svcs, Stream, Stream->ConfigParams, Stream->Rank);
 
     if (!Stream->DP_Interface)
     {
-        CP_verbose(Stream, "Failed to load DataPlane %s for Stream \"%s\"\n",
+        CP_verbose(Stream, CriticalVerbose,
+                   "Failed to load DataPlane %s for Stream \"%s\"\n",
                    Params->DataTransport, Filename);
         return NULL;
     }
@@ -1306,7 +1310,7 @@ SstStream SstWriterOpen(const char *Name, SstParams Params, SMPI_Comm comm)
     if (Stream->RendezvousReaderCount > 0)
     {
         Stream->FirstReaderCondition =
-            CMCondition_get(Stream->CPInfo->cm, NULL);
+            CMCondition_get(Stream->CPInfo->SharedCM->cm, NULL);
     }
     else
     {
@@ -1315,18 +1319,17 @@ SstStream SstWriterOpen(const char *Name, SstParams Params, SMPI_Comm comm)
 
     attr_list DPAttrs = create_attr_list();
     Stream->DP_Stream = Stream->DP_Interface->initWriter(
-        &Svcs, Stream, Stream->ConfigParams, DPAttrs);
+        &Svcs, Stream, Stream->ConfigParams, DPAttrs, &Stream->Stats);
 
     if (Stream->Rank == 0)
     {
         registerContactInfo(Filename, Stream, DPAttrs);
     }
 
-    CP_verbose(Stream, "Opening Stream \"%s\"\n", Filename);
-
     if (Stream->Rank == 0)
     {
-        CP_verbose(Stream, "Writer stream params are:\n");
+        CP_verbose(Stream, SummaryVerbose, "Opening Stream \"%s\"\n", Filename);
+        CP_verbose(Stream, SummaryVerbose, "Writer stream params are:\n");
         CP_dumpParams(Stream, Stream->ConfigParams, 0 /* writer side */);
     }
 
@@ -1339,7 +1342,8 @@ SstStream SstWriterOpen(const char *Name, SstParams Params, SMPI_Comm comm)
     while (Stream->RendezvousReaderCount > 0)
     {
         WS_ReaderInfo reader;
-        CP_verbose(Stream, "Stream \"%s\" waiting for %d readers\n", Filename,
+        CP_verbose(Stream, PerStepVerbose,
+                   "Stream \"%s\" waiting for %d readers\n", Filename,
                    Stream->RendezvousReaderCount);
         if (Stream->Rank == 0)
         {
@@ -1353,8 +1357,6 @@ SstStream SstWriterOpen(const char *Name, SstParams Params, SMPI_Comm comm)
         }
         SMPI_Barrier(Stream->mpiComm);
 
-        struct timeval Start;
-        gettimeofday(&Start, NULL);
         reader = WriterParticipateInReaderOpen(Stream);
         if (!reader)
         {
@@ -1382,9 +1384,11 @@ SstStream SstWriterOpen(const char *Name, SstParams Params, SMPI_Comm comm)
         }
         Stream->RendezvousReaderCount--;
     }
+    gettimeofday(&Stream->ValidStartTime, NULL);
     Stream->Filename = Filename;
     Stream->Status = Established;
-    CP_verbose(Stream, "Finish opening Stream \"%s\"\n", Filename);
+    CP_verbose(Stream, PerStepVerbose, "Finish opening Stream \"%s\"\n",
+               Filename);
     AddToLastCallFreeList(Stream);
     return Stream;
 }
@@ -1398,11 +1402,12 @@ void sendOneToEachReaderRank(SstStream Stream, CMFormat f, void *Msg,
         WS_ReaderInfo CP_WSR_Stream = Stream->Readers[i];
         if (CP_WSR_Stream->ReaderStatus == Established)
         {
-            CP_verbose(Stream, "Working on reader cohort %d\n", i);
+            CP_verbose(Stream, TraceVerbose, "Working on reader cohort %d\n",
+                       i);
         }
         else
         {
-            CP_verbose(Stream, "Skipping reader cohort %d\n", i);
+            CP_verbose(Stream, TraceVerbose, "Skipping reader cohort %d\n", i);
             continue;
         }
         sendOneToWSRCohort(CP_WSR_Stream, f, Msg, RS_StreamPtr);
@@ -1418,7 +1423,7 @@ static void CloseWSRStream(CManager cm, void *WSR_Stream_v)
     SstStream ParentStream = CP_WSR_Stream->ParentStream;
 
     STREAM_MUTEX_LOCK(ParentStream);
-    CP_verbose(ParentStream,
+    CP_verbose(ParentStream, PerRankVerbose,
                "Delayed task Moving Reader stream %p to status %s\n",
                CP_WSR_Stream, SSTStreamStatusStr[PeerClosed]);
     CP_PeerFailCloseWSReader(CP_WSR_Stream, PeerClosed);
@@ -1433,7 +1438,7 @@ static void CP_PeerFailCloseWSReader(WS_ReaderInfo CP_WSR_Stream,
     if (ParentStream->Status != Established)
     {
         CP_verbose(
-            ParentStream,
+            ParentStream, TraceVerbose,
             "In PeerFailCloseWSReader, but Parent status not Established, %d\n",
             ParentStream->Status);
         return;
@@ -1441,7 +1446,7 @@ static void CP_PeerFailCloseWSReader(WS_ReaderInfo CP_WSR_Stream,
 
     if (CP_WSR_Stream->ReaderStatus == NewState)
     {
-        CP_verbose(ParentStream,
+        CP_verbose(ParentStream, TraceVerbose,
                    "In PeerFailCloseWSReader, but status is already set% d\n",
                    ParentStream->Status);
         return;
@@ -1454,7 +1459,7 @@ static void CP_PeerFailCloseWSReader(WS_ReaderInfo CP_WSR_Stream,
         (NewState == PeerFailed))
     {
         // enter this on fail or deliberate close
-        CP_verbose(ParentStream,
+        CP_verbose(ParentStream, PerRankVerbose,
                    "In PeerFailCloseWSReader, releasing sent timesteps\n");
         DerefAllSentTimesteps(CP_WSR_Stream->ParentStream, CP_WSR_Stream);
         CP_WSR_Stream->OldestUnreleasedTimestep =
@@ -1470,12 +1475,13 @@ static void CP_PeerFailCloseWSReader(WS_ReaderInfo CP_WSR_Stream,
         if (NewState == PeerFailed)
         {
             // move to fully closed state later
-            CMfree(CMadd_delayed_task(ParentStream->CPInfo->cm, 2, 0,
+            CMfree(CMadd_delayed_task(ParentStream->CPInfo->SharedCM->cm, 2, 0,
                                       CloseWSRStream, CP_WSR_Stream));
         }
     }
-    CP_verbose(ParentStream, "Moving Reader stream %p to status %s\n",
-               CP_WSR_Stream, SSTStreamStatusStr[NewState]);
+    CP_verbose(ParentStream, PerStepVerbose,
+               "Moving Reader stream %p to status %s\n", CP_WSR_Stream,
+               SSTStreamStatusStr[NewState]);
 
     QueueMaintenance(ParentStream);
 }
@@ -1501,12 +1507,12 @@ void SstWriterClose(SstStream Stream)
     STREAM_MUTEX_LOCK(Stream);
     Msg.FinalTimestep = Stream->LastProvidedTimestep;
     CP_verbose(
-        Stream,
+        Stream, PerStepVerbose,
         "SstWriterClose, Sending Close at Timestep %d, one to each reader\n",
         Msg.FinalTimestep);
 
-    sendOneToEachReaderRank(Stream, Stream->CPInfo->WriterCloseFormat, &Msg,
-                            &Msg.RS_Stream);
+    sendOneToEachReaderRank(Stream, Stream->CPInfo->SharedCM->WriterCloseFormat,
+                            &Msg, &Msg.RS_Stream);
 
     UntagPreciousTimesteps(Stream);
     Stream->ConfigParams->ReserveQueueLimit = 0;
@@ -1535,9 +1541,9 @@ void SstWriterClose(SstStream Stream)
         }
         while (Stream->QueuedTimesteps)
         {
-            CP_verbose(Stream,
+            CP_verbose(Stream, PerStepVerbose,
                        "Waiting for timesteps to be released in WriterClose\n");
-            if (Stream->CPVerbose)
+            if (Stream->CPVerbosityLevel >= TraceVerbose)
             {
                 CPTimestepList List = Stream->QueuedTimesteps;
                 char *StringList = malloc(1);
@@ -1546,7 +1552,7 @@ void SstWriterClose(SstStream Stream)
                 while (List)
                 {
                     char tmp[20];
-                    CP_verbose(Stream,
+                    CP_verbose(Stream, TraceVerbose,
                                "IN TS WAIT, ENTRIES are Timestep %ld (exp %d, "
                                "Prec %d, Ref %d), Count now %d\n",
                                List->Timestep, List->Expired,
@@ -1558,15 +1564,16 @@ void SstWriterClose(SstStream Stream)
                     strcat(StringList, tmp);
                     List = List->Next;
                 }
-                CP_verbose(Stream, "The timesteps still queued are: %s\n",
-                           StringList);
+                CP_verbose(Stream, TraceVerbose,
+                           "The timesteps still queued are: %s\n", StringList);
                 free(StringList);
             }
-            CP_verbose(Stream, "Reader Count is %d\n", Stream->ReaderCount);
+            CP_verbose(Stream, TraceVerbose, "Reader Count is %d\n",
+                       Stream->ReaderCount);
             for (int i = 0; i < Stream->ReaderCount; i++)
             {
                 CP_verbose(
-                    Stream, "Reader [%d] status is %s\n", i,
+                    Stream, TraceVerbose, "Reader [%d] status is %s\n", i,
                     SSTStreamStatusStr[Stream->Readers[i]->ReaderStatus]);
             }
             /* NEED TO HANDLE FAILURE HERE */
@@ -1636,10 +1643,15 @@ void SstWriterClose(SstStream Stream)
     STREAM_MUTEX_UNLOCK(Stream);
     gettimeofday(&CloseTime, NULL);
     timersub(&CloseTime, &Stream->ValidStartTime, &Diff);
-    if (Stream->Stats)
-        Stream->Stats->ValidTimeSecs = (double)Diff.tv_usec / 1e6 + Diff.tv_sec;
+    Stream->Stats.StreamValidTimeSecs =
+        (double)Diff.tv_usec / 1e6 + Diff.tv_sec;
 
-    CP_verbose(Stream, "All timesteps are released in WriterClose\n");
+    if (Stream->CPVerbosityLevel >= (int)SummaryVerbose)
+    {
+        DoStreamSummary(Stream);
+    }
+    CP_verbose(Stream, PerStepVerbose,
+               "All timesteps are released in WriterClose\n");
 
     /*
      *  Only rank 0 removes contact info, and only when everything is closed.
@@ -1827,7 +1839,8 @@ static void ProcessReaderStatusList(SstStream Stream,
     {
         if (Stream->Readers[i]->ReaderStatus != Metadata->ReaderStatus[i])
         {
-            CP_verbose(Stream, "Adjusting reader %d status from %s to %s\n", i,
+            CP_verbose(Stream, PerRankVerbose,
+                       "Adjusting reader %d status from %s to %s\n", i,
                        SSTStreamStatusStr[Stream->Readers[i]->ReaderStatus],
                        SSTStreamStatusStr[Metadata->ReaderStatus[i]]);
             CP_PeerFailCloseWSReader(Stream->Readers[i],
@@ -1857,12 +1870,13 @@ static void ActOnTSLockStatus(SstStream Stream, long Timestep)
             }
             Msg.Timestep = Timestep;
             SomethingSent++;
-            sendOneToWSRCohort(Stream->Readers[i],
-                               Stream->CPInfo->CommPatternLockedFormat, &Msg,
-                               &Msg.RS_Stream);
+            sendOneToWSRCohort(
+                Stream->Readers[i],
+                Stream->CPInfo->SharedCM->CommPatternLockedFormat, &Msg,
+                &Msg.RS_Stream);
             Stream->Readers[i]->PreloadMode = SstPreloadLearned;
             Stream->Readers[i]->PreloadModeActiveTimestep = Timestep;
-            CP_verbose(Stream,
+            CP_verbose(Stream, PerStepVerbose,
                        "Setting preload mode Learned for reader %d, active at "
                        "timestep %ld\n",
                        i, Timestep);
@@ -1871,7 +1885,7 @@ static void ActOnTSLockStatus(SstStream Stream, long Timestep)
     if (SomethingSent)
     {
         CP_verbose(
-            Stream,
+            Stream, TraceVerbose,
             "Doing a barrier after notifying DP of preload mode changes\n");
         SMPI_Barrier(Stream->mpiComm);
     }
@@ -1885,7 +1899,7 @@ static void ProcessReleaseList(SstStream Stream, ReturnMetadataInfo Metadata)
     for (int i = 0; i < Metadata->ReleaseCount; i++)
     {
         CPTimestepList List = Stream->QueuedTimesteps;
-        CP_verbose(Stream, "Release List, TS %ld\n",
+        CP_verbose(Stream, TraceVerbose, "Release List, TS %ld\n",
                    Metadata->ReleaseList[i].Timestep);
         while (List)
         {
@@ -1905,12 +1919,12 @@ static void ProcessReleaseList(SstStream Stream, ReturnMetadataInfo Metadata)
                 assert(j < Stream->ReaderCount);
                 if (List->Timestep > Stream->Readers[j]->LastReleasedTimestep)
                 {
-                    CP_verbose(Stream,
+                    CP_verbose(Stream, TraceVerbose,
                                "Updating reader %d last released to %ld\n", j,
                                List->Timestep);
                     Stream->Readers[j]->LastReleasedTimestep = List->Timestep;
                 }
-                CP_verbose(Stream,
+                CP_verbose(Stream, TraceVerbose,
                            "Release List, and set ref count of timestep %ld\n",
                            Metadata->ReleaseList[i].Timestep);
                 /* per reader release here */
@@ -1949,7 +1963,7 @@ static void ProcessLockDefnsList(SstStream Stream, ReturnMetadataInfo Metadata)
         WS_ReaderInfo Reader = (WS_ReaderInfo)Stream->Readers[j];
 
         Reader->FullCommPatternLocked = 1;
-        CP_verbose(Stream, "LockDefns List, FOUND TS %ld\n",
+        CP_verbose(Stream, TraceVerbose, "LockDefns List, FOUND TS %ld\n",
                    Metadata->LockDefnsList[i].Timestep);
     }
     STREAM_MUTEX_UNLOCK(Stream);
@@ -2107,6 +2121,14 @@ extern void SstInternalProvideTimestep(
     Stream->DP_Interface->provideTimestep(&Svcs, Stream->DP_Stream, Data,
                                           LocalMetadata, Timestep,
                                           &DP_TimestepInfo);
+    if (Formats)
+    {
+        FFSFormatList tmp = Formats;
+        while (tmp)
+        {
+            tmp = tmp->Next;
+        }
+    }
     STREAM_MUTEX_LOCK(Stream);
 
     /* Md is the local contribution to MetaData */
@@ -2141,6 +2163,7 @@ extern void SstInternalProvideTimestep(
     Entry->Next = Stream->QueuedTimesteps;
     Stream->QueuedTimesteps = Entry;
     Stream->QueuedTimestepCount++;
+    Stream->Stats.TimestepsCreated++;
     /* no one waits on timesteps being added, so no condition signal to note
      * change */
 
@@ -2161,7 +2184,7 @@ extern void SstInternalProvideTimestep(
         QueueMaintenance(Stream);
         if (Stream->QueueFullPolicy == SstQueueFullDiscard)
         {
-            CP_verbose(Stream,
+            CP_verbose(Stream, TraceVerbose,
                        "Testing Discard Condition, Queued Timestep Count %d, "
                        "QueueLimit %d\n",
                        Stream->QueuedTimestepCount, Stream->QueueLimit);
@@ -2176,7 +2199,8 @@ extern void SstInternalProvideTimestep(
             while ((Stream->QueueLimit > 0) &&
                    (Stream->QueuedTimestepCount > Stream->QueueLimit))
             {
-                CP_verbose(Stream, "Blocking on QueueFull condition\n");
+                CP_verbose(Stream, PerStepVerbose,
+                           "Blocking on QueueFull condition\n");
                 STREAM_CONDITION_WAIT(Stream);
             }
         }
@@ -2267,15 +2291,15 @@ extern void SstInternalProvideTimestep(
         Msg->Metadata = NULL;
         Msg->DP_TimestepInfo = NULL;
 
-        CP_verbose(Stream,
+        CP_verbose(Stream, PerStepVerbose,
                    "Sending Empty TimestepMetadata for Discarded "
                    "timestep %d, one to each reader\n",
                    Timestep);
 
         STREAM_MUTEX_LOCK(Stream);
-        sendOneToEachReaderRank(Stream,
-                                Stream->CPInfo->DeliverTimestepMetadataFormat,
-                                Msg, &Msg->RS_Stream);
+        sendOneToEachReaderRank(
+            Stream, Stream->CPInfo->SharedCM->DeliverTimestepMetadataFormat,
+            Msg, &Msg->RS_Stream);
 
         Entry->Expired = 1;
         Entry->ReferenceCount = 0;
@@ -2285,7 +2309,7 @@ extern void SstInternalProvideTimestep(
     else
     {
 
-        CP_verbose(Stream,
+        CP_verbose(Stream, PerStepVerbose,
                    "Sending TimestepMetadata for timestep %d (ref count "
                    "%d), one to each reader\n",
                    Timestep, Entry->ReferenceCount);
@@ -2299,8 +2323,12 @@ extern void SstInternalProvideTimestep(
     while (PendingReaderCount--)
     {
         WS_ReaderInfo reader;
-        CP_verbose(Stream,
-                   "Writer side ReaderLateArrival accepting incoming reader\n");
+        if (Stream->Rank == 0)
+        {
+            CP_verbose(
+                Stream, SummaryVerbose,
+                "Writer side ReaderLateArrival accepting incoming reader\n");
+        }
         reader = WriterParticipateInReaderOpen(Stream);
         if (!reader)
         {
@@ -2364,7 +2392,8 @@ extern void SstWriterDefinitionLock(SstStream Stream, long EffectiveTimestep)
         }
     }
     STREAM_MUTEX_UNLOCK(Stream);
-    CP_verbose(Stream, "Writer-side definitions lock as of timestep %d\n",
+    CP_verbose(Stream, PerStepVerbose,
+               "Writer-side definitions lock as of timestep %d\n",
                EffectiveTimestep);
 }
 
@@ -2421,7 +2450,7 @@ void CP_ReaderCloseHandler(CManager cm, CMConnection conn, void *Msg_v,
         return;
     }
 
-    CP_verbose(CP_WSR_Stream->ParentStream,
+    CP_verbose(CP_WSR_Stream->ParentStream, PerStepVerbose,
                "Reader Close message received for stream %p.  Setting state to "
                "PeerClosed and releasing timesteps.\n",
                CP_WSR_Stream);
@@ -2470,12 +2499,12 @@ void CP_ReaderActivateHandler(CManager cm, CMConnection conn, void *Msg_v,
     struct _ReaderActivateMsg *Msg = (struct _ReaderActivateMsg *)Msg_v;
 
     WS_ReaderInfo CP_WSR_Stream = Msg->WSR_Stream;
-    CP_verbose(CP_WSR_Stream->ParentStream,
+    CP_verbose(CP_WSR_Stream->ParentStream, PerStepVerbose,
                "Reader Activate message received "
                "for Stream %p.  Setting state to "
                "Established.\n",
                CP_WSR_Stream);
-    CP_verbose(CP_WSR_Stream->ParentStream,
+    CP_verbose(CP_WSR_Stream->ParentStream, PerStepVerbose,
                "Parent stream reader count is now %d.\n",
                CP_WSR_Stream->ParentStream->ReaderCount);
     STREAM_MUTEX_LOCK(CP_WSR_Stream->ParentStream);
@@ -2506,13 +2535,14 @@ extern void CP_ReleaseTimestepHandler(CManager cm, CMConnection conn,
             ReaderNum = i;
         }
     }
-    CP_verbose(ParentStream,
+    CP_verbose(ParentStream, TraceVerbose,
                "Received a release timestep message "
                "for timestep %d from reader cohort %d\n",
                Msg->Timestep, ReaderNum);
 
     /* decrement the reference count for the released timestep */
-    CP_verbose(ParentStream, "Got the lock in release timestep\n");
+    CP_verbose(ParentStream, TraceVerbose,
+               "Got the lock in release timestep\n");
     Reader->LastReleasedTimestep = Msg->Timestep;
     if ((ParentStream->Rank == 0) &&
         (ParentStream->ConfigParams->CPCommPattern == SstCPCommMin))
@@ -2525,13 +2555,14 @@ extern void CP_ReleaseTimestepHandler(CManager cm, CMConnection conn,
         ParentStream->ReleaseList[ParentStream->ReleaseCount].Reader = Reader;
         ParentStream->ReleaseCount++;
     }
-    CP_verbose(ParentStream, "Doing dereference sent\n");
+    CP_verbose(ParentStream, TraceVerbose, "Doing dereference sent\n");
     DerefSentTimestep(ParentStream, Reader, Msg->Timestep);
-    CP_verbose(ParentStream, "Doing QueueMaint\n");
+    CP_verbose(ParentStream, TraceVerbose, "Doing QueueMaint\n");
     QueueMaintenance(ParentStream);
     Reader->OldestUnreleasedTimestep = Msg->Timestep + 1;
     STREAM_CONDITION_SIGNAL(ParentStream);
-    CP_verbose(ParentStream, "Releasing the lock in release timestep\n");
+    CP_verbose(ParentStream, TraceVerbose,
+               "Releasing the lock in release timestep\n");
     STREAM_MUTEX_UNLOCK(ParentStream);
     TAU_STOP_FUNC();
 }
@@ -2553,7 +2584,7 @@ extern void CP_LockReaderDefinitionsHandler(CManager cm, CMConnection conn,
             ReaderNum = i;
         }
     }
-    CP_verbose(ParentStream,
+    CP_verbose(ParentStream, TraceVerbose,
                "Received a lock reader definitions message "
                "for timestep %d from reader cohort %d\n",
                Msg->Timestep, ReaderNum);
diff --git a/source/adios2/toolkit/sst/cp/ffs_marshal.c b/source/adios2/toolkit/sst/cp/ffs_marshal.c
index 389ed31c64..56d2bd8477 100644
--- a/source/adios2/toolkit/sst/cp/ffs_marshal.c
+++ b/source/adios2/toolkit/sst/cp/ffs_marshal.c
@@ -48,233 +48,155 @@ static char *ConcatName(const char *base_name, const char *postfix)
     return Ret;
 }
 
-static char *BuildVarName(const char *base_name, const char *type,
+static char *BuildVarName(const char *base_name, const int type,
                           const int element_size)
 {
-    int Len = strlen(base_name) + strlen(type) + strlen("SST_") + 16;
+    int Len = strlen(base_name) + 2 + strlen("SST_") + 16;
     char *Ret = malloc(Len);
-    sprintf(Ret, "SST%d_%d_", element_size, (int)strlen(type));
-    strcat(Ret, type);
-    strcat(Ret, "_");
+    sprintf(Ret, "SST%d_%d_", element_size, type);
     strcat(Ret, base_name);
     return Ret;
 }
 
-static void BreakdownVarName(const char *Name, char **base_name_p,
-                             char **type_p, int *element_size_p)
+static void BreakdownVarName(const char *Name, char **base_name_p, int *type_p,
+                             int *element_size_p)
 {
-    int TypeLen;
+    int Type;
     int ElementSize;
-    const char *NameStart;
-    char *TypeStart = strchr(Name, '_') + 1;
-    TypeStart = strchr(TypeStart, '_') + 1;
-    sscanf(Name, "SST%d_%d_", &ElementSize, &TypeLen);
-    NameStart = TypeStart + TypeLen + 1;
+    const char *NameStart = strchr(strchr(Name, '_') + 1, '_') + 1;
+    sscanf(Name, "SST%d_%d_", &ElementSize, &Type);
     *element_size_p = ElementSize;
-    *type_p = malloc(TypeLen + 1);
-    strncpy(*type_p, TypeStart, TypeLen);
-    (*type_p)[TypeLen] = 0;
+    *type_p = Type;
     *base_name_p = strdup(NameStart);
 }
 
-static char *BuildArrayName(const char *base_name, const char *type,
-                            const int element_size)
+static char *BuildArrayDimsName(const char *base_name, const int type,
+                                const int element_size)
 {
-    int Len = strlen(base_name) + strlen(type) + strlen("SST_") + 16;
+    int Len = strlen(base_name) + 3 + strlen("SST_") + 16;
     char *Ret = malloc(Len);
-    sprintf(Ret, "SST%d_%d_", element_size, (int)strlen(type));
-    strcat(Ret, type);
-    strcat(Ret, "_");
+    sprintf(Ret, "SST%d_%d_", element_size, type);
     strcat(Ret, base_name);
     strcat(Ret, "Dims");
     return Ret;
 }
 
+static char *BuildArrayDBCountName(const char *base_name, const int type,
+                                   const int element_size)
+{
+    int Len = strlen(base_name) + 3 + strlen("SST_") + 16;
+    char *Ret = malloc(Len);
+    sprintf(Ret, "SST%d_%d_", element_size, type);
+    strcat(Ret, base_name);
+    strcat(Ret, "DBCount");
+    return Ret;
+}
+
 static void BreakdownArrayName(const char *Name, char **base_name_p,
-                               char **type_p, int *element_size_p)
+                               int *type_p, int *element_size_p)
 {
-    int TypeLen;
+    int Type;
     int ElementSize;
-    const char *NameStart;
-    char *TypeStart = strchr(Name, '_') + 1;
-    TypeStart = strchr(TypeStart, '_') + 1;
-    sscanf(Name, "SST%d_%d_", &ElementSize, &TypeLen);
-    NameStart = TypeStart + TypeLen + 1;
+    const char *NameStart = strchr(strchr(Name, '_') + 1, '_') + 1;
+    sscanf(Name, "SST%d_%d_", &ElementSize, &Type);
     *element_size_p = ElementSize;
-    *type_p = malloc(TypeLen + 1);
-    strncpy(*type_p, TypeStart, TypeLen);
-    (*type_p)[TypeLen] = 0;
+    *type_p = Type;
     *base_name_p = strdup(NameStart);
     (*base_name_p)[strlen(*base_name_p) - 4] = 0; // kill "Dims"
 }
 
-static char *TranslateADIOS2Type2FFS(const char *Type)
+static char *TranslateADIOS2Type2FFS(const int Type)
 {
-    if (strcmp(Type, "char") == 0)
-    {
-        return strdup("integer");
-    }
-    else if (strcmp(Type, "signed char") == 0)
+    switch (Type)
     {
+    case Int8:
+    case Int16:
+    case Int32:
+    case Int64:
         return strdup("integer");
-    }
-    else if (strcmp(Type, "unsigned char") == 0)
-    {
+    case UInt8:
+    case UInt16:
+    case UInt32:
+    case UInt64:
         return strdup("unsigned integer");
-    }
-    else if (strcmp(Type, "short") == 0)
-    {
-        return strdup("integer");
-    }
-    else if (strcmp(Type, "unsigned short") == 0)
-    {
-        return strdup("unsigned integer");
-    }
-    else if (strcmp(Type, "int") == 0)
-    {
-        return strdup("integer");
-    }
-    else if (strcmp(Type, "unsigned int") == 0)
-    {
-        return strdup("unsigned integer");
-    }
-    else if (strcmp(Type, "long int") == 0)
-    {
-        return strdup("integer");
-    }
-    else if (strcmp(Type, "long long int") == 0)
-    {
-        return strdup("integer");
-    }
-    else if (strcmp(Type, "unsigned long int") == 0)
-    {
-        return strdup("unsigned integer");
-    }
-    else if (strcmp(Type, "unsigned long long int") == 0)
-    {
-        return strdup("unsigned integer");
-    }
-    else if (strcmp(Type, "float") == 0)
-    {
-        return strdup("float");
-    }
-    else if (strcmp(Type, "double") == 0)
-    {
+    case Float:
+    case Double:
         return strdup("float");
-    }
-    else if (strcmp(Type, "long double") == 0)
-    {
-        return strdup("float");
-    }
-    else if (strcmp(Type, "float complex") == 0)
-    {
+    case FloatComplex:
         return strdup("complex4");
-    }
-    else if (strcmp(Type, "double complex") == 0)
-    {
+    case DoubleComplex:
         return strdup("complex8");
+    case String:
+        return strdup("string");
     }
-    else if (strcmp(Type, "int8_t") == 0)
-    {
-        return strdup("integer");
-    }
-    else if (strcmp(Type, "int16_t") == 0)
-    {
-        return strdup("integer");
-    }
-    else if (strcmp(Type, "int32_t") == 0)
-    {
-        return strdup("integer");
-    }
-    else if (strcmp(Type, "int64_t") == 0)
-    {
-        return strdup("integer");
-    }
-    else if (strcmp(Type, "uint8_t") == 0)
-    {
-        return strdup("unsigned integer");
-    }
-    else if (strcmp(Type, "uint16_t") == 0)
-    {
-        return strdup("unsigned integer");
-    }
-    else if (strcmp(Type, "uint32_t") == 0)
-    {
-        return strdup("unsigned integer");
-    }
-    else if (strcmp(Type, "uint64_t") == 0)
-    {
-        return strdup("unsigned integer");
-    }
-
-    return strdup(Type);
+    return 0;
 }
 
-static char *TranslateFFSType2ADIOS(const char *Type, int size)
+static int TranslateFFSType2ADIOS(const char *Type, int size)
 {
     if (strcmp(Type, "integer") == 0)
     {
         if (size == 1)
         {
-            return strdup("int8_t");
+            return Int8;
         }
         else if (size == 2)
         {
-            return strdup("int16_t");
+            return Int16;
         }
         else if (size == 4)
         {
-            return strdup("int32_t");
+            return Int32;
         }
         else if (size == 8)
         {
-            return strdup("int64_t");
+            return Int64;
         }
     }
     else if (strcmp(Type, "unsigned integer") == 0)
     {
         if (size == 1)
         {
-            return strdup("uint8_t");
+            return UInt8;
         }
         else if (size == 2)
         {
-            return strdup("uint16_t");
+            return UInt16;
         }
         else if (size == 4)
         {
-            return strdup("uint32_t");
+            return UInt32;
         }
         else if (size == 8)
         {
-            return strdup("uint64_t");
+            return UInt64;
         }
     }
     else if ((strcmp(Type, "double") == 0) || (strcmp(Type, "float") == 0))
     {
         if (size == sizeof(float))
         {
-            return strdup("float");
+            return Float;
         }
         else if ((sizeof(long double) != sizeof(double)) &&
                  (size == sizeof(long double)))
         {
-            return strdup("long double");
+            return Double;
         }
         else
         {
-            return strdup("double");
+            return Double;
         }
     }
     else if (strcmp(Type, "complex4") == 0)
     {
-        return strdup("float complex");
+        return FloatComplex;
     }
     else if (strcmp(Type, "complex8") == 0)
     {
-        return strdup("double complex");
+        return DoubleComplex;
     }
-    return strdup(Type);
+    return 0;
 }
 
 static void RecalcMarshalStorageSize(SstStream Stream)
@@ -288,7 +210,8 @@ static void RecalcMarshalStorageSize(SstStream Stream)
         NewDataSize =
             (LastDataField->field_offset + LastDataField->field_size + 7) & ~7;
         Stream->D = realloc(Stream->D, NewDataSize + 8);
-        memset(Stream->D + Stream->DataSize, 0, NewDataSize - Stream->DataSize);
+        memset((char *)(Stream->D) + Stream->DataSize, 0,
+               NewDataSize - Stream->DataSize);
         Stream->DataSize = NewDataSize;
     }
     if (Info->MetaFieldCount)
@@ -299,7 +222,7 @@ static void RecalcMarshalStorageSize(SstStream Stream)
         NewMetaSize =
             (LastMetaField->field_offset + LastMetaField->field_size + 7) & ~7;
         Stream->M = realloc(Stream->M, NewMetaSize + 8);
-        memset(Stream->M + Stream->MetadataSize, 0,
+        memset((char *)(Stream->M) + Stream->MetadataSize, 0,
                NewMetaSize - Stream->MetadataSize);
         Stream->MetadataSize = NewMetaSize;
     }
@@ -319,7 +242,7 @@ static void RecalcAttributeStorageSize(SstStream Stream)
                            ~7;
         Info->AttributeData =
             realloc(Info->AttributeData, NewAttributeSize + 8);
-        memset(Info->AttributeData + Info->AttributeSize, 0,
+        memset((char *)(Info->AttributeData) + Info->AttributeSize, 0,
                NewAttributeSize - Info->AttributeSize);
         Info->AttributeSize = NewAttributeSize;
     }
@@ -364,7 +287,7 @@ static void AddSimpleField(FMFieldList *FieldP, int *CountP, const char *Name,
 }
 
 static void AddField(FMFieldList *FieldP, int *CountP, const char *Name,
-                     const char *Type, int ElementSize)
+                     const int Type, int ElementSize)
 {
     char *TransType = TranslateADIOS2Type2FFS(Type);
     AddSimpleField(FieldP, CountP, Name, TransType, ElementSize);
@@ -372,7 +295,7 @@ static void AddField(FMFieldList *FieldP, int *CountP, const char *Name,
 }
 
 static void AddFixedArrayField(FMFieldList *FieldP, int *CountP,
-                               const char *Name, const char *Type,
+                               const char *Name, const int Type,
                                int ElementSize, int DimCount)
 {
     const char *TransType = TranslateADIOS2Type2FFS(Type);
@@ -384,7 +307,7 @@ static void AddFixedArrayField(FMFieldList *FieldP, int *CountP,
     (*FieldP)[*CountP - 1].field_size = ElementSize;
 }
 static void AddVarArrayField(FMFieldList *FieldP, int *CountP, const char *Name,
-                             const char *Type, int ElementSize, char *SizeField)
+                             const int Type, int ElementSize, char *SizeField)
 {
     char *TransType = TranslateADIOS2Type2FFS(Type);
     char *TypeWithArray = malloc(strlen(TransType) + strlen(SizeField) + 8);
@@ -444,7 +367,7 @@ extern void FFSFreeMarshalData(SstStream Stream)
 
         for (int i = 0; i < Info->RecCount; i++)
         {
-            free(Info->RecList[i].Type);
+            //            free(Info->RecList[i].Type);
         }
         if (Info->RecList)
             free(Info->RecList);
@@ -485,17 +408,27 @@ extern void FFSFreeMarshalData(SstStream Stream)
                 free(Info->DataFieldLists);
             for (int i = 0; i < Info->VarCount; i++)
             {
-                free(Info->VarList[i].VarName);
-                free(Info->VarList[i].PerWriterMetaFieldDesc);
-                free(Info->VarList[i].PerWriterDataFieldDesc);
-                free(Info->VarList[i].PerWriterStart);
-                free(Info->VarList[i].PerWriterCounts);
-                free(Info->VarList[i].PerWriterIncomingData);
-                free(Info->VarList[i].PerWriterIncomingSize);
+                free(Info->VarList[i]->VarName);
+                free(Info->VarList[i]->PerWriterMetaFieldOffset);
+                free(Info->VarList[i]->PerWriterBlockCount);
+                free(Info->VarList[i]->PerWriterBlockStart);
+                free(Info->VarList[i]->PerWriterStart);
+                free(Info->VarList[i]->PerWriterCounts);
+                free(Info->VarList[i]->PerWriterIncomingData);
+                free(Info->VarList[i]->PerWriterIncomingSize);
+                free(Info->VarList[i]);
             }
             if (Info->VarList)
                 free(Info->VarList);
 
+            struct ControlInfo *tmp = Info->ControlBlocks;
+            Info->ControlBlocks = NULL;
+            while (tmp)
+            {
+                struct ControlInfo *next = tmp->Next;
+                free(tmp);
+                tmp = next;
+            }
             free(Info);
             Stream->ReaderMarshalData = NULL;
         }
@@ -507,8 +440,8 @@ extern void FFSFreeMarshalData(SstStream Stream)
 #endif
 
 static FFSWriterRec CreateWriterRec(SstStream Stream, void *Variable,
-                                    const char *Name, const char *Type,
-                                    size_t ElemSize, size_t DimCount)
+                                    const char *Name, int Type, size_t ElemSize,
+                                    size_t DimCount)
 {
     if (!Stream->WriterMarshalData)
     {
@@ -522,7 +455,7 @@ static FFSWriterRec CreateWriterRec(SstStream Stream, void *Variable,
     Rec->Key = Variable;
     Rec->FieldID = Info->RecCount;
     Rec->DimCount = DimCount;
-    Rec->Type = strdup(Type);
+    Rec->Type = Type;
     if (DimCount == 0)
     {
         // simple field, only add base value FMField to metadata
@@ -541,8 +474,9 @@ static FFSWriterRec CreateWriterRec(SstStream Stream, void *Variable,
     {
         // Array field.  To Metadata, add FMFields for DimCount, Shape, Count
         // and Offsets matching _MetaArrayRec
-        char *ArrayName = BuildArrayName(Name, Type, ElemSize);
-        AddField(&Info->MetaFields, &Info->MetaFieldCount, ArrayName, "integer",
+        char *ArrayName = BuildArrayDimsName(Name, Type, ElemSize);
+        char *ArrayDBCount = BuildArrayDBCountName(Name, Type, ElemSize);
+        AddField(&Info->MetaFields, &Info->MetaFieldCount, ArrayName, Int64,
                  sizeof(size_t));
         free(ArrayName);
         Rec->MetaOffset =
@@ -550,12 +484,15 @@ static FFSWriterRec CreateWriterRec(SstStream Stream, void *Variable,
         char *ShapeName = ConcatName(Name, "Shape");
         char *CountName = ConcatName(Name, "Count");
         char *OffsetsName = ConcatName(Name, "Offsets");
+        AddField(&Info->MetaFields, &Info->MetaFieldCount, ArrayDBCount, Int64,
+                 sizeof(size_t));
         AddFixedArrayField(&Info->MetaFields, &Info->MetaFieldCount, ShapeName,
-                           "integer", sizeof(size_t), DimCount);
-        AddFixedArrayField(&Info->MetaFields, &Info->MetaFieldCount, CountName,
-                           "integer", sizeof(size_t), DimCount);
-        AddFixedArrayField(&Info->MetaFields, &Info->MetaFieldCount,
-                           OffsetsName, "integer", sizeof(size_t), DimCount);
+                           Int64, sizeof(size_t), DimCount);
+        AddVarArrayField(&Info->MetaFields, &Info->MetaFieldCount, CountName,
+                         Int64, sizeof(size_t), ArrayDBCount);
+        AddVarArrayField(&Info->MetaFields, &Info->MetaFieldCount, OffsetsName,
+                         Int64, sizeof(size_t), ArrayDBCount);
+        free(ArrayDBCount);
         free(ShapeName);
         free(CountName);
         free(OffsetsName);
@@ -564,13 +501,13 @@ static FFSWriterRec CreateWriterRec(SstStream Stream, void *Variable,
         if ((Stream->ConfigParams->CompressionMethod == SstCompressZFP) &&
             ZFPcompressionPossible(Type, DimCount))
         {
-            Type = "char";
+            Type = Int8;
             ElemSize = 1;
         }
         // To Data, add FMFields for ElemCount and Array matching _ArrayRec
         char *ElemCountName = ConcatName(Name, "ElemCount");
-        AddField(&Info->DataFields, &Info->DataFieldCount, ElemCountName,
-                 "integer", sizeof(size_t));
+        AddField(&Info->DataFields, &Info->DataFieldCount, ElemCountName, Int64,
+                 sizeof(size_t));
         Rec->DataOffset =
             Info->DataFields[Info->DataFieldCount - 1].field_offset;
         char *SstName = ConcatName(Name, "");
@@ -595,10 +532,11 @@ typedef struct _ArrayRec
 
 typedef struct _MetaArrayRec
 {
-    size_t Dims;
-    size_t *Shape;
-    size_t *Count;
-    size_t *Offsets;
+    size_t Dims;     // How many dimensions does this array have
+    size_t DBCount;  // Dimens * BlockCount
+    size_t *Shape;   // Global dimensionality  [Dims]	NULL for local
+    size_t *Count;   // Per-block Counts	  [DBCount]
+    size_t *Offsets; // Per-block Offsets	  [DBCount]	NULL for local
 } MetaArrayRec;
 
 typedef struct _FFSTimestepInfo
@@ -653,6 +591,14 @@ static size_t *CopyDims(const size_t Count, const size_t *Vals)
     return Ret;
 }
 
+static size_t *AppendDims(size_t *OldDims, const size_t OldCount,
+                          const size_t Count, const size_t *Vals)
+{
+    size_t *Ret = realloc(OldDims, (OldCount + Count) * sizeof(Ret[0]));
+    memcpy(Ret + OldCount, Vals, Count * sizeof(Ret[0]));
+    return Ret;
+}
+
 static size_t CalcSize(const size_t Count, const size_t *Vals)
 {
     size_t i;
@@ -688,9 +634,9 @@ static FFSVarRec LookupVarByKey(SstStream Stream, void *Key)
 
     for (int i = 0; i < Info->VarCount; i++)
     {
-        if (Info->VarList[i].Variable == Key)
+        if (Info->VarList[i]->Variable == Key)
         {
-            return &Info->VarList[i];
+            return Info->VarList[i];
         }
     }
 
@@ -703,9 +649,9 @@ static FFSVarRec LookupVarByName(SstStream Stream, const char *Name)
 
     for (int i = 0; i < Info->VarCount; i++)
     {
-        if (strcmp(Info->VarList[i].VarName, Name) == 0)
+        if (strcmp(Info->VarList[i]->VarName, Name) == 0)
         {
-            return &Info->VarList[i];
+            return Info->VarList[i];
         }
     }
 
@@ -717,25 +663,24 @@ static FFSVarRec CreateVarRec(SstStream Stream, const char *ArrayName)
     struct FFSReaderMarshalBase *Info = Stream->ReaderMarshalData;
     Info->VarList =
         realloc(Info->VarList, sizeof(Info->VarList[0]) * (Info->VarCount + 1));
-    memset(&Info->VarList[Info->VarCount], 0, sizeof(Info->VarList[0]));
-    Info->VarList[Info->VarCount].VarName = strdup(ArrayName);
-    Info->VarList[Info->VarCount].PerWriterMetaFieldDesc =
-        calloc(sizeof(FMFieldList), Stream->WriterCohortSize);
-    Info->VarList[Info->VarCount].PerWriterDataFieldDesc =
-        calloc(sizeof(FMFieldList), Stream->WriterCohortSize);
-    Info->VarList[Info->VarCount].PerWriterStart =
+    FFSVarRec Ret = calloc(1, sizeof(struct FFSVarRec));
+    Ret->VarName = strdup(ArrayName);
+    Ret->PerWriterMetaFieldOffset =
+        calloc(sizeof(size_t), Stream->WriterCohortSize);
+    Ret->PerWriterStart = calloc(sizeof(size_t *), Stream->WriterCohortSize);
+    Ret->PerWriterBlockStart =
         calloc(sizeof(size_t *), Stream->WriterCohortSize);
-    Info->VarList[Info->VarCount].PerWriterCounts =
+    Ret->PerWriterBlockCount =
         calloc(sizeof(size_t *), Stream->WriterCohortSize);
-    Info->VarList[Info->VarCount].PerWriterIncomingData =
+    Ret->PerWriterCounts = calloc(sizeof(size_t *), Stream->WriterCohortSize);
+    Ret->PerWriterIncomingData =
         calloc(sizeof(void *), Stream->WriterCohortSize);
-    Info->VarList[Info->VarCount].PerWriterIncomingSize =
+    Ret->PerWriterIncomingSize =
         calloc(sizeof(size_t), Stream->WriterCohortSize);
-    return &Info->VarList[Info->VarCount++];
+    Info->VarList[Info->VarCount++] = Ret;
+    return Ret;
 }
 
-extern void CP_verbose(SstStream Stream, char *Format, ...);
-
 extern int SstFFSWriterBeginStep(SstStream Stream, int mode,
                                  const float timeout_sec)
 {
@@ -774,13 +719,15 @@ extern int SstFFSGetDeferred(SstStream Stream, void *Variable, const char *Name,
     {
         void *IncomingDataBase =
             ((char *)Info->MetadataBaseAddrs[GetFromWriter]) +
-            Var->PerWriterMetaFieldDesc[GetFromWriter]->field_offset;
-        memcpy(Data, IncomingDataBase,
-               Var->PerWriterMetaFieldDesc[GetFromWriter]->field_size);
+            Var->PerWriterMetaFieldOffset[GetFromWriter];
+        memcpy(Data, IncomingDataBase, Var->ElementSize);
         return 0; // No Sync needed
     }
     else
     {
+        CP_verbose(Stream, TraceVerbose,
+                   "Get request, Name %s, Start %zu, Count %zu\n", Name,
+                   Start[0], Count[0]);
         // Build request structure and enter it into requests list
         FFSArrayRequest Req = malloc(sizeof(*Req));
         Req->VarRec = Var;
@@ -812,9 +759,8 @@ extern int SstFFSGetLocalDeferred(SstStream Stream, void *Variable,
     {
         void *IncomingDataBase =
             ((char *)Info->MetadataBaseAddrs[GetFromWriter]) +
-            Var->PerWriterMetaFieldDesc[GetFromWriter]->field_offset;
-        memcpy(Data, IncomingDataBase,
-               Var->PerWriterMetaFieldDesc[GetFromWriter]->field_size);
+            Var->PerWriterMetaFieldOffset[GetFromWriter];
+        memcpy(Data, IncomingDataBase, Var->ElementSize);
         return 0; // No Sync needed
     }
     else
@@ -824,8 +770,11 @@ extern int SstFFSGetLocalDeferred(SstStream Stream, void *Variable,
         memset(Req, 0, sizeof(*Req));
         Req->VarRec = Var;
         Req->RequestType = Local;
-        Req->NodeID = BlockID;
+        Req->BlockID = BlockID;
         // make a copy of Count request
+        CP_verbose(Stream, TraceVerbose,
+                   "Get request local, Name %s, BlockID %d, Count %zu\n", Name,
+                   BlockID, Count[0]);
         Req->Count = malloc(sizeof(Count[0]) * Var->DimCount);
         memcpy(Req->Count, Count, sizeof(Count[0]) * Var->DimCount);
         Req->Data = Data;
@@ -839,7 +788,9 @@ static int NeedWriter(FFSArrayRequest Req, int i)
 {
     if (Req->RequestType == Local)
     {
-        return (Req->NodeID == i);
+        size_t NodeFirst = Req->VarRec->PerWriterBlockStart[i];
+        size_t NodeLast = Req->VarRec->PerWriterBlockCount[i] + NodeFirst - 1;
+        return (NodeFirst <= Req->BlockID) && (NodeLast >= Req->BlockID);
     }
     // else Global case
     for (int j = 0; j < Req->VarRec->DimCount; j++)
@@ -877,7 +828,7 @@ static void IssueReadRequests(SstStream Stream, FFSArrayRequest Reqs)
     {
         for (int i = 0; i < Stream->WriterCohortSize; i++)
         {
-            if (NeedWriter(Reqs, i))
+            if ((Info->WriterInfo[i].Status != Needed) && (NeedWriter(Reqs, i)))
             {
                 Info->WriterInfo[i].Status = Needed;
             }
@@ -992,7 +943,6 @@ static void DecodeAndPrepareData(SstStream Stream, int Writer)
         {
             VarRec->PerWriterIncomingData[Writer] = data_base->Array;
             VarRec->PerWriterIncomingSize[Writer] = data_base->ElemCount;
-            VarRec->PerWriterDataFieldDesc[Writer] = &FieldList[i + 1];
         }
         i += 2;
     }
@@ -1011,17 +961,19 @@ static SstStatusValue WaitForReadRequests(SstStream Stream)
             if (Result == SstSuccess)
             {
                 Info->WriterInfo[i].Status = Full;
-                DecodeAndPrepareData(Stream, i);
+                if (!Stream->ConfigParams->ReaderShortCircuitReads)
+                    DecodeAndPrepareData(Stream, i);
             }
             else
             {
-                CP_verbose(Stream, "Wait for remote read completion failed, "
-                                   "returning failure\n");
+                CP_verbose(Stream, CriticalVerbose,
+                           "Wait for remote read completion failed, "
+                           "returning failure\n");
                 return Result;
             }
         }
     }
-    CP_verbose(Stream, "All remote memory reads completed\n");
+    CP_verbose(Stream, TraceVerbose, "All remote memory reads completed\n");
     return SstSuccess;
 }
 
@@ -1390,12 +1342,25 @@ static void FillReadRequests(SstStream Stream, FFSArrayRequest Reqs)
                 size_t *SelOffset = Reqs->Start;
                 size_t *SelOffsetFree = NULL;
                 size_t *SelSize = Reqs->Count;
-                char *Type = Reqs->VarRec->Type;
+                int Type = Reqs->VarRec->Type;
                 void *IncomingData = Reqs->VarRec->PerWriterIncomingData[i];
                 int FreeIncoming = 0;
 
                 if (Reqs->RequestType == Local)
                 {
+                    int LocalBlockID =
+                        Reqs->BlockID - Reqs->VarRec->PerWriterBlockStart[i];
+                    size_t DataOffset = 0;
+                    for (int i = 0; i < LocalBlockID; i++)
+                    {
+                        int BlockElemCount = 1;
+                        for (int j = 0; j < DimCount; j++)
+                        {
+                            BlockElemCount *= RankSize[j];
+                        }
+                        DataOffset += BlockElemCount * ElementSize;
+                        RankSize += DimCount;
+                    }
                     RankOffset = calloc(DimCount, sizeof(RankOffset[0]));
                     RankOffsetFree = RankOffset;
                     GlobalDimensions =
@@ -1410,6 +1375,7 @@ static void FillReadRequests(SstStream Stream, FFSArrayRequest Reqs)
                     {
                         GlobalDimensions[i] = RankSize[i];
                     }
+                    IncomingData = (char *)IncomingData + DataOffset;
                 }
                 if ((Stream->WriterConfigParams->CompressionMethod ==
                      SstCompressZFP) &&
@@ -1465,12 +1431,14 @@ extern SstStatusValue SstFFSPerformGets(SstStream Stream)
 
     if (Ret == SstSuccess)
     {
-        FillReadRequests(Stream, Info->PendingVarRequests);
+        if (!Stream->ConfigParams->ReaderShortCircuitReads)
+            FillReadRequests(Stream, Info->PendingVarRequests);
     }
     else
     {
-        CP_verbose(Stream, "Some memory read failed, not filling requests and "
-                           "returning failure\n");
+        CP_verbose(Stream, CriticalVerbose,
+                   "Some memory read failed, not filling requests and "
+                   "returning failure\n");
     }
     ClearReadRequests(Stream);
 
@@ -1485,7 +1453,7 @@ extern void SstFFSWriterEndStep(SstStream Stream, size_t Timestep)
 
     TAU_START("Marshaling overhead in SstFFSWriterEndStep");
 
-    CP_verbose(Stream, "Calling SstWriterEndStep\n");
+    CP_verbose(Stream, PerStepVerbose, "Calling SstWriterEndStep\n");
     // if field lists have changed, register formats with FFS local context, add
     // to format chain
     if (!Stream->WriterMarshalData)
@@ -1505,13 +1473,15 @@ extern void SstFFSWriterEndStep(SstStream Stream, size_t Timestep)
         struct_list[0].field_list = Info->MetaFields;
         struct_list[0].struct_size =
             FMstruct_size_field_list(Info->MetaFields, sizeof(char *));
+
         FMFormat Format =
             register_data_format(Info->LocalFMContext, &struct_list[0]);
         Info->MetaFormat = Format;
-        Block->FormatServerRep =
-            get_server_rep_FMformat(Format, &Block->FormatServerRepLen);
-        Block->FormatIDRep =
-            get_server_ID_FMformat(Format, &Block->FormatIDRepLen);
+        int size;
+        Block->FormatServerRep = get_server_rep_FMformat(Format, &size);
+        Block->FormatServerRepLen = size;
+        Block->FormatIDRep = get_server_ID_FMformat(Format, &size);
+        Block->FormatIDRepLen = size;
         Block->Next = NULL;
         Formats = Block;
     }
@@ -1530,10 +1500,11 @@ extern void SstFFSWriterEndStep(SstStream Stream, size_t Timestep)
         FMFormat Format =
             register_data_format(Info->LocalFMContext, &struct_list[0]);
         Info->DataFormat = Format;
-        Block->FormatServerRep =
-            get_server_rep_FMformat(Format, &Block->FormatServerRepLen);
-        Block->FormatIDRep =
-            get_server_ID_FMformat(Format, &Block->FormatIDRepLen);
+        int size;
+        Block->FormatServerRep = get_server_rep_FMformat(Format, &size);
+        Block->FormatServerRepLen = size;
+        Block->FormatIDRep = get_server_ID_FMformat(Format, &size);
+        Block->FormatIDRepLen = size;
         Block->Next = NULL;
         if (Formats)
         {
@@ -1552,10 +1523,11 @@ extern void SstFFSWriterEndStep(SstStream Stream, size_t Timestep)
             Info->LocalFMContext, "Attributes", Info->AttributeFields,
             FMstruct_size_field_list(Info->AttributeFields, sizeof(char *)));
         AttributeFormat = Format;
-        Block->FormatServerRep =
-            get_server_rep_FMformat(Format, &Block->FormatServerRepLen);
-        Block->FormatIDRep =
-            get_server_ID_FMformat(Format, &Block->FormatIDRepLen);
+        int size;
+        Block->FormatServerRep = get_server_rep_FMformat(Format, &size);
+        Block->FormatServerRepLen = size;
+        Block->FormatIDRep = get_server_ID_FMformat(Format, &size);
+        Block->FormatIDRepLen = size;
         Block->Next = NULL;
         if (Formats)
         {
@@ -1670,7 +1642,7 @@ extern void SstFFSWriterEndStep(SstStream Stream, size_t Timestep)
 static void LoadAttributes(SstStream Stream, TSMetadataMsg MetaData)
 {
     static int DumpMetadata = -1;
-    Stream->AttrSetupUpcall(Stream->SetupUpcallReader, NULL, NULL, NULL);
+    Stream->AttrSetupUpcall(Stream->SetupUpcallReader, NULL, 0, NULL);
     for (int WriterRank = 0; WriterRank < Stream->WriterCohortSize;
          WriterRank++)
     {
@@ -1735,13 +1707,12 @@ static void LoadAttributes(SstStream Stream, TSMetadataMsg MetaData)
             char *FieldName;
             void *field_data = (char *)BaseData + FieldList[i].field_offset;
 
-            char *Type;
+            int Type;
             int ElemSize;
             BreakdownVarName(FieldList[i].field_name, &FieldName, &Type,
                              &ElemSize);
             Stream->AttrSetupUpcall(Stream->SetupUpcallReader, FieldName, Type,
                                     field_data);
-            free(Type);
             free(FieldName);
             i++;
         }
@@ -1793,25 +1764,106 @@ extern void FFSClearTimestepData(SstStream Stream)
            sizeof(Info->DataFieldLists[0]) * Stream->WriterCohortSize);
     for (int i = 0; i < Info->VarCount; i++)
     {
-        free(Info->VarList[i].VarName);
-        free(Info->VarList[i].PerWriterMetaFieldDesc);
-        free(Info->VarList[i].PerWriterDataFieldDesc);
-        free(Info->VarList[i].PerWriterStart);
-        free(Info->VarList[i].PerWriterCounts);
-        free(Info->VarList[i].PerWriterIncomingData);
-        free(Info->VarList[i].PerWriterIncomingSize);
-        if (Info->VarList[i].Type)
-            free(Info->VarList[i].Type);
-    }
-    Info->VarCount = 0;
+        Info->VarList[i]->Variable = NULL;
+    }
+}
+
+static struct ControlInfo *BuildControl(SstStream Stream, FMFormat Format)
+{
+    struct FFSReaderMarshalBase *Info = Stream->ReaderMarshalData;
+    FMStructDescList FormatList = format_list_of_FMFormat(Format);
+    FMFieldList FieldList = FormatList[0].field_list;
+    while (strncmp(FieldList->field_name, "BitField", 8) == 0)
+        FieldList++;
+    while (FieldList->field_name &&
+           (strncmp(FieldList->field_name, "DataBlockSize", 8) == 0))
+        FieldList++;
+    int i = 0;
+    int ControlCount = 0;
+    struct ControlInfo *ret = malloc(sizeof(*ret));
+    ret->Format = Format;
+    while (FieldList[i].field_name)
+    {
+        ret = realloc(ret,
+                      sizeof(*ret) + ControlCount * sizeof(struct ControlInfo));
+        struct ControlStruct *C = &(ret->Controls[ControlCount]);
+        ControlCount++;
+
+        C->FieldIndex = i;
+        C->FieldOffset = FieldList[i].field_offset;
+
+        if (NameIndicatesArray(FieldList[i].field_name))
+        {
+            char *ArrayName;
+            int Type;
+            FFSVarRec VarRec = NULL;
+            int ElementSize;
+            C->IsArray = 1;
+            BreakdownArrayName(FieldList[i].field_name, &ArrayName, &Type,
+                               &ElementSize);
+            //            if (WriterRank != 0)
+            //            {
+            VarRec = LookupVarByName(Stream, ArrayName);
+            //            }
+            if (!VarRec)
+            {
+                VarRec = CreateVarRec(Stream, ArrayName);
+                VarRec->Type = Type;
+                VarRec->ElementSize = ElementSize;
+                C->ElementSize = ElementSize;
+            }
+            i += 5; // number of fields in MetaArrayRec
+            free(ArrayName);
+            C->VarRec = VarRec;
+        }
+        else
+        {
+            /* simple field */
+            char *FieldName = strdup(FieldList[i].field_name + 4); // skip SST_
+            FFSVarRec VarRec = NULL;
+            C->IsArray = 0;
+            VarRec = LookupVarByName(Stream, FieldName);
+            if (!VarRec)
+            {
+                int Type = TranslateFFSType2ADIOS(FieldList[i].field_type,
+                                                  FieldList[i].field_size);
+                VarRec = CreateVarRec(Stream, FieldName);
+                VarRec->DimCount = 0;
+                C->Type = Type;
+                VarRec->Type = Type;
+            }
+            VarRec->ElementSize = FieldList[i].field_size;
+            C->ElementSize = FieldList[i].field_size;
+            C->VarRec = VarRec;
+            free(FieldName);
+            i++;
+        }
+    }
+    ret->ControlCount = ControlCount;
+    ret->Next = Info->ControlBlocks;
+    Info->ControlBlocks = ret;
+    return ret;
+}
+
+static struct ControlInfo *GetPriorControl(SstStream Stream, FMFormat Format)
+{
+    struct FFSReaderMarshalBase *Info = Stream->ReaderMarshalData;
+    struct ControlInfo *tmp = Info->ControlBlocks;
+    while (tmp)
+    {
+        if (tmp->Format == Format)
+        {
+            return tmp;
+        }
+        tmp = tmp->Next;
+    }
+    return NULL;
 }
 
 static void BuildVarList(SstStream Stream, TSMetadataMsg MetaData,
                          int WriterRank)
 {
     FFSTypeHandle FFSformat;
-    FMFieldList FieldList;
-    FMStructDescList FormatList;
     void *BaseData;
     static int DumpMetadata = -1;
 
@@ -1884,9 +1936,8 @@ static void BuildVarList(SstStream Stream, TSMetadataMsg MetaData,
             Stream->ReaderFFSContext, MetaData->Metadata[WriterRank].block,
             MetaData->Metadata[WriterRank].DataSize);
         BaseData = malloc(DecodedLength);
-        FFSBuffer decode_buf = create_fixed_FFSBuffer(BaseData, DecodedLength);
         FFSdecode_to_buffer(Stream->ReaderFFSContext,
-                            MetaData->Metadata[WriterRank].block, decode_buf);
+                            MetaData->Metadata[WriterRank].block, BaseData);
     }
     if (DumpMetadata == -1)
     {
@@ -1899,39 +1950,28 @@ static void BuildVarList(SstStream Stream, TSMetadataMsg MetaData,
         FMdump_data(FMFormat_of_original(FFSformat), BaseData, 1024000);
         printf("\n\n");
     }
+    struct ControlInfo *Control;
+    struct ControlStruct *ControlArray;
+    Control = GetPriorControl(Stream, FMFormat_of_original(FFSformat));
+    if (!Control)
+    {
+        Control = BuildControl(Stream, FMFormat_of_original(FFSformat));
+    }
+    ControlArray = &Control->Controls[0];
+
     Info->MetadataBaseAddrs[WriterRank] = BaseData;
-    FormatList = format_list_of_FMFormat(FMFormat_of_original(FFSformat));
-    FieldList = FormatList[0].field_list;
-    while (strncmp(FieldList->field_name, "BitField", 8) == 0)
-        FieldList++;
-    while (FieldList->field_name &&
-           (strncmp(FieldList->field_name, "DataBlockSize", 8) == 0))
-        FieldList++;
-    int i = 0;
-    int j = 0;
-    while (FieldList[i].field_name)
+    for (int i = 0; i < Control->ControlCount; i++)
     {
-        void *field_data = (char *)BaseData + FieldList[i].field_offset;
-        if (NameIndicatesArray(FieldList[i].field_name))
+        int FieldOffset = ControlArray[i].FieldOffset;
+        FFSVarRec VarRec = ControlArray[i].VarRec;
+        void *field_data = (char *)BaseData + FieldOffset;
+        if (!FFSBitfieldTest(BaseData, i))
+        {
+            continue;
+        }
+        if (ControlArray[i].IsArray)
         {
             MetaArrayRec *meta_base = field_data;
-            char *ArrayName;
-            char *Type;
-            FFSVarRec VarRec = NULL;
-            int ElementSize;
-            if (!FFSBitfieldTest(BaseData, j))
-            {
-                /* only work with fields that were written */
-                i += 4;
-                j++;
-                continue;
-            }
-            BreakdownArrayName(FieldList[i].field_name, &ArrayName, &Type,
-                               &ElementSize);
-            if (WriterRank != 0)
-            {
-                VarRec = LookupVarByName(Stream, ArrayName);
-            }
             if ((meta_base->Dims > 1) &&
                 (Stream->WriterConfigParams->IsRowMajor !=
                  Stream->ConfigParams->IsRowMajor))
@@ -1942,64 +1982,53 @@ static void BuildVarList(SstStream Stream, TSMetadataMsg MetaData,
                 ReverseDimensions(meta_base->Count, meta_base->Dims);
                 ReverseDimensions(meta_base->Offsets, meta_base->Dims);
             }
-            if (!VarRec)
-            {
-                VarRec = CreateVarRec(Stream, ArrayName);
-                VarRec->DimCount = meta_base->Dims;
-                VarRec->Type = Type;
-                VarRec->ElementSize = ElementSize;
-                VarRec->Variable = Stream->ArraySetupUpcall(
-                    Stream->SetupUpcallReader, ArrayName, Type, meta_base->Dims,
-                    meta_base->Shape, meta_base->Offsets, meta_base->Count);
-            }
             if (WriterRank == 0)
             {
                 VarRec->GlobalDims = meta_base->Shape;
             }
+            if (!VarRec->Variable)
+            {
+                VarRec->Variable = Stream->ArraySetupUpcall(
+                    Stream->SetupUpcallReader, VarRec->VarName, VarRec->Type,
+                    meta_base->Dims, meta_base->Shape, meta_base->Offsets,
+                    meta_base->Count);
+            }
+            VarRec->DimCount = meta_base->Dims;
+            VarRec->PerWriterBlockCount[WriterRank] =
+                meta_base->Dims ? meta_base->DBCount / meta_base->Dims : 1;
             VarRec->PerWriterStart[WriterRank] = meta_base->Offsets;
             VarRec->PerWriterCounts[WriterRank] = meta_base->Count;
-            VarRec->PerWriterMetaFieldDesc[WriterRank] = &FieldList[i];
-            VarRec->PerWriterDataFieldDesc[WriterRank] = NULL;
-            Stream->ArrayBlocksInfoUpcall(Stream->SetupUpcallReader,
-                                          VarRec->Variable, Type, WriterRank,
-                                          meta_base->Dims, meta_base->Shape,
-                                          meta_base->Offsets, meta_base->Count);
-            i += 4;
-            free(ArrayName);
-        }
-        else
-        {
-            /* simple field */
-            char *FieldName = strdup(FieldList[i].field_name + 4); // skip SST_
-            FFSVarRec VarRec = NULL;
-            if (!FFSBitfieldTest(BaseData, j))
+            if (WriterRank == 0)
             {
-                /* only work with fields that were written */
-                i++;
-                j++;
-                continue;
+                VarRec->PerWriterBlockStart[WriterRank] = 0;
             }
-            if (WriterRank != 0)
+            if (WriterRank < Stream->WriterCohortSize - 1)
             {
-                VarRec = LookupVarByName(Stream, FieldName);
+                VarRec->PerWriterBlockStart[WriterRank + 1] =
+                    VarRec->PerWriterBlockStart[WriterRank] +
+                    VarRec->PerWriterBlockCount[WriterRank];
             }
-            if (!VarRec)
+            for (int i = 0; i < VarRec->PerWriterBlockCount[WriterRank]; i++)
+            {
+                size_t *Offsets = NULL;
+                if (meta_base->Offsets)
+                    Offsets = meta_base->Offsets + (i * meta_base->Dims);
+                Stream->ArrayBlocksInfoUpcall(
+                    Stream->SetupUpcallReader, VarRec->Variable, VarRec->Type,
+                    WriterRank, meta_base->Dims, meta_base->Shape, Offsets,
+                    meta_base->Count);
+            }
+        }
+        else
+        {
+            if (!VarRec->Variable)
             {
-                char *Type = TranslateFFSType2ADIOS(FieldList[i].field_type,
-                                                    FieldList[i].field_size);
-                VarRec = CreateVarRec(Stream, FieldName);
-                VarRec->DimCount = 0;
                 VarRec->Variable = Stream->VarSetupUpcall(
-                    Stream->SetupUpcallReader, FieldName, Type, field_data);
-                free(Type);
+                    Stream->SetupUpcallReader, VarRec->VarName, VarRec->Type,
+                    field_data);
             }
-            VarRec->PerWriterMetaFieldDesc[WriterRank] = &FieldList[i];
-            VarRec->PerWriterDataFieldDesc[WriterRank] = NULL;
-            free(FieldName);
-            i++;
+            VarRec->PerWriterMetaFieldOffset[WriterRank] = FieldOffset;
         }
-        /* real variable count is in j, i tracks the entries in the metadata */
-        j++;
     }
 }
 
@@ -2030,13 +2059,13 @@ extern void FFSMarshalInstallMetadata(SstStream Stream, TSMetadataMsg MetaData)
 
 static void FFSBitfieldSet(struct FFSMetadataInfoStruct *MBase, int Bit)
 {
-    int Element = Bit / 64;
-    int ElementBit = Bit % 64;
+    int Element = Bit / (sizeof(size_t) * 8);
+    int ElementBit = Bit % (sizeof(size_t) * 8);
     if (Element >= MBase->BitFieldCount)
     {
         MBase->BitField =
             realloc(MBase->BitField, sizeof(size_t) * (Element + 1));
-        memset(MBase->BitField + MBase->BitFieldCount * sizeof(size_t), 0,
+        memset(MBase->BitField + MBase->BitFieldCount, 0,
                (Element - MBase->BitFieldCount + 1) * sizeof(size_t));
         MBase->BitFieldCount = Element + 1;
     }
@@ -2045,13 +2074,13 @@ static void FFSBitfieldSet(struct FFSMetadataInfoStruct *MBase, int Bit)
 
 static int FFSBitfieldTest(struct FFSMetadataInfoStruct *MBase, int Bit)
 {
-    int Element = Bit / 64;
-    int ElementBit = Bit % 64;
+    int Element = Bit / (sizeof(size_t) * 8);
+    int ElementBit = Bit % (sizeof(size_t) * 8);
     if (Element >= MBase->BitFieldCount)
     {
         MBase->BitField =
             realloc(MBase->BitField, sizeof(size_t) * (Element + 1));
-        memset(MBase->BitField + MBase->BitFieldCount * sizeof(size_t), 0,
+        memset(MBase->BitField + MBase->BitFieldCount, 0,
                (Element - MBase->BitFieldCount + 1) * sizeof(size_t));
         MBase->BitFieldCount = Element + 1;
     }
@@ -2072,7 +2101,7 @@ extern void SstFFSSetZFPParams(SstStream Stream, attr_list Attrs)
 }
 
 extern void SstFFSMarshal(SstStream Stream, void *Variable, const char *Name,
-                          const char *Type, size_t ElemSize, size_t DimCount,
+                          const int Type, size_t ElemSize, size_t DimCount,
                           const size_t *Shape, const size_t *Count,
                           const size_t *Offsets, const void *Data)
 {
@@ -2086,29 +2115,50 @@ extern void SstFFSMarshal(SstStream Stream, void *Variable, const char *Name,
     }
 
     MBase = Stream->M;
+    int AlreadyWritten = FFSBitfieldTest(MBase, Rec->FieldID);
     FFSBitfieldSet(MBase, Rec->FieldID);
 
     if (Rec->DimCount == 0)
     {
-        char *base = Stream->M + Rec->MetaOffset;
-        memcpy(base, Data, ElemSize);
+        memcpy((char *)(Stream->M) + Rec->MetaOffset, Data, ElemSize);
     }
     else
     {
-        MetaArrayRec *MetaEntry = Stream->M + Rec->MetaOffset;
-        ArrayRec *DataEntry = Stream->D + Rec->DataOffset;
+        MetaArrayRec *MetaEntry =
+            (MetaArrayRec *)((char *)(Stream->M) + Rec->MetaOffset);
+        ArrayRec *DataEntry =
+            (ArrayRec *)((char *)(Stream->D) + Rec->DataOffset);
 
         /* handle metadata */
         MetaEntry->Dims = DimCount;
-        if (Shape)
-            MetaEntry->Shape = CopyDims(DimCount, Shape);
-        else
-            MetaEntry->Shape = NULL;
-        MetaEntry->Count = CopyDims(DimCount, Count);
-        if (Offsets)
-            MetaEntry->Offsets = CopyDims(DimCount, Offsets);
+        if (!AlreadyWritten)
+        {
+            if (Shape)
+                MetaEntry->Shape = CopyDims(DimCount, Shape);
+            else
+                MetaEntry->Shape = NULL;
+            MetaEntry->DBCount = DimCount;
+            MetaEntry->Count = CopyDims(DimCount, Count);
+            if (Offsets)
+                MetaEntry->Offsets = CopyDims(DimCount, Offsets);
+            else
+                MetaEntry->Offsets = NULL;
+        }
         else
-            MetaEntry->Offsets = NULL;
+        {
+            /* already got some metadata, add blocks */
+            size_t PreviousDBCount = MetaEntry->DBCount;
+            //  Assume shape is still valid   (modify this if shape /global
+            //  dimensions can change )
+            // Also assume Dims is always right and consistent, otherwise, bad
+            // things
+            MetaEntry->DBCount += DimCount;
+            MetaEntry->Count =
+                AppendDims(MetaEntry->Count, PreviousDBCount, DimCount, Count);
+            if (Offsets)
+                MetaEntry->Offsets = AppendDims(
+                    MetaEntry->Offsets, PreviousDBCount, DimCount, Offsets);
+        }
 
         if ((Stream->ConfigParams->CompressionMethod == SstCompressZFP) &&
             ZFPcompressionPossible(Type, DimCount))
@@ -2124,33 +2174,48 @@ extern void SstFFSMarshal(SstStream Stream, void *Variable, const char *Name,
         }
         else
         {
-            /* normal array case */
-            size_t ElemCount = CalcSize(DimCount, Count);
-            DataEntry->ElemCount = ElemCount;
-            /* this is PutSync case, so we have to copy the data NOW */
-            DataEntry->Array = malloc(ElemCount * ElemSize);
-            memcpy(DataEntry->Array, Data, ElemCount * ElemSize);
+            if (!AlreadyWritten)
+            {
+                /* normal array case */
+                size_t ElemCount = CalcSize(DimCount, Count);
+                DataEntry->ElemCount = ElemCount;
+                /* this is PutSync case, so we have to copy the data NOW */
+                DataEntry->Array = malloc(ElemCount * ElemSize);
+                memcpy(DataEntry->Array, Data, ElemCount * ElemSize);
+            }
+            else
+            {
+                size_t ElemCount = CalcSize(DimCount, Count);
+                /* this is PutSync case, so we have to copy the data NOW */
+                DataEntry->Array =
+                    realloc(DataEntry->Array,
+                            (DataEntry->ElemCount + ElemCount) * ElemSize);
+                memcpy((char *)DataEntry->Array +
+                           DataEntry->ElemCount * ElemSize,
+                       Data, ElemCount * ElemSize);
+                DataEntry->ElemCount += ElemCount;
+            }
         }
     }
 }
 
 extern void SstFFSMarshalAttribute(SstStream Stream, const char *Name,
-                                   const char *Type, size_t ElemSize,
+                                   const int Type, size_t ElemSize,
                                    size_t ElemCount, const void *Data)
 {
 
     struct FFSWriterMarshalBase *Info;
     Info = (struct FFSWriterMarshalBase *)Stream->WriterMarshalData;
-    const char *String = NULL;
+    const char *AttrString = NULL;
     const char *DataAddress = Data;
 
-    if (strcmp(Type, "string") == 0)
+    if (Type == String)
     {
         ElemSize = sizeof(char *);
-        String = Data;
-        DataAddress = (const char *)&String;
+        AttrString = Data;
+        DataAddress = (const char *)&AttrString;
     }
-    if (ElemCount == -1)
+    if (ElemCount == (size_t)(-1))
     {
         // simple field, only simple attribute name and value
         char *SstName = BuildVarName(Name, Type, ElemSize);
@@ -2160,7 +2225,8 @@ extern void SstFFSMarshalAttribute(SstStream Stream, const char *Name,
         RecalcAttributeStorageSize(Stream);
         int DataOffset =
             Info->AttributeFields[Info->AttributeFieldCount - 1].field_offset;
-        memcpy(Info->AttributeData + DataOffset, DataAddress, ElemSize);
+        memcpy((char *)(Info->AttributeData) + DataOffset, DataAddress,
+               ElemSize);
     }
     else
     {
diff --git a/source/adios2/toolkit/sst/cp/ffs_marshal.h b/source/adios2/toolkit/sst/cp/ffs_marshal.h
index 9ccb022b82..9969480d46 100644
--- a/source/adios2/toolkit/sst/cp/ffs_marshal.h
+++ b/source/adios2/toolkit/sst/cp/ffs_marshal.h
@@ -1,3 +1,23 @@
+enum DataType
+{
+    None,
+    Int8,
+    Int16,
+    Int32,
+    Int64,
+    UInt8,
+    UInt16,
+    UInt32,
+    UInt64,
+    Float,
+    Double,
+    LongDouble,
+    FloatComplex,
+    DoubleComplex,
+    String,
+    Compound
+};
+
 typedef struct _FFSWriterRec
 {
     void *Key;
@@ -5,7 +25,7 @@ typedef struct _FFSWriterRec
     size_t DataOffset;
     size_t MetaOffset;
     int DimCount;
-    char *Type;
+    int Type;
 } * FFSWriterRec;
 
 struct FFSWriterMarshalBase
@@ -32,12 +52,13 @@ typedef struct FFSVarRec
 {
     void *Variable;
     char *VarName;
-    FMFieldList *PerWriterMetaFieldDesc;
-    FMFieldList *PerWriterDataFieldDesc;
+    size_t *PerWriterMetaFieldOffset;
     size_t DimCount;
-    char *Type;
+    int Type;
     int ElementSize;
     size_t *GlobalDims;
+    size_t *PerWriterBlockStart;
+    size_t *PerWriterBlockCount;
     size_t **PerWriterStart;
     size_t **PerWriterCounts;
     void **PerWriterIncomingData;
@@ -54,7 +75,7 @@ typedef struct FFSArrayRequest
 {
     FFSVarRec VarRec;
     enum FFSRequestTypeEnum RequestType;
-    size_t NodeID;
+    size_t BlockID;
     size_t *Start;
     size_t *Count;
     void *Data;
@@ -76,10 +97,28 @@ typedef struct FFSReaderPerWriterRec
     DP_CompletionHandle ReadHandle;
 } FFSReaderPerWriterRec;
 
+struct ControlStruct
+{
+    int FieldIndex;
+    int FieldOffset;
+    FFSVarRec VarRec;
+    int IsArray;
+    int Type;
+    int ElementSize;
+};
+
+struct ControlInfo
+{
+    FMFormat Format;
+    int ControlCount;
+    struct ControlInfo *Next;
+    struct ControlStruct Controls[1];
+};
+
 struct FFSReaderMarshalBase
 {
     int VarCount;
-    FFSVarRec VarList;
+    FFSVarRec *VarList;
     FMContext LocalFMContext;
     FFSArrayRequest PendingVarRequests;
 
@@ -90,12 +129,13 @@ struct FFSReaderMarshalBase
     FMFieldList *DataFieldLists;
 
     FFSReaderPerWriterRec *WriterInfo;
+    struct ControlInfo *ControlBlocks;
 };
 
-extern char *FFS_ZFPCompress(SstStream Stream, const size_t DimCount,
-                             char *Type, void *Data, const size_t *Count,
+extern char *FFS_ZFPCompress(SstStream Stream, const size_t DimCount, int Type,
+                             void *Data, const size_t *Count,
                              size_t *ByteCountP);
 extern void *FFS_ZFPDecompress(SstStream Stream, const size_t DimCount,
-                               char *Type, void *bufferIn, const size_t sizeIn,
+                               int Type, void *bufferIn, const size_t sizeIn,
                                const size_t *Dimensions, attr_list Parameters);
-extern int ZFPcompressionPossible(const char *Type, const int DimCount);
+extern int ZFPcompressionPossible(const int Type, const int DimCount);
diff --git a/source/adios2/toolkit/sst/cp/ffs_zfp.c b/source/adios2/toolkit/sst/cp/ffs_zfp.c
index 7f4737a2d0..6f09688a46 100644
--- a/source/adios2/toolkit/sst/cp/ffs_zfp.c
+++ b/source/adios2/toolkit/sst/cp/ffs_zfp.c
@@ -14,53 +14,53 @@
 #include "ffs_marshal.h"
 #include "zfp.h"
 
-static zfp_type GetZFPType(const char *Type)
+static zfp_type GetZFPType(int Type)
 {
-    if (strcmp(Type, "int") == 0)
+    if (Type == Int32)
     {
         return zfp_type_int32;
     }
-    else if (strcmp(Type, "unsigned int") == 0)
+    else if (Type == UInt32)
     {
         return zfp_type_int32;
     }
-    else if (strcmp(Type, "long int") == 0)
+    else if (Type == Int64)
     {
         return zfp_type_int64;
     }
-    else if (strcmp(Type, "long long int") == 0)
+    else if (Type == Int64)
     {
         return zfp_type_int64;
     }
-    else if (strcmp(Type, "unsigned long int") == 0)
+    else if (Type == UInt64)
     {
         return zfp_type_int64;
     }
-    else if (strcmp(Type, "unsigned long long int") == 0)
+    else if (Type == UInt64)
     {
         return zfp_type_int64;
     }
-    else if (strcmp(Type, "float") == 0)
+    else if (Type == Float)
     {
         return zfp_type_float;
     }
-    else if (strcmp(Type, "double") == 0)
+    else if (Type == Double)
     {
         return zfp_type_double;
     }
-    else if (strcmp(Type, "long double") == 0)
+    else if (Type == Double)
     {
         return zfp_type_double;
     }
     return zfp_type_none;
 }
 
-extern int ZFPcompressionPossible(const char *Type, int DimCount)
+extern int ZFPcompressionPossible(const int Type, int DimCount)
 {
     return ((GetZFPType(Type) != zfp_type_none) && (DimCount < 4));
 }
 
-static zfp_field *GetZFPField(void *Data, const size_t DimCount, char *Type,
+static zfp_field *GetZFPField(void *Data, const size_t DimCount, int Type,
                               const size_t *Dimensions)
 {
     zfp_type zfpType = GetZFPType(Type);
@@ -94,8 +94,7 @@ static atom_t ZFPToleranceAtom = -1;
 static atom_t ZFPRateAtom = -1;
 static atom_t ZFPPrecisionAtom = -1;
 
-zfp_stream *GetZFPStream(const size_t DimCount, char *Type,
-                         attr_list Parameters)
+zfp_stream *GetZFPStream(const size_t DimCount, int Type, attr_list Parameters)
 {
 
     zfp_stream *zstream = zfp_stream_open(NULL);
@@ -135,8 +134,8 @@ zfp_stream *GetZFPStream(const size_t DimCount, char *Type,
     return zstream;
 }
 
-extern char *FFS_ZFPCompress(SstStream Stream, const size_t DimCount,
-                             char *Type, void *Data, const size_t *Count,
+extern char *FFS_ZFPCompress(SstStream Stream, const size_t DimCount, int Type,
+                             void *Data, const size_t *Count,
                              size_t *ByteCountP)
 {
     struct FFSWriterMarshalBase *Info = Stream->WriterMarshalData;
@@ -158,7 +157,7 @@ extern char *FFS_ZFPCompress(SstStream Stream, const size_t DimCount,
     return bufferOut;
 }
 
-void *FFS_ZFPDecompress(SstStream Stream, const size_t DimCount, char *Type,
+void *FFS_ZFPDecompress(SstStream Stream, const size_t DimCount, int Type,
                         void *bufferIn, const size_t sizeIn,
                         const size_t *Dimensions, attr_list Parameters)
 {
diff --git a/source/adios2/toolkit/sst/dp/dp.c b/source/adios2/toolkit/sst/dp/dp.c
index 7f91a7522d..e17fbfb045 100644
--- a/source/adios2/toolkit/sst/dp/dp.c
+++ b/source/adios2/toolkit/sst/dp/dp.c
@@ -52,7 +52,7 @@ static DPlist AddDPPossibility(CP_Services Svcs, void *CP_Stream, DPlist List,
 }
 
 CP_DP_Interface SelectDP(CP_Services Svcs, void *CP_Stream,
-                         struct _SstParams *Params)
+                         struct _SstParams *Params, int Rank)
 {
     CP_DP_Interface Ret;
     DPlist List = NULL;
@@ -75,15 +75,18 @@ CP_DP_Interface SelectDP(CP_Services Svcs, void *CP_Stream,
     int FoundPreferred = 0;
     if (Params->DataTransport)
     {
-        Svcs->verbose(CP_Stream, "Prefered dataplane name is \"%s\"\n",
-                      Params->DataTransport);
+        if (Rank == 0)
+            Svcs->verbose(CP_Stream, DPPerStepVerbose,
+                          "Prefered dataplane name is \"%s\"\n",
+                          Params->DataTransport);
     }
     while (List[i].Interface)
     {
-        Svcs->verbose(
-            CP_Stream,
-            "Considering DataPlane \"%s\" for possible use, priority is %d\n",
-            List[i].Name, List[i].Priority);
+        if (Rank == 0)
+            Svcs->verbose(CP_Stream, DPPerStepVerbose,
+                          "Considering DataPlane \"%s\" for possible use, "
+                          "priority is %d\n",
+                          List[i].Name, List[i].Priority);
         if (Params->DataTransport)
         {
             if (strcasecmp(List[i].Name, Params->DataTransport) == 0)
@@ -96,10 +99,11 @@ CP_DP_Interface SelectDP(CP_Services Svcs, void *CP_Stream,
                 }
                 else
                 {
-                    fprintf(stderr,
-                            "Warning:  Perferred DataPlane \"%s\" is "
-                            "not available.",
-                            List[i].Name);
+                    if (Rank == 0)
+                        fprintf(stderr,
+                                "Warning:  Perferred DataPlane \"%s\" is "
+                                "not available.\n",
+                                List[i].Name);
                 }
             }
         }
@@ -112,20 +116,23 @@ CP_DP_Interface SelectDP(CP_Services Svcs, void *CP_Stream,
     }
     if (Params->DataTransport && (FoundPreferred == 0))
     {
-        fprintf(stderr, "Warning:  Preferred DataPlane \"%s\" not found.",
-                Params->DataTransport);
+        if (Rank == 0)
+            fprintf(stderr, "Warning:  Preferred DataPlane \"%s\" not found.\n",
+                    Params->DataTransport);
     }
     if (SelectedDP != -1)
     {
-        Svcs->verbose(CP_Stream,
-                      "Selecting DataPlane \"%s\" (preferred) for use\n",
-                      List[SelectedDP].Name);
+        if (Rank == 0)
+            Svcs->verbose(CP_Stream, DPSummaryVerbose,
+                          "Selecting DataPlane \"%s\" (preferred) for use\n",
+                          List[SelectedDP].Name);
     }
     else
     {
-        Svcs->verbose(CP_Stream,
-                      "Selecting DataPlane \"%s\", priority %d for use\n",
-                      List[BestPrioDP].Name, List[BestPrioDP].Priority);
+        if (Rank == 0)
+            Svcs->verbose(CP_Stream, DPSummaryVerbose,
+                          "Selecting DataPlane \"%s\", priority %d for use\n",
+                          List[BestPrioDP].Name, List[BestPrioDP].Priority);
         SelectedDP = BestPrioDP;
     }
     i = 0;
diff --git a/source/adios2/toolkit/sst/dp/dummy_dp.c b/source/adios2/toolkit/sst/dp/dummy_dp.c
deleted file mode 100644
index 4b1a4b97e5..0000000000
--- a/source/adios2/toolkit/sst/dp/dummy_dp.c
+++ /dev/null
@@ -1,603 +0,0 @@
-#include 
-#include 
-#include 
-#include 
-
-#include 
-#include 
-
-#include "sst_data.h"
-
-#include "adios2/toolkit/profiling/taustubs/taustubs.h"
-#include "dp_interface.h"
-
-/*
- *  Some conventions:
- *    `RS` indicates a reader-side item.
- *    `WS` indicates a writer-side item.
- *    `WSR` indicates a writer-side per-reader item.
- *
- *   We keep different "stream" structures for the reader side and for the
- *   writer side.  On the writer side, there's actually a "stream"
- *   per-connected-reader (a WSR_Stream), with the idea that some (many?)
- *   RDMA transports will require connections/pairing, so we'd need to track
- *   resources per reader.
- *
- *   Generally the 'contact information' we exchange at init time includes
- *   the address of the local 'stream' data structure.  This address isn't
- *   particularly useful to the far side, but it can be returned with
- *   requests to indicate what resource is targeted.  For example, when a
- *   remote memory read request arrives at the writer from the reader, it
- *   includes the WSR_Stream value that is the address of the writer-side
- *   per-reader data structure.  Upon message arrival, we just cast that
- *   value back into a pointer.
- *
- *   By design, neither the data plane nor the control plane reference the
- *   other's symbols directly.  The interface between the control plane and
- *   the data plane is represented by the types and structures defined in
- *   dp_interface.h and is a set of function pointers and FFS-style
- *   descriptions of the data structures to be communicated at init time.
- *   This allows for the future possibility of loading planes at run-time, etc.
- *
- *   This "dummy" data plane uses control plane functionality to implement
- *   the ReadRemoteMemory functionality.  That is, it both the request to
- *   read memory and the response which carries the data are actually
- *   accomplished using the connections and message delivery facilities of
- *   the control plane, made available here via CP_Services.  A real data
- *   plane would replace one or both of these with RDMA functionality.
- */
-
-typedef struct _Dummy_RS_Stream
-{
-    CManager cm;
-    void *CP_Stream;
-    CMFormat ReadRequestFormat;
-    int Rank;
-
-    /* writer info */
-    int WriterCohortSize;
-    CP_PeerCohort PeerCohort;
-    struct _DummyWriterContactInfo *WriterContactInfo;
-} * Dummy_RS_Stream;
-
-typedef struct _Dummy_WSR_Stream
-{
-    struct _Dummy_WS_Stream *WS_Stream;
-    CP_PeerCohort PeerCohort;
-    int ReaderCohortSize;
-    struct _DummyReaderContactInfo *ReaderContactInfo;
-} * Dummy_WSR_Stream;
-
-typedef struct _TimestepEntry
-{
-    long Timestep;
-    struct _SstData *Data;
-    struct _DummyPerTimestepInfo *DP_TimestepInfo;
-    struct _TimestepEntry *Next;
-
-} * TimestepList;
-
-typedef struct _Dummy_WS_Stream
-{
-    CManager cm;
-    void *CP_Stream;
-    int Rank;
-
-    TimestepList Timesteps;
-    CMFormat ReadReplyFormat;
-
-    int ReaderCount;
-    Dummy_WSR_Stream *Readers;
-} * Dummy_WS_Stream;
-
-typedef struct _DummyReaderContactInfo
-{
-    char *ContactString;
-    void *RS_Stream;
-} * DummyReaderContactInfo;
-
-typedef struct _DummyWriterContactInfo
-{
-    char *ContactString;
-    void *WS_Stream;
-} * DummyWriterContactInfo;
-
-typedef struct _DummyReadRequestMsg
-{
-    long Timestep;
-    size_t Offset;
-    size_t Length;
-    void *WS_Stream;
-    void *RS_Stream;
-    int RequestingRank;
-    int NotifyCondition;
-} * DummyReadRequestMsg;
-
-static FMField DummyReadRequestList[] = {
-    {"Timestep", "integer", sizeof(long),
-     FMOffset(DummyReadRequestMsg, Timestep)},
-    {"Offset", "integer", sizeof(size_t),
-     FMOffset(DummyReadRequestMsg, Offset)},
-    {"Length", "integer", sizeof(size_t),
-     FMOffset(DummyReadRequestMsg, Length)},
-    {"WS_Stream", "integer", sizeof(void *),
-     FMOffset(DummyReadRequestMsg, WS_Stream)},
-    {"RS_Stream", "integer", sizeof(void *),
-     FMOffset(DummyReadRequestMsg, RS_Stream)},
-    {"RequestingRank", "integer", sizeof(int),
-     FMOffset(DummyReadRequestMsg, RequestingRank)},
-    {"NotifyCondition", "integer", sizeof(int),
-     FMOffset(DummyReadRequestMsg, NotifyCondition)},
-    {NULL, NULL, 0, 0}};
-
-static FMStructDescRec DummyReadRequestStructs[] = {
-    {"DummyReadRequest", DummyReadRequestList,
-     sizeof(struct _DummyReadRequestMsg), NULL},
-    {NULL, NULL, 0, NULL}};
-
-typedef struct _DummyReadReplyMsg
-{
-    long Timestep;
-    size_t DataLength;
-    void *RS_Stream;
-    char *Data;
-    int NotifyCondition;
-} * DummyReadReplyMsg;
-
-static FMField DummyReadReplyList[] = {
-    {"Timestep", "integer", sizeof(long),
-     FMOffset(DummyReadReplyMsg, Timestep)},
-    {"RS_Stream", "integer", sizeof(void *),
-     FMOffset(DummyReadReplyMsg, RS_Stream)},
-    {"DataLength", "integer", sizeof(size_t),
-     FMOffset(DummyReadReplyMsg, DataLength)},
-    {"Data", "char[DataLength]", sizeof(char),
-     FMOffset(DummyReadReplyMsg, Data)},
-    {"NotifyCondition", "integer", sizeof(int),
-     FMOffset(DummyReadReplyMsg, NotifyCondition)},
-    {NULL, NULL, 0, 0}};
-
-static FMStructDescRec DummyReadReplyStructs[] = {
-    {"DummyReadReply", DummyReadReplyList, sizeof(struct _DummyReadReplyMsg),
-     NULL},
-    {NULL, NULL, 0, NULL}};
-
-static void DummyReadReplyHandler(CManager cm, CMConnection conn, void *msg_v,
-                                  void *client_Data, attr_list attrs);
-
-static DP_RS_Stream DummyInitReader(CP_Services Svcs, void *CP_Stream,
-                                    void **ReaderContactInfoPtr)
-{
-    Dummy_RS_Stream Stream = malloc(sizeof(struct _Dummy_RS_Stream));
-    DummyReaderContactInfo Contact =
-        malloc(sizeof(struct _DummyReaderContactInfo));
-    CManager cm = Svcs->getCManager(CP_Stream);
-    char *DummyContactString = malloc(64);
-    SMPI_Comm comm = Svcs->getMPIComm(CP_Stream);
-    CMFormat F;
-
-    memset(Stream, 0, sizeof(*Stream));
-    memset(Contact, 0, sizeof(*Contact));
-
-    /*
-     * save the CP_stream value of later use
-     */
-    Stream->CP_Stream = CP_Stream;
-
-    SMPI_Comm_rank(comm, &Stream->Rank);
-
-    sprintf(DummyContactString, "Reader Rank %d, test contact", Stream->Rank);
-
-    /*
-     * add a handler for read reply messages
-     */
-    Stream->ReadRequestFormat = CMregister_format(cm, DummyReadRequestStructs);
-    F = CMregister_format(cm, DummyReadReplyStructs);
-    CMregister_handler(F, DummyReadReplyHandler, Svcs);
-
-    Contact->ContactString = DummyContactString;
-    Contact->RS_Stream = Stream;
-
-    *ReaderContactInfoPtr = Contact;
-
-    return Stream;
-}
-
-static void DummyReadRequestHandler(CManager cm, CMConnection conn, void *msg_v,
-                                    void *client_Data, attr_list attrs)
-{
-    TAU_START_FUNC();
-    DummyReadRequestMsg ReadRequestMsg = (DummyReadRequestMsg)msg_v;
-    Dummy_WSR_Stream WSR_Stream = ReadRequestMsg->WS_Stream;
-
-    Dummy_WS_Stream WS_Stream = WSR_Stream->WS_Stream;
-    TimestepList tmp = WS_Stream->Timesteps;
-    CP_Services Svcs = (CP_Services)client_Data;
-
-    Svcs->verbose(WS_Stream->CP_Stream,
-                  "Got a request to read remote memory "
-                  "from reader rank %d: timestep %d, "
-                  "offset %d, length %d\n",
-                  ReadRequestMsg->RequestingRank, ReadRequestMsg->Timestep,
-                  ReadRequestMsg->Offset, ReadRequestMsg->Length);
-    while (tmp != NULL)
-    {
-        if (tmp->Timestep == ReadRequestMsg->Timestep)
-        {
-            struct _DummyReadReplyMsg ReadReplyMsg;
-            /* memset avoids uninit byte warnings from valgrind */
-            memset(&ReadReplyMsg, 0, sizeof(ReadReplyMsg));
-            ReadReplyMsg.Timestep = ReadRequestMsg->Timestep;
-            ReadReplyMsg.DataLength = ReadRequestMsg->Length;
-            ReadReplyMsg.Data = tmp->Data->block + ReadRequestMsg->Offset;
-            ReadReplyMsg.RS_Stream = ReadRequestMsg->RS_Stream;
-            ReadReplyMsg.NotifyCondition = ReadRequestMsg->NotifyCondition;
-            Svcs->verbose(
-                WS_Stream->CP_Stream,
-                "Sending a reply to reader rank %d for remote memory read\n",
-                ReadRequestMsg->RequestingRank);
-            Svcs->sendToPeer(WS_Stream->CP_Stream, WSR_Stream->PeerCohort,
-                             ReadRequestMsg->RequestingRank,
-                             WS_Stream->ReadReplyFormat, &ReadReplyMsg);
-            TAU_STOP_FUNC();
-            return;
-        }
-        tmp = tmp->Next;
-    }
-    /*
-     * Shouldn't ever get here because we should never get a request for a
-     * timestep that we don't have.
-     */
-    fprintf(stderr, "Failed to read Timestep %ld, not found\n",
-            ReadRequestMsg->Timestep);
-    /*
-     * in the interest of not failing a writer on a reader failure, don't
-     * assert(0) here.  Probably this sort of error should close the link to
-     * a reader though.
-     */
-    TAU_STOP_FUNC();
-}
-
-typedef struct _DummyCompletionHandle
-{
-    int CMcondition;
-    CManager cm;
-    void *CPStream;
-    void *Buffer;
-    int Rank;
-} * DummyCompletionHandle;
-
-static void DummyReadReplyHandler(CManager cm, CMConnection conn, void *msg_v,
-                                  void *client_Data, attr_list attrs)
-{
-    TAU_START_FUNC();
-    DummyReadReplyMsg ReadReplyMsg = (DummyReadReplyMsg)msg_v;
-    Dummy_RS_Stream RS_Stream = ReadReplyMsg->RS_Stream;
-    CP_Services Svcs = (CP_Services)client_Data;
-    DummyCompletionHandle Handle =
-        CMCondition_get_client_data(cm, ReadReplyMsg->NotifyCondition);
-
-    Svcs->verbose(
-        RS_Stream->CP_Stream,
-        "Got a reply to remote memory read from rank %d, condition is %d\n",
-        Handle->Rank, ReadReplyMsg->NotifyCondition);
-
-    /*
-     * `Handle` contains the full request info and is `client_data`
-     * associated with the CMCondition.  Once we get it, copy the incoming
-     * data to the buffer area given by the request
-     */
-    memcpy(Handle->Buffer, ReadReplyMsg->Data, ReadReplyMsg->DataLength);
-
-    /*
-     * Signal the condition to wake the reader if they are waiting.
-     */
-    CMCondition_signal(cm, ReadReplyMsg->NotifyCondition);
-    TAU_STOP_FUNC();
-}
-
-static DP_WS_Stream DummyInitWriter(CP_Services Svcs, void *CP_Stream)
-{
-    Dummy_WS_Stream Stream = malloc(sizeof(struct _Dummy_WS_Stream));
-    CManager cm = Svcs->getCManager(CP_Stream);
-    SMPI_Comm comm = Svcs->getMPIComm(CP_Stream);
-    CMFormat F;
-
-    memset(Stream, 0, sizeof(struct _Dummy_WS_Stream));
-
-    SMPI_Comm_rank(comm, &Stream->Rank);
-
-    /*
-     * save the CP_stream value of later use
-     */
-    Stream->CP_Stream = CP_Stream;
-
-    /*
-     * add a handler for read request messages
-     */
-    F = CMregister_format(cm, DummyReadRequestStructs);
-    CMregister_handler(F, DummyReadRequestHandler, Svcs);
-
-    /*
-     * register read reply message structure so we can send later
-     */
-    Stream->ReadReplyFormat = CMregister_format(cm, DummyReadReplyStructs);
-
-    return (void *)Stream;
-}
-
-static DP_WSR_Stream DummyInitWriterPerReader(CP_Services Svcs,
-                                              DP_WS_Stream WS_Stream_v,
-                                              int readerCohortSize,
-                                              CP_PeerCohort PeerCohort,
-                                              void **providedReaderInfo_v,
-                                              void **WriterContactInfoPtr)
-{
-    Dummy_WS_Stream WS_Stream = (Dummy_WS_Stream)WS_Stream_v;
-    Dummy_WSR_Stream WSR_Stream = malloc(sizeof(*WSR_Stream));
-    DummyWriterContactInfo ContactInfo;
-    SMPI_Comm comm = Svcs->getMPIComm(WS_Stream->CP_Stream);
-    int Rank;
-    char *DummyContactString = malloc(64);
-    DummyReaderContactInfo *providedReaderInfo =
-        (DummyReaderContactInfo *)providedReaderInfo_v;
-
-    SMPI_Comm_rank(comm, &Rank);
-    sprintf(DummyContactString, "Writer Rank %d, test contact", Rank);
-
-    WSR_Stream->WS_Stream = WS_Stream; /* pointer to writer struct */
-    WSR_Stream->PeerCohort = PeerCohort;
-
-    /*
-     * make a copy of writer contact information (original will not be
-     * preserved)
-     */
-    WSR_Stream->ReaderContactInfo =
-        malloc(sizeof(struct _DummyReaderContactInfo) * readerCohortSize);
-    for (int i = 0; i < readerCohortSize; i++)
-    {
-        WSR_Stream->ReaderContactInfo[i].ContactString =
-            strdup(providedReaderInfo[i]->ContactString);
-        WSR_Stream->ReaderContactInfo[i].RS_Stream =
-            providedReaderInfo[i]->RS_Stream;
-        Svcs->verbose(
-            WS_Stream->CP_Stream,
-            "Received contact info \"%s\", RD_Stream %p for Reader Rank %d\n",
-            WSR_Stream->ReaderContactInfo[i].ContactString,
-            WSR_Stream->ReaderContactInfo[i].RS_Stream, i);
-    }
-
-    /*
-     * add this writer-side reader-specific stream to the parent writer stream
-     * structure
-     */
-    WS_Stream->Readers = realloc(
-        WS_Stream->Readers, sizeof(*WSR_Stream) * (WS_Stream->ReaderCount + 1));
-    WS_Stream->Readers[WS_Stream->ReaderCount] = WSR_Stream;
-    WS_Stream->ReaderCount++;
-
-    ContactInfo = malloc(sizeof(struct _DummyWriterContactInfo));
-    memset(ContactInfo, 0, sizeof(struct _DummyWriterContactInfo));
-    ContactInfo->ContactString = DummyContactString;
-    ContactInfo->WS_Stream = WSR_Stream;
-    *WriterContactInfoPtr = ContactInfo;
-
-    return WSR_Stream;
-}
-
-static void DummyProvideWriterDataToReader(CP_Services Svcs,
-                                           DP_RS_Stream RS_Stream_v,
-                                           int writerCohortSize,
-                                           CP_PeerCohort PeerCohort,
-                                           void **providedWriterInfo_v)
-{
-    Dummy_RS_Stream RS_Stream = (Dummy_RS_Stream)RS_Stream_v;
-    DummyWriterContactInfo *providedWriterInfo =
-        (DummyWriterContactInfo *)providedWriterInfo_v;
-
-    RS_Stream->PeerCohort = PeerCohort;
-    RS_Stream->WriterCohortSize = writerCohortSize;
-
-    /*
-     * make a copy of writer contact information (original will not be
-     * preserved)
-     */
-    RS_Stream->WriterContactInfo =
-        malloc(sizeof(struct _DummyWriterContactInfo) * writerCohortSize);
-    for (int i = 0; i < writerCohortSize; i++)
-    {
-        RS_Stream->WriterContactInfo[i].ContactString =
-            strdup(providedWriterInfo[i]->ContactString);
-        RS_Stream->WriterContactInfo[i].WS_Stream =
-            providedWriterInfo[i]->WS_Stream;
-        Svcs->verbose(
-            RS_Stream->CP_Stream,
-            "Received contact info \"%s\", WS_stream %p for WSR Rank %d\n",
-            RS_Stream->WriterContactInfo[i].ContactString,
-            RS_Stream->WriterContactInfo[i].WS_Stream, i);
-    }
-}
-
-typedef struct _DummyPerTimestepInfo
-{
-    char *CheckString;
-    int CheckInt;
-} * DummyPerTimestepInfo;
-
-static void *DummyReadRemoteMemory(CP_Services Svcs, DP_RS_Stream Stream_v,
-                                   int Rank, long Timestep, size_t Offset,
-                                   size_t Length, void *Buffer,
-                                   void *DP_TimestepInfo)
-{
-    Dummy_RS_Stream Stream = (Dummy_RS_Stream)
-        Stream_v; /* DP_RS_Stream is the return from InitReader */
-    CManager cm = Svcs->getCManager(Stream->CP_Stream);
-    DummyCompletionHandle ret = malloc(sizeof(struct _DummyCompletionHandle));
-    struct _DummyReadRequestMsg ReadRequestMsg;
-
-    ret->CMcondition = CMCondition_get(cm, NULL);
-    ret->CPStream = Stream->CP_Stream;
-    ret->cm = cm;
-    ret->Buffer = Buffer;
-    ret->Rank = Rank;
-    /*
-     * set the completion handle as client Data on the condition so that
-     * handler has access to it.
-     */
-    CMCondition_set_client_data(cm, ret->CMcondition, ret);
-
-    Svcs->verbose(Stream->CP_Stream,
-                  "Adios requesting to read remote memory for Timestep %d "
-                  "from Rank %d, WSR_Stream = %p\n",
-                  Timestep, Rank, Stream->WriterContactInfo[Rank].WS_Stream);
-
-    /* send request to appropriate writer */
-    /* memset avoids uninit byte warnings from valgrind */
-    memset(&ReadRequestMsg, 0, sizeof(ReadRequestMsg));
-    ReadRequestMsg.Timestep = Timestep;
-    ReadRequestMsg.Offset = Offset;
-    ReadRequestMsg.Length = Length;
-    ReadRequestMsg.WS_Stream = Stream->WriterContactInfo[Rank].WS_Stream;
-    ReadRequestMsg.RS_Stream = Stream;
-    ReadRequestMsg.RequestingRank = Stream->Rank;
-    ReadRequestMsg.NotifyCondition = ret->CMcondition;
-    Svcs->sendToPeer(Stream->CP_Stream, Stream->PeerCohort, Rank,
-                     Stream->ReadRequestFormat, &ReadRequestMsg);
-
-    return ret;
-}
-
-static void DummyWaitForCompletion(CP_Services Svcs, void *Handle_v)
-{
-    DummyCompletionHandle Handle = (DummyCompletionHandle)Handle_v;
-    Svcs->verbose(
-        Handle->CPStream,
-        "Waiting for completion of memory read to rank %d, condition %d\n",
-        Handle->Rank, Handle->CMcondition);
-    /*
-     * Wait for the CM condition to be signalled.  If it has been already,
-     * this returns immediately.  Copying the incoming data to the waiting
-     * buffer has been done by the reply handler.
-     */
-    CMCondition_wait(Handle->cm, Handle->CMcondition);
-    Svcs->verbose(
-        Handle->CPStream,
-        "Remote memory read to rank %d with condition %d has completed\n",
-        Handle->Rank, Handle->CMcondition);
-    free(Handle);
-}
-
-static void DummyProvideTimestep(CP_Services Svcs, DP_WS_Stream Stream_v,
-                                 struct _SstData *Data,
-                                 struct _SstData *LocalMetadata, long Timestep,
-                                 void **TimestepInfoPtr)
-{
-    Dummy_WS_Stream Stream = (Dummy_WS_Stream)Stream_v;
-    TimestepList Entry = malloc(sizeof(struct _TimestepEntry));
-    struct _DummyPerTimestepInfo *Info =
-        malloc(sizeof(struct _DummyPerTimestepInfo));
-
-    Info->CheckString = malloc(64);
-    sprintf(Info->CheckString, "Dummy info for timestep %ld from rank %d",
-            Timestep, Stream->Rank);
-    Info->CheckInt = Stream->Rank * 1000 + Timestep;
-    Entry->Data = Data;
-    Entry->Timestep = Timestep;
-    Entry->DP_TimestepInfo = Info;
-
-    Entry->Next = Stream->Timesteps;
-    Stream->Timesteps = Entry;
-    *TimestepInfoPtr = Info;
-}
-
-static void DummyReleaseTimestep(CP_Services Svcs, DP_WS_Stream Stream_v,
-                                 long Timestep)
-{
-    Dummy_WS_Stream Stream = (Dummy_WS_Stream)Stream_v;
-    TimestepList List = Stream->Timesteps;
-
-    Svcs->verbose(Stream->CP_Stream, "Releasing timestep %ld\n", Timestep);
-    if (Stream->Timesteps->Timestep == Timestep)
-    {
-        Stream->Timesteps = List->Next;
-        free(List);
-    }
-    else
-    {
-        TimestepList last = List;
-        List = List->Next;
-        while (List != NULL)
-        {
-            if (List->Timestep == Timestep)
-            {
-                last->Next = List->Next;
-                free(List);
-                return;
-            }
-            last = List;
-            List = List->Next;
-        }
-        /*
-         * Shouldn't ever get here because we should never release a
-         * timestep that we don't have.
-         */
-        fprintf(stderr, "Failed to release Timestep %ld, not found\n",
-                Timestep);
-        assert(0);
-    }
-}
-
-static FMField DummyReaderContactList[] = {
-    {"ContactString", "string", sizeof(char *),
-     FMOffset(DummyReaderContactInfo, ContactString)},
-    {"reader_ID", "integer", sizeof(void *),
-     FMOffset(DummyReaderContactInfo, RS_Stream)},
-    {NULL, NULL, 0, 0}};
-
-static FMStructDescRec DummyReaderContactStructs[] = {
-    {"DummyReaderContactInfo", DummyReaderContactList,
-     sizeof(struct _DummyReaderContactInfo), NULL},
-    {NULL, NULL, 0, NULL}};
-
-static FMField DummyWriterContactList[] = {
-    {"ContactString", "string", sizeof(char *),
-     FMOffset(DummyWriterContactInfo, ContactString)},
-    {"writer_ID", "integer", sizeof(void *),
-     FMOffset(DummyWriterContactInfo, WS_Stream)},
-    {NULL, NULL, 0, 0}};
-
-static FMStructDescRec DummyWriterContactStructs[] = {
-    {"DummyWriterContactInfo", DummyWriterContactList,
-     sizeof(struct _DummyWriterContactInfo), NULL},
-    {NULL, NULL, 0, NULL}};
-
-static FMField DummyTimestepInfoList[] = {
-    {"CheckString", "string", sizeof(char *),
-     FMOffset(DummyPerTimestepInfo, CheckString)},
-    {"CheckInt", "integer", sizeof(void *),
-     FMOffset(DummyPerTimestepInfo, CheckInt)},
-    {NULL, NULL, 0, 0}};
-
-static FMStructDescRec DummyTimestepInfoStructs[] = {
-    {"DummyTimestepInfo", DummyTimestepInfoList,
-     sizeof(struct _DummyPerTimestepInfo), NULL},
-    {NULL, NULL, 0, NULL}};
-
-static struct _CP_DP_Interface dummyDPInterface;
-
-extern CP_DP_Interface LoadDummyDP()
-{
-    memset(&dummyDPInterface, 0, sizeof(dummyDPInterface));
-    dummyDPInterface.ReaderContactFormats = DummyReaderContactStructs;
-    dummyDPInterface.WriterContactFormats = DummyWriterContactStructs;
-    dummyDPInterface.TimestepInfoFormats = NULL; // DummyTimestepInfoStructs;
-    dummyDPInterface.initReader = DummyInitReader;
-    dummyDPInterface.initWriter = DummyInitWriter;
-    dummyDPInterface.initWriterPerReader = DummyInitWriterPerReader;
-    dummyDPInterface.provideWriterDataToReader = DummyProvideWriterDataToReader;
-    dummyDPInterface.readRemoteMemory = DummyReadRemoteMemory;
-    dummyDPInterface.waitForCompletion = DummyWaitForCompletion;
-    dummyDPInterface.provideTimestep = DummyProvideTimestep;
-    dummyDPInterface.releaseTimestep = DummyReleaseTimestep;
-    return &dummyDPInterface;
-}
diff --git a/source/adios2/toolkit/sst/dp/evpath_dp.c b/source/adios2/toolkit/sst/dp/evpath_dp.c
index c02a4b5e63..8042a646f3 100644
--- a/source/adios2/toolkit/sst/dp/evpath_dp.c
+++ b/source/adios2/toolkit/sst/dp/evpath_dp.c
@@ -13,6 +13,16 @@
 #include "adios2/toolkit/profiling/taustubs/taustubs.h"
 #include "dp_interface.h"
 
+#if defined(__has_feature)
+#if __has_feature(thread_sanitizer)
+#define NO_SANITIZE_THREAD __attribute__((no_sanitize("thread")))
+#endif
+#endif
+
+#ifndef NO_SANITIZE_THREAD
+#define NO_SANITIZE_THREAD
+#endif
+
 /*
  *  Some conventions:
  *    `RS` indicates a reader-side item.
@@ -71,6 +81,8 @@ typedef struct _Evpath_RS_Stream
     long PreloadActiveTimestep;
     long TotalReadRequests;
     long ReadRequestsFromPreload;
+    SstStats Stats;
+    long LastPreloadTimestep;
 } * Evpath_RS_Stream;
 
 typedef struct _Evpath_WSR_Stream
@@ -125,6 +137,7 @@ typedef struct _Evpath_WS_Stream
 
     int ReaderCount;
     Evpath_WSR_Stream *Readers;
+    SstStats Stats;
 } * Evpath_WS_Stream;
 
 typedef struct _EvpathReaderContactInfo
@@ -243,7 +256,7 @@ static void SendSpeculativePreloadMsgs(CP_Services Svcs,
 static DP_RS_Stream EvpathInitReader(CP_Services Svcs, void *CP_Stream,
                                      void **ReaderContactInfoPtr,
                                      struct _SstParams *Params,
-                                     attr_list WriterContact)
+                                     attr_list WriterContact, SstStats Stats)
 {
     Evpath_RS_Stream Stream = malloc(sizeof(struct _Evpath_RS_Stream));
     EvpathReaderContactInfo Contact =
@@ -262,13 +275,23 @@ static DP_RS_Stream EvpathInitReader(CP_Services Svcs, void *CP_Stream,
      * save the CP_stream value of later use
      */
     Stream->CP_Stream = CP_Stream;
+    Stream->Stats = Stats;
+    Stream->LastPreloadTimestep = -1;
 
     pthread_mutex_init(&Stream->DataLock, NULL);
 
     SMPI_Comm_rank(comm, &Stream->Rank);
 
-    set_string_attr(ListenAttrs, attr_atom_from_string("CM_TRANSPORT"),
-                    strdup("sockets"));
+    if (Params->WANDataTransport)
+    {
+        set_string_attr(ListenAttrs, attr_atom_from_string("CM_TRANSPORT"),
+                        strdup(Params->WANDataTransport));
+    }
+    else
+    {
+        set_string_attr(ListenAttrs, attr_atom_from_string("CM_TRANSPORT"),
+                        strdup("sockets"));
+    }
 
     if (Params->DataInterface)
     {
@@ -369,7 +392,7 @@ static void EvpathReadRequestHandler(CManager cm, CMConnection incoming_conn,
     CP_Services Svcs = (CP_Services)client_Data;
     int RequestingRank = ReadRequestMsg->RequestingRank;
 
-    Svcs->verbose(WS_Stream->CP_Stream,
+    Svcs->verbose(WS_Stream->CP_Stream, DPTraceVerbose,
                   "Got a request to read remote memory "
                   "from reader rank %d: timestep %d, "
                   "offset %d, length %d\n",
@@ -392,7 +415,7 @@ static void EvpathReadRequestHandler(CManager cm, CMConnection incoming_conn,
             ReadReplyMsg.RS_Stream = ReadRequestMsg->RS_Stream;
             ReadReplyMsg.NotifyCondition = ReadRequestMsg->NotifyCondition;
             Svcs->verbose(
-                WS_Stream->CP_Stream,
+                WS_Stream->CP_Stream, DPTraceVerbose,
                 "Sending a reply to reader rank %d for remote memory read\n",
                 RequestingRank);
             ReplyConn = WSR_Stream->ReaderContactInfo[RequestingRank].Conn;
@@ -473,9 +496,10 @@ static void EvpathReadReplyHandler(CManager cm, CMConnection conn, void *msg_v,
 
     if (CMCondition_has_signaled(cm, ReadReplyMsg->NotifyCondition))
     {
-        Svcs->verbose(RS_Stream->CP_Stream, "Got a reply to remote memory "
-                                            "read, but the condition is "
-                                            "already signalled, returning\n");
+        Svcs->verbose(RS_Stream->CP_Stream, DPTraceVerbose,
+                      "Got a reply to remote memory "
+                      "read, but the condition is "
+                      "already signalled, returning\n");
         TAU_STOP_FUNC();
         return;
     }
@@ -484,13 +508,13 @@ static void EvpathReadReplyHandler(CManager cm, CMConnection conn, void *msg_v,
     if (!Handle)
     {
         Svcs->verbose(
-            RS_Stream->CP_Stream,
+            RS_Stream->CP_Stream, DPCriticalVerbose,
             "Got a reply to remote memory read, but condition not found\n");
         TAU_STOP_FUNC();
         return;
     }
     Svcs->verbose(
-        RS_Stream->CP_Stream,
+        RS_Stream->CP_Stream, DPTraceVerbose,
         "Got a reply to remote memory read from rank %d, condition is %d\n",
         Handle->Rank, ReadReplyMsg->NotifyCondition);
 
@@ -501,6 +525,8 @@ static void EvpathReadReplyHandler(CManager cm, CMConnection conn, void *msg_v,
      */
     memcpy(Handle->Buffer, ReadReplyMsg->Data, ReadReplyMsg->DataLength);
 
+    RS_Stream->Stats->DataBytesReceived += ReadReplyMsg->DataLength;
+
     /*
      * Signal the condition to wake the reader if they are waiting.
      */
@@ -508,6 +534,35 @@ static void EvpathReadReplyHandler(CManager cm, CMConnection conn, void *msg_v,
     TAU_STOP_FUNC();
 }
 
+/*
+ * To "fingerprint" a data block, take 8 sample bytes in it and put
+ * them in a long.  If a sample point is zero, use the next non-zero
+ * byte.  This is only used in verbose mode as an indication that
+ * we're dealing with a copy of the memory region between reader and
+ * writer, so we're not being pedantic about anything here.
+ */
+static unsigned long writeBlockFingerprint(char *Page, size_t Size)
+{
+    size_t Start = Size / 16;
+    size_t Stride = Size / 8;
+    unsigned long Print = 0;
+    if (!Page)
+        return 0;
+    for (int i = 0; i < 8; i++)
+    {
+        size_t Index = Start + Stride * i;
+        unsigned char Component = 0;
+        while ((Page[Index] == 0) && (Index < (Size - 1)))
+        {
+            Component++;
+            Index++;
+        }
+        Component += (unsigned char)Page[Index];
+        Print |= (((unsigned long)Component) << (8 * i));
+    }
+    return Print;
+}
+
 // reader-side routine, called from the main program
 static int HandleRequestWithPreloaded(CP_Services Svcs,
                                       Evpath_RS_Stream RS_Stream, int Rank,
@@ -525,10 +580,11 @@ static int HandleRequestWithPreloaded(CP_Services Svcs,
     {
         return 0;
     }
-    Svcs->verbose(RS_Stream->CP_Stream,
+    Svcs->verbose(RS_Stream->CP_Stream, DPTraceVerbose,
                   "Satisfying remote memory read with preload from writer rank "
-                  "%d for timestep %ld\n",
-                  Rank, Timestep);
+                  "%d for timestep %ld, fprint %lx\n",
+                  Rank, Timestep,
+                  writeBlockFingerprint(Entry->Data, Entry->DataSize));
     memcpy(Buffer, Entry->Data + Offset, Length);
     return 1;
 }
@@ -558,6 +614,11 @@ static void DiscardPriorPreloaded(CP_Services Svcs, Evpath_RS_Stream RS_Stream,
             /* free item */
             if (ItemToFree->Data)
             {
+                Svcs->verbose(RS_Stream->CP_Stream, DPPerRankVerbose,
+                              "Discarding prior, TS %ld, data %p, fprint %lx\n",
+                              ItemToFree->Timestep, ItemToFree->Data,
+                              writeBlockFingerprint(ItemToFree->Data,
+                                                    ItemToFree->DataSize));
                 CMreturn_buffer(cm, ItemToFree->Data);
             }
 
@@ -584,10 +645,11 @@ static void EvpathPreloadHandler(CManager cm, CMConnection conn, void *msg_v,
     RSTimestepList Entry = calloc(1, sizeof(*Entry));
 
     Svcs->verbose(
-        RS_Stream->CP_Stream,
-        "Got a preload message from writer rank %d for timestep %ld\n",
-        PreloadMsg->WriterRank, PreloadMsg->Timestep);
-
+        RS_Stream->CP_Stream, DPPerStepVerbose,
+        "Got a preload message from writer rank %d for timestep %ld, fprint "
+        "%lx\n",
+        PreloadMsg->WriterRank, PreloadMsg->Timestep,
+        writeBlockFingerprint(PreloadMsg->Data, PreloadMsg->DataLength));
     /* arrange for this message data to stay around */
     CMtake_buffer(cm, msg_v);
 
@@ -596,7 +658,15 @@ static void EvpathPreloadHandler(CManager cm, CMConnection conn, void *msg_v,
     Entry->Data = PreloadMsg->Data;
     Entry->DataSize = PreloadMsg->DataLength;
     Entry->DataStart = 0;
-
+    RS_Stream->Stats->DataBytesReceived += PreloadMsg->DataLength;
+    RS_Stream->Stats->PreloadBytesReceived += PreloadMsg->DataLength;
+    if (PreloadMsg->Timestep > RS_Stream->LastPreloadTimestep)
+    {
+        // only incremenet PreloadTimesteps once, even if we get multiple
+        // preloads on a timestep (from different ranks)
+        RS_Stream->LastPreloadTimestep = PreloadMsg->Timestep;
+        RS_Stream->Stats->PreloadTimestepsReceived++;
+    }
     pthread_mutex_lock(&RS_Stream->DataLock);
     Entry->Next = RS_Stream->QueuedTimesteps;
     RS_Stream->QueuedTimesteps = Entry;
@@ -622,7 +692,7 @@ static void EvpathPreloadHandler(CManager cm, CMConnection conn, void *msg_v,
 // writer-side routine, called from the main program
 static DP_WS_Stream EvpathInitWriter(CP_Services Svcs, void *CP_Stream,
                                      struct _SstParams *Params,
-                                     attr_list DPAttrs)
+                                     attr_list DPAttrs, SstStats Stats)
 {
     Evpath_WS_Stream Stream = malloc(sizeof(struct _Evpath_WS_Stream));
     CManager cm = Svcs->getCManager(CP_Stream);
@@ -639,6 +709,7 @@ static DP_WS_Stream EvpathInitWriter(CP_Services Svcs, void *CP_Stream,
      * save the CP_stream value of later use
      */
     Stream->CP_Stream = CP_Stream;
+    Stream->Stats = Stats;
 
     /*
      * add a handler for read request messages
@@ -733,7 +804,7 @@ static DP_WSR_Stream EvpathInitWriterPerReader(CP_Services Svcs,
         WSR_Stream->ReaderContactInfo[i].RS_Stream =
             providedReaderInfo[i]->RS_Stream;
         Svcs->verbose(
-            WS_Stream->CP_Stream,
+            WS_Stream->CP_Stream, DPTraceVerbose,
             "Received contact info \"%s\", RD_Stream %p for Reader Rank %d\n",
             WSR_Stream->ReaderContactInfo[i].ContactString,
             WSR_Stream->ReaderContactInfo[i].RS_Stream, i);
@@ -798,7 +869,7 @@ static void EvpathProvideWriterDataToReader(CP_Services Svcs,
         RS_Stream->WriterContactInfo[i].WS_Stream =
             providedWriterInfo[i]->WS_Stream;
         Svcs->verbose(
-            RS_Stream->CP_Stream,
+            RS_Stream->CP_Stream, DPTraceVerbose,
             "Received contact info \"%s\", WS_stream %p for WSR Rank %d\n",
             RS_Stream->WriterContactInfo[i].ContactString,
             RS_Stream->WriterContactInfo[i].WS_Stream, i);
@@ -845,7 +916,7 @@ static void FailRequestsToRank(CP_Services Svcs, CManager cm,
 {
     EvpathCompletionHandle Tmp;
     int FailedSomethingToRank = 0;
-    Svcs->verbose(Stream->CP_Stream,
+    Svcs->verbose(Stream->CP_Stream, DPTraceVerbose,
                   "Fail pending requests to rank %d on stream %p\n", FailedRank,
                   Stream);
     pthread_mutex_lock(&Stream->DataLock);
@@ -856,21 +927,22 @@ static void FailRequestsToRank(CP_Services Svcs, CManager cm,
         {
             Tmp->Failed = 1;
             FailedSomethingToRank = 1;
-            Svcs->verbose(Tmp->CPStream,
+            Svcs->verbose(Tmp->CPStream, DPTraceVerbose,
                           "Found a pending remote memory read "
                           "to writer rank %d, marking as "
                           "failed and signalling condition %d\n",
                           Tmp->Rank, Tmp->CMcondition);
             CMCondition_signal(cm, Tmp->CMcondition);
-            Svcs->verbose(Tmp->CPStream, "Did the signal of condition %d\n",
-                          Tmp->Rank, Tmp->CMcondition);
+            Svcs->verbose(Tmp->CPStream, DPTraceVerbose,
+                          "Did the signal of condition %d\n", Tmp->Rank,
+                          Tmp->CMcondition);
         }
         Tmp = Tmp->Next;
     }
     if (FailedSomethingToRank)
     {
         Tmp = Stream->PendingReadRequests;
-        Svcs->verbose(Stream->CP_Stream,
+        Svcs->verbose(Stream->CP_Stream, DPTraceVerbose,
                       "We were waiting for requests on rank %d, fail *all* "
                       "pending requests on stream %p\n",
                       FailedRank, Stream);
@@ -881,20 +953,21 @@ static void FailRequestsToRank(CP_Services Svcs, CManager cm,
             {
                 Tmp->Failed = 1;
                 FailedSomethingToRank = 1;
-                Svcs->verbose(Tmp->CPStream,
+                Svcs->verbose(Tmp->CPStream, DPTraceVerbose,
                               "Found a pending remote memory read "
                               "to writer rank %d, marking as "
                               "failed and signalling condition %d\n",
                               Tmp->Rank, Tmp->CMcondition);
                 CMCondition_signal(cm, Tmp->CMcondition);
-                Svcs->verbose(Tmp->CPStream, "Did the signal of condition %d\n",
-                              Tmp->Rank, Tmp->CMcondition);
+                Svcs->verbose(Tmp->CPStream, DPTraceVerbose,
+                              "Did the signal of condition %d\n", Tmp->Rank,
+                              Tmp->CMcondition);
             }
             Tmp = Tmp->Next;
         }
     }
     pthread_mutex_unlock(&Stream->DataLock);
-    Svcs->verbose(Stream->CP_Stream,
+    Svcs->verbose(Stream->CP_Stream, DPPerRankVerbose,
                   "Done Failing requests to writer %d from stream %p\n",
                   FailedRank, Stream);
 }
@@ -980,14 +1053,14 @@ static void *EvpathReadRemoteMemory(CP_Services Svcs, DP_RS_Stream Stream_v,
     {
         // we're in some kind of preload mode, but we don't have the data yet,
         // wait for it without sending a request
-        Svcs->verbose(Stream->CP_Stream,
+        Svcs->verbose(Stream->CP_Stream, DPTraceVerbose,
                       "Adios waiting for preload data for Timestep %d "
                       "from Rank %d, WSR_Stream = %p, DP_TimestepInfo %p\n",
                       Timestep, Rank, Stream->WriterContactInfo[Rank].WS_Stream,
                       DP_TimestepInfo);
         return ret;
     }
-    Svcs->verbose(Stream->CP_Stream,
+    Svcs->verbose(Stream->CP_Stream, DPTraceVerbose,
                   "Adios requesting to read remote memory for Timestep %d "
                   "from Rank %d, WSR_Stream = %p, DP_TimestepInfo %p\n",
                   Timestep, Rank, Stream->WriterContactInfo[Rank].WS_Stream,
@@ -1020,7 +1093,7 @@ static int EvpathWaitForCompletion(CP_Services Svcs, void *Handle_v)
     int Ret = 1;
     if (Handle->CMcondition != -1)
         Svcs->verbose(
-            Handle->CPStream,
+            Handle->CPStream, DPTraceVerbose,
             "Waiting for completion of memory read to rank %d, condition %d\n",
             Handle->Rank, Handle->CMcondition);
     /*
@@ -1032,7 +1105,7 @@ static int EvpathWaitForCompletion(CP_Services Svcs, void *Handle_v)
         CMCondition_wait(Handle->cm, Handle->CMcondition);
     if (Handle->Failed)
     {
-        Svcs->verbose(Handle->CPStream,
+        Svcs->verbose(Handle->CPStream, DPTraceVerbose,
                       "Remote memory read to rank %d with "
                       "condition %d has FAILED because of "
                       "writer failure\n",
@@ -1042,7 +1115,7 @@ static int EvpathWaitForCompletion(CP_Services Svcs, void *Handle_v)
     else
     {
         if (Handle->CMcondition != -1)
-            Svcs->verbose(Handle->CPStream,
+            Svcs->verbose(Handle->CPStream, DPTraceVerbose,
                           "Remote memory read to rank %d with condition %d has "
                           "completed\n",
                           Handle->Rank, Handle->CMcondition);
@@ -1061,7 +1134,7 @@ static void EvpathNotifyConnFailure(CP_Services Svcs, DP_RS_Stream Stream_v,
     Evpath_RS_Stream Stream = (Evpath_RS_Stream)
         Stream_v; /* DP_RS_Stream is the return from InitReader */
     CManager cm = Svcs->getCManager(Stream->CP_Stream);
-    Svcs->verbose(Stream->CP_Stream,
+    Svcs->verbose(Stream->CP_Stream, DPPerRankVerbose,
                   "received notification that writer peer "
                   "%d has failed, failing any pending "
                   "requests\n",
@@ -1109,7 +1182,7 @@ static void EvpathWSReaderRegisterTimestep(CP_Services Svcs,
         return;
     }
 
-    Svcs->verbose(WS_Stream->CP_Stream,
+    Svcs->verbose(WS_Stream->CP_Stream, DPPerRankVerbose,
                   "Per reader registration for timestep %ld, preload mode %d\n",
                   Timestep, PreloadMode);
     if (PreloadMode == SstPreloadLearned)
@@ -1121,16 +1194,18 @@ static void EvpathWSReaderRegisterTimestep(CP_Services Svcs,
         if (WSR_Stream->ReaderRequestArray)
         {
             Svcs->verbose(
-                WS_Stream->CP_Stream,
-                "Sending Learned Preload messages, reader %p, timestep %ld\n",
-                WSR_Stream, Timestep);
+                WS_Stream->CP_Stream, DPPerRankVerbose,
+                "Sending Learned Preload messages, reader %p, timestep %ld, "
+                "fprint %lx\n",
+                WSR_Stream, Timestep,
+                writeBlockFingerprint(Entry->Data.block, Entry->Data.DataSize));
             SendPreloadMsgs(Svcs, WSR_Stream, Entry);
         }
     }
     else if (PreloadMode == SstPreloadSpeculative)
     {
         Svcs->verbose(
-            WS_Stream->CP_Stream,
+            WS_Stream->CP_Stream, DPPerRankVerbose,
             "Sending Speculative Preload messages, reader %p, timestep %ld\n",
             WSR_Stream, Timestep);
         SendSpeculativePreloadMsgs(Svcs, WSR_Stream, Entry);
@@ -1145,7 +1220,7 @@ static void EvpathRSTimestepArrived(CP_Services Svcs, DP_RS_Stream RS_Stream_v,
 {
     Evpath_RS_Stream RS_Stream = (Evpath_RS_Stream)RS_Stream_v;
     Svcs->verbose(
-        RS_Stream->CP_Stream,
+        RS_Stream->CP_Stream, DPPerRankVerbose,
         "EVPATH registering reader arrival of TS %ld metadata, preload mode "
         "%d\n",
         Timestep, PreloadMode);
@@ -1163,7 +1238,7 @@ static void SendPreloadMsgs(CP_Services Svcs, Evpath_WSR_Stream WSR_Stream,
     Evpath_WS_Stream WS_Stream =
         WSR_Stream->WS_Stream; /* pointer to writer struct */
     struct _EvpathPreloadMsg PreloadMsg;
-    Svcs->verbose(WS_Stream->CP_Stream,
+    Svcs->verbose(WS_Stream->CP_Stream, DPPerRankVerbose,
                   "EVPATH Sending preload messages for timestep %ld\n",
                   TS->Timestep);
     memset(&PreloadMsg, 0, sizeof(PreloadMsg));
@@ -1178,7 +1253,7 @@ static void SendPreloadMsgs(CP_Services Svcs, Evpath_WSR_Stream WSR_Stream,
         {
             PreloadMsg.RS_Stream = WSR_Stream->ReaderContactInfo[i].RS_Stream;
             Svcs->verbose(
-                WS_Stream->CP_Stream,
+                WS_Stream->CP_Stream, DPTraceVerbose,
                 "EVPATH Preload message for timestep %ld, going to rank %d\n",
                 TS->Timestep, i);
             CMwrite(WSR_Stream->ReaderContactInfo[i].Conn,
@@ -1212,7 +1287,7 @@ static void SendSpeculativePreloadMsgs(CP_Services Svcs,
             if (!Conn)
             {
                 Svcs->verbose(
-                    WS_Stream->CP_Stream,
+                    WS_Stream->CP_Stream, DPCriticalVerbose,
                     "Failed to connect to reader rank %d for response to "
                     "remote read, assume failure, no response sent\n",
                     i);
@@ -1239,7 +1314,7 @@ static void EvpathReaderReleaseTimestep(CP_Services Svcs,
     if ((!WSR_Stream->ReaderRequestArray) &&
         (Timestep == WSR_Stream->ReadPatternLockTimestep))
     {
-        Svcs->verbose(WS_Stream->CP_Stream,
+        Svcs->verbose(WS_Stream->CP_Stream, DPPerRankVerbose,
                       "EVPATH Saving the read pattern for timestep %ld\n",
                       Timestep);
         /* save the pattern */
@@ -1255,7 +1330,7 @@ static void EvpathReaderReleaseTimestep(CP_Services Svcs,
                         WSR_Stream->ReaderRequestArray = ReqList->RequestList;
                         /* so it doesn't get free'd */
                         ReqList->RequestList = NULL;
-                        Svcs->verbose(WS_Stream->CP_Stream,
+                        Svcs->verbose(WS_Stream->CP_Stream, DPTraceVerbose,
                                       "EVPATH Found timestep\n", Timestep);
                     }
                     ReqList = ReqList->Next;
@@ -1264,7 +1339,7 @@ static void EvpathReaderReleaseTimestep(CP_Services Svcs,
             tmp = tmp->Next;
         }
         /* send stored timesteps based on learned pattern */
-        Svcs->verbose(WS_Stream->CP_Stream,
+        Svcs->verbose(WS_Stream->CP_Stream, DPPerRankVerbose,
                       "EVPATH Sending learned preloads for queued messages\n");
         tmp = WS_Stream->Timesteps;
         while (tmp != NULL)
@@ -1303,6 +1378,11 @@ static void EvpathProvideTimestep(CP_Services Svcs, DP_WS_Stream Stream_v,
     Entry->Timestep = Timestep;
     Entry->Next = NULL;
 
+    Svcs->verbose(
+        WS_Stream->CP_Stream, DPPerRankVerbose,
+        "ProvideTimestep, registering timestep %ld, data %p, fprint %lx\n",
+        Timestep, Data->block,
+        writeBlockFingerprint(Data->block, Data->DataSize));
     pthread_mutex_lock(&WS_Stream->DataLock);
     if (WS_Stream->Timesteps)
     {
@@ -1327,7 +1407,8 @@ static void EvpathReleaseTimestep(CP_Services Svcs, DP_WS_Stream Stream_v,
     Evpath_WS_Stream WS_Stream = (Evpath_WS_Stream)Stream_v;
     TimestepList List;
 
-    Svcs->verbose(WS_Stream->CP_Stream, "Releasing timestep %ld\n", Timestep);
+    Svcs->verbose(WS_Stream->CP_Stream, DPPerRankVerbose,
+                  "Releasing timestep %ld\n", Timestep);
     pthread_mutex_lock(&WS_Stream->DataLock);
     List = WS_Stream->Timesteps;
     if (WS_Stream->Timesteps && (WS_Stream->Timesteps->Timestep == Timestep))
@@ -1432,21 +1513,22 @@ static FMStructDescRec EvpathWriterContactStructs[] = {
 //     sizeof(struct _EvpathPerTimestepInfo), NULL},
 //    {NULL, NULL, 0, NULL}};
 
-static struct _CP_DP_Interface evpathDPInterface;
+static struct _CP_DP_Interface evpathDPInterface = {
+    NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
+    NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL};
 
 static int EvpathGetPriority(CP_Services Svcs, void *CP_Stream,
                              struct _SstParams *Params)
 {
     // Define any unique attributes here
-    (void)attr_atom_from_string("EVPATH_DP_Attr");
+    //    (void)attr_atom_from_string("EVPATH_DP_Attr");
 
     /* The evpath DP should be a lower priority than any RDMA dp, so return 1 */
     return 1;
 }
 
-extern CP_DP_Interface LoadEVpathDP()
+extern NO_SANITIZE_THREAD CP_DP_Interface LoadEVpathDP()
 {
-    memset(&evpathDPInterface, 0, sizeof(evpathDPInterface));
     evpathDPInterface.ReaderContactFormats = EvpathReaderContactStructs;
     evpathDPInterface.WriterContactFormats = EvpathWriterContactStructs;
     evpathDPInterface.TimestepInfoFormats = NULL; // EvpathTimestepInfoStructs;
diff --git a/source/adios2/toolkit/sst/dp/nvstream_dp.c b/source/adios2/toolkit/sst/dp/nvstream_dp.c
deleted file mode 100644
index 71e20bb3d7..0000000000
--- a/source/adios2/toolkit/sst/dp/nvstream_dp.c
+++ /dev/null
@@ -1,842 +0,0 @@
-#include 
-#include 
-#include 
-#include 
-#include 
-
-#include 
-#include 
-
-#include "sst_data.h"
-
-#include "adios2/toolkit/profiling/taustubs/taustubs.h"
-#include "dp_interface.h"
-#include "nvswrapper.h"
-
-/*
- *  Some conventions:
- *    `RS` indicates a reader-side item.
- *    `WS` indicates a writer-side item.
- *    `WSR` indicates a writer-side per-reader item.
- *
- *   We keep different "stream" structures for the reader side and for the
- *   writer side.  On the writer side, there's actually a "stream"
- *   per-connected-reader (a WSR_Stream), with the idea that some (many?)
- *   RDMA transports will require connections/pairing, so we'd need to track
- *   resources per reader.
- *
- *   Generally the 'contact information' we exchange at init time includes
- *   the address of the local 'stream' data structure.  This address isn't
- *   particularly useful to the far side, but it can be returned with
- *   requests to indicate what resource is targeted.  For example, when a
- *   remote memory read request arrives at the writer from the reader, it
- *   includes the WSR_Stream value that is the address of the writer-side
- *   per-reader data structure.  Upon message arrival, we just cast that
- *   value back into a pointer.
- *
- *   By design, neither the data plane nor the control plane reference the
- *   other's symbols directly.  The interface between the control plane and
- *   the data plane is represented by the types and structures defined in
- *   dp_interface.h and is a set of function pointers and FFS-style
- *   descriptions of the data structures to be communicated at init time.
- *   This allows for the future possibility of loading planes at run-time, etc.
- *
- *   This "nvstream" data plane uses control plane functionality to implement
- *   the ReadRemoteMemory functionality.  That is, it both the request to
- *   read memory and the response which carries the data are actually
- *   accomplished using the connections and message delivery facilities of
- *   the control plane, made available here via CP_Services.  A real data
- *   plane would replace one or both of these with RDMA functionality.
- */
-
-typedef struct _Nvstream_RS_Stream
-{
-    CManager cm;
-    void *CP_Stream;
-    CMFormat ReadRequestFormat;
-    pthread_mutex_t DataLock;
-    int Rank;
-
-    /* writer info */
-    int WriterCohortSize;
-    CP_PeerCohort PeerCohort;
-    void **writer_nvs;
-    struct _NvstreamWriterContactInfo *WriterContactInfo;
-    struct _NvstreamCompletionHandle *PendingReadRequests;
-
-    /* queued timestep info */
-    struct _RSTimestepEntry *QueuedTimesteps;
-} * Nvstream_RS_Stream;
-
-typedef struct _Nvstream_WSR_Stream
-{
-    struct _Nvstream_WS_Stream *WS_Stream;
-    CP_PeerCohort PeerCohort;
-    int ReaderCohortSize;
-    char *ReaderRequests;
-    struct _NvstreamReaderContactInfo *ReaderContactInfo;
-    struct _NvstreamWriterContactInfo
-        *WriterContactInfo; /* included so we can free on destroy */
-} * Nvstream_WSR_Stream;
-
-typedef struct _TimestepEntry
-{
-    long Timestep;
-    struct _SstData Data;
-    struct _NvstreamPerTimestepInfo *DP_TimestepInfo;
-    struct _TimestepEntry *Next;
-} * TimestepList;
-
-typedef struct _RSTimestepEntry
-{
-    long Timestep;
-    int WriterRank;
-    char *Data;
-    long DataSize;
-    long DataStart;
-    struct _RSTimestepEntry *Next;
-} * RSTimestepList;
-
-typedef struct _Nvstream_WS_Stream
-{
-    CManager cm;
-    void *CP_Stream;
-    int Rank;
-    void *nvs;
-
-    TimestepList Timesteps;
-    CMFormat ReadReplyFormat;
-
-    int ReaderCount;
-    Nvstream_WSR_Stream *Readers;
-} * Nvstream_WS_Stream;
-
-typedef struct _NvstreamReaderContactInfo
-{
-    char *ContactString;
-    CMConnection Conn;
-    void *RS_Stream;
-} * NvstreamReaderContactInfo;
-
-typedef struct _NvstreamWriterContactInfo
-{
-    char *ContactString;
-    void *WS_Stream;
-} * NvstreamWriterContactInfo;
-
-typedef struct _NvstreamReadRequestMsg
-{
-    long Timestep;
-    size_t Offset;
-    size_t Length;
-    void *WS_Stream;
-    void *RS_Stream;
-    int RequestingRank;
-    int NotifyCondition;
-} * NvstreamReadRequestMsg;
-
-static FMField NvstreamReadRequestList[] = {
-    {"Timestep", "integer", sizeof(long),
-     FMOffset(NvstreamReadRequestMsg, Timestep)},
-    {"Offset", "integer", sizeof(size_t),
-     FMOffset(NvstreamReadRequestMsg, Offset)},
-    {"Length", "integer", sizeof(size_t),
-     FMOffset(NvstreamReadRequestMsg, Length)},
-    {"WS_Stream", "integer", sizeof(void *),
-     FMOffset(NvstreamReadRequestMsg, WS_Stream)},
-    {"RS_Stream", "integer", sizeof(void *),
-     FMOffset(NvstreamReadRequestMsg, RS_Stream)},
-    {"RequestingRank", "integer", sizeof(int),
-     FMOffset(NvstreamReadRequestMsg, RequestingRank)},
-    {"NotifyCondition", "integer", sizeof(int),
-     FMOffset(NvstreamReadRequestMsg, NotifyCondition)},
-    {NULL, NULL, 0, 0}};
-
-static FMStructDescRec NvstreamReadRequestStructs[] = {
-    {"NvstreamReadRequest", NvstreamReadRequestList,
-     sizeof(struct _NvstreamReadRequestMsg), NULL},
-    {NULL, NULL, 0, NULL}};
-
-typedef struct _NvstreamReadReplyMsg
-{
-    long Timestep;
-    size_t DataLength;
-    void *RS_Stream;
-    char *Data;
-    int NotifyCondition;
-} * NvstreamReadReplyMsg;
-
-static FMField NvstreamReadReplyList[] = {
-    {"Timestep", "integer", sizeof(long),
-     FMOffset(NvstreamReadReplyMsg, Timestep)},
-    {"RS_Stream", "integer", sizeof(void *),
-     FMOffset(NvstreamReadReplyMsg, RS_Stream)},
-    {"DataLength", "integer", sizeof(size_t),
-     FMOffset(NvstreamReadReplyMsg, DataLength)},
-    {"Data", "char[DataLength]", sizeof(char),
-     FMOffset(NvstreamReadReplyMsg, Data)},
-    {"NotifyCondition", "integer", sizeof(int),
-     FMOffset(NvstreamReadReplyMsg, NotifyCondition)},
-    {NULL, NULL, 0, 0}};
-
-static FMStructDescRec NvstreamReadReplyStructs[] = {
-    {"NvstreamReadReply", NvstreamReadReplyList,
-     sizeof(struct _NvstreamReadReplyMsg), NULL},
-    {NULL, NULL, 0, NULL}};
-
-static void NvstreamReadReplyHandler(CManager cm, CMConnection conn,
-                                     void *msg_v, void *client_Data,
-                                     attr_list attrs);
-
-static DP_RS_Stream NvstreamInitReader(CP_Services Svcs, void *CP_Stream,
-                                       void **ReaderContactInfoPtr,
-                                       struct _SstParams *Params)
-{
-    Nvstream_RS_Stream Stream = malloc(sizeof(struct _Nvstream_RS_Stream));
-    NvstreamReaderContactInfo Contact =
-        malloc(sizeof(struct _NvstreamReaderContactInfo));
-    CManager cm = Svcs->getCManager(CP_Stream);
-    char *NvstreamContactString;
-    SMPI_Comm comm = Svcs->getMPIComm(CP_Stream);
-    CMFormat F;
-    CManager CM = Svcs->getCManager(CP_Stream);
-    attr_list ListenAttrs = create_attr_list();
-
-    memset(Stream, 0, sizeof(*Stream));
-    memset(Contact, 0, sizeof(*Contact));
-
-    /*
-     * save the CP_stream value of later use
-     */
-    Stream->CP_Stream = CP_Stream;
-
-    pthread_mutex_init(&Stream->DataLock, NULL);
-
-    SMPI_Comm_rank(comm, &Stream->Rank);
-
-    set_string_attr(ListenAttrs, attr_atom_from_string("CM_TRANSPORT"),
-                    "sockets");
-
-    if (Params->DataInterface)
-    {
-        set_string_attr(ListenAttrs, attr_atom_from_string("IP_INTERFACE"),
-                        strdup(Params->DataInterface));
-    }
-    else if (Params->NetworkInterface)
-    {
-        set_string_attr(ListenAttrs, attr_atom_from_string("IP_INTERFACE"),
-                        strdup(Params->NetworkInterface));
-    }
-
-    CMlisten_specific(CM, ListenAttrs);
-    attr_list ContactList = CMget_specific_contact_list(CM, ListenAttrs);
-
-    NvstreamContactString = attr_list_to_string(ContactList);
-
-    Contact->ContactString = NvstreamContactString;
-    Contact->RS_Stream = Stream;
-
-    *ReaderContactInfoPtr = Contact;
-
-    return Stream;
-}
-
-static void NvstreamDestroyReader(CP_Services Svcs, DP_RS_Stream RS_Stream_v)
-{
-    Nvstream_RS_Stream RS_Stream = (Nvstream_RS_Stream)RS_Stream_v;
-    free(RS_Stream);
-}
-
-static void NvstreamReadRequestHandler(CManager cm, CMConnection conn,
-                                       void *msg_v, void *client_Data,
-                                       attr_list attrs)
-{
-    TAU_START_FUNC();
-    NvstreamReadRequestMsg ReadRequestMsg = (NvstreamReadRequestMsg)msg_v;
-    Nvstream_WSR_Stream WSR_Stream = ReadRequestMsg->WS_Stream;
-
-    Nvstream_WS_Stream WS_Stream = WSR_Stream->WS_Stream;
-    TimestepList tmp = WS_Stream->Timesteps;
-    CP_Services Svcs = (CP_Services)client_Data;
-    int RequestingRank = ReadRequestMsg->RequestingRank;
-
-    Svcs->verbose(WS_Stream->CP_Stream,
-                  "Got a request to read remote memory "
-                  "from reader rank %d: timestep %d, "
-                  "offset %d, length %d\n",
-                  RequestingRank, ReadRequestMsg->Timestep,
-                  ReadRequestMsg->Offset, ReadRequestMsg->Length);
-    while (tmp != NULL)
-    {
-        if (tmp->Timestep == ReadRequestMsg->Timestep)
-        {
-            struct _NvstreamReadReplyMsg ReadReplyMsg;
-            /* memset avoids uninit byte warnings from valgrind */
-            memset(&ReadReplyMsg, 0, sizeof(ReadReplyMsg));
-            ReadReplyMsg.Timestep = ReadRequestMsg->Timestep;
-            ReadReplyMsg.DataLength = ReadRequestMsg->Length;
-            ReadReplyMsg.Data = tmp->Data.block + ReadRequestMsg->Offset;
-            ReadReplyMsg.RS_Stream = ReadRequestMsg->RS_Stream;
-            ReadReplyMsg.NotifyCondition = ReadRequestMsg->NotifyCondition;
-            Svcs->verbose(
-                WS_Stream->CP_Stream,
-                "Sending a reply to reader rank %d for remote memory read\n",
-                RequestingRank);
-            if (!WSR_Stream->ReaderContactInfo[RequestingRank].Conn)
-            {
-                attr_list List = attr_list_from_string(
-                    WSR_Stream->ReaderContactInfo[RequestingRank]
-                        .ContactString);
-                CMConnection Conn = CMget_conn(cm, List);
-                free_attr_list(List);
-                WSR_Stream->ReaderContactInfo[RequestingRank].Conn = Conn;
-            }
-            CMwrite(WSR_Stream->ReaderContactInfo[RequestingRank].Conn,
-                    WS_Stream->ReadReplyFormat, &ReadReplyMsg);
-
-            TAU_STOP_FUNC();
-            return;
-        }
-        tmp = tmp->Next;
-    }
-    /*
-     * Shouldn't ever get here because we should never get a request for a
-     * timestep that we don't have.
-     */
-    fprintf(stderr, "Writer rank %d - Failed to read Timestep %ld, not found\n",
-            WSR_Stream->WS_Stream->Rank, ReadRequestMsg->Timestep);
-    /*
-     * in the interest of not failing a writer on a reader failure, don't
-     * assert(0) here.  Probably this sort of error should close the link to
-     * a reader though.
-     */
-    TAU_STOP_FUNC();
-}
-
-typedef struct _NvstreamCompletionHandle
-{
-    int CMcondition;
-    CManager cm;
-    void *CPStream;
-    void *DPStream;
-    void *Buffer;
-    int Failed;
-    int Rank;
-    struct _NvstreamCompletionHandle *Next;
-} * NvstreamCompletionHandle;
-
-static void NvstreamReadReplyHandler(CManager cm, CMConnection conn,
-                                     void *msg_v, void *client_Data,
-                                     attr_list attrs)
-{
-    TAU_START_FUNC();
-    NvstreamReadReplyMsg ReadReplyMsg = (NvstreamReadReplyMsg)msg_v;
-    Nvstream_RS_Stream RS_Stream = ReadReplyMsg->RS_Stream;
-    CP_Services Svcs = (CP_Services)client_Data;
-    NvstreamCompletionHandle Handle = NULL;
-
-    if (CMCondition_has_signaled(cm, ReadReplyMsg->NotifyCondition))
-    {
-        Svcs->verbose(RS_Stream->CP_Stream, "Got a reply to remote memory "
-                                            "read, but the condition is "
-                                            "already signalled, returning\n");
-        TAU_STOP_FUNC();
-        return;
-    }
-    Handle = CMCondition_get_client_data(cm, ReadReplyMsg->NotifyCondition);
-
-    if (!Handle)
-    {
-        Svcs->verbose(
-            RS_Stream->CP_Stream,
-            "Got a reply to remote memory read, but condition not found\n");
-        TAU_STOP_FUNC();
-        return;
-    }
-    Svcs->verbose(
-        RS_Stream->CP_Stream,
-        "Got a reply to remote memory read from rank %d, condition is %d\n",
-        Handle->Rank, ReadReplyMsg->NotifyCondition);
-
-    /*
-     * `Handle` contains the full request info and is `client_data`
-     * associated with the CMCondition.  Once we get it, copy the incoming
-     * data to the buffer area given by the request
-     */
-    memcpy(Handle->Buffer, ReadReplyMsg->Data, ReadReplyMsg->DataLength);
-
-    /*
-     * Signal the condition to wake the reader if they are waiting.
-     */
-    CMCondition_signal(cm, ReadReplyMsg->NotifyCondition);
-    TAU_STOP_FUNC();
-}
-
-static DP_WS_Stream NvstreamInitWriter(CP_Services Svcs, void *CP_Stream,
-                                       struct _SstParams *Params)
-{
-    Nvstream_WS_Stream Stream = malloc(sizeof(struct _Nvstream_WS_Stream));
-    CManager cm = Svcs->getCManager(CP_Stream);
-    SMPI_Comm comm = Svcs->getMPIComm(CP_Stream);
-    CMFormat F;
-
-    memset(Stream, 0, sizeof(struct _Nvstream_WS_Stream));
-
-    SMPI_Comm_rank(comm, &Stream->Rank);
-
-    /*
-     * save the CP_stream value of later use
-     */
-    Stream->CP_Stream = CP_Stream;
-
-    Stream->nvs = nvs_create_store();
-
-    Svcs->verbose(Stream->CP_Stream,
-                  "Initializing NVSTREAM writer, create store returned %p\n",
-                  Stream->nvs);
-
-    return (void *)Stream;
-}
-
-static void NvstreamDestroyWriter(CP_Services Svcs, DP_WS_Stream WS_Stream_v)
-{
-    Nvstream_WS_Stream WS_Stream = (Nvstream_WS_Stream)WS_Stream_v;
-    nvs_finalize_(WS_Stream->nvs);
-    for (int i = 0; i < WS_Stream->ReaderCount; i++)
-    {
-        if (WS_Stream->Readers[i])
-        {
-            free(WS_Stream->Readers[i]->WriterContactInfo->ContactString);
-            free(WS_Stream->Readers[i]->WriterContactInfo);
-            free(WS_Stream->Readers[i]->ReaderContactInfo->ContactString);
-            if (WS_Stream->Readers[i]->ReaderContactInfo->Conn)
-                CMConnection_close(
-                    WS_Stream->Readers[i]->ReaderContactInfo->Conn);
-            free(WS_Stream->Readers[i]->ReaderContactInfo);
-            free(WS_Stream->Readers[i]);
-        }
-    }
-    free(WS_Stream->Readers);
-    free(WS_Stream);
-}
-
-static DP_WSR_Stream NvstreamInitWriterPerReader(CP_Services Svcs,
-                                                 DP_WS_Stream WS_Stream_v,
-                                                 int readerCohortSize,
-                                                 CP_PeerCohort PeerCohort,
-                                                 void **providedReaderInfo_v,
-                                                 void **WriterContactInfoPtr)
-{
-    Nvstream_WS_Stream WS_Stream = (Nvstream_WS_Stream)WS_Stream_v;
-    Nvstream_WSR_Stream WSR_Stream = malloc(sizeof(*WSR_Stream));
-    NvstreamWriterContactInfo ContactInfo;
-    SMPI_Comm comm = Svcs->getMPIComm(WS_Stream->CP_Stream);
-    int Rank;
-    NvstreamReaderContactInfo *providedReaderInfo =
-        (NvstreamReaderContactInfo *)providedReaderInfo_v;
-
-    SMPI_Comm_rank(comm, &Rank);
-
-    WSR_Stream->WS_Stream = WS_Stream; /* pointer to writer struct */
-    WSR_Stream->PeerCohort = PeerCohort;
-
-    /*
-     * make a copy of writer contact information (original will not be
-     * preserved)
-     */
-    WSR_Stream->ReaderCohortSize = readerCohortSize;
-    WSR_Stream->ReaderContactInfo =
-        malloc(sizeof(struct _NvstreamReaderContactInfo) * readerCohortSize);
-    for (int i = 0; i < readerCohortSize; i++)
-    {
-        WSR_Stream->ReaderContactInfo[i].ContactString = NULL;
-        WSR_Stream->ReaderContactInfo[i].Conn = NULL;
-        WSR_Stream->ReaderContactInfo[i].RS_Stream =
-            providedReaderInfo[i]->RS_Stream;
-        Svcs->verbose(
-            WS_Stream->CP_Stream,
-            "Received contact info \"%s\", RD_Stream %p for Reader Rank %d\n",
-            WSR_Stream->ReaderContactInfo[i].ContactString,
-            WSR_Stream->ReaderContactInfo[i].RS_Stream, i);
-    }
-
-    /*
-     * add this writer-side reader-specific stream to the parent writer stream
-     * structure
-     */
-    WS_Stream->Readers = realloc(
-        WS_Stream->Readers, sizeof(*WSR_Stream) * (WS_Stream->ReaderCount + 1));
-    WS_Stream->Readers[WS_Stream->ReaderCount] = WSR_Stream;
-    WS_Stream->ReaderCount++;
-
-    ContactInfo = malloc(sizeof(struct _NvstreamWriterContactInfo));
-    memset(ContactInfo, 0, sizeof(struct _NvstreamWriterContactInfo));
-    ContactInfo->ContactString = strdup(nvs_get_store_name(WS_Stream->nvs));
-    ContactInfo->WS_Stream = WSR_Stream;
-    *WriterContactInfoPtr = ContactInfo;
-    WSR_Stream->WriterContactInfo = ContactInfo;
-
-    return WSR_Stream;
-}
-
-static void NvstreamDestroyWriterPerReader(CP_Services Svcs,
-                                           DP_WSR_Stream WSR_Stream_v)
-{
-    Nvstream_WSR_Stream WSR_Stream = (Nvstream_WSR_Stream)WSR_Stream_v;
-    free(WSR_Stream);
-}
-
-static void NvstreamProvideWriterDataToReader(CP_Services Svcs,
-                                              DP_RS_Stream RS_Stream_v,
-                                              int writerCohortSize,
-                                              CP_PeerCohort PeerCohort,
-                                              void **providedWriterInfo_v)
-{
-    Nvstream_RS_Stream RS_Stream = (Nvstream_RS_Stream)RS_Stream_v;
-    NvstreamWriterContactInfo *providedWriterInfo =
-        (NvstreamWriterContactInfo *)providedWriterInfo_v;
-
-    RS_Stream->PeerCohort = PeerCohort;
-    RS_Stream->WriterCohortSize = writerCohortSize;
-
-    /*
-     * make a copy of writer contact information (original will not be
-     * preserved)
-     */
-    RS_Stream->WriterContactInfo =
-        malloc(sizeof(struct _NvstreamWriterContactInfo) * writerCohortSize);
-    for (int i = 0; i < writerCohortSize; i++)
-    {
-        RS_Stream->WriterContactInfo[i].ContactString =
-            strdup(providedWriterInfo[i]->ContactString);
-        RS_Stream->WriterContactInfo[i].WS_Stream =
-            providedWriterInfo[i]->WS_Stream;
-        Svcs->verbose(
-            RS_Stream->CP_Stream,
-            "Received contact info \"%s\", WS_stream %p for WSR Rank %d\n",
-            RS_Stream->WriterContactInfo[i].ContactString,
-            RS_Stream->WriterContactInfo[i].WS_Stream, i);
-    }
-    RS_Stream->writer_nvs = malloc(sizeof(void *) * writerCohortSize);
-    for (int i = 0; i < writerCohortSize; i++)
-    {
-        RS_Stream->writer_nvs[i] =
-            nvs_open_store(RS_Stream->WriterContactInfo[i].ContactString);
-    }
-}
-
-static void AddRequestToList(CP_Services Svcs, Nvstream_RS_Stream Stream,
-                             NvstreamCompletionHandle Handle)
-{
-    Handle->Next = Stream->PendingReadRequests;
-    Stream->PendingReadRequests = Handle;
-}
-
-static void RemoveRequestFromList(CP_Services Svcs, Nvstream_RS_Stream Stream,
-                                  NvstreamCompletionHandle Handle)
-{
-    NvstreamCompletionHandle Tmp = Stream->PendingReadRequests;
-
-    if (Stream->PendingReadRequests == Handle)
-    {
-        Stream->PendingReadRequests = Handle->Next;
-        return;
-    }
-
-    while (Tmp != NULL && Tmp->Next != Handle)
-    {
-        Tmp = Tmp->Next;
-    }
-
-    if (Tmp == NULL)
-        return;
-
-    // Tmp->Next must be the handle to remove
-    Tmp->Next = Tmp->Next->Next;
-}
-
-static void FailRequestsToRank(CP_Services Svcs, CManager cm,
-                               Nvstream_RS_Stream Stream, int FailedRank)
-{
-    NvstreamCompletionHandle Tmp = Stream->PendingReadRequests;
-    Svcs->verbose(Stream->CP_Stream,
-                  "Fail pending requests to writer rank %d\n", FailedRank);
-    while (Tmp != NULL)
-    {
-        if (Tmp->Rank == FailedRank)
-        {
-            Tmp->Failed = 1;
-            Svcs->verbose(Tmp->CPStream,
-                          "Found a pending remote memory read "
-                          "to failed writer rank %d, marking as "
-                          "failed and signalling condition %d\n",
-                          Tmp->Rank, Tmp->CMcondition);
-            CMCondition_signal(cm, Tmp->CMcondition);
-            Svcs->verbose(Tmp->CPStream, "Did the signal of condition %d\n",
-                          Tmp->Rank, Tmp->CMcondition);
-        }
-        Tmp = Tmp->Next;
-    }
-    Svcs->verbose(Stream->CP_Stream,
-                  "Done Failing requests to writer rank %d\n", FailedRank);
-}
-
-typedef struct _NvstreamPerTimestepInfo
-{
-    char *CheckString;
-    int CheckInt;
-} * NvstreamPerTimestepInfo;
-
-static void *NvstreamReadRemoteMemory(CP_Services Svcs, DP_RS_Stream Stream_v,
-                                      int Rank, long Timestep, size_t Offset,
-                                      size_t Length, void *Buffer,
-                                      void *DP_TimestepInfo)
-{
-    Nvstream_RS_Stream Stream = (Nvstream_RS_Stream)
-        Stream_v; /* DP_RS_Stream is the return from InitReader */
-    CManager cm = Svcs->getCManager(Stream->CP_Stream);
-    NvstreamCompletionHandle ret =
-        malloc(sizeof(struct _NvstreamCompletionHandle));
-    NvstreamPerTimestepInfo TimestepInfo =
-        (NvstreamPerTimestepInfo)DP_TimestepInfo;
-    struct _NvstreamReadRequestMsg ReadRequestMsg;
-
-    static long LastRequestedTimestep = -1;
-
-    LastRequestedTimestep = Timestep;
-
-    ret->CPStream = Stream->CP_Stream;
-    ret->DPStream = Stream;
-    ret->Failed = 0;
-    ret->cm = cm;
-    ret->Buffer = Buffer;
-    ret->Rank = Rank;
-    ret->CMcondition = -1;
-
-    Svcs->verbose(Stream->CP_Stream,
-                  "Adios requesting to read remote memory for Timestep %d "
-                  "from Rank %d, WSR_Stream = %p, nvs = %p\n",
-                  Timestep, Rank, Stream->WriterContactInfo[Rank].WS_Stream,
-                  Stream->writer_nvs[Rank]);
-
-    char *StringName = malloc(20);
-    void *NVBlock;
-    char *BaseAddr;
-    sprintf(StringName, "Timestep_%ld", Timestep);
-    BaseAddr = nvs_get_with_malloc(Stream->writer_nvs[Rank], StringName, 1);
-
-    if (BaseAddr)
-    {
-        memcpy(Buffer, BaseAddr + Offset, Length);
-    }
-    else
-    {
-        fprintf(stderr, "remote memory read failed\n");
-    }
-    return ret;
-}
-
-static int NvstreamWaitForCompletion(CP_Services Svcs, void *Handle_v)
-{
-    NvstreamCompletionHandle Handle = (NvstreamCompletionHandle)Handle_v;
-    int Ret = 1;
-    Svcs->verbose(
-        Handle->CPStream,
-        "Waiting for completion of memory read to rank %d, condition %d\n",
-        Handle->Rank, Handle->CMcondition);
-    /*
-     * Wait for the CM condition to be signalled.  If it has been already,
-     * this returns immediately.  Copying the incoming data to the waiting
-     * buffer has been done by the reply handler.
-     */
-    if (Handle->CMcondition != -1)
-        CMCondition_wait(Handle->cm, Handle->CMcondition);
-    if (Handle->Failed)
-    {
-        Svcs->verbose(Handle->CPStream,
-                      "Remote memory read to rank %d with "
-                      "condition %d has FAILED because of "
-                      "writer failure\n",
-                      Handle->Rank, Handle->CMcondition);
-        Ret = 0;
-    }
-    else
-    {
-        Svcs->verbose(
-            Handle->CPStream,
-            "Remote memory read to rank %d with condition %d has completed\n",
-            Handle->Rank, Handle->CMcondition);
-    }
-    RemoveRequestFromList(Svcs, Handle->DPStream, Handle);
-    free(Handle);
-    return Ret;
-}
-
-static void NvstreamNotifyConnFailure(CP_Services Svcs, DP_RS_Stream Stream_v,
-                                      int FailedPeerRank)
-{
-    Nvstream_RS_Stream Stream = (Nvstream_RS_Stream)
-        Stream_v; /* DP_RS_Stream is the return from InitReader */
-    CManager cm = Svcs->getCManager(Stream->CP_Stream);
-    Svcs->verbose(Stream->CP_Stream,
-                  "received notification that writer peer "
-                  "%d has failed, failing any pending "
-                  "requests\n",
-                  FailedPeerRank);
-    FailRequestsToRank(Svcs, cm, Stream, FailedPeerRank);
-}
-
-static void NvstreamProvideTimestep(CP_Services Svcs, DP_WS_Stream Stream_v,
-                                    struct _SstData *Data,
-                                    struct _SstData *LocalMetadata,
-                                    long Timestep, void **TimestepInfoPtr)
-{
-    Nvstream_WS_Stream Stream = (Nvstream_WS_Stream)Stream_v;
-    TimestepList Entry = malloc(sizeof(struct _TimestepEntry));
-    struct _NvstreamPerTimestepInfo *Info = NULL;
-    //        malloc(sizeof(struct _NvstreamPerTimestepInfo));
-
-    //  This section exercised the CP's ability to distribute DP per timestep
-    //  info. Commenting out as needed for Nvstream for now
-    //    Info->CheckString = malloc(64);
-    //    sprintf(Info->CheckString, "Nvstream info for timestep %ld from rank
-    //    %d",
-    //            Timestep, Stream->Rank);
-    //    Info->CheckInt = Stream->Rank * 1000 + Timestep;
-    //    *TimestepInfoPtr = Info;
-    memset(Entry, 0, sizeof(*Entry));
-    Entry->DP_TimestepInfo = NULL;
-    Entry->Data = *Data;
-    Entry->Timestep = Timestep;
-
-    Entry->Next = Stream->Timesteps;
-    Stream->Timesteps = Entry;
-    *TimestepInfoPtr = NULL;
-
-    char *StringName = malloc(20);
-    void *NVBlock;
-    sprintf(StringName, "Timestep_%ld", Timestep);
-    NVBlock = nvs_alloc(Stream->nvs, &Data->DataSize, StringName);
-    memcpy(NVBlock, Data->block, Data->DataSize);
-    nvs_snapshot_(Stream->nvs, &Stream->Rank);
-    fprintf(stderr, "Did alloc, memcpy and snapshot with name %s\n",
-            StringName);
-}
-
-static void NvstreamReleaseTimestep(CP_Services Svcs, DP_WS_Stream Stream_v,
-                                    long Timestep)
-{
-    Nvstream_WS_Stream Stream = (Nvstream_WS_Stream)Stream_v;
-    TimestepList List = Stream->Timesteps;
-
-    Svcs->verbose(Stream->CP_Stream, "Releasing timestep %ld\n", Timestep);
-    if (Stream->Timesteps->Timestep == Timestep)
-    {
-        Stream->Timesteps = List->Next;
-        if (List->DP_TimestepInfo && List->DP_TimestepInfo->CheckString)
-            free(List->DP_TimestepInfo->CheckString);
-        if (List->DP_TimestepInfo)
-            free(List->DP_TimestepInfo);
-        free(List);
-    }
-    else
-    {
-        TimestepList last = List;
-        List = List->Next;
-        while (List != NULL)
-        {
-            if (List->Timestep == Timestep)
-            {
-                last->Next = List->Next;
-                if (List->DP_TimestepInfo && List->DP_TimestepInfo->CheckString)
-                    free(List->DP_TimestepInfo->CheckString);
-                if (List->DP_TimestepInfo)
-                    free(List->DP_TimestepInfo);
-                free(List);
-                return;
-            }
-            last = List;
-            List = List->Next;
-        }
-        /*
-         * Shouldn't ever get here because we should never release a
-         * timestep that we don't have.
-         */
-        fprintf(stderr, "Failed to release Timestep %ld, not found\n",
-                Timestep);
-        assert(0);
-    }
-}
-
-static FMField NvstreamReaderContactList[] = {
-    {"ContactString", "string", sizeof(char *),
-     FMOffset(NvstreamReaderContactInfo, ContactString)},
-    {"reader_ID", "integer", sizeof(void *),
-     FMOffset(NvstreamReaderContactInfo, RS_Stream)},
-    {NULL, NULL, 0, 0}};
-
-static FMStructDescRec NvstreamReaderContactStructs[] = {
-    {"NvstreamReaderContactInfo", NvstreamReaderContactList,
-     sizeof(struct _NvstreamReaderContactInfo), NULL},
-    {NULL, NULL, 0, NULL}};
-
-static FMField NvstreamWriterContactList[] = {
-    {"ContactString", "string", sizeof(char *),
-     FMOffset(NvstreamWriterContactInfo, ContactString)},
-    {"writer_ID", "integer", sizeof(void *),
-     FMOffset(NvstreamWriterContactInfo, WS_Stream)},
-    {NULL, NULL, 0, 0}};
-
-static FMStructDescRec NvstreamWriterContactStructs[] = {
-    {"NvstreamWriterContactInfo", NvstreamWriterContactList,
-     sizeof(struct _NvstreamWriterContactInfo), NULL},
-    {NULL, NULL, 0, NULL}};
-
-static FMField NvstreamTimestepInfoList[] = {
-    {"CheckString", "string", sizeof(char *),
-     FMOffset(NvstreamPerTimestepInfo, CheckString)},
-    {"CheckInt", "integer", sizeof(void *),
-     FMOffset(NvstreamPerTimestepInfo, CheckInt)},
-    {NULL, NULL, 0, 0}};
-
-static FMStructDescRec NvstreamTimestepInfoStructs[] = {
-    {"NvstreamTimestepInfo", NvstreamTimestepInfoList,
-     sizeof(struct _NvstreamPerTimestepInfo), NULL},
-    {NULL, NULL, 0, NULL}};
-
-static struct _CP_DP_Interface nvstreamDPInterface;
-
-static int NvstreamGetPriority(CP_Services Svcs, void *CP_Stream,
-                               struct _SstParams *Params)
-{
-    /* The nvstream DP priority 10 */
-    return 10;
-}
-
-extern CP_DP_Interface LoadNvstreamDP()
-{
-    memset(&nvstreamDPInterface, 0, sizeof(nvstreamDPInterface));
-    nvstreamDPInterface.ReaderContactFormats = NvstreamReaderContactStructs;
-    nvstreamDPInterface.WriterContactFormats = NvstreamWriterContactStructs;
-    nvstreamDPInterface.TimestepInfoFormats =
-        NULL; // NvstreamTimestepInfoStructs;
-    nvstreamDPInterface.initReader = NvstreamInitReader;
-    nvstreamDPInterface.initWriter = NvstreamInitWriter;
-    nvstreamDPInterface.initWriterPerReader = NvstreamInitWriterPerReader;
-    nvstreamDPInterface.provideWriterDataToReader =
-        NvstreamProvideWriterDataToReader;
-    nvstreamDPInterface.readRemoteMemory = NvstreamReadRemoteMemory;
-    nvstreamDPInterface.waitForCompletion = NvstreamWaitForCompletion;
-    nvstreamDPInterface.notifyConnFailure = NvstreamNotifyConnFailure;
-    nvstreamDPInterface.provideTimestep = NvstreamProvideTimestep;
-    nvstreamDPInterface.releaseTimestep = NvstreamReleaseTimestep;
-    nvstreamDPInterface.readerRegisterTimestep = NULL;
-    nvstreamDPInterface.readerReleaseTimestep = NULL;
-    nvstreamDPInterface.readPatternLocked = NULL;
-    nvstreamDPInterface.destroyReader = NvstreamDestroyReader;
-    nvstreamDPInterface.destroyWriter = NvstreamDestroyWriter;
-    nvstreamDPInterface.destroyWriterPerReader = NvstreamDestroyWriterPerReader;
-    nvstreamDPInterface.getPriority = NvstreamGetPriority;
-    nvstreamDPInterface.unGetPriority = NULL;
-    return &nvstreamDPInterface;
-}
diff --git a/source/adios2/toolkit/sst/dp/nvswrapper.cpp b/source/adios2/toolkit/sst/dp/nvswrapper.cpp
deleted file mode 100644
index 052e3ac439..0000000000
--- a/source/adios2/toolkit/sst/dp/nvswrapper.cpp
+++ /dev/null
@@ -1,64 +0,0 @@
-#include "nvs/store.h"
-#include "nvs/store_manager.h"
-#include 
-#include 
-#include 
-
-#include "nvswrapper.h"
-
-extern "C" {
-void *nvs_create_store()
-{
-    nvs::Store *st = nvs::StoreManager::GetInstance("");
-    std::string store_name = st->get_store_id();
-    return (void *)st;
-}
-
-void *nvs_open_store(char *STORE_ID)
-{
-    std::string store_name = STORE_ID;
-    nvs::Store *st = nvs::StoreManager::GetInstance(store_name);
-    return (void *)st;
-}
-
-char *nvs_get_store_name(void *vst)
-{
-    nvs::Store *st = (nvs::Store *)vst;
-    const char *tmp = st->get_store_id().c_str();
-    return strdup(tmp);
-}
-
-void *nvs_alloc(void *vst, unsigned long *size, char *s)
-{
-    void *ptr;
-    nvs::Store *st = (nvs::Store *)vst;
-    st->create_obj(std::string(s), *size, &ptr);
-    return ptr;
-}
-
-void *nvs_get_with_malloc(void *vst, char *key, long version)
-{
-    void *ptr;
-    nvs::Store *st = (nvs::Store *)vst;
-    nvs::ErrorCode tmp = st->get_with_malloc(std::string(key), version, &ptr);
-    if (tmp != nvs::ErrorCode::NO_ERROR)
-    {
-        std::cout << "get with malloc error" << std::endl;
-    }
-    return ptr;
-}
-
-void nvs_free_(void *vst, void *ptr) {}
-
-void nvs_snapshot_(void *vst, int *proc_id)
-{
-    nvs::Store *st = (nvs::Store *)vst;
-    st->put_all();
-}
-
-int nvs_finalize_(void *vst)
-{
-    nvs::Store *st = (nvs::Store *)vst;
-    delete st;
-}
-}
diff --git a/source/adios2/toolkit/sst/dp/nvswrapper.h b/source/adios2/toolkit/sst/dp/nvswrapper.h
deleted file mode 100644
index 8b9a0091e8..0000000000
--- a/source/adios2/toolkit/sst/dp/nvswrapper.h
+++ /dev/null
@@ -1,34 +0,0 @@
-//
-// Created by pradeep on 1/11/18.
-// Adapted by eisen July 2019
-//
-
-#ifndef NVSWRAPPER_H
-#define NVSWRAPPER_H
-
-/* C interface for fortran apps */
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-void *nvs_open_store(char *store_name);
-
-void *nvs_create_store();
-
-char *nvs_get_store_name(void *vst);
-
-void *nvs_alloc(void *vst, unsigned long *n, char *s);
-
-void *nvs_get_with_malloc(void *vst, char *key, long version);
-
-void nvs_free_(void *vst, void *ptr);
-
-void nvs_snapshot_(void *vst, int *proc_id);
-
-int nvs_finalize_(void *vst);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif // NVSWRAPPER_H
diff --git a/source/adios2/toolkit/sst/dp/rdma_dp.c b/source/adios2/toolkit/sst/dp/rdma_dp.c
index 2776daa8a3..149b24539e 100644
--- a/source/adios2/toolkit/sst/dp/rdma_dp.c
+++ b/source/adios2/toolkit/sst/dp/rdma_dp.c
@@ -1,4 +1,6 @@
 #include 
+#include 
+#include 
 #include 
 #include 
 #include 
@@ -17,6 +19,16 @@
 #include 
 #include 
 
+#if defined(__has_feature)
+#if __has_feature(thread_sanitizer)
+#define NO_SANITIZE_THREAD __attribute__((no_sanitize("thread")))
+#endif
+#endif
+
+#ifndef NO_SANITIZE_THREAD
+#define NO_SANITIZE_THREAD
+#endif
+
 #ifdef SST_HAVE_FI_GNI
 #include 
 #ifdef SST_HAVE_CRAY_DRC
@@ -34,8 +46,10 @@
 #include "dp_interface.h"
 
 #define DP_AV_DEF_SIZE 512
+#define REQ_LIST_GRAN 8
+#define DP_DATA_RECV_SIZE 64
 
-pthread_mutex_t fabric_mutex = PTHREAD_MUTEX_INITIALIZER;
+static pthread_mutex_t fabric_mutex = PTHREAD_MUTEX_INITIALIZER;
 pthread_mutex_t wsr_mutex = PTHREAD_MUTEX_INITIALIZER;
 pthread_mutex_t ts_mutex = PTHREAD_MUTEX_INITIALIZER;
 
@@ -46,9 +60,9 @@ struct fabric_state
     struct fi_info *info;
     // struct fi_info *linfo;
     int local_mr_req;
+    int rx_cq_data;
     size_t addr_len;
     size_t msg_prefix_size;
-    // size_t lmsg_prefix_size;
     struct fid_fabric *fabric;
     struct fid_domain *domain;
     struct fid_ep *signal;
@@ -98,12 +112,14 @@ struct fabric_state
  *   plane would replace one or both of these with RDMA functionality.
  */
 
-static void init_fabric(struct fabric_state *fabric, struct _SstParams *Params)
+static void init_fabric(struct fabric_state *fabric, struct _SstParams *Params,
+                        CP_Services Svcs, void *CP_Stream)
 {
     struct fi_info *hints, *info, *originfo, *useinfo;
     struct fi_av_attr av_attr = {0};
     struct fi_cq_attr cq_attr = {0};
     char *ifname;
+    int result;
 
     hints = fi_allocinfo();
     hints->caps = FI_MSG | FI_SEND | FI_RECV | FI_REMOTE_READ |
@@ -126,9 +142,13 @@ static void init_fabric(struct fabric_state *fabric, struct _SstParams *Params)
 
     fabric->info = NULL;
 
+    pthread_mutex_lock(&fabric_mutex);
     fi_getinfo(FI_VERSION(1, 5), NULL, NULL, 0, hints, &info);
+    pthread_mutex_unlock(&fabric_mutex);
     if (!info)
     {
+        Svcs->verbose(CP_Stream, DPCriticalVerbose, "no fabrics detected.\n");
+        fabric->info = NULL;
         return;
     }
     fi_freeinfo(hints);
@@ -142,6 +162,8 @@ static void init_fabric(struct fabric_state *fabric, struct _SstParams *Params)
 
         if (ifname && strcmp(ifname, domain_name) == 0)
         {
+            Svcs->verbose(CP_Stream, DPTraceVerbose,
+                          "using interface set by FABRIC_IFACE.\n");
             useinfo = info;
             break;
         }
@@ -151,14 +173,31 @@ static void init_fabric(struct fabric_state *fabric, struct _SstParams *Params)
             (!useinfo || !ifname ||
              (strcmp(useinfo->domain_attr->name, ifname) != 0)))
         {
+            Svcs->verbose(CP_Stream, DPTraceVerbose,
+                          "seeing candidate fabric %s, will use this unless we "
+                          "see something better.\n",
+                          prov_name);
             useinfo = info;
         }
         else if (((strstr(prov_name, "verbs") && info->src_addr) ||
                   strstr(prov_name, "gni") || strstr(prov_name, "psm2")) &&
                  !useinfo)
         {
+            Svcs->verbose(CP_Stream, DPTraceVerbose,
+                          "seeing candidate fabric %s, will use this unless we "
+                          "see something better.\n",
+                          prov_name);
             useinfo = info;
         }
+        else
+        {
+            Svcs->verbose(
+                CP_Stream, DPTraceVerbose,
+                "ignoring fabric %s because it's not of a supported type. It "
+                "may work to force this fabric to be used by setting "
+                "FABRIC_IFACE to %s, but it may not be stable or performant.\n",
+                prov_name, domain_name);
+        }
         info = info->next;
     }
 
@@ -166,6 +205,14 @@ static void init_fabric(struct fabric_state *fabric, struct _SstParams *Params)
 
     if (!info)
     {
+        Svcs->verbose(
+            CP_Stream, DPCriticalVerbose,
+            "none of the usable system fabrics are supported high speed "
+            "interfaces (verbs, gni, psm2.) To use a compatible fabric that is "
+            "being ignored (probably sockets), set the environment variable "
+            "FABRIC_IFACE to the interface name. Check the output of fi_info "
+            "to troubleshoot this message.\n");
+        fabric->info = NULL;
         return;
     }
 
@@ -200,6 +247,15 @@ static void init_fabric(struct fabric_state *fabric, struct _SstParams *Params)
         fabric->msg_prefix_size = 0;
     }
 
+    if (info->mode & FI_RX_CQ_DATA)
+    {
+        fabric->rx_cq_data = 1;
+    }
+    else
+    {
+        fabric->rx_cq_data = 0;
+    }
+
     fabric->addr_len = info->src_addrlen;
 
     info->domain_attr->mr_mode = FI_MR_BASIC;
@@ -211,46 +267,154 @@ static void init_fabric(struct fabric_state *fabric, struct _SstParams *Params)
     }
 #endif /* SST_HAVE_CRAY_DRC */
     fabric->info = fi_dupinfo(info);
+    if (!fabric->info)
+    {
+        Svcs->verbose(CP_Stream, DPCriticalVerbose,
+                      "copying the fabric info failed.\n");
+        return;
+    }
 
-    fi_fabric(info->fabric_attr, &fabric->fabric, fabric->ctx);
-    fi_domain(fabric->fabric, info, &fabric->domain, fabric->ctx);
+    Svcs->verbose(CP_Stream, DPTraceVerbose,
+                  "Fabric parameters to use at fabric initialization: %s\n",
+                  fi_tostr(fabric->info, FI_TYPE_INFO));
+
+    result = fi_fabric(info->fabric_attr, &fabric->fabric, fabric->ctx);
+    if (result != FI_SUCCESS)
+    {
+        Svcs->verbose(
+            CP_Stream, DPCriticalVerbose,
+            "opening fabric access failed with %d (%s). This is fatal.\n",
+            result, fi_strerror(result));
+        return;
+    }
+    result = fi_domain(fabric->fabric, info, &fabric->domain, fabric->ctx);
+    if (result != FI_SUCCESS)
+    {
+        Svcs->verbose(CP_Stream, DPCriticalVerbose,
+                      "accessing domain failed with %d (%s). This is fatal.\n",
+                      result, fi_strerror(result));
+        return;
+    }
     info->ep_attr->type = FI_EP_RDM;
-    fi_endpoint(fabric->domain, info, &fabric->signal, fabric->ctx);
+    result = fi_endpoint(fabric->domain, info, &fabric->signal, fabric->ctx);
+    if (result != FI_SUCCESS || !fabric->signal)
+    {
+        Svcs->verbose(CP_Stream, DPCriticalVerbose,
+                      "opening endpoint failed with %d (%s). This is fatal.\n",
+                      result, fi_strerror(result));
+        return;
+    }
 
     av_attr.type = FI_AV_MAP;
     av_attr.count = DP_AV_DEF_SIZE;
     av_attr.ep_per_node = 0;
-    fi_av_open(fabric->domain, &av_attr, &fabric->av, fabric->ctx);
-    fi_ep_bind(fabric->signal, &fabric->av->fid, 0);
+    result = fi_av_open(fabric->domain, &av_attr, &fabric->av, fabric->ctx);
+    if (result != FI_SUCCESS)
+    {
+        Svcs->verbose(CP_Stream, DPCriticalVerbose,
+                      "could not initialize address vector, failed with %d "
+                      "(%s). This is fatal.\n",
+                      result, fi_strerror(result));
+        return;
+    }
+    result = fi_ep_bind(fabric->signal, &fabric->av->fid, 0);
+    if (result != FI_SUCCESS)
+    {
+        Svcs->verbose(CP_Stream, DPCriticalVerbose,
+                      "could not bind endpoint to address vector, failed with "
+                      "%d (%s). This is fatal.\n",
+                      result, fi_strerror(result));
+        return;
+    }
 
     cq_attr.size = 0;
     cq_attr.format = FI_CQ_FORMAT_DATA;
     cq_attr.wait_obj = FI_WAIT_UNSPEC;
     cq_attr.wait_cond = FI_CQ_COND_NONE;
-    fi_cq_open(fabric->domain, &cq_attr, &fabric->cq_signal, fabric->ctx);
-    fi_ep_bind(fabric->signal, &fabric->cq_signal->fid, FI_TRANSMIT | FI_RECV);
+    result =
+        fi_cq_open(fabric->domain, &cq_attr, &fabric->cq_signal, fabric->ctx);
+    if (result != FI_SUCCESS)
+    {
+        Svcs->verbose(
+            CP_Stream, DPCriticalVerbose,
+            "opening completion queue failed with %d (%s). This is fatal.\n",
+            result, fi_strerror(result));
+        return;
+    }
+
+    result = fi_ep_bind(fabric->signal, &fabric->cq_signal->fid,
+                        FI_TRANSMIT | FI_RECV);
+    if (result != FI_SUCCESS)
+    {
+        Svcs->verbose(CP_Stream, DPCriticalVerbose,
+                      "could not bind endpoint to completion queue, failed "
+                      "with %d (%s). This is fatal.\n",
+                      result, fi_strerror(result));
+        return;
+    }
 
-    fi_enable(fabric->signal);
+    result = fi_enable(fabric->signal);
+    if (result != FI_SUCCESS)
+    {
+        Svcs->verbose(CP_Stream, DPCriticalVerbose,
+                      "enable endpoint, failed with %d (%s). This is fatal.\n",
+                      result, fi_strerror(result));
+        return;
+    }
 
     fi_freeinfo(originfo);
 }
 
-static void fini_fabric(struct fabric_state *fabric)
+static void fini_fabric(struct fabric_state *fabric, CP_Services Svcs,
+                        void *CP_Stream)
 {
 
-    int status;
+    int res;
 
     do
     {
-        status = fi_close((struct fid *)fabric->cq_signal);
-    } while (status == FI_EBUSY);
+        res = fi_close((struct fid *)fabric->signal);
+    } while (res == -FI_EBUSY);
+
+    if (res != FI_SUCCESS)
+    {
+        Svcs->verbose(CP_Stream, DPCriticalVerbose,
+                      "could not close ep, failed with %d (%s).\n", res,
+                      fi_strerror(res));
+        return;
+    }
+
+    res = fi_close((struct fid *)fabric->cq_signal);
+    if (res != FI_SUCCESS)
+    {
+        Svcs->verbose(CP_Stream, DPCriticalVerbose,
+                      "could not close cq, failed with %d (%s).\n", res,
+                      fi_strerror(res));
+    }
 
-    fi_close((struct fid *)fabric->domain);
-    fi_close((struct fid *)fabric->fabric);
+    res = fi_close((struct fid *)fabric->av);
+    if (res != FI_SUCCESS)
+    {
+        Svcs->verbose(CP_Stream, DPCriticalVerbose,
+                      "could not close av, failed with %d (%s).\n", res,
+                      fi_strerror(res));
+    }
+    res = fi_close((struct fid *)fabric->domain);
+    if (res != FI_SUCCESS)
+    {
+        Svcs->verbose(CP_Stream, DPCriticalVerbose,
+                      "could not close domain, failed with %d (%s).\n", res,
+                      fi_strerror(res));
+        return;
+    }
 
-    if (status)
+    res = fi_close((struct fid *)fabric->fabric);
+    if (res != FI_SUCCESS)
     {
-        // TODO: error handling
+        Svcs->verbose(CP_Stream, DPCriticalVerbose,
+                      "could not close fabric, failed with %d (%s).\n", res,
+                      fi_strerror(res));
+        return;
     }
 
     fi_freeinfo(fabric->info);
@@ -259,32 +423,115 @@ static void fini_fabric(struct fabric_state *fabric)
     {
         free(fabric->ctx);
     }
+
+#ifdef SST_HAVE_CRAY_DRC
+    if (Fabric->auth_key)
+    {
+        free(Fabric->auth_key);
+    }
+#endif /* SST_HAVE_CRAY_DRC */
 }
 
 typedef struct fabric_state *FabricState;
 
+typedef struct _RdmaCompletionHandle
+{
+    struct fid_mr *LocalMR;
+    void *CPStream;
+    void *Buffer;
+    size_t Length;
+    int Rank;
+    int Pending;
+    void *PreloadBuffer;
+} * RdmaCompletionHandle;
+
+typedef struct _RdmaBufferHandle
+{
+    uint8_t *Block;
+    uint64_t Key;
+} * RdmaBufferHandle;
+
+typedef struct _RdmaBuffer
+{
+    struct _RdmaBufferHandle Handle;
+    uint64_t BufferLen;
+    uint64_t Offset;
+} * RdmaBuffer;
+
+typedef struct _RdmaReqLogEntry
+{
+    size_t Offset;
+    size_t Length;
+} * RdmaReqLogEntry;
+
+typedef struct _RdmaRankReqLog
+{
+    RdmaBuffer ReqLog;
+    int Entries;
+    union
+    {
+        int MaxEntries; // Reader Side
+        int Rank;       // Writer Side
+    };
+    union
+    {
+        void *Buffer;               // Reader side
+        uint64_t PreloadBufferSize; // Writer side
+    };
+    long BufferSize;
+    union
+    {
+        struct fid_mr *preqbmr;       // Reader side
+        struct _RdmaRankReqLog *next; // Writer side
+    };
+    RdmaCompletionHandle *PreloadHandles;
+} * RdmaRankReqLog;
+
+typedef struct _RdmaStepLogEntry
+{
+    long Timestep;
+    RdmaRankReqLog RankLog;
+    struct _RdmaStepLogEntry *Next;
+    int Entries;
+    long BufferSize;
+    int WRanks;
+} * RdmaStepLogEntry;
+
 typedef struct _Rdma_RS_Stream
 {
     CManager cm;
     void *CP_Stream;
     int Rank;
     FabricState Fabric;
+
+    long PreloadStep;
+    int PreloadPosted;
+    RdmaStepLogEntry StepLog;
+    RdmaStepLogEntry PreloadStepLog;
+    struct _RdmaBuffer PreloadBuffer;
+    struct fid_mr *pbmr;
+    uint64_t *RecvCounter;
+    int PreloadAvail;
+
     struct _SstParams *Params;
+    struct _RdmaReaderContactInfo *ContactInfo;
 
     /* writer info */
     int WriterCohortSize;
     CP_PeerCohort PeerCohort;
     struct _RdmaWriterContactInfo *WriterContactInfo;
+
     fi_addr_t *WriterAddr;
-} * Rdma_RS_Stream;
+    struct _RdmaBufferHandle *WriterRoll;
 
-typedef struct _Rdma_WSR_Stream
-{
-    struct _Rdma_WS_Stream *WS_Stream;
-    CP_PeerCohort PeerCohort;
-    int ReaderCohortSize;
-    struct _RdmaWriterContactInfo *WriterContactInfo;
-} * Rdma_WSR_Stream;
+    int PendingReads;
+    long EarlyReads;
+    long TotalReads;
+
+    void *RecvDataBuffer;
+    struct fid_mr *rbmr;
+    void *rbdesc;
+} * Rdma_RS_Stream;
 
 typedef struct _RdmaPerTimestepInfo
 {
@@ -296,21 +543,41 @@ typedef struct _TimestepEntry
 {
     long Timestep;
     struct _SstData *Data;
-    struct _RdmaPerTimestepInfo *DP_TimestepInfo;
-    struct _TimestepEntry *Next;
+    struct _RdmaBufferHandle *DP_TimestepInfo;
+    struct _TimestepEntry *Prev, *Next;
     struct fid_mr *mr;
+    void *Desc;
     uint64_t Key;
+    uint64_t OutstandingWrites;
+    int BufferSlot;
 } * TimestepList;
 
+typedef struct _Rdma_WSR_Stream
+{
+    struct _Rdma_WS_Stream *WS_Stream;
+    CP_PeerCohort PeerCohort;
+    int ReaderCohortSize;
+    struct _RdmaWriterContactInfo *WriterContactInfo;
+    struct _RdmaBuffer *ReaderRoll;
+    struct fid_mr *rrmr;
+    fi_addr_t *ReaderAddr;
+    int SelectLocked;
+    int Preload;
+    int SelectionPulled;
+    RdmaRankReqLog PreloadReq;
+    TimestepList LastReleased;
+    int PreloadUsed[2];
+} * Rdma_WSR_Stream;
+
 typedef struct _Rdma_WS_Stream
 {
     CManager cm;
     void *CP_Stream;
     int Rank;
     FabricState Fabric;
-
+    int DefLocked;
+    int PreloadAvail;
     TimestepList Timesteps;
-
     int ReaderCount;
     Rdma_WSR_Stream *Readers;
 } * Rdma_WS_Stream;
@@ -318,6 +585,8 @@ typedef struct _Rdma_WS_Stream
 typedef struct _RdmaReaderContactInfo
 {
     void *RS_Stream;
+    size_t Length;
+    void *Address;
 } * RdmaReaderContactInfo;
 
 typedef struct _RdmaWriterContactInfo
@@ -325,22 +594,42 @@ typedef struct _RdmaWriterContactInfo
     void *WS_Stream;
     size_t Length;
     void *Address;
-#ifdef SST_HAVE_CRAY_DRC
-    int Credential;
-#endif /* SST_HAVE_CRAY_DRC */
+    struct _RdmaBufferHandle ReaderRollHandle;
 } * RdmaWriterContactInfo;
 
+static TimestepList GetStep(Rdma_WS_Stream Stream, long Timestep)
+{
+    TimestepList Step;
+
+    pthread_mutex_lock(&ts_mutex);
+    Step = Stream->Timesteps;
+    while (Step && Step->Timestep != Timestep)
+    {
+        Step = Step->Prev;
+    }
+    pthread_mutex_unlock(&ts_mutex);
+
+    return (Step);
+}
+
 static DP_RS_Stream RdmaInitReader(CP_Services Svcs, void *CP_Stream,
                                    void **ReaderContactInfoPtr,
                                    struct _SstParams *Params,
-                                   attr_list WriterContact)
+                                   attr_list WriterContact, SstStats Stats)
 {
     Rdma_RS_Stream Stream = malloc(sizeof(struct _Rdma_RS_Stream));
     CManager cm = Svcs->getCManager(CP_Stream);
     SMPI_Comm comm = Svcs->getMPIComm(CP_Stream);
+    RdmaReaderContactInfo ContactInfo =
+        malloc(sizeof(struct _RdmaReaderContactInfo));
     FabricState Fabric;
+    char *PreloadEnv = NULL;
+    int rc;
 
     memset(Stream, 0, sizeof(*Stream));
+    Stream->Fabric = calloc(1, sizeof(*Fabric));
+
+    Fabric = Stream->Fabric;
 
     /*
      * save the CP_stream value of later use
@@ -351,41 +640,147 @@ static DP_RS_Stream RdmaInitReader(CP_Services Svcs, void *CP_Stream,
 
     *ReaderContactInfoPtr = NULL;
 
+    ContactInfo->RS_Stream = Stream;
+
     if (Params)
     {
         Stream->Params = malloc(sizeof(*Stream->Params));
         memcpy(Stream->Params, Params, sizeof(*Params));
     }
 
+    PreloadEnv = getenv("SST_DP_PRELOAD");
+    if (PreloadEnv &&
+        (strcmp(PreloadEnv, "1") == 0 || strcmp(PreloadEnv, "yes") == 0 ||
+         strcmp(PreloadEnv, "Yes") == 0 || strcmp(PreloadEnv, "YES") == 0))
+    {
+        Svcs->verbose(CP_Stream, DPTraceVerbose,
+                      "making preload available in RDMA DP based on "
+                      "environment variable value.\n");
+        Stream->PreloadAvail = 1;
+    }
+    else
+    {
+        Stream->PreloadAvail = 0;
+    }
+
 #ifdef SST_HAVE_CRAY_DRC
-    int protection_key;
+    int attr_cred, try_left;
     if (!get_int_attr(WriterContact, attr_atom_from_string("RDMA_DRC_KEY"),
-                      &protection_key))
+                      &attr_cred))
     {
-        Svcs->verbose(CP_Stream, "Didn't find DRC credential\n");
+        Svcs->verbose(CP_Stream, DPCriticalVerbose,
+                      "Didn't find DRC credential for Cray RDMA\n");
         return NULL;
     }
-    // do something with protection key?
-#endif
+    Fabric->credential = attr_cred;
+
+    try_left = DP_DRC_MAX_TRY;
+    rc = drc_access(Fabric->credential, 0, &Fabric->drc_info);
+    while (rc != DRC_SUCCESS && try_left--)
+    {
+        usleep(DP_DRC_WAIT_USEC);
+        rc = drc_access(Fabric->credential, 0, &Fabric->drc_info);
+    }
+    if (rc != DRC_SUCCESS)
+    {
+        Svcs->verbose(CP_Stream, DPCriticalVerbose,
+                      "Could not access DRC credential. Last failed with %d.\n",
+                      rc);
+    }
+
+    Fabric->auth_key = malloc(sizeof(*Fabric->auth_key));
+    Fabric->auth_key->type = GNIX_AKT_RAW;
+    Fabric->auth_key->raw.protection_key =
+        drc_get_first_cookie(Fabric->drc_info);
+    Svcs->verbose(CP_Stream, "Using protection key %08x.\n", DPSummaryVerbose,
+                  Fabric->auth_key->raw.protection_key);
+
+#endif /* SST_HAVE_CRAY_DRC */
+
+    init_fabric(Stream->Fabric, Stream->Params, Svcs, CP_Stream);
+    if (!Fabric->info)
+    {
+        Svcs->verbose(CP_Stream, DPCriticalVerbose,
+                      "Could not find a valid transport fabric.\n");
+        return NULL;
+    }
+
+    ContactInfo->Length = Fabric->info->src_addrlen;
+    ContactInfo->Address = malloc(ContactInfo->Length);
+    fi_getname((fid_t)Fabric->signal, ContactInfo->Address,
+               &ContactInfo->Length);
+
+    Stream->PreloadStep = -1;
+    Stream->ContactInfo = ContactInfo;
+    Stream->PreloadPosted = 0;
+    Stream->PendingReads = 0;
+    Stream->RecvCounter = NULL;
+    Stream->EarlyReads = 0;
+    Stream->TotalReads = 0;
+
+    *ReaderContactInfoPtr = ContactInfo;
+
     return Stream;
 }
 
-typedef struct _RdmaCompletionHandle
+static void RdmaReadPatternLocked(CP_Services Svcs, DP_WSR_Stream WSRStream_v,
+                                  long EffectiveTimestep)
 {
-    struct fid_mr *LocalMR;
-    void *CPStream;
-    void *Buffer;
-    size_t Length;
-    int Rank;
-    int Pending;
-} * RdmaCompletionHandle;
+    Rdma_WSR_Stream WSR_Stream = (Rdma_WSR_Stream)WSRStream_v;
+    Rdma_WS_Stream WS_Stream = WSR_Stream->WS_Stream;
+
+    if (WS_Stream->PreloadAvail)
+    {
+        if (WS_Stream->Rank == 0)
+        {
+            Svcs->verbose(WS_Stream->CP_Stream, DPTraceVerbose,
+                          "read pattern is locked\n");
+        }
+        WSR_Stream->SelectLocked = EffectiveTimestep;
+        WSR_Stream->Preload = 1;
+    }
+    else if (WS_Stream->Rank == 0)
+    {
+        Svcs->verbose(
+            WS_Stream->CP_Stream, DPSummaryVerbose,
+            "RDMA dataplane is ignoring a read pattern lock notification "
+            "because preloading is disabled. Enable by setting the environment "
+            "variable SST_DP_PRELOAD to 'yes'\n");
+    }
+}
+
+static void RdmaWritePatternLocked(CP_Services Svcs, DP_RS_Stream Stream_v,
+                                   long EffectiveTimestep)
+{
+    Rdma_RS_Stream Stream = (Rdma_RS_Stream)Stream_v;
+
+    if (Stream->PreloadAvail)
+    {
+        Stream->PreloadStep = EffectiveTimestep;
+        if (Stream->Rank == 0)
+        {
+            Svcs->verbose(Stream->CP_Stream, DPSummaryVerbose,
+                          "write pattern is locked.\n");
+        }
+    }
+    else if (Stream->Rank == 0)
+    {
+        Svcs->verbose(
+            Stream->CP_Stream, DPSummaryVerbose,
+            "RDMA dataplane is ignoring a write pattern lock notification "
+            "because preloading is disabled. Enable by setting the environment "
+            "variable SST_DP_PRELOAD to 'yes'\n");
+    }
+}
 
 static DP_WS_Stream RdmaInitWriter(CP_Services Svcs, void *CP_Stream,
-                                   struct _SstParams *Params, attr_list DPAttrs)
+                                   struct _SstParams *Params, attr_list DPAttrs,
+                                   SstStats Stats)
 {
     Rdma_WS_Stream Stream = malloc(sizeof(struct _Rdma_WS_Stream));
     CManager cm = Svcs->getCManager(CP_Stream);
     SMPI_Comm comm = Svcs->getMPIComm(CP_Stream);
+    char *PreloadEnv;
     FabricState Fabric;
     int rc;
     int try_left;
@@ -394,6 +789,24 @@ static DP_WS_Stream RdmaInitWriter(CP_Services Svcs, void *CP_Stream,
 
     SMPI_Comm_rank(comm, &Stream->Rank);
 
+    PreloadEnv = getenv("SST_DP_PRELOAD");
+    if (PreloadEnv &&
+        (strcmp(PreloadEnv, "1") == 0 || strcmp(PreloadEnv, "yes") == 0 ||
+         strcmp(PreloadEnv, "Yes") == 0 || strcmp(PreloadEnv, "YES") == 0))
+    {
+        if (Stream->Rank == 0)
+        {
+            Svcs->verbose(CP_Stream, DPSummaryVerbose,
+                          "making preload available in RDMA DP based on "
+                          "environment variable value.\n");
+        }
+        Stream->PreloadAvail = 1;
+    }
+    else
+    {
+        Stream->PreloadAvail = 0;
+    }
+
     Stream->Fabric = calloc(1, sizeof(struct fabric_state));
     Fabric = Stream->Fabric;
 #ifdef SST_HAVE_CRAY_DRC
@@ -402,14 +815,15 @@ static DP_WS_Stream RdmaInitWriter(CP_Services Svcs, void *CP_Stream,
         rc = drc_acquire(&Fabric->credential, DRC_FLAGS_FLEX_CREDENTIAL);
         if (rc != DRC_SUCCESS)
         {
-            Svcs->verbose(CP_Stream,
+            Svcs->verbose(CP_Stream, DPCriticalVerbose,
                           "Could not acquire DRC credential. Failed with %d.\n",
                           rc);
             goto err_out;
         }
         else
         {
-            Svcs->verbose(CP_Stream, "DRC acquired credential id %d.\n",
+            Svcs->verbose(CP_Stream, DPTraceVerbose,
+                          "DRC acquired credential id %d.\n",
                           Fabric->credential);
         }
     }
@@ -426,7 +840,7 @@ static DP_WS_Stream RdmaInitWriter(CP_Services Svcs, void *CP_Stream,
     }
     if (rc != DRC_SUCCESS)
     {
-        Svcs->verbose(CP_Stream,
+        Svcs->verbose(CP_Stream, DPCriticalVerbose,
                       "Could not access DRC credential. Last failed with %d.\n",
                       rc);
         goto err_out;
@@ -436,23 +850,22 @@ static DP_WS_Stream RdmaInitWriter(CP_Services Svcs, void *CP_Stream,
     Fabric->auth_key->type = GNIX_AKT_RAW;
     Fabric->auth_key->raw.protection_key =
         drc_get_first_cookie(Fabric->drc_info);
-    Svcs->verbose(CP_Stream, "Using protection key %08x.\n",
+    Svcs->verbose(CP_Stream, DPTraceVerbose, "Using protection key %08x.\n",
                   Fabric->auth_key->raw.protection_key);
-
-    set_int_attr(DPAttrs, attr_atom_from_string("RDMA_DRC_KEY"),
-                 Fabric->auth_key->raw.protection_key);
-
+    long attr_cred = Fabric->credential;
+    set_long_attr(DPAttrs, attr_atom_from_string("RDMA_DRC_CRED"), attr_cred);
 #endif /* SST_HAVE_CRAY_DRC */
 
-    init_fabric(Stream->Fabric, Params);
+    init_fabric(Stream->Fabric, Params, Svcs, CP_Stream);
     Fabric = Stream->Fabric;
     if (!Fabric->info)
     {
-        Svcs->verbose(CP_Stream, "Could not find a valid transport fabric.\n");
+        Svcs->verbose(CP_Stream, DPTraceVerbose,
+                      "Could not find a valid transport fabric.\n");
         goto err_out;
     }
 
-    Svcs->verbose(CP_Stream, "Fabric Parameters:\n%s\n",
+    Svcs->verbose(CP_Stream, DPTraceVerbose, "Fabric Parameters:\n%s\n",
                   fi_tostr(Fabric->info, FI_TYPE_INFO));
 
     /*
@@ -460,6 +873,8 @@ static DP_WS_Stream RdmaInitWriter(CP_Services Svcs, void *CP_Stream,
      */
     Stream->CP_Stream = CP_Stream;
 
+    Stream->DefLocked = -1;
+
     return (void *)Stream;
 
 err_out:
@@ -485,16 +900,29 @@ static DP_WSR_Stream RdmaInitWriterPerReader(CP_Services Svcs,
     Rdma_WSR_Stream WSR_Stream = malloc(sizeof(*WSR_Stream));
     FabricState Fabric = WS_Stream->Fabric;
     RdmaWriterContactInfo ContactInfo;
+    RdmaReaderContactInfo *providedReaderInfo =
+        (RdmaReaderContactInfo *)providedReaderInfo_v;
     SMPI_Comm comm = Svcs->getMPIComm(WS_Stream->CP_Stream);
-    int Rank;
-
-    SMPI_Comm_rank(comm, &Rank);
+    RdmaBufferHandle ReaderRollHandle;
+    int i;
 
     WSR_Stream->WS_Stream = WS_Stream; /* pointer to writer struct */
     WSR_Stream->PeerCohort = PeerCohort;
 
     WSR_Stream->ReaderCohortSize = readerCohortSize;
 
+    WSR_Stream->ReaderAddr =
+        calloc(readerCohortSize, sizeof(*WSR_Stream->ReaderAddr));
+
+    for (i = 0; i < readerCohortSize; i++)
+    {
+        fi_av_insert(Fabric->av, providedReaderInfo[i]->Address, 1,
+                     &WSR_Stream->ReaderAddr[i], 0, NULL);
+        Svcs->verbose(WS_Stream->CP_Stream, DPTraceVerbose,
+                      "Received contact info for RS_Stream %p, WSR Rank %d\n",
+                      providedReaderInfo[i]->RS_Stream, i);
+    }
+
     /*
      * add this writer-side reader-specific stream to the parent writer stream
      * structure
@@ -506,18 +934,35 @@ static DP_WSR_Stream RdmaInitWriterPerReader(CP_Services Svcs,
     WS_Stream->ReaderCount++;
     pthread_mutex_unlock(&wsr_mutex);
 
-    ContactInfo = malloc(sizeof(struct _RdmaWriterContactInfo));
-    memset(ContactInfo, 0, sizeof(struct _RdmaWriterContactInfo));
+    ContactInfo = calloc(1, sizeof(struct _RdmaWriterContactInfo));
     ContactInfo->WS_Stream = WSR_Stream;
 
     ContactInfo->Length = Fabric->info->src_addrlen;
     ContactInfo->Address = malloc(ContactInfo->Length);
     fi_getname((fid_t)Fabric->signal, ContactInfo->Address,
                &ContactInfo->Length);
-#ifdef SST_HAVE_CRAY_DRC
-    ContactInfo->Credential = Fabric->credential;
-#endif /* SST_HAVE_CRAY_DRC */
+
+    ReaderRollHandle = &ContactInfo->ReaderRollHandle;
+    ReaderRollHandle->Block =
+        calloc(readerCohortSize, sizeof(struct _RdmaBuffer));
+    fi_mr_reg(Fabric->domain, ReaderRollHandle->Block,
+              readerCohortSize * sizeof(struct _RdmaBuffer), FI_REMOTE_WRITE, 0,
+              0, 0, &WSR_Stream->rrmr, Fabric->ctx);
+    ReaderRollHandle->Key = fi_mr_key(WSR_Stream->rrmr);
+
     WSR_Stream->WriterContactInfo = ContactInfo;
+
+    WSR_Stream->ReaderRoll = malloc(sizeof(struct _RdmaBuffer));
+    WSR_Stream->ReaderRoll->Handle = *ReaderRollHandle;
+    WSR_Stream->ReaderRoll->BufferLen =
+        readerCohortSize * sizeof(struct _RdmaBuffer);
+
+    WSR_Stream->Preload = 0;
+    WSR_Stream->SelectionPulled = 0;
+    WSR_Stream->SelectLocked = -1;
+
+    WSR_Stream->LastReleased = NULL;
+
     *WriterContactInfoPtr = ContactInfo;
 
     return WSR_Stream;
@@ -541,111 +986,104 @@ static void RdmaProvideWriterDataToReader(CP_Services Svcs,
     RS_Stream->WriterCohortSize = writerCohortSize;
     RS_Stream->WriterAddr =
         calloc(writerCohortSize, sizeof(*RS_Stream->WriterAddr));
+    RS_Stream->WriterRoll =
+        calloc(writerCohortSize, sizeof(*RS_Stream->WriterRoll));
 
-    RS_Stream->Fabric = calloc(1, sizeof(struct fabric_state));
+    /*
+     * make a copy of writer contact information (original will not be
+     * preserved)
+     */
+    RS_Stream->WriterContactInfo =
+        malloc(sizeof(struct _RdmaWriterContactInfo) * writerCohortSize);
 
-    Fabric = RS_Stream->Fabric;
-#ifdef SST_HAVE_CRAY_DRC
-    if (providedWriterInfo)
+    for (int i = 0; i < writerCohortSize; i++)
     {
-        Fabric->credential = (*providedWriterInfo)->Credential;
+        RS_Stream->WriterContactInfo[i].WS_Stream =
+            providedWriterInfo[i]->WS_Stream;
+        fi_av_insert(Fabric->av, providedWriterInfo[i]->Address, 1,
+                     &RS_Stream->WriterAddr[i], 0, NULL);
+        RS_Stream->WriterRoll[i] = providedWriterInfo[i]->ReaderRollHandle;
+        Svcs->verbose(RS_Stream->CP_Stream, DPTraceVerbose,
+                      "Received contact info for WS_stream %p, WSR Rank %d\n",
+                      RS_Stream->WriterContactInfo[i].WS_Stream, i);
     }
-    else
+}
+
+static void LogRequest(CP_Services Svcs, Rdma_RS_Stream RS_Stream, int Rank,
+                       long Timestep, size_t Offset, size_t Length)
+{
+    RdmaStepLogEntry *StepLog_p;
+    RdmaStepLogEntry StepLog;
+    RdmaBuffer LogEntry;
+    size_t ReqLogSize;
+    int LogIdx;
+
+    StepLog_p = &(RS_Stream->StepLog);
+    while (*StepLog_p && Timestep < (*StepLog_p)->Timestep)
     {
-        Svcs->verbose(CP_Stream,
-                      "Writer contact info needed to access DRC credentials.\n",
-                      rc);
+        StepLog_p = &((*StepLog_p)->Next);
     }
 
-    // at large scale the servers backing drc can become overwhelmed and
-    // timeout. The return value is -22 (EINVAL). a work around is to wait
-    // and try again.
-
-    try_left = DP_DRC_MAX_TRY;
-    rc = drc_access(Fabric->credential, 0, &Fabric->drc_info);
-    while (rc != DRC_SUCCESS && try_left--)
+    if (!(*StepLog_p) || (*StepLog_p)->Timestep != Timestep)
     {
-        usleep(DP_DRC_WAIT_USEC);
-        rc = drc_access(Fabric->credential, 0, &Fabric->drc_info);
+        StepLog = malloc(sizeof(*StepLog));
+        StepLog->RankLog =
+            calloc(RS_Stream->WriterCohortSize, sizeof(*StepLog->RankLog));
+        StepLog->Timestep = Timestep;
+        StepLog->Next = *StepLog_p;
+        StepLog->BufferSize = 0;
+        StepLog->WRanks = 0;
+        *StepLog_p = StepLog;
     }
-    if (rc != DRC_SUCCESS)
+    else
     {
-        Svcs->verbose(CP_Stream,
-                      "Could not access DRC credential. Last failed with %d.\n",
-                      rc);
+        StepLog = *StepLog_p;
     }
 
-    Fabric->auth_key = malloc(sizeof(*Fabric->auth_key));
-    Fabric->auth_key->type = GNIX_AKT_RAW;
-    Fabric->auth_key->raw.protection_key =
-        drc_get_first_cookie(Fabric->drc_info);
-    Svcs->verbose(CP_Stream, "Using protection key %08x.\n",
-                  Fabric->auth_key->raw.protection_key);
-#endif /* SST_HAVE_CRAY_DRC */
-
-    init_fabric(RS_Stream->Fabric, RS_Stream->Params);
-    if (!Fabric->info)
+    StepLog->BufferSize += Length;
+    StepLog->Entries++;
+    if (!StepLog->RankLog[Rank].ReqLog)
     {
-        Svcs->verbose(CP_Stream, "Could not find a valid transport fabric.\n");
+        ReqLogSize =
+            (REQ_LIST_GRAN * sizeof(struct _RdmaRankReqLog)) +
+            sizeof(uint64_t); // extra uint64_t for the preload buffer key
+        StepLog->RankLog[Rank].ReqLog = calloc(1, ReqLogSize);
+        StepLog->RankLog[Rank].MaxEntries = REQ_LIST_GRAN;
+        StepLog->WRanks++;
     }
-
-    Svcs->verbose(CP_Stream, "Fabric Parameters:\n%s\n",
-                  fi_tostr(Fabric->info, FI_TYPE_INFO));
-
-    /*
-     * make a copy of writer contact information (original will not be
-     * preserved)
-     */
-    RS_Stream->WriterContactInfo =
-        malloc(sizeof(struct _RdmaWriterContactInfo) * writerCohortSize);
-    for (int i = 0; i < writerCohortSize; i++)
+    if (StepLog->RankLog[Rank].MaxEntries == StepLog->RankLog[Rank].Entries)
     {
-        RS_Stream->WriterContactInfo[i].WS_Stream =
-            providedWriterInfo[i]->WS_Stream;
-        fi_av_insert(Fabric->av, providedWriterInfo[i]->Address, 1,
-                     &RS_Stream->WriterAddr[i], 0, NULL);
-        Svcs->verbose(RS_Stream->CP_Stream,
-                      "Received contact info for WS_stream %p, WSR Rank %d\n",
-                      RS_Stream->WriterContactInfo[i].WS_Stream, i);
+        StepLog->RankLog[Rank].MaxEntries *= 2;
+        ReqLogSize = (StepLog->RankLog[Rank].MaxEntries *
+                      sizeof(struct _RdmaRankReqLog)) +
+                     sizeof(uint64_t);
+        StepLog->RankLog[Rank].ReqLog =
+            realloc(StepLog->RankLog[Rank].ReqLog, ReqLogSize);
     }
+    StepLog->RankLog[Rank].BufferSize += Length;
+    LogIdx = StepLog->RankLog[Rank].Entries++;
+    LogEntry = &StepLog->RankLog[Rank].ReqLog[LogIdx];
+    LogEntry->BufferLen = Length;
+    LogEntry->Offset = Offset;
+    LogEntry->Handle.Block = NULL;
 }
 
-static void *RdmaReadRemoteMemory(CP_Services Svcs, DP_RS_Stream Stream_v,
-                                  int Rank, long Timestep, size_t Offset,
-                                  size_t Length, void *Buffer,
-                                  void *DP_TimestepInfo)
+static ssize_t PostRead(CP_Services Svcs, Rdma_RS_Stream RS_Stream, int Rank,
+                        long Timestep, size_t Offset, size_t Length,
+                        void *Buffer, RdmaBufferHandle Info,
+                        RdmaCompletionHandle *ret_v)
 {
-    Rdma_RS_Stream RS_Stream = (Rdma_RS_Stream)Stream_v;
     FabricState Fabric = RS_Stream->Fabric;
-    RdmaPerTimestepInfo Info = (RdmaPerTimestepInfo)DP_TimestepInfo;
-    RdmaCompletionHandle ret = malloc(sizeof(struct _RdmaCompletionHandle));
-    fi_addr_t SrcAddress;
+    fi_addr_t SrcAddress = RS_Stream->WriterAddr[Rank];
     void *LocalDesc = NULL;
     uint8_t *Addr;
+    RdmaCompletionHandle ret;
     ssize_t rc;
 
-    Svcs->verbose(RS_Stream->CP_Stream,
-                  "Performing remote read of Writer Rank %d\n", Rank);
-
-    if (Info)
-    {
-        Svcs->verbose(RS_Stream->CP_Stream,
-                      "Block address is %p, with a key of %d\n", Info->Block,
-                      Info->Key);
-    }
-    else
-    {
-        Svcs->verbose(RS_Stream->CP_Stream, "Timestep info is null\n");
-    }
-
-    ret->CPStream = RS_Stream;
-    ret->Buffer = Buffer;
-    ret->Rank = Rank;
-    ret->Length = Length;
+    *ret_v = malloc(sizeof(struct _RdmaCompletionHandle));
+    ret = *ret_v;
     ret->Pending = 1;
 
-    SrcAddress = RS_Stream->WriterAddr[Rank];
-
     if (Fabric->local_mr_req)
     {
         // register dest buffer
@@ -656,9 +1094,10 @@ static void *RdmaReadRemoteMemory(CP_Services Svcs, DP_RS_Stream Stream_v,
 
     Addr = Info->Block + Offset;
 
-    Svcs->verbose(RS_Stream->CP_Stream,
-                  "Target of remote read on Writer Rank %d is %p\n", Rank,
-                  Addr);
+    Svcs->verbose(
+        RS_Stream->CP_Stream, DPTraceVerbose,
+        "Remote read target is Rank %d (Offset = %zi, Length = %zi)\n", Rank,
+        Offset, Length);
 
     do
     {
@@ -668,15 +1107,137 @@ static void *RdmaReadRemoteMemory(CP_Services Svcs, DP_RS_Stream Stream_v,
 
     if (rc != 0)
     {
-        Svcs->verbose(RS_Stream->CP_Stream, "fi_read failed with code %d.\n",
-                      rc);
+        Svcs->verbose(RS_Stream->CP_Stream, DPCriticalVerbose,
+                      "fi_read failed with code %d.\n", rc);
+        return (rc);
+    }
+    else
+    {
+
+        Svcs->verbose(RS_Stream->CP_Stream, DPTraceVerbose,
+                      "Posted RDMA get for Writer Rank %d for handle %p\n",
+                      Rank, (void *)ret);
+        RS_Stream->PendingReads++;
+    }
+
+    return (rc);
+}
+
+static RdmaBuffer GetRequest(Rdma_RS_Stream Stream, RdmaStepLogEntry StepLog,
+                             int Rank, size_t Offset, size_t Length)
+{
+
+    RdmaRankReqLog RankLog = &StepLog->RankLog[Rank];
+    RdmaBuffer Req;
+    int i;
+
+    for (i = 0; i < RankLog->Entries; i++)
+    {
+        Req = &RankLog->ReqLog[i];
+        if (Req->BufferLen == Length && Req->Offset == Offset)
+        {
+            return (Req);
+        }
+    }
+
+    return (NULL);
+}
+
+static void *RdmaReadRemoteMemory(CP_Services Svcs, DP_RS_Stream Stream_v,
+                                  int Rank, long Timestep, size_t Offset,
+                                  size_t Length, void *Buffer,
+                                  void *DP_TimestepInfo)
+{
+    RdmaCompletionHandle ret;
+    Rdma_RS_Stream RS_Stream = (Rdma_RS_Stream)Stream_v;
+    RdmaBufferHandle Info = (RdmaBufferHandle)DP_TimestepInfo;
+    RdmaStepLogEntry StepLog;
+    RdmaRankReqLog RankLog;
+    RdmaBuffer Req;
+    int BufferSlot;
+    int WRidx;
+
+    Svcs->verbose(RS_Stream->CP_Stream, DPTraceVerbose,
+                  "Performing remote read of Writer Rank %d at step %d\n", Rank,
+                  Timestep);
+
+    if (Info)
+    {
+        Svcs->verbose(RS_Stream->CP_Stream, DPTraceVerbose,
+                      "Block address is %p, with a key of %d\n", Info->Block,
+                      Info->Key);
+    }
+    else
+    {
+        Svcs->verbose(RS_Stream->CP_Stream, DPCriticalVerbose,
+                      "Timestep info is null\n");
         free(ret);
-        return NULL;
+        return (NULL);
+    }
+
+    pthread_mutex_lock(&ts_mutex);
+    if (RS_Stream->PreloadPosted)
+    {
+        RS_Stream->TotalReads++;
+        Req = GetRequest(RS_Stream, RS_Stream->PreloadStepLog, Rank, Offset,
+                         Length);
+        if (Req)
+        {
+            BufferSlot = Timestep & 1;
+            StepLog = RS_Stream->PreloadStepLog;
+            RankLog = &StepLog->RankLog[Rank];
+            WRidx = Req - RankLog->ReqLog;
+            ret = &((RankLog->PreloadHandles[BufferSlot])[WRidx]);
+            ret->PreloadBuffer =
+                Req->Handle.Block + (BufferSlot * StepLog->BufferSize);
+            ret->Pending++;
+            if (ret->Pending == 0)
+            {
+                // the data has already been preloaded, safe to copy
+                memcpy(Buffer, ret->PreloadBuffer, Length);
+                RS_Stream->EarlyReads++;
+            }
+            else if (ret->Pending != 1)
+            {
+                Svcs->verbose(
+                    RS_Stream->CP_Stream, DPCriticalVerbose,
+                    "rank %d, wrank %d, entry %d, buffer slot %d, bad "
+                    "handle pending value.\n",
+                    RS_Stream->Rank, Rank, WRidx, BufferSlot);
+            }
+        }
+        else
+        {
+            Svcs->verbose(RS_Stream->CP_Stream, DPPerRankVerbose,
+                          "read patterns are fixed, but new request to rank %d "
+                          "(Offset = %zi, Length = %zi \n",
+                          Rank, Offset, Length);
+            ret->PreloadBuffer = NULL;
+            if (PostRead(Svcs, RS_Stream, Rank, Timestep, Offset, Length,
+                         Buffer, Info, &ret) != 0)
+            {
+                free(ret);
+                return (NULL);
+            }
+        }
     }
+    else
+    {
+        LogRequest(Svcs, RS_Stream, Rank, Timestep, Offset, Length);
+        if (PostRead(Svcs, RS_Stream, Rank, Timestep, Offset, Length, Buffer,
+                     Info, &ret) != 0)
+        {
+            free(ret);
+            return (NULL);
+        }
+        ret->PreloadBuffer = NULL;
+    }
+    pthread_mutex_unlock(&ts_mutex);
 
-    Svcs->verbose(RS_Stream->CP_Stream,
-                  "Posted RDMA get for Writer Rank %d for handle %p\n", Rank,
-                  (void *)ret);
+    ret->CPStream = RS_Stream;
+    ret->Buffer = Buffer;
+    ret->Rank = Rank;
+    ret->Length = Length;
 
     return (ret);
 }
@@ -687,7 +1248,7 @@ static void RdmaNotifyConnFailure(CP_Services Svcs, DP_RS_Stream Stream_v,
     Rdma_RS_Stream Stream = (Rdma_RS_Stream)
         Stream_v; /* DP_RS_Stream is the return from InitReader */
     CManager cm = Svcs->getCManager(Stream->CP_Stream);
-    Svcs->verbose(Stream->CP_Stream,
+    Svcs->verbose(Stream->CP_Stream, DPTraceVerbose,
                   "received notification that writer peer "
                   "%d has failed, failing any pending "
                   "requests\n",
@@ -696,17 +1257,19 @@ static void RdmaNotifyConnFailure(CP_Services Svcs, DP_RS_Stream Stream_v,
     //   FailRequestsToRank(Svcs, cm, Stream, FailedPeerRank);
 }
 
-/*
- * RdmaWaitForCompletion should return 1 if successful, but 0 if the reads
- * failed for some reason or were aborted by RdmaNotifyConnFailure()
+/* We still have to handle Pull completions while waiting for push to complete
  */
-static int RdmaWaitForCompletion(CP_Services Svcs, void *Handle_v)
+static int DoPushWait(CP_Services Svcs, Rdma_RS_Stream Stream,
+                      RdmaCompletionHandle Handle)
 {
-    RdmaCompletionHandle Handle = (RdmaCompletionHandle)Handle_v;
-    Rdma_RS_Stream Stream = Handle->CPStream;
     FabricState Fabric = Stream->Fabric;
+    RdmaStepLogEntry StepLog = Stream->PreloadStepLog;
+    RdmaRankReqLog RankLog;
+    RdmaBuffer Req;
     RdmaCompletionHandle Handle_t;
     struct fi_cq_data_entry CQEntry = {0};
+    int WRank, WRidx;
+    int BufferSlot;
 
     while (Handle->Pending > 0)
     {
@@ -714,11 +1277,58 @@ static int RdmaWaitForCompletion(CP_Services Svcs, void *Handle_v)
         rc = fi_cq_sread(Fabric->cq_signal, (void *)(&CQEntry), 1, NULL, -1);
         if (rc < 1)
         {
-            // Handle errrors
+            Svcs->verbose(Stream->CP_Stream, DPCriticalVerbose,
+                          "failure while waiting for completions (%d).\n", rc);
+            return 0;
+        }
+        else if (CQEntry.flags & FI_REMOTE_CQ_DATA)
+        {
+            BufferSlot = CQEntry.data >> 31;
+            WRidx = (CQEntry.data >> 20) & 0x3FF;
+            WRank = CQEntry.data & 0x0FFFFF;
+
+            Svcs->verbose(Stream->CP_Stream, DPTraceVerbose,
+                          "got completion for Rank %d, push request %d.\n",
+                          WRank, WRidx);
+            RankLog = &StepLog->RankLog[WRank];
+            Svcs->verbose(Stream->CP_Stream, DPTraceVerbose,
+                          "CQEntry.data = %" PRIu64
+                          ", BufferSlot = %d, WRank = %d, WRidx = %d\n",
+                          CQEntry.data, BufferSlot, WRank, WRidx);
+            Handle_t = (RdmaCompletionHandle) &
+                       ((RankLog->PreloadHandles[BufferSlot])[WRidx]);
+            if (Handle_t)
+            {
+                pthread_mutex_lock(&ts_mutex);
+                Handle_t->Pending--;
+                if (Handle_t->Pending == 0)
+                {
+                    // already saw a ReadRemote for this data, safe to copy
+                    memcpy(Handle_t->Buffer, CQEntry.buf, CQEntry.len);
+                }
+                else if (Handle_t->Pending != -1)
+                {
+                    Svcs->verbose(
+                        Stream->CP_Stream, DPCriticalVerbose,
+                        "rank %d, wrank %d, entry %d, buffer slot %d, bad "
+                        "handle pending value.\n",
+                        Stream->Rank, WRank, WRidx, BufferSlot);
+                }
+                else
+                {
+                }
+                pthread_mutex_unlock(&ts_mutex);
+            }
+            else
+            {
+                Svcs->verbose(
+                    Stream->CP_Stream, DPCriticalVerbose,
+                    "Got push completion without a known handle...\n");
+            }
         }
         else
         {
-            Svcs->verbose(Stream->CP_Stream,
+            Svcs->verbose(Stream->CP_Stream, DPTraceVerbose,
                           "got completion for request with handle %p.\n",
                           CQEntry.op_context);
             Handle_t = (RdmaCompletionHandle)CQEntry.op_context;
@@ -726,6 +1336,43 @@ static int RdmaWaitForCompletion(CP_Services Svcs, void *Handle_v)
         }
     }
 
+    if (Handle->LocalMR && Fabric->local_mr_req)
+    {
+        fi_close((struct fid *)Handle->LocalMR);
+    }
+
+    return (1);
+}
+
+static int DoPullWait(CP_Services Svcs, Rdma_RS_Stream Stream,
+                      RdmaCompletionHandle Handle)
+{
+    FabricState Fabric = Stream->Fabric;
+    RdmaCompletionHandle Handle_t;
+    struct fi_cq_data_entry CQEntry = {0};
+
+    while (Handle->Pending > 0)
+    {
+        ssize_t rc;
+        rc = fi_cq_sread(Fabric->cq_signal, (void *)(&CQEntry), 1, NULL, -1);
+        if (rc < 1)
+        {
+            Svcs->verbose(Stream->CP_Stream, DPCriticalVerbose,
+                          "failure while waiting for completions (%d).\n", rc);
+            return 0;
+        }
+        else
+        {
+            Svcs->verbose(
+                Stream->CP_Stream, DPTraceVerbose,
+                "got completion for request with handle %p (flags %li).\n",
+                CQEntry.op_context, CQEntry.flags);
+            Handle_t = (RdmaCompletionHandle)CQEntry.op_context;
+            Handle_t->Pending--;
+        }
+    }
+
+    // TODO: maybe reuse this memory registration
     if (Fabric->local_mr_req)
     {
         fi_close((struct fid *)Handle->LocalMR);
@@ -734,6 +1381,28 @@ static int RdmaWaitForCompletion(CP_Services Svcs, void *Handle_v)
     return (1);
 }
 
+/*
+ * RdmaWaitForCompletion should return 1 if successful, but 0 if the reads
+ * failed for some reason or were aborted by RdmaNotifyConnFailure()
+ */
+static int RdmaWaitForCompletion(CP_Services Svcs, void *Handle_v)
+{
+    RdmaCompletionHandle Handle = (RdmaCompletionHandle)Handle_v;
+    Rdma_RS_Stream Stream = Handle->CPStream;
+
+    Svcs->verbose(Stream->CP_Stream, DPTraceVerbose, "Rank %d, %s\n",
+                  Stream->Rank, __func__);
+
+    if (Stream->PreloadPosted && Handle->PreloadBuffer)
+    {
+        return (DoPushWait(Svcs, Stream, Handle));
+    }
+    else
+    {
+        return (DoPullWait(Svcs, Stream, Handle));
+    }
+}
+
 static void RdmaProvideTimestep(CP_Services Svcs, DP_WS_Stream Stream_v,
                                 struct _SstData *Data,
                                 struct _SstData *LocalMetadata, long Timestep,
@@ -741,19 +1410,30 @@ static void RdmaProvideTimestep(CP_Services Svcs, DP_WS_Stream Stream_v,
 {
     Rdma_WS_Stream Stream = (Rdma_WS_Stream)Stream_v;
     TimestepList Entry = malloc(sizeof(struct _TimestepEntry));
-    RdmaPerTimestepInfo Info = malloc(sizeof(struct _RdmaPerTimestepInfo));
+    RdmaBufferHandle Info = malloc(sizeof(struct _RdmaBufferHandle));
     FabricState Fabric = Stream->Fabric;
 
     Entry->Data = malloc(sizeof(*Data));
     memcpy(Entry->Data, Data, sizeof(*Data));
     Entry->Timestep = Timestep;
     Entry->DP_TimestepInfo = Info;
+    Entry->BufferSlot = -1;
+    Entry->Desc = NULL;
 
-    fi_mr_reg(Fabric->domain, Data->block, Data->DataSize, FI_REMOTE_READ, 0, 0,
-              0, &Entry->mr, Fabric->ctx);
+    fi_mr_reg(Fabric->domain, Data->block, Data->DataSize,
+              FI_WRITE | FI_REMOTE_READ, 0, 0, 0, &Entry->mr, Fabric->ctx);
     Entry->Key = fi_mr_key(Entry->mr);
+    if (Fabric->local_mr_req)
+    {
+        Entry->Desc = fi_mr_desc(Entry->mr);
+    }
     pthread_mutex_lock(&ts_mutex);
-    Entry->Next = Stream->Timesteps;
+    if (Stream->Timesteps)
+    {
+        Stream->Timesteps->Next = Entry;
+    }
+    Entry->Prev = Stream->Timesteps;
+    Entry->Next = NULL;
     Stream->Timesteps = Entry;
     // Probably doesn't need to be in the lock
     // |
@@ -762,7 +1442,7 @@ static void RdmaProvideTimestep(CP_Services Svcs, DP_WS_Stream Stream_v,
     pthread_mutex_unlock(&ts_mutex);
     Info->Block = (uint8_t *)Data->block;
 
-    Svcs->verbose(Stream->CP_Stream,
+    Svcs->verbose(Stream->CP_Stream, DPTraceVerbose,
                   "Providing timestep data with block %p and access key %d\n",
                   Info->Block, Info->Key);
 
@@ -773,16 +1453,19 @@ static void RdmaReleaseTimestep(CP_Services Svcs, DP_WS_Stream Stream_v,
                                 long Timestep)
 {
     Rdma_WS_Stream Stream = (Rdma_WS_Stream)Stream_v;
+    Rdma_WSR_Stream WSR_Stream;
     TimestepList *List = &Stream->Timesteps;
     TimestepList ReleaseTSL;
-    RdmaPerTimestepInfo Info;
+    RdmaBufferHandle Info;
+    int i;
 
-    Svcs->verbose(Stream->CP_Stream, "Releasing timestep %ld\n", Timestep);
+    Svcs->verbose(Stream->CP_Stream, DPTraceVerbose, "Releasing timestep %ld\n",
+                  Timestep);
 
     pthread_mutex_lock(&ts_mutex);
     while ((*List) && (*List)->Timestep != Timestep)
     {
-        List = &((*List)->Next);
+        List = &((*List)->Prev);
     }
 
     if ((*List) == NULL)
@@ -791,13 +1474,13 @@ static void RdmaReleaseTimestep(CP_Services Svcs, DP_WS_Stream Stream_v,
          * Shouldn't ever get here because we should never release a
          * timestep that we don't have.
          */
-        fprintf(stderr, "Failed to release Timestep %ld, not found\n",
-                Timestep);
+        Svcs->verbose(Stream->CP_Stream, DPCriticalVerbose,
+                      "Failed to release Timestep %ld, not found\n", Timestep);
         assert(0);
     }
 
     ReleaseTSL = *List;
-    *List = ReleaseTSL->Next;
+    *List = ReleaseTSL->Prev;
     pthread_mutex_unlock(&ts_mutex);
     fi_close((struct fid *)ReleaseTSL->mr);
     if (ReleaseTSL->Data)
@@ -812,16 +1495,60 @@ static void RdmaReleaseTimestep(CP_Services Svcs, DP_WS_Stream Stream_v,
     free(ReleaseTSL);
 }
 
+static void RdmaDestroyRankReqLog(Rdma_RS_Stream RS_Stream,
+                                  RdmaRankReqLog RankReqLog)
+{
+    RdmaReqLogEntry tmp;
+    int i;
+
+    for (i = 0; i < RS_Stream->WriterCohortSize; i++)
+    {
+        if (RankReqLog[i].ReqLog)
+        {
+            free(RankReqLog[i].ReqLog);
+        }
+    }
+    free(RankReqLog);
+}
+
 static void RdmaDestroyReader(CP_Services Svcs, DP_RS_Stream RS_Stream_v)
 {
     Rdma_RS_Stream RS_Stream = (Rdma_RS_Stream)RS_Stream_v;
+    RdmaStepLogEntry StepLog = RS_Stream->StepLog;
+    RdmaStepLogEntry tStepLog;
+
+    if (RS_Stream->PreloadStep > -1)
+    {
+        Svcs->verbose(
+            RS_Stream->CP_Stream, DPSummaryVerbose,
+            "Reader Rank %d: %li early reads of %li total reads (where preload "
+            "was possible.)\n",
+            RS_Stream->Rank, RS_Stream->EarlyReads, RS_Stream->TotalReads);
+    }
 
-    Svcs->verbose(RS_Stream->CP_Stream, "Tearing down RDMA state on reader.\n");
-    fini_fabric(RS_Stream->Fabric);
+    Svcs->verbose(RS_Stream->CP_Stream, DPTraceVerbose,
+                  "Tearing down RDMA state on reader.\n");
+    if (RS_Stream->Fabric)
+    {
+        fini_fabric(RS_Stream->Fabric, Svcs, RS_Stream->CP_Stream);
+    }
+
+    while (StepLog)
+    {
+        RdmaDestroyRankReqLog(RS_Stream, StepLog->RankLog);
+        tStepLog = StepLog;
+        StepLog = StepLog->Next;
+        free(tStepLog);
+    }
 
     free(RS_Stream->WriterContactInfo);
     free(RS_Stream->WriterAddr);
-    free(RS_Stream->Fabric);
+    free(RS_Stream->WriterRoll);
+    if (RS_Stream->ContactInfo)
+    {
+        free(RS_Stream->ContactInfo->Address);
+        free(RS_Stream->ContactInfo);
+    }
     free(RS_Stream);
 }
 
@@ -842,6 +1569,11 @@ static void RdmaDestroyWriterPerReader(CP_Services Svcs,
             break;
         }
     }
+    fi_close((struct fid *)WSR_Stream->rrmr);
+    if (WSR_Stream->ReaderAddr)
+    {
+        free(WSR_Stream->ReaderAddr);
+    }
     WS_Stream->Readers = realloc(
         WS_Stream->Readers, sizeof(*WSR_Stream) * (WS_Stream->ReaderCount - 1));
     WS_Stream->ReaderCount--;
@@ -852,13 +1584,25 @@ static void RdmaDestroyWriterPerReader(CP_Services Svcs,
         WriterContactInfo = WSR_Stream->WriterContactInfo;
         free(WriterContactInfo->Address);
     }
+    if (WriterContactInfo->ReaderRollHandle.Block)
+    {
+        free(WriterContactInfo->ReaderRollHandle.Block);
+    }
     free(WSR_Stream->WriterContactInfo);
+
+    if (WSR_Stream->ReaderRoll)
+    {
+        free(WSR_Stream->ReaderRoll);
+    }
     free(WSR_Stream);
 }
 
 static FMField RdmaReaderContactList[] = {
     {"reader_ID", "integer", sizeof(void *),
      FMOffset(RdmaReaderContactInfo, RS_Stream)},
+    {"Length", "integer", sizeof(int), FMOffset(RdmaReaderContactInfo, Length)},
+    {"Address", "integer[Length]", sizeof(char),
+     FMOffset(RdmaReaderContactInfo, Address)},
     {NULL, NULL, 0, 0}};
 
 static FMStructDescRec RdmaReaderContactStructs[] = {
@@ -866,6 +1610,16 @@ static FMStructDescRec RdmaReaderContactStructs[] = {
      sizeof(struct _RdmaReaderContactInfo), NULL},
     {NULL, NULL, 0, NULL}};
 
+static FMField RdmaBufferHandleList[] = {
+    {"Block", "integer", sizeof(void *), FMOffset(RdmaBufferHandle, Block)},
+    {"Key", "integer", sizeof(uint64_t), FMOffset(RdmaBufferHandle, Key)},
+    {NULL, NULL, 0, 0}};
+
+static FMStructDescRec RdmaBufferHandleStructs[] = {
+    {"RdmaBufferHandle", RdmaBufferHandleList, sizeof(struct _RdmaBufferHandle),
+     NULL},
+    {NULL, NULL, 0, NULL}};
+
 static void RdmaDestroyWriter(CP_Services Svcs, DP_WS_Stream WS_Stream_v)
 {
     Rdma_WS_Stream WS_Stream = (Rdma_WS_Stream)WS_Stream_v;
@@ -876,21 +1630,16 @@ static void RdmaDestroyWriter(CP_Services Svcs, DP_WS_Stream WS_Stream_v)
     Credential = WS_Stream->Fabric->credential;
 #endif /* SST_HAVE_CRAY_DRC */
 
-    Svcs->verbose(WS_Stream->CP_Stream, "Tearing down RDMA state on writer.\n");
-    fini_fabric(WS_Stream->Fabric);
-
-#ifdef SST_HAVE_CRAY_DRC
-    if (WS_Stream->Rank == 0)
-    {
-        drc_release(Credential, 0);
-    }
-#endif /* SST_HAVE_CRAY_DRC */
-
+    Svcs->verbose(WS_Stream->CP_Stream, DPTraceVerbose,
+                  "Releasing reader-specific state for remaining readers.\n");
     while (WS_Stream->ReaderCount > 0)
     {
         RdmaDestroyWriterPerReader(Svcs, WS_Stream->Readers[0]);
     }
 
+    Svcs->verbose(WS_Stream->CP_Stream, DPTraceVerbose,
+                  "Releasing remaining timesteps.\n");
+
     pthread_mutex_lock(&ts_mutex);
     while (WS_Stream->Timesteps)
     {
@@ -901,6 +1650,20 @@ static void RdmaDestroyWriter(CP_Services Svcs, DP_WS_Stream WS_Stream_v)
     }
     pthread_mutex_unlock(&ts_mutex);
 
+    Svcs->verbose(WS_Stream->CP_Stream, DPTraceVerbose,
+                  "Tearing down RDMA state on writer.\n");
+    if (WS_Stream->Fabric)
+    {
+        fini_fabric(WS_Stream->Fabric, Svcs, WS_Stream->CP_Stream);
+    }
+
+#ifdef SST_HAVE_CRAY_DRC
+    if (WS_Stream->Rank == 0)
+    {
+        drc_release(Credential, 0);
+    }
+#endif /* SST_HAVE_CRAY_DRC */
+
     free(WS_Stream->Fabric);
     free(WS_Stream);
 }
@@ -911,28 +1674,18 @@ static FMField RdmaWriterContactList[] = {
     {"Length", "integer", sizeof(int), FMOffset(RdmaWriterContactInfo, Length)},
     {"Address", "integer[Length]", sizeof(char),
      FMOffset(RdmaWriterContactInfo, Address)},
-#ifdef SST_HAVE_CRAY_DRC
-    {"Credential", "integer", sizeof(int),
-     FMOffset(RdmaWriterContactInfo, Credential)},
-#endif /* SST_HAVE_CRAY_DRC */
+    {"ReaderRollHandle", "RdmaBufferHandle", sizeof(struct _RdmaBufferHandle),
+     FMOffset(RdmaWriterContactInfo, ReaderRollHandle)},
     {NULL, NULL, 0, 0}};
 
 static FMStructDescRec RdmaWriterContactStructs[] = {
     {"RdmaWriterContactInfo", RdmaWriterContactList,
      sizeof(struct _RdmaWriterContactInfo), NULL},
+    {"RdmaBufferHandle", RdmaBufferHandleList, sizeof(struct _RdmaBufferHandle),
+     NULL},
     {NULL, NULL, 0, NULL}};
 
-static FMField RdmaTimestepInfoList[] = {
-    {"Block", "integer", sizeof(void *), FMOffset(RdmaPerTimestepInfo, Block)},
-    {"Key", "integer", sizeof(uint64_t), FMOffset(RdmaPerTimestepInfo, Key)},
-    {NULL, NULL, 0, 0}};
-
-static FMStructDescRec RdmaTimestepInfoStructs[] = {
-    {"RdmaTimestepInfo", RdmaTimestepInfoList,
-     sizeof(struct _RdmaPerTimestepInfo), NULL},
-    {NULL, NULL, 0, NULL}};
-
-static struct _CP_DP_Interface RdmaDPInterface;
+static struct _CP_DP_Interface RdmaDPInterface = {0};
 
 /* In RdmaGetPriority, the Rdma DP should do whatever is necessary to test to
  * see if it
@@ -954,9 +1707,6 @@ static int RdmaGetPriority(CP_Services Svcs, void *CP_Stream,
     char *forkunsafe;
     int Ret = -1;
 
-    (void)attr_atom_from_string(
-        "RDMA_DRC_KEY"); // make sure this attribute is translatable
-
     hints = fi_allocinfo();
     hints->caps = FI_MSG | FI_SEND | FI_RECV | FI_REMOTE_READ |
                   FI_REMOTE_WRITE | FI_RMA | FI_READ | FI_WRITE;
@@ -982,12 +1732,14 @@ static int RdmaGetPriority(CP_Services Svcs, void *CP_Stream,
         putenv("FI_FORK_UNSAFE=Yes");
     }
 
+    pthread_mutex_lock(&fabric_mutex);
     fi_getinfo(FI_VERSION(1, 5), NULL, NULL, 0, hints, &info);
+    pthread_mutex_unlock(&fabric_mutex);
     fi_freeinfo(hints);
 
     if (!info)
     {
-        Svcs->verbose(CP_Stream,
+        Svcs->verbose(CP_Stream, DPTraceVerbose,
                       "RDMA Dataplane could not find any viable fabrics.\n");
     }
 
@@ -1001,7 +1753,7 @@ static int RdmaGetPriority(CP_Services Svcs, void *CP_Stream,
         domain_name = info->domain_attr->name;
         if (ifname && strcmp(ifname, domain_name) == 0)
         {
-            Svcs->verbose(CP_Stream,
+            Svcs->verbose(CP_Stream, DPPerStepVerbose,
                           "RDMA Dataplane found the requested "
                           "interface %s, provider type %s.\n",
                           ifname, prov_name);
@@ -1012,7 +1764,7 @@ static int RdmaGetPriority(CP_Services Svcs, void *CP_Stream,
             strstr(prov_name, "gni") || strstr(prov_name, "psm2"))
         {
 
-            Svcs->verbose(CP_Stream,
+            Svcs->verbose(CP_Stream, DPPerStepVerbose,
                           "RDMA Dataplane sees interface %s, "
                           "provider type %s, which should work.\n",
                           domain_name, prov_name);
@@ -1024,7 +1776,7 @@ static int RdmaGetPriority(CP_Services Svcs, void *CP_Stream,
     if (Ret == -1)
     {
         Svcs->verbose(
-            CP_Stream,
+            CP_Stream, DPPerStepVerbose,
             "RDMA Dataplane could not find an RDMA-compatible fabric.\n");
     }
 
@@ -1034,7 +1786,7 @@ static int RdmaGetPriority(CP_Services Svcs, void *CP_Stream,
     }
 
     Svcs->verbose(
-        CP_Stream,
+        CP_Stream, DPPerStepVerbose,
         "RDMA Dataplane evaluating viability, returning priority %d\n", Ret);
     return Ret;
 }
@@ -1045,15 +1797,482 @@ static int RdmaGetPriority(CP_Services Svcs, void *CP_Stream,
  */
 static void RdmaUnGetPriority(CP_Services Svcs, void *CP_Stream)
 {
-    Svcs->verbose(CP_Stream, "RDMA Dataplane unloading\n");
+    Svcs->verbose(CP_Stream, DPPerStepVerbose, "RDMA Dataplane unloading\n");
+}
+
+static void PushData(CP_Services Svcs, Rdma_WSR_Stream Stream,
+                     TimestepList Step, int BufferSlot)
+{
+    Rdma_WS_Stream WS_Stream = Stream->WS_Stream;
+    FabricState Fabric = WS_Stream->Fabric;
+    RdmaRankReqLog RankReq = Stream->PreloadReq;
+    RdmaBuffer Req, ReaderRoll, RollBuffer;
+    uint64_t RecvCounter;
+    uint8_t *StepBuffer;
+    int i, rc;
+
+    StepBuffer = (uint8_t *)Step->Data->block;
+    ReaderRoll = (RdmaBuffer)Stream->ReaderRoll->Handle.Block;
+
+    Step->OutstandingWrites = 0;
+    while (RankReq)
+    {
+        RollBuffer = &ReaderRoll[RankReq->Rank];
+        for (i = 0; i < RankReq->Entries; i++)
+        {
+            // TODO: this can only handle 4096 requests per reader rank. Fix.
+            uint64_t Data = ((uint64_t)i << 20) | WS_Stream->Rank;
+            Data |= BufferSlot << 31;
+            Data &= 0xFFFFFFFF;
+            Svcs->verbose(WS_Stream->CP_Stream, DPTraceVerbose,
+                          "Sending Data = %" PRIu64
+                          " ; BufferSlot = %d, Rank = %d, Entry = %d\n",
+                          Data, BufferSlot, WS_Stream->Rank, i);
+            Req = &RankReq->ReqLog[i];
+            do
+            {
+                rc = fi_writedata(Fabric->signal, StepBuffer + Req->Offset,
+                                  Req->BufferLen, Step->Desc, Data,
+                                  Stream->ReaderAddr[RankReq->Rank],
+                                  (uint64_t)Req->Handle.Block +
+                                      (BufferSlot * RankReq->PreloadBufferSize),
+                                  RollBuffer->Offset, (void *)(Step->Timestep));
+            } while (rc == -EAGAIN);
+            if (rc != 0)
+            {
+                Svcs->verbose(WS_Stream->CP_Stream, DPCriticalVerbose,
+                              "fi_read failed with code %d.\n", rc);
+            }
+        }
+        Step->OutstandingWrites += RankReq->Entries;
+        RankReq = RankReq->next;
+    }
+}
+
+static void RdmaReaderRegisterTimestep(CP_Services Svcs,
+                                       DP_WSR_Stream WSRStream_v, long Timestep,
+                                       SstPreloadModeType PreloadMode)
+{
+    Rdma_WSR_Stream WSR_Stream = (Rdma_WSR_Stream)WSRStream_v;
+    Rdma_WS_Stream WS_Stream = WSR_Stream->WS_Stream;
+    TimestepList Step;
+
+    if (PreloadMode == SstPreloadLearned && WS_Stream->DefLocked < 0)
+    {
+        WS_Stream->DefLocked = Timestep;
+        if (WSR_Stream->SelectLocked >= 0)
+        {
+            Svcs->verbose(WS_Stream->CP_Stream, DPTraceVerbose,
+                          "enabling preload.\n");
+            WSR_Stream->Preload = 1;
+        }
+    }
+
+    Step = GetStep(WS_Stream, Timestep);
+    pthread_mutex_lock(&ts_mutex);
+    if (WSR_Stream->SelectionPulled &&
+        WSR_Stream->PreloadUsed[Step->Timestep & 1] == 0)
+    {
+        PushData(Svcs, WSR_Stream, Step, Step->Timestep & 1);
+        WSR_Stream->PreloadUsed[Step->Timestep & 1] = 1;
+        Step->BufferSlot = Step->Timestep & 1;
+    }
+    pthread_mutex_unlock(&ts_mutex);
+}
+
+static void PostPreload(CP_Services Svcs, Rdma_RS_Stream Stream, long Timestep)
+{
+    RdmaStepLogEntry StepLog;
+    FabricState Fabric = Stream->Fabric;
+    RdmaBuffer PreloadBuffer = &Stream->PreloadBuffer;
+    RdmaRankReqLog RankLog;
+    RdmaBuffer SendBuffer;
+    struct fid_mr *sbmr = NULL;
+    void *sbdesc = NULL;
+    size_t SBSize;
+    RdmaBuffer ReqLog;
+    uint64_t PreloadKey;
+    uint8_t *RawPLBuffer;
+    int WRidx = 0;
+    uint64_t RollDest;
+    struct fi_cq_data_entry CQEntry = {0};
+    uint8_t *RecvBuffer;
+    RdmaBuffer CQBuffer;
+    size_t RBLen;
+    uint64_t *BLenHolder;
+    int rc;
+    int i, j;
+
+    Svcs->verbose(Stream->CP_Stream, DPTraceVerbose, "rank %d: %s\n",
+                  Stream->Rank, __func__);
+
+    StepLog = Stream->StepLog;
+    while (StepLog)
+    {
+        if (StepLog->Timestep == Timestep)
+        {
+            break;
+        }
+        StepLog = StepLog->Next;
+    }
+    if (!StepLog)
+    {
+        Svcs->verbose(Stream->CP_Stream, DPCriticalVerbose,
+                      "trying to post preload data for a "
+                      "timestep with no access history.");
+        return;
+    }
+
+    Stream->PreloadStepLog = StepLog;
+
+    PreloadBuffer->BufferLen = 2 * StepLog->BufferSize;
+    PreloadBuffer->Handle.Block = malloc(PreloadBuffer->BufferLen);
+    fi_mr_reg(Fabric->domain, PreloadBuffer->Handle.Block,
+              PreloadBuffer->BufferLen, FI_REMOTE_WRITE, 0, 0, 0, &Stream->pbmr,
+              Fabric->ctx);
+    PreloadKey = fi_mr_key(Stream->pbmr);
+
+    SBSize = sizeof(*SendBuffer) * StepLog->WRanks;
+    SendBuffer = malloc(SBSize);
+    if (Fabric->local_mr_req)
+    {
+        fi_mr_reg(Fabric->domain, SendBuffer, SBSize, FI_WRITE, 0, 0, 0, &sbmr,
+                  Fabric->ctx);
+        sbdesc = fi_mr_desc(sbmr);
+    }
+
+    if (Fabric->rx_cq_data)
+    {
+        RBLen = 2 * StepLog->Entries * DP_DATA_RECV_SIZE;
+        Stream->RecvDataBuffer = malloc(RBLen);
+        fi_mr_reg(Fabric->domain, Stream->RecvDataBuffer, RBLen, FI_RECV, 0, 0,
+                  0, &Stream->rbmr, Fabric->ctx);
+        Stream->rbdesc = fi_mr_desc(Stream->rbmr);
+        RecvBuffer = (uint8_t *)Stream->RecvDataBuffer;
+        for (i = 0; i < 2 * StepLog->Entries; i++)
+        {
+            rc = fi_recv(Fabric->signal, RecvBuffer, DP_DATA_RECV_SIZE,
+                         Stream->rbdesc, FI_ADDR_UNSPEC, Fabric->ctx);
+            if (rc)
+            {
+                Svcs->verbose(Stream->CP_Stream, DPCriticalVerbose,
+                              "Rank %d, fi_recv failed.\n", Stream->Rank);
+            }
+            RecvBuffer += DP_DATA_RECV_SIZE;
+        }
+    }
+
+    RawPLBuffer = PreloadBuffer->Handle.Block;
+    for (i = 0; i < Stream->WriterCohortSize; i++)
+    {
+        RankLog = &StepLog->RankLog[i];
+        if (RankLog->Entries > 0)
+        {
+            RankLog->Buffer = (void *)RawPLBuffer;
+            fi_mr_reg(Fabric->domain, RankLog->ReqLog,
+                      (sizeof(struct _RdmaBuffer) * RankLog->Entries) +
+                          sizeof(uint64_t),
+                      FI_REMOTE_READ, 0, 0, 0, &RankLog->preqbmr, Fabric->ctx);
+            for (j = 0; j < RankLog->Entries; j++)
+            {
+                ReqLog = &RankLog->ReqLog[j];
+                ReqLog->Handle.Block = RawPLBuffer;
+                ReqLog->Handle.Key = PreloadKey;
+                RawPLBuffer += ReqLog->BufferLen;
+            }
+            /* We always allocate and extra sizeof(uint64_t) in the ReqLog. We
+             * use this to make the size of the Preload Buffer available to each
+             * writer. */
+            BLenHolder = (uint64_t *)(&RankLog->ReqLog[RankLog->Entries]);
+            *BLenHolder = StepLog->BufferSize;
+
+            SendBuffer[WRidx].BufferLen =
+                (RankLog->Entries * sizeof(struct _RdmaBuffer)) +
+                sizeof(uint64_t);
+            SendBuffer[WRidx].Offset = (uint64_t)PreloadKey;
+            SendBuffer[WRidx].Handle.Block = (void *)RankLog->ReqLog;
+            SendBuffer[WRidx].Handle.Key = fi_mr_key(RankLog->preqbmr);
+            RollDest = (uint64_t)Stream->WriterRoll[i].Block +
+                       (sizeof(struct _RdmaBuffer) * Stream->Rank);
+            fi_write(Fabric->signal, &SendBuffer[WRidx],
+                     sizeof(struct _RdmaBuffer), sbdesc, Stream->WriterAddr[i],
+                     RollDest, Stream->WriterRoll[i].Key, &SendBuffer[WRidx]);
+            RankLog->PreloadHandles = malloc(sizeof(void *) * 2);
+            RankLog->PreloadHandles[0] =
+                calloc(sizeof(struct _RdmaCompletionHandle), RankLog->Entries);
+            RankLog->PreloadHandles[1] =
+                calloc(sizeof(struct _RdmaCompletionHandle), RankLog->Entries);
+            WRidx++;
+        }
+    }
+
+    while (WRidx > 0)
+    {
+        fi_cq_sread(Fabric->cq_signal, (void *)(&CQEntry), 1, NULL, -1);
+        CQBuffer = CQEntry.op_context;
+        if (CQBuffer >= SendBuffer && CQBuffer < (SendBuffer + StepLog->WRanks))
+        {
+            WRidx--;
+        }
+        else
+        {
+            Svcs->verbose(Stream->CP_Stream, DPCriticalVerbose,
+                          "got unexpected completion while posting preload "
+                          "pattern. This is probably an error.\n");
+        }
+    }
+
+    if (Fabric->local_mr_req)
+    {
+        fi_close((struct fid *)sbmr);
+    }
+    free(SendBuffer);
+}
+
+static void RdmaTimestepArrived(CP_Services Svcs, DP_RS_Stream Stream_v,
+                                long Timestep, SstPreloadModeType PreloadMode)
+{
+    Rdma_RS_Stream Stream = (Rdma_RS_Stream)Stream_v;
+
+    Svcs->verbose(Stream->CP_Stream, DPTraceVerbose,
+                  "%s with Timestep = %li, PreloadMode = %d\n", __func__,
+                  Timestep, PreloadMode);
+    if (PreloadMode == SstPreloadLearned && Stream->PreloadStep == -1)
+    {
+        if (Stream->PreloadAvail)
+        {
+            Stream->PreloadStep = Timestep;
+            if (Stream->Rank == 0)
+            {
+                Svcs->verbose(Stream->CP_Stream, DPSummaryVerbose,
+                              "write pattern is locked.\n");
+            }
+        }
+        else if (Stream->Rank == 0)
+        {
+            Svcs->verbose(
+                Stream->CP_Stream, DPSummaryVerbose,
+                "RDMA dataplane is ignoring a write pattern lock notification "
+                "because preloading is disabled. Enable by setting the "
+                "environment "
+                "variable SST_DP_PRELOAD to 'yes'\n");
+        }
+    }
+}
+
+static void RdmaReaderReleaseTimestep(CP_Services Svcs, DP_RS_Stream Stream_v,
+                                      long Timestep)
+{
+    Rdma_RS_Stream Stream = (Rdma_RS_Stream)Stream_v;
+
+    pthread_mutex_lock(&ts_mutex);
+    if (Stream->PreloadStep > -1 && Timestep >= Stream->PreloadStep &&
+        !Stream->PreloadPosted)
+    {
+        // TODO: Destroy all StepLog entries other than the one used for Preload
+        PostPreload(Svcs, Stream, Timestep);
+        Stream->PreloadPosted = 1;
+    }
+    pthread_mutex_unlock(&ts_mutex);
+
+    // This might be be a good spot to flush the Step list if we aren't doing
+    // preload (yet.)
+}
+
+static void PullSelection(CP_Services Svcs, Rdma_WSR_Stream Stream)
+{
+    Rdma_WS_Stream WS_Stream = Stream->WS_Stream;
+    FabricState Fabric = WS_Stream->Fabric;
+    RdmaBuffer ReaderRoll = (RdmaBuffer)Stream->ReaderRoll->Handle.Block;
+    struct _RdmaBuffer ReqBuffer = {0};
+    struct fi_cq_data_entry CQEntry = {0};
+    struct fid_mr *rrmr = NULL;
+    void *rrdesc = NULL;
+    RdmaRankReqLog *RankReq_p = &Stream->PreloadReq;
+    RdmaRankReqLog RankReq, CQRankReq;
+    RdmaBuffer CQReqLog;
+    uint8_t *ReadBuffer;
+    int i;
+
+    for (i = 0; i < Stream->ReaderCohortSize; i++)
+    {
+        if (ReaderRoll[i].BufferLen > 0)
+        {
+            RankReq = malloc(sizeof(struct _RdmaRankReqLog));
+            RankReq->Entries =
+                (ReaderRoll[i].BufferLen - sizeof(uint64_t)) /
+                sizeof(struct _RdmaBuffer); // piggyback the RecvCounter address
+            RankReq->ReqLog = malloc(ReaderRoll[i].BufferLen);
+            RankReq->BufferSize = ReaderRoll[i].BufferLen;
+            RankReq->next = NULL;
+            RankReq->Rank = i;
+            *RankReq_p = RankReq;
+            RankReq_p = &RankReq->next;
+            ReqBuffer.BufferLen += ReaderRoll[i].BufferLen;
+        }
+    }
+
+    ReqBuffer.Handle.Block = ReadBuffer = malloc(ReqBuffer.BufferLen);
+    if (Fabric->local_mr_req)
+    {
+        fi_mr_reg(Fabric->domain, ReqBuffer.Handle.Block, ReqBuffer.BufferLen,
+                  FI_READ, 0, 0, 0, &rrmr, Fabric->ctx);
+        rrdesc = fi_mr_desc(rrmr);
+    }
+
+    for (RankReq = Stream->PreloadReq; RankReq; RankReq = RankReq->next)
+    {
+        RankReq->ReqLog = (RdmaBuffer)ReadBuffer;
+        fi_read(Fabric->signal, RankReq->ReqLog, RankReq->BufferSize, rrdesc,
+                Stream->ReaderAddr[RankReq->Rank],
+                (uint64_t)ReaderRoll[RankReq->Rank].Handle.Block,
+                ReaderRoll[RankReq->Rank].Handle.Key, RankReq);
+        ReadBuffer += RankReq->BufferSize;
+    }
+
+    RankReq = Stream->PreloadReq;
+    while (RankReq)
+    {
+        fi_cq_sread(Fabric->cq_signal, (void *)(&CQEntry), 1, NULL, -1);
+        CQRankReq = CQEntry.op_context;
+        if (CQEntry.flags & FI_READ)
+        {
+            CQReqLog = CQRankReq->ReqLog;
+            CQRankReq->PreloadBufferSize =
+                *((uint64_t *)&CQReqLog[CQRankReq->Entries]);
+            RankReq = RankReq->next;
+        }
+        else
+        {
+            Svcs->verbose(
+                WS_Stream->CP_Stream, DPCriticalVerbose,
+                "got unexpected completion while fetching preload patterns."
+                " This is probably an error.\n");
+        }
+    }
+
+    if (Fabric->local_mr_req)
+    {
+        fi_close((struct fid *)rrmr);
+    }
+}
+
+static void CompletePush(CP_Services Svcs, Rdma_WSR_Stream Stream,
+                         TimestepList Step)
+{
+    Rdma_WS_Stream WS_Stream = Stream->WS_Stream;
+    FabricState Fabric = WS_Stream->Fabric;
+    TimestepList CQStep;
+    struct fi_cq_data_entry CQEntry = {0};
+    long CQTimestep;
+
+    while (Step->OutstandingWrites > 0)
+    {
+        fi_cq_sread(Fabric->cq_signal, (void *)(&CQEntry), 1, NULL, -1);
+        if (CQEntry.flags & FI_WRITE)
+        {
+            CQTimestep = (long)CQEntry.op_context;
+            if (CQTimestep == Step->Timestep)
+            {
+                CQStep = Step;
+            }
+            else
+            {
+                Svcs->verbose(WS_Stream->CP_Stream, DPCriticalVerbose,
+                              "while completing step %d, saw completion notice "
+                              "for step %d.\n",
+                              Step->Timestep, CQTimestep);
+
+                CQStep = GetStep(WS_Stream, CQTimestep);
+
+                if (!CQStep)
+                {
+                    Svcs->verbose(WS_Stream->CP_Stream, DPCriticalVerbose,
+                                  "received completion for step %d, which we "
+                                  "don't know about.\n",
+                                  CQTimestep);
+                }
+            }
+            CQStep->OutstandingWrites--;
+        }
+        else
+        {
+            Svcs->verbose(WS_Stream->CP_Stream, DPCriticalVerbose,
+                          "while waiting for push to complete, saw an unknown "
+                          "completion. This is probably an error.\n");
+        }
+    }
+}
+
+static void RdmaReleaseTimestepPerReader(CP_Services Svcs,
+                                         DP_WSR_Stream Stream_v, long Timestep)
+{
+    Rdma_WSR_Stream Stream = (Rdma_WSR_Stream)Stream_v;
+    Rdma_WS_Stream WS_Stream = Stream->WS_Stream;
+    TimestepList Step = GetStep(WS_Stream, Timestep);
+
+    if (!Step)
+    {
+        return;
+    }
+
+    if (Stream->Preload)
+    {
+        if (Stream->SelectionPulled)
+        {
+            // Make sure all writes for this timestep have completed
+            CompletePush(Svcs, Stream, Step);
+            // If data have been provided for the next timestep, push it
+            pthread_mutex_lock(&ts_mutex);
+            Stream->PreloadUsed[Step->Timestep & 1] = 0;
+            for (Step = Step->Next; Step; Step = Step->Next)
+            {
+                if (Step->BufferSlot == -1)
+                {
+                    if (Stream->PreloadUsed[Step->Timestep & 1] == 1)
+                    {
+                        Svcs->verbose(
+                            WS_Stream->CP_Stream, DPPerStepVerbose,
+                            "rank %d, RX preload buffers full, deferring"
+                            " preload of step %li.\n",
+                            WS_Stream->Rank, Step->Timestep);
+                    }
+                    else
+                    {
+                        PushData(Svcs, Stream, Step, Step->Timestep & 1);
+                        Stream->PreloadUsed[Step->Timestep & 1] = 1;
+                        Step->BufferSlot = Step->Timestep & 1;
+                    }
+                    break;
+                }
+            }
+            pthread_mutex_unlock(&ts_mutex);
+        }
+        else
+        {
+            PullSelection(Svcs, Stream);
+            Stream->PreloadUsed[0] = Stream->PreloadUsed[1] = 0;
+            Stream->SelectionPulled = 1;
+
+            pthread_mutex_lock(&ts_mutex);
+            Step = Step->Next;
+            while (Step && Stream->PreloadUsed[Step->Timestep & 1] == 0)
+            {
+                PushData(Svcs, Stream, Step, Step->Timestep & 1);
+                Stream->PreloadUsed[Step->Timestep & 1] = 1;
+                Step->BufferSlot = Step->Timestep & 1;
+                Step = Step->Next;
+            }
+            pthread_mutex_unlock(&ts_mutex);
+        }
+    }
 }
 
-extern CP_DP_Interface LoadRdmaDP()
+extern NO_SANITIZE_THREAD CP_DP_Interface LoadRdmaDP()
 {
-    memset(&RdmaDPInterface, 0, sizeof(RdmaDPInterface));
     RdmaDPInterface.ReaderContactFormats = RdmaReaderContactStructs;
     RdmaDPInterface.WriterContactFormats = RdmaWriterContactStructs;
-    RdmaDPInterface.TimestepInfoFormats = RdmaTimestepInfoStructs;
+    RdmaDPInterface.TimestepInfoFormats = RdmaBufferHandleStructs;
     RdmaDPInterface.initReader = RdmaInitReader;
     RdmaDPInterface.initWriter = RdmaInitWriter;
     RdmaDPInterface.initWriterPerReader = RdmaInitWriterPerReader;
@@ -1062,9 +2281,13 @@ extern CP_DP_Interface LoadRdmaDP()
     RdmaDPInterface.waitForCompletion = RdmaWaitForCompletion;
     RdmaDPInterface.notifyConnFailure = RdmaNotifyConnFailure;
     RdmaDPInterface.provideTimestep = RdmaProvideTimestep;
-    RdmaDPInterface.readerRegisterTimestep = NULL;
+    RdmaDPInterface.readerRegisterTimestep = RdmaReaderRegisterTimestep;
     RdmaDPInterface.releaseTimestep = RdmaReleaseTimestep;
-    RdmaDPInterface.readerReleaseTimestep = NULL;
+    RdmaDPInterface.readerReleaseTimestep = RdmaReleaseTimestepPerReader;
+    RdmaDPInterface.WSRreadPatternLocked = RdmaReadPatternLocked;
+    RdmaDPInterface.RSreadPatternLocked = RdmaWritePatternLocked;
+    RdmaDPInterface.RSReleaseTimestep = RdmaReaderReleaseTimestep;
+    RdmaDPInterface.timestepArrived = RdmaTimestepArrived;
     RdmaDPInterface.destroyReader = RdmaDestroyReader;
     RdmaDPInterface.destroyWriter = RdmaDestroyWriter;
     RdmaDPInterface.destroyWriterPerReader = RdmaDestroyWriterPerReader;
diff --git a/source/adios2/toolkit/sst/dp_interface.h b/source/adios2/toolkit/sst/dp_interface.h
index 74d6aa4c9a..f8906f4229 100644
--- a/source/adios2/toolkit/sst/dp_interface.h
+++ b/source/adios2/toolkit/sst/dp_interface.h
@@ -77,7 +77,8 @@ typedef void *CP_PeerCohort;
 typedef DP_RS_Stream (*CP_DP_InitReaderFunc)(CP_Services Svcs, void *CP_Stream,
                                              void **ReaderContactInfoPtr,
                                              struct _SstParams *Params,
-                                             attr_list WriterContactAttributes);
+                                             attr_list WriterContactAttributes,
+                                             SstStats Stats);
 
 /*!
  * CP_DP_DestroyReaderFunc is the type of a dataplane reader-side
@@ -100,7 +101,7 @@ typedef void (*CP_DP_DestroyReaderFunc)(CP_Services Svcs, DP_RS_Stream Reader);
  */
 typedef DP_WS_Stream (*CP_DP_InitWriterFunc)(CP_Services Svcs, void *CP_Stream,
                                              struct _SstParams *Params,
-                                             attr_list DPAttrs);
+                                             attr_list DPAttrs, SstStats Stats);
 
 /*!
  * CP_DP_DestroyWriterFunc is the type of a dataplane writer-side
@@ -383,8 +384,13 @@ struct _CP_DP_Interface
         getPriority; // both sides, part of DP selection process.
     CP_DP_UnGetPriorityFunc unGetPriority;
 };
+#define DPTraceVerbose 5
+#define DPPerRankVerbose 4
+#define DPPerStepVerbose 3
+#define DPSummaryVerbose 2
+#define DPCriticalVerbose 1
 
-typedef void (*CP_VerboseFunc)(void *CP_Stream, char *Format, ...);
+typedef void (*CP_VerboseFunc)(void *CP_Stream, int Level, char *Format, ...);
 typedef CManager (*CP_GetCManagerFunc)(void *CP_stream);
 typedef SMPI_Comm (*CP_GetMPICommFunc)(void *CP_Stream);
 typedef int (*CP_SendToPeerFunc)(void *CP_Stream, CP_PeerCohort PeerCohort,
@@ -398,6 +404,6 @@ struct _CP_Services
 };
 
 CP_DP_Interface SelectDP(CP_Services Svcs, void *CP_Stream,
-                         struct _SstParams *Params);
+                         struct _SstParams *Params, int Rank);
 
 #endif
diff --git a/source/adios2/toolkit/sst/sst.h b/source/adios2/toolkit/sst/sst.h
index 8ae6f77765..6f4fb64562 100644
--- a/source/adios2/toolkit/sst/sst.h
+++ b/source/adios2/toolkit/sst/sst.h
@@ -47,17 +47,6 @@ typedef enum
     SstLatestAvailable // reader advance mode
 } SstStepMode;
 
-/*
- * Struct that represents statistics tracked by SST
- */
-typedef struct _SstStats
-{
-    double OpenTimeSecs;
-    double CloseTimeSecs;
-    double ValidTimeSecs;
-    size_t BytesTransferred;
-} * SstStats;
-
 typedef struct _SstParams *SstParams;
 
 typedef enum
@@ -128,15 +117,15 @@ extern void SstReaderDefinitionLock(SstStream stream, long EffectiveTimestep);
  *  Calls that support FFS-based marshaling, source code in cp/ffs_marshal.c
  */
 typedef void *(*VarSetupUpcallFunc)(void *Reader, const char *Name,
-                                    const char *Type, void *Data);
+                                    const int Type, void *Data);
 typedef void (*AttrSetupUpcallFunc)(void *Reader, const char *Name,
-                                    const char *Type, void *Data);
+                                    const int Type, void *Data);
 typedef void *(*ArraySetupUpcallFunc)(void *Reader, const char *Name,
-                                      const char *Type, int DimsCount,
+                                      const int Type, int DimsCount,
                                       size_t *Shape, size_t *Start,
                                       size_t *Count);
 typedef void (*ArrayBlocksInfoUpcallFunc)(void *Reader, void *Variable,
-                                          const char *Type, int WriterRank,
+                                          const int Type, int WriterRank,
                                           int DimsCount, size_t *Shape,
                                           size_t *Start, size_t *Count);
 extern void SstReaderInitFFSCallback(
@@ -159,11 +148,11 @@ SstWriterInitMetadataCallback(SstStream stream, void *Writer,
                               FreeMetadataUpcallFunc FreeCallback);
 
 extern void SstFFSMarshal(SstStream Stream, void *Variable, const char *Name,
-                          const char *Type, size_t ElemSize, size_t DimCount,
+                          const int Type, size_t ElemSize, size_t DimCount,
                           const size_t *Shape, const size_t *Count,
                           const size_t *Offsets, const void *data);
 extern void SstFFSMarshalAttribute(SstStream Stream, const char *Name,
-                                   const char *Type, size_t ElemSize,
+                                   const int Type, size_t ElemSize,
                                    size_t ElemCount, const void *data);
 /* GetDeferred calls return true if need later sync */
 extern int SstFFSGetDeferred(SstStream Stream, void *Variable, const char *Name,
@@ -181,11 +170,6 @@ extern int SstFFSWriterBeginStep(SstStream Stream, int mode,
                                  const float timeout_sec);
 extern void SstFFSWriterEndStep(SstStream Stream, size_t Step);
 
-/*
- *  General Operations
- */
-extern void SstSetStatsSave(SstStream Stream, SstStats Save);
-
 #include "sst_data.h"
 
 #define SST_POSTFIX ".sst"
diff --git a/source/adios2/toolkit/sst/sst_data.h b/source/adios2/toolkit/sst/sst_data.h
index 1b988b53a8..e3490fdc03 100644
--- a/source/adios2/toolkit/sst/sst_data.h
+++ b/source/adios2/toolkit/sst/sst_data.h
@@ -25,10 +25,32 @@ struct _SstBlock
     char *BlockData;
 };
 
+/*
+ * Struct that represents statistics tracked by SST
+ */
+typedef struct _SstStats
+{
+    double StreamValidTimeSecs;
+    size_t BytesTransferred;
+    size_t TimestepsCreated;
+    size_t TimestepsDelivered;
+
+    size_t TimestepMetadataReceived;
+    size_t TimestepsConsumed;
+    size_t MetadataBytesReceived;
+    size_t DataBytesReceived;
+    size_t PreloadBytesReceived;
+    size_t PreloadTimestepsReceived;
+    size_t BytesRead;
+    double RunningFanIn;
+} * SstStats;
+
 #define SST_FOREACH_PARAMETER_TYPE_4ARGS(MACRO)                                \
     MACRO(MarshalMethod, MarshalMethod, size_t, SstMarshalBP)                  \
+    MACRO(verbose, Int, int, 0)                                                \
     MACRO(RegistrationMethod, RegMethod, size_t, 0)                            \
     MACRO(DataTransport, String, char *, NULL)                                 \
+    MACRO(WANDataTransport, String, char *, NULL)                              \
     MACRO(OpenTimeoutSecs, Int, int, 60)                                       \
     MACRO(RendezvousReaderCount, Int, int, 1)                                  \
     MACRO(QueueLimit, Int, int, 0)                                             \
@@ -45,6 +67,7 @@ struct _SstBlock
     MACRO(AlwaysProvideLatestTimestep, Bool, int, 0)                           \
     MACRO(SpeculativePreloadMode, SpecPreloadMode, int, SpecPreloadAuto)       \
     MACRO(SpecAutoNodeThreshold, Int, int, 1)                                  \
+    MACRO(ReaderShortCircuitReads, Bool, int, 0)                               \
     MACRO(ControlModule, String, char *, NULL)
 
 typedef enum
diff --git a/source/adios2/toolkit/sst/util/sst_conn_tool.c b/source/adios2/toolkit/sst/util/sst_conn_tool.c
index 53ca23e5da..f49ceb26d3 100644
--- a/source/adios2/toolkit/sst/util/sst_conn_tool.c
+++ b/source/adios2/toolkit/sst/util/sst_conn_tool.c
@@ -58,10 +58,10 @@ extern SMPI_Comm SMPI_COMM_WORLD;
 
 static atom_t TRANSPORT = -1;
 static atom_t IP_PORT = -1;
-static atom_t IP_HOSTNAME = -1;
+/* static atom_t IP_HOSTNAME = -1; */
 static atom_t IP_ADDR = -1;
 static atom_t ENET_PORT = -1;
-static atom_t ENET_HOSTNAME = -1;
+/* static atom_t ENET_HOSTNAME = -1; */
 static atom_t ENET_ADDR = -1;
 
 struct option options[] = {{"help", no_argument, NULL, 'h'},
@@ -117,10 +117,10 @@ static void do_connect();
 static void init_atoms()
 {
     TRANSPORT = attr_atom_from_string("CM_TRANSPORT");
-    IP_HOSTNAME = attr_atom_from_string("IP_HOST");
+    /* IP_HOSTNAME = attr_atom_from_string("IP_HOST"); */
     IP_PORT = attr_atom_from_string("IP_PORT");
     IP_ADDR = attr_atom_from_string("IP_ADDR");
-    ENET_HOSTNAME = attr_atom_from_string("CM_ENET_HOST");
+    /* ENET_HOSTNAME = attr_atom_from_string("CM_ENET_HOST"); */
     ENET_PORT = attr_atom_from_string("CM_ENET_PORT");
     ENET_ADDR = attr_atom_from_string("CM_ENET_ADDR");
 }
diff --git a/source/adios2/toolkit/transport/file/FilePOSIX.cpp b/source/adios2/toolkit/transport/file/FilePOSIX.cpp
index 271b3ac858..24540d2ff8 100644
--- a/source/adios2/toolkit/transport/file/FilePOSIX.cpp
+++ b/source/adios2/toolkit/transport/file/FilePOSIX.cpp
@@ -10,6 +10,8 @@
 #include "FilePOSIX.h"
 
 #include       // remove
+#include      // strerror
+#include      // errno
 #include      // open
 #include     // write output
 #include   // open, fstat
@@ -47,9 +49,7 @@ void FilePOSIX::WaitForOpen()
             m_FileDescriptor = m_OpenFuture.get();
         }
         m_IsOpening = false;
-        CheckFile(
-            "couldn't open file " + m_Name +
-            ", check permissions or path existence, in call to POSIX open");
+        CheckFile("couldn't open file " + m_Name + ", in call to POSIX open");
         m_IsOpen = true;
     }
 }
@@ -59,7 +59,9 @@ void FilePOSIX::Open(const std::string &name, const Mode openMode,
 {
     auto lf_AsyncOpenWrite = [&](const std::string &name) -> int {
         ProfilerStart("open");
+        errno = 0;
         int FD = open(m_Name.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0666);
+        m_Errno = errno;
         ProfilerStop("open");
         return FD;
     };
@@ -80,23 +82,29 @@ void FilePOSIX::Open(const std::string &name, const Mode openMode,
         else
         {
             ProfilerStart("open");
+            errno = 0;
             m_FileDescriptor =
                 open(m_Name.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0666);
+            m_Errno = errno;
             ProfilerStop("open");
         }
         break;
 
     case (Mode::Append):
         ProfilerStart("open");
+        errno = 0;
         // m_FileDescriptor = open(m_Name.c_str(), O_RDWR);
         m_FileDescriptor = open(m_Name.c_str(), O_RDWR | O_CREAT, 0777);
         lseek(m_FileDescriptor, 0, SEEK_END);
+        m_Errno = errno;
         ProfilerStop("open");
         break;
 
     case (Mode::Read):
         ProfilerStart("open");
+        errno = 0;
         m_FileDescriptor = open(m_Name.c_str(), O_RDONLY);
+        m_Errno = errno;
         ProfilerStop("open");
         break;
 
@@ -107,9 +115,7 @@ void FilePOSIX::Open(const std::string &name, const Mode openMode,
 
     if (!m_IsOpening)
     {
-        CheckFile(
-            "couldn't open file " + m_Name +
-            ", check permissions or path existence, in call to POSIX open");
+        CheckFile("couldn't open file " + m_Name + ", in call to POSIX open");
         m_IsOpen = true;
     }
 }
@@ -120,7 +126,9 @@ void FilePOSIX::Write(const char *buffer, size_t size, size_t start)
         while (size > 0)
         {
             ProfilerStart("write");
+            errno = 0;
             const auto writtenSize = write(m_FileDescriptor, buffer, size);
+            m_Errno = errno;
             ProfilerStop("write");
 
             if (writtenSize == -1)
@@ -132,7 +140,7 @@ void FilePOSIX::Write(const char *buffer, size_t size, size_t start)
 
                 throw std::ios_base::failure(
                     "ERROR: couldn't write to file " + m_Name +
-                    ", in call to FileDescriptor Write\n");
+                    ", in call to POSIX Write" + SysErrMsg());
             }
 
             buffer += writtenSize;
@@ -143,14 +151,16 @@ void FilePOSIX::Write(const char *buffer, size_t size, size_t start)
     WaitForOpen();
     if (start != MaxSizeT)
     {
+        errno = 0;
         const auto newPosition = lseek(m_FileDescriptor, start, SEEK_SET);
+        m_Errno = errno;
 
         if (static_cast(newPosition) != start)
         {
             throw std::ios_base::failure(
                 "ERROR: couldn't move to start position " +
                 std::to_string(start) + " in file " + m_Name +
-                ", in call to POSIX lseek\n");
+                ", in call to POSIX lseek" + SysErrMsg());
         }
     }
 
@@ -179,7 +189,9 @@ void FilePOSIX::Read(char *buffer, size_t size, size_t start)
         while (size > 0)
         {
             ProfilerStart("read");
+            errno = 0;
             const auto readSize = read(m_FileDescriptor, buffer, size);
+            m_Errno = errno;
             ProfilerStop("read");
 
             if (readSize == -1)
@@ -189,9 +201,9 @@ void FilePOSIX::Read(char *buffer, size_t size, size_t start)
                     continue;
                 }
 
-                throw std::ios_base::failure("ERROR: couldn't read from file " +
-                                             m_Name +
-                                             ", in call to POSIX IO read\n");
+                throw std::ios_base::failure(
+                    "ERROR: couldn't read from file " + m_Name +
+                    ", in call to POSIX IO read" + SysErrMsg());
             }
 
             buffer += readSize;
@@ -203,15 +215,16 @@ void FilePOSIX::Read(char *buffer, size_t size, size_t start)
 
     if (start != MaxSizeT)
     {
+        errno = 0;
         const auto newPosition = lseek(m_FileDescriptor, start, SEEK_SET);
+        m_Errno = errno;
 
         if (static_cast(newPosition) != start)
         {
             throw std::ios_base::failure(
                 "ERROR: couldn't move to start position " +
                 std::to_string(start) + " in file " + m_Name +
-                ", in call to POSIX lseek errno " + std::to_string(errno) +
-                "\n");
+                ", in call to POSIX lseek" + SysErrMsg());
         }
     }
 
@@ -238,11 +251,14 @@ size_t FilePOSIX::GetSize()
 {
     struct stat fileStat;
     WaitForOpen();
+    errno = 0;
     if (fstat(m_FileDescriptor, &fileStat) == -1)
     {
+        m_Errno = errno;
         throw std::ios_base::failure("ERROR: couldn't get size of file " +
-                                     m_Name + "\n");
+                                     m_Name + SysErrMsg());
     }
+    m_Errno = errno;
     return static_cast(fileStat.st_size);
 }
 
@@ -252,13 +268,16 @@ void FilePOSIX::Close()
 {
     WaitForOpen();
     ProfilerStart("close");
+    errno = 0;
     const int status = close(m_FileDescriptor);
+    m_Errno = errno;
     ProfilerStop("close");
 
     if (status == -1)
     {
         throw std::ios_base::failure("ERROR: couldn't close file " + m_Name +
-                                     ", in call to POSIX IO close\n");
+                                     ", in call to POSIX IO close" +
+                                     SysErrMsg());
     }
 
     m_IsOpen = false;
@@ -278,31 +297,41 @@ void FilePOSIX::CheckFile(const std::string hint) const
 {
     if (m_FileDescriptor == -1)
     {
-        throw std::ios_base::failure("ERROR: " + hint + "\n");
+        throw std::ios_base::failure("ERROR: " + hint + SysErrMsg());
     }
 }
 
+std::string FilePOSIX::SysErrMsg() const
+{
+    return std::string(": errno = " + std::to_string(m_Errno) + ": " +
+                       strerror(m_Errno));
+}
+
 void FilePOSIX::SeekToEnd()
 {
     WaitForOpen();
+    errno = 0;
     const int status = lseek(m_FileDescriptor, 0, SEEK_END);
+    m_Errno = 0;
     if (status == -1)
     {
         throw std::ios_base::failure(
             "ERROR: couldn't seek to the end of file " + m_Name +
-            ", in call to POSIX IO lseek\n");
+            ", in call to POSIX IO lseek" + SysErrMsg());
     }
 }
 
 void FilePOSIX::SeekToBegin()
 {
     WaitForOpen();
+    errno = 0;
     const int status = lseek(m_FileDescriptor, 0, SEEK_SET);
+    m_Errno = errno;
     if (status == -1)
     {
         throw std::ios_base::failure(
             "ERROR: couldn't seek to the begin of file " + m_Name +
-            ", in call to POSIX IO lseek\n");
+            ", in call to POSIX IO lseek" + SysErrMsg());
     }
 }
 
diff --git a/source/adios2/toolkit/transport/file/FilePOSIX.h b/source/adios2/toolkit/transport/file/FilePOSIX.h
index cc3e361a6c..4834d02b8d 100644
--- a/source/adios2/toolkit/transport/file/FilePOSIX.h
+++ b/source/adios2/toolkit/transport/file/FilePOSIX.h
@@ -57,6 +57,7 @@ class FilePOSIX : public Transport
 private:
     /** POSIX file handle returned by Open */
     int m_FileDescriptor = -1;
+    int m_Errno = 0;
     bool m_IsOpening = false;
     std::future m_OpenFuture;
 
@@ -66,6 +67,7 @@ class FilePOSIX : public Transport
      */
     void CheckFile(const std::string hint) const;
     void WaitForOpen();
+    std::string SysErrMsg() const;
 };
 
 } // end namespace transport
diff --git a/source/adios2/toolkit/transportman/TransportMan.cpp b/source/adios2/toolkit/transportman/TransportMan.cpp
index 0207bef7be..a094c0f44d 100644
--- a/source/adios2/toolkit/transportman/TransportMan.cpp
+++ b/source/adios2/toolkit/transportman/TransportMan.cpp
@@ -39,19 +39,26 @@ namespace transportman
 TransportMan::TransportMan(helper::Comm &comm) : m_Comm(comm) {}
 
 void TransportMan::MkDirsBarrier(const std::vector &fileNames,
+                                 const std::vector ¶metersVector,
                                  const bool nodeLocal)
 {
     auto lf_CreateDirectories = [&](const std::vector &fileNames) {
-        for (const std::string fileName : fileNames)
+        for (size_t i = 0; i < fileNames.size(); ++i)
         {
-            const auto lastPathSeparator(fileName.find_last_of(PathSeparator));
+            const auto lastPathSeparator(
+                fileNames[i].find_last_of(PathSeparator));
             if (lastPathSeparator == std::string::npos)
             {
                 continue;
             }
-
-            const std::string path(fileName.substr(0, lastPathSeparator));
-            helper::CreateDirectory(path);
+            const Params ¶meters = parametersVector[i];
+            const std::string type = parameters.at("transport");
+            if (type == "File" || type == "file")
+            {
+                const std::string path(
+                    fileNames[i].substr(0, lastPathSeparator));
+                helper::CreateDirectory(path);
+            }
         }
     };
 
@@ -194,26 +201,68 @@ void TransportMan::WriteFiles(const char *buffer, const size_t size,
 void TransportMan::WriteFileAt(const char *buffer, const size_t size,
                                const size_t start, const int transportIndex)
 {
-    auto itTransport = m_Transports.find(transportIndex);
-    CheckFile(itTransport, ", in call to WriteFileAt with index " +
-                               std::to_string(transportIndex));
-    itTransport->second->Write(buffer, size, start);
+    if (transportIndex == -1)
+    {
+        for (auto &transportPair : m_Transports)
+        {
+            auto &transport = transportPair.second;
+            if (transport->m_Type == "File")
+            {
+                transport->Write(buffer, size, start);
+            }
+        }
+    }
+    else
+    {
+        auto itTransport = m_Transports.find(transportIndex);
+        CheckFile(itTransport, ", in call to WriteFileAt with index " +
+                                   std::to_string(transportIndex));
+        itTransport->second->Write(buffer, size, start);
+    }
 }
 
 void TransportMan::SeekToFileEnd(const int transportIndex)
 {
-    auto itTransport = m_Transports.find(transportIndex);
-    CheckFile(itTransport, ", in call to SeekToFileEnd with index " +
-                               std::to_string(transportIndex));
-    itTransport->second->SeekToEnd();
+    if (transportIndex == -1)
+    {
+        for (auto &transportPair : m_Transports)
+        {
+            auto &transport = transportPair.second;
+            if (transport->m_Type == "File")
+            {
+                transport->SeekToEnd();
+            }
+        }
+    }
+    else
+    {
+        auto itTransport = m_Transports.find(transportIndex);
+        CheckFile(itTransport, ", in call to SeekToFileEnd with index " +
+                                   std::to_string(transportIndex));
+        itTransport->second->SeekToEnd();
+    }
 }
 
 void TransportMan::SeekToFileBegin(const int transportIndex)
 {
-    auto itTransport = m_Transports.find(transportIndex);
-    CheckFile(itTransport, ", in call to SeekToFileBegin with index " +
-                               std::to_string(transportIndex));
-    itTransport->second->SeekToBegin();
+    if (transportIndex == -1)
+    {
+        for (auto &transportPair : m_Transports)
+        {
+            auto &transport = transportPair.second;
+            if (transport->m_Type == "File")
+            {
+                transport->SeekToBegin();
+            }
+        }
+    }
+    else
+    {
+        auto itTransport = m_Transports.find(transportIndex);
+        CheckFile(itTransport, ", in call to SeekToFileBegin with index " +
+                                   std::to_string(transportIndex));
+        itTransport->second->SeekToBegin();
+    }
 }
 
 size_t TransportMan::GetFileSize(const size_t transportIndex) const
diff --git a/source/adios2/toolkit/transportman/TransportMan.h b/source/adios2/toolkit/transportman/TransportMan.h
index ffdb917e2e..35544cf70c 100644
--- a/source/adios2/toolkit/transportman/TransportMan.h
+++ b/source/adios2/toolkit/transportman/TransportMan.h
@@ -56,6 +56,7 @@ class TransportMan
      * @param nodeLocal true: all ranks create a directory
      */
     void MkDirsBarrier(const std::vector &fileNames,
+                       const std::vector ¶metersVector,
                        const bool nodeLocal);
 
     /**
@@ -161,9 +162,9 @@ class TransportMan
     /** Checks if all transports are closed */
     bool AllTransportsClosed() const noexcept;
 
-    void SeekToFileEnd(const int transportIndex = 0);
+    void SeekToFileEnd(const int transportIndex = -1);
 
-    void SeekToFileBegin(const int transportIndex = 0);
+    void SeekToFileBegin(const int transportIndex = -1);
 
     /**
      * Check if a file exists.
diff --git a/source/utils/adios_iotest/adios_iotest.cpp b/source/utils/adios_iotest/adios_iotest.cpp
index 330137e77c..3ecdacb2f5 100644
--- a/source/utils/adios_iotest/adios_iotest.cpp
+++ b/source/utils/adios_iotest/adios_iotest.cpp
@@ -327,8 +327,8 @@ int main(int argc, char *argv[])
     timeEnd = MPI_Wtime();
     if (!settings.myRank)
     {
-        std::cout << "ADIOS IOTEST test time " << timeEnd - timeStart
-                  << "  seconds " << std::endl;
+        std::cout << "ADIOS IOTEST App " << settings.appId << " total time "
+                  << timeEnd - timeStart << " seconds " << std::endl;
     }
 
     MPI_Finalize();
diff --git a/source/utils/adios_iotest/settings.cpp b/source/utils/adios_iotest/settings.cpp
index bab37393a8..77dccb606d 100644
--- a/source/utils/adios_iotest/settings.cpp
+++ b/source/utils/adios_iotest/settings.cpp
@@ -10,6 +10,8 @@
 
 #include "settings.h"
 
+#include 
+#include 
 #include 
 #include 
 #include 
@@ -20,6 +22,7 @@ struct option options[] = {{"help", no_argument, NULL, 'h'},
                            {"appid", required_argument, NULL, 'a'},
                            {"config", required_argument, NULL, 'c'},
                            {"decomp", required_argument, NULL, 'd'},
+                           {"decomp-ratio", required_argument, NULL, 'D'},
                            {"xml", required_argument, NULL, 'x'},
                            {"strong-scaling", no_argument, NULL, 's'},
                            {"weak-scaling", no_argument, NULL, 'w'},
@@ -30,7 +33,7 @@ struct option options[] = {{"help", no_argument, NULL, 'h'},
 #endif
                            {NULL, 0, NULL, 0}};
 
-static const char *optstring = "-hvswtFHa:c:d:x:";
+static const char *optstring = "-hvswtFHa:c:d:D:x:";
 
 size_t Settings::ndigits(size_t n) const
 {
@@ -42,7 +45,8 @@ size_t Settings::ndigits(size_t n) const
 void Settings::displayHelp()
 {
     std::cout
-        << "Usage: adios_iotest -a appid -c config {-s | -w} -d d1 [d2 .. dN] "
+        << "Usage: adios_iotest -a appid -c config {-s | -w} {-d d1[,d2,..,dN] "
+           "| -D r1[,r2,..,rN]}"
            "[-x "
            "file]\n"
         << "  -a appID:  unique number for each application in the workflow\n"
@@ -51,6 +55,13 @@ void Settings::displayHelp()
         << "      d1:        number of processes in 1st (slowest) dimension\n"
         << "      dN:        number of processes in Nth dimension\n"
         << "                 d1*d2*..*dN must equal the number of processes\n"
+        << "   -D ...    define process decomposition ratio:\n"
+        << "      r1:        ratio of process decomposition in the 1st "
+           "(slowest) dimension\n"
+        << "      rN:        ratio of process decomposition in the Nth "
+           "dimension\n"
+        << "                 r1xr2x..xrN must scale up to process count"
+           "count without remainder\n"
         << "  -s OR -w:  strong or weak scaling. \n"
         << "             Dimensions in config are treated accordingly\n"
         << "  -x file    ADIOS configuration XML file\n"
@@ -77,10 +88,56 @@ size_t Settings::stringToNumber(const std::string &varName,
     return retval;
 }
 
+int Settings::parseCSDecomp(const char *arg)
+{
+    char *argCopy = (char *)malloc(strlen(arg) * sizeof(*argCopy));
+    char *ratio;
+
+    strcpy(argCopy, arg);
+    ratio = strtok(argCopy, ",");
+    while (ratio)
+    {
+        processDecomp[nDecomp++] = stringToNumber("decomposition ratio", ratio);
+        ratio = strtok(NULL, ",");
+    }
+
+    free(argCopy);
+
+    return (0);
+}
+
+int Settings::rescaleDecomp()
+{
+    size_t ratioProd = 1;
+    size_t scaleFactor;
+
+    for (size_t i = 0; i < nDecomp; i++)
+    {
+        ratioProd *= processDecomp[i];
+    }
+
+    for (scaleFactor = 1; ratioProd * pow(scaleFactor, nDecomp) <= nProc;
+         scaleFactor++)
+    {
+        if (ratioProd * pow(scaleFactor, nDecomp) == nProc)
+        {
+            for (size_t i = 0; i < nDecomp; i++)
+            {
+                processDecomp[i] *= scaleFactor;
+            }
+            return (0);
+        }
+    }
+
+    throw std::invalid_argument(
+        "decomposition ratios must scale up to process count");
+}
+
 int Settings::processArgs(int argc, char *argv[])
 {
     bool appIdDefined = false;
     bool scalingDefined = false;
+    bool decompDefined = false;
     int c;
     int last_c = '_';
 
@@ -97,9 +154,41 @@ int Settings::processArgs(int argc, char *argv[])
             configFileName = optarg;
             break;
         case 'd':
-            processDecomp[nDecomp] =
-                stringToNumber("decomposition in dimension 1", optarg);
-            ++nDecomp;
+            if (decompDefined && isRatioDecomp)
+            {
+                throw std::invalid_argument(
+                    "Cannot have -D and -d used at the same time ");
+            }
+            if (strchr(optarg, ','))
+            {
+                parseCSDecomp(optarg);
+            }
+            else
+            {
+                processDecomp[nDecomp] =
+                    stringToNumber("decomposition in dimension 1", optarg);
+                ++nDecomp;
+            }
+            decompDefined = true;
+            break;
+        case 'D':
+            if (decompDefined && !isRatioDecomp)
+            {
+                throw std::invalid_argument(
+                    "Cannot have -D and -d used at the same time ");
+            }
+            if (strchr(optarg, ','))
+            {
+                parseCSDecomp(optarg);
+            }
+            else
+            {
+                processDecomp[nDecomp] =
+                    stringToNumber("decomposition in dimension 1", optarg);
+                ++nDecomp;
+            }
+            decompDefined = true;
+            isRatioDecomp = true;
             break;
         case 'F':
             fixedPattern = true;
@@ -146,7 +235,7 @@ int Settings::processArgs(int argc, char *argv[])
             /* This means a field is unknown, or could be multiple arg or bad
              * arg*/
 
-            if (last_c == 'd')
+            if (last_c == 'd' || last_c == 'D')
             { // --decomp extra arg (or not if not a number)
                 processDecomp[nDecomp] = stringToNumber(
                     "decomposition in dimension " + std::to_string(nDecomp + 1),
@@ -220,6 +309,11 @@ int Settings::processArguments(int argc, char *argv[], MPI_Comm worldComm)
         MPI_Comm_size(appComm, &nproc);
         myRank = static_cast(rank);
         nProc = static_cast(nproc);
+
+        if (isRatioDecomp)
+        {
+            rescaleDecomp();
+        }
     }
     catch (std::exception &e) // command-line argument errors
     {
diff --git a/source/utils/adios_iotest/settings.h b/source/utils/adios_iotest/settings.h
index 647dd7aa11..c3b2a4775c 100644
--- a/source/utils/adios_iotest/settings.h
+++ b/source/utils/adios_iotest/settings.h
@@ -40,6 +40,7 @@ class Settings
     bool isStrongScaling = true; // strong or weak scaling
     bool ioTimer = false;        // used to measure io time
     bool fixedPattern = false;   // should Lock definitions?
+    bool isRatioDecomp = false;
     IOLib iolib = IOLib::ADIOS;
     //   process decomposition
     std::vector processDecomp = {1, 1, 1, 1, 1, 1, 1, 1,
@@ -57,6 +58,8 @@ class Settings
     int processArguments(int argc, char *argv[], MPI_Comm worldComm);
     int extraArgumentChecks();
     size_t stringToNumber(const std::string &varName, const char *arg) const;
+    int parseCSDecomp(const char *arg);
+    int rescaleDecomp();
     size_t ndigits(size_t n) const;
 
 private:
diff --git a/source/utils/adios_reorganize/Reorganize.cpp b/source/utils/adios_reorganize/Reorganize.cpp
index 7ebb3fec41..d09372a2b1 100644
--- a/source/utils/adios_reorganize/Reorganize.cpp
+++ b/source/utils/adios_reorganize/Reorganize.cpp
@@ -194,8 +194,8 @@ void Reorganize::Run()
         }
 
         curr_step = static_cast(rStream.CurrentStep());
-        const core::DataMap &variables = io.GetVariablesDataMap();
-        const core::DataMap &attributes = io.GetAttributesDataMap();
+        const core::VarMap &variables = io.GetVariables();
+        const core::AttrMap &attributes = io.GetAttributes();
 
         print0("____________________\n\nFile info:");
         print0("  current step:   ", curr_step);
@@ -480,8 +480,8 @@ Reorganize::Decompose(int numproc, int rank, VarInfo &vi,
 }
 
 int Reorganize::ProcessMetadata(core::Engine &rStream, core::IO &io,
-                                const core::DataMap &variables,
-                                const core::DataMap &attributes, int step)
+                                const core::VarMap &variables,
+                                const core::AttrMap &attributes, int step)
 {
     int retval = 0;
 
@@ -494,17 +494,17 @@ int Reorganize::ProcessMetadata(core::Engine &rStream, core::IO &io,
     for (const auto &variablePair : variables)
     {
         const std::string &name(variablePair.first);
-        const std::string &type(variablePair.second.first);
+        const DataType type(variablePair.second->m_Type);
         core::VariableBase *variable = nullptr;
         print0("Get info on variable ", varidx, ": ", name);
         size_t nBlocks = 1;
 
-        if (type == "compound")
+        if (type == DataType::Compound)
         {
             // not supported
         }
 #define declare_template_instantiation(T)                                      \
-    else if (type == helper::GetType())                                     \
+    else if (type == helper::GetDataType())                                 \
     {                                                                          \
         core::Variable *v = io.InquireVariable(variablePair.first);      \
         if (v->m_ShapeID == adios2::ShapeID::LocalArray)                       \
@@ -526,7 +526,7 @@ int Reorganize::ProcessMetadata(core::Engine &rStream, core::IO &io,
             // print variable type and dimensions
             if (!m_Rank)
             {
-                std::cout << "    " << type << " " << name;
+                std::cout << "    " << ToString(type) << " " << name;
             }
             // if (variable->GetShape().size() > 0)
             if (variable->m_ShapeID == adios2::ShapeID::GlobalArray)
@@ -608,8 +608,7 @@ int Reorganize::ProcessMetadata(core::Engine &rStream, core::IO &io,
 }
 
 int Reorganize::ReadWrite(core::Engine &rStream, core::Engine &wStream,
-                          core::IO &io, const core::DataMap &variables,
-                          int step)
+                          core::IO &io, const core::VarMap &variables, int step)
 {
     int retval = 0;
 
@@ -639,13 +638,13 @@ int Reorganize::ReadWrite(core::Engine &rStream, core::Engine &wStream,
                 // read variable subset
                 std::cout << "rank " << m_Rank << ": Read variable " << name
                           << std::endl;
-                const std::string &type = variables.at(name).first;
-                if (type == "compound")
+                const DataType type = variables.at(name)->m_Type;
+                if (type == DataType::Compound)
                 {
                     // not supported
                 }
 #define declare_template_instantiation(T)                                      \
-    else if (type == helper::GetType())                                     \
+    else if (type == helper::GetDataType())                                 \
     {                                                                          \
         varinfo[varidx].readbuf = calloc(1, varinfo[varidx].writesize);        \
         if (varinfo[varidx].count.size() == 0)                                 \
@@ -683,13 +682,13 @@ int Reorganize::ReadWrite(core::Engine &rStream, core::Engine &wStream,
                 // Write variable subset
                 std::cout << "rank " << m_Rank << ": Write variable " << name
                           << std::endl;
-                const std::string &type = variables.at(name).first;
-                if (type == "compound")
+                const DataType type = variables.at(name)->m_Type;
+                if (type == DataType::Compound)
                 {
                     // not supported
                 }
 #define declare_template_instantiation(T)                                      \
-    else if (type == helper::GetType())                                     \
+    else if (type == helper::GetDataType())                                 \
     {                                                                          \
         if (varinfo[varidx].count.size() == 0)                                 \
         {                                                                      \
diff --git a/source/utils/adios_reorganize/Reorganize.h b/source/utils/adios_reorganize/Reorganize.h
index 948f617194..9a41827bb2 100644
--- a/source/utils/adios_reorganize/Reorganize.h
+++ b/source/utils/adios_reorganize/Reorganize.h
@@ -11,7 +11,7 @@
 #ifndef UTILS_REORGANIZE_REORGANIZE_H_
 #define UTILS_REORGANIZE_REORGANIZE_H_
 
-#include "adios2/core/IO.h" // DataMap
+#include "adios2/core/IO.h"
 #include "adios2/helper/adiosComm.h"
 #include "utils/Utils.h"
 
@@ -20,7 +20,7 @@ namespace adios2
 namespace utils
 {
 
-typedef struct
+struct VarInfo
 {
     core::VariableBase *v = nullptr;
     std::string type;
@@ -28,7 +28,7 @@ typedef struct
     Dims count;
     size_t writesize = 0; // size of subset this process writes, 0: do not write
     void *readbuf = nullptr; // read in buffer
-} VarInfo;
+};
 
 class Reorganize : public Utils
 {
@@ -61,10 +61,10 @@ class Reorganize : public Utils
                      const int *np // number of processes in each dimension
     );
     int ProcessMetadata(core::Engine &rStream, core::IO &io,
-                        const core::DataMap &variables,
-                        const core::DataMap &attributes, int step);
+                        const core::VarMap &variables,
+                        const core::AttrMap &attributes, int step);
     int ReadWrite(core::Engine &rStream, core::Engine &wStream, core::IO &io,
-                  const core::DataMap &variables, int step);
+                  const core::VarMap &variables, int step);
     Params parseParams(const std::string ¶m_str);
 
     // Input arguments
diff --git a/source/utils/bp4dbg/adios2/bp4dbg/data.py b/source/utils/bp4dbg/adios2/bp4dbg/data.py
index b4077bf7ac..3b1f54e9c5 100644
--- a/source/utils/bp4dbg/adios2/bp4dbg/data.py
+++ b/source/utils/bp4dbg/adios2/bp4dbg/data.py
@@ -1,5 +1,3 @@
-from __future__ import (absolute_import, division, print_function,
-                        unicode_literals)
 import numpy as np
 from os import fstat
 from .utils import *
@@ -16,7 +14,7 @@ def ReadEncodedString(f, ID, limit, lensize=2):
         print("CODING ERROR: bp4dbp_data.ReadEncodedString: "
               "lensize must be 2 or 4")
         return False, ""
-    if (namelen > limit):
+    if namelen > limit:
         print("ERROR: " + ID + " string length ({0}) is longer than the "
               "limit to stay inside the block ({1})".format(
                   namelen, limit))
@@ -31,7 +29,7 @@ def ReadEncodedStringArray(f, ID, limit, nStrings):
         # 2 bytes length + string
         # !!! String here INCLUDES Terminating \0 !!!
         namelen = np.fromfile(f, dtype=np.uint32, count=1)[0]
-        if (namelen > limit - 4):
+        if namelen > limit - 4:
             print("ERROR: " + ID + " string length ({0}) is longer than the "
                   "limit to stay inside the block ({1})".format(
                       namelen, limit - 4))
@@ -169,7 +167,7 @@ def ReadVarData(f, nElements, typeID, ldims, varLen,
     if typeID == 9:  # string type
         return ReadStringVarData(f, varLen, varsStartPosition)
     typeSize = GetTypeSize(typeID)
-    if (typeSize == 0):
+    if typeSize == 0:
         print("ERROR: Cannot process variable data block with "
               "unknown type size")
         return False
@@ -177,7 +175,7 @@ def ReadVarData(f, nElements, typeID, ldims, varLen,
     currentPosition = f.tell()
     print("      Payload offset  : {0}".format(currentPosition))
 
-    if (currentPosition + varLen > varsStartPosition + varsTotalLength):
+    if currentPosition + varLen > varsStartPosition + varsTotalLength:
         print("ERROR: Variable data block of size would reach beyond all "
               "variable blocks")
         print("VarsStartPosition = {0} varsTotalLength = {1}".format(
@@ -213,7 +211,7 @@ def ReadVMD(f, varidx, varsStartPosition, varsTotalLength):
     print("      Starting offset : {0}".format(startPosition))
     # 4 bytes TAG
     tag = f.read(4)
-    if (tag != b"[VMD"):
+    if tag != b"[VMD":
         print("  Tag: " + str(tag))
         print("ERROR: VAR group does not start with [VMD")
         return False
@@ -224,8 +222,8 @@ def ReadVMD(f, varidx, varsStartPosition, varsTotalLength):
     print("      Var block size  : {0} bytes (+4 for Tag)".format(vmdlen))
     expectedVarBlockLength = vmdlen + 4  # [VMD is not included in vmdlen
 
-    if (startPosition + expectedVarBlockLength >
-            varsStartPosition + varsTotalLength):
+    if startPosition + expectedVarBlockLength > \
+            varsStartPosition + varsTotalLength:
         print("ERROR: There is not enough bytes inside this PG to read "
               "this Var block")
         print("VarsStartPosition = {0} varsTotalLength = {1}".format(
@@ -299,13 +297,13 @@ def ReadVMD(f, varidx, varsStartPosition, varsTotalLength):
         # Read Local Dimensions (1 byte flag + 8 byte value)
         # Is Dimension a variable ID 1 byte, 'y' or 'n' or '\0'
         isDimensionVarID = f.read(1)
-        if (isDimensionVarID != b'y' and isDimensionVarID != b'n' and
-                isDimensionVarID != b'\0'):
+        if isDimensionVarID != b'y' and isDimensionVarID != b'n' and \
+                isDimensionVarID != b'\0':
             print(
                 "ERROR: Next byte for isDimensionVarID must be 'y' or 'n' "
                 "but it isn't = {0}".format(isDimensionVarID))
             return False
-        if (isDimensionVarID == b'\0'):
+        if isDimensionVarID == b'\0':
             isDimensionVarID = b'n'
         ldims[i] = np.fromfile(f, dtype=np.uint64, count=1)[0]
         print("           local  dim : {0}".format(ldims[i]))
@@ -313,13 +311,13 @@ def ReadVMD(f, varidx, varsStartPosition, varsTotalLength):
         # Read Global Dimensions (1 byte flag + 8 byte value)
         # Is Dimension a variable ID 1 byte, 'y' or 'n' or '\0'
         isDimensionVarID = f.read(1)
-        if (isDimensionVarID != b'y' and isDimensionVarID != b'n' and
-                isDimensionVarID != b'\0'):
+        if isDimensionVarID != b'y' and isDimensionVarID != b'n' \
+                and isDimensionVarID != b'\0':
             print(
                 "ERROR: Next byte for isDimensionVarID must be 'y' or 'n' "
                 "but it isn't = {0}".format(isDimensionVarID))
             return False
-        if (isDimensionVarID == b'\0'):
+        if isDimensionVarID == b'\0':
             isDimensionVarID = b'n'
         gdim = np.fromfile(f, dtype=np.uint64, count=1)[0]
         if i == 0 and ldims[i] == 0 and gdim == LocalValueDim:
@@ -331,13 +329,13 @@ def ReadVMD(f, varidx, varsStartPosition, varsTotalLength):
         # Read Offset Dimensions (1 byte flag + 8 byte value)
         # Is Dimension a variable ID 1 byte, 'y' or 'n' or '\0'
         isDimensionVarID = f.read(1)
-        if (isDimensionVarID != b'y' and isDimensionVarID != b'n' and
-                isDimensionVarID != b'\0'):
+        if isDimensionVarID != b'y' and isDimensionVarID != b'n' and \
+                isDimensionVarID != b'\0':
             print(
                 "ERROR: Next byte for isDimensionVarID must be 'y' or 'n' "
                 "but it isn't = {0}".format(isDimensionVarID))
             return False
-        if (isDimensionVarID == b'\0'):
+        if isDimensionVarID == b'\0':
             isDimensionVarID = b'n'
         offset = np.fromfile(f, dtype=np.uint64, count=1)[0]
         print("           offset dim : {0}".format(offset))
@@ -351,7 +349,7 @@ def ReadVMD(f, varidx, varsStartPosition, varsTotalLength):
     # 1 byte length of tag
     endTagLen = np.fromfile(f, dtype=np.uint8, count=1)[0]
     tag = f.read(endTagLen)
-    if (not tag.endswith(b"VMD]")):
+    if not tag.endswith(b"VMD]"):
         print("  Tag: " + str(tag))
         print("ERROR: VAR group metadata does not end with VMD]")
         return False
@@ -382,7 +380,7 @@ def ReadAMD(f, attridx, attrsStartPosition, attrsTotalLength):
     print("      Starting offset : {0}".format(startPosition))
     # 4 bytes TAG
     tag = f.read(4)
-    if (tag != b"[AMD"):
+    if tag != b"[AMD":
         print("  Tag: " + str(tag))
         print("ERROR: ATTR group does not start with [AMD")
         return False
@@ -392,8 +390,8 @@ def ReadAMD(f, attridx, attrsStartPosition, attrsTotalLength):
     amdlen = np.fromfile(f, dtype=np.uint32, count=1)[0]
     print("      Attr block size : {0} bytes (+4 for Tag)".format(amdlen))
     expectedAttrBlockLength = amdlen + 4  # [AMD is not included in amdlen
-    if (startPosition + expectedAttrBlockLength >
-            attrsStartPosition + attrsTotalLength):
+    if startPosition + expectedAttrBlockLength > \
+            attrsStartPosition + attrsTotalLength:
         print("ERROR: There is not enough bytes inside this PG "
               "to read this Attr block")
         print("AttrsStartPosition = {0} attrsTotalLength = {1}".format(
@@ -423,7 +421,7 @@ def ReadAMD(f, attridx, attrsStartPosition, attrsTotalLength):
 
     # isAttrAVar 1 byte, 'y' or 'n'
     isAttrAVar = f.read(1)
-    if (isAttrAVar != b'y' and isAttrAVar != b'n'):
+    if isAttrAVar != b'y' and isAttrAVar != b'n':
         print(
             "ERROR: Next byte for isAttrAVar must be 'y' or 'n' "
             "but it isn't = {0}".format(isAttrAVar))
@@ -471,7 +469,7 @@ def ReadAMD(f, attridx, attrsStartPosition, attrsTotalLength):
 
     # End TAG AMD]
     tag = f.read(4)
-    if (tag != b"AMD]"):
+    if tag != b"AMD]":
         print("  Tag: " + str(tag))
         print("ERROR: PG group metadata does not end with AMD]")
         return False
@@ -490,7 +488,7 @@ def ReadPG(f, fileSize, pgidx):
     print("Process Group {0}: ".format(pgidx))
     print("  Starting offset : {0}".format(pgStartPosition))
     tag = f.read(4)
-    if (tag != b"[PGI"):
+    if tag != b"[PGI":
         print("  Tag: " + str(tag))
         print("ERROR: PG group does not start with [PGI")
         return False
@@ -502,13 +500,13 @@ def ReadPG(f, fileSize, pgidx):
     print("  PG length       : {0} bytes (+4 for Tag)".format(pglen))
     # pglen does not include the opening tag 4 bytes:
     expectedPGLength = pglen + 4
-    if (pgStartPosition + expectedPGLength > fileSize):
+    if pgStartPosition + expectedPGLength > fileSize:
         print("ERROR: There is not enough bytes in file to read this PG")
         return False
 
     # ColumnMajor (host language Fortran) 1 byte, 'y' or 'n'
     isColumnMajor = f.read(1)
-    if (isColumnMajor != b'y' and isColumnMajor != b'n'):
+    if isColumnMajor != b'y' and isColumnMajor != b'n':
         print(
             "ERROR: Next byte for isColumnMajor must be 'y' or 'n' "
             "but it isn't = {0}".format(isColumnMajor))
@@ -568,7 +566,7 @@ def ReadPG(f, fileSize, pgidx):
     print("  Vars length     : {0} bytes".format(varlen))
     sizeLimit = expectedPGLength - (f.tell() - pgStartPosition)
     expectedVarsLength = varlen  # need to read this more
-    if (expectedVarsLength > sizeLimit):
+    if expectedVarsLength > sizeLimit:
         print("ERROR: There is not enough bytes in PG to read the variables")
         return False
 
@@ -593,7 +591,7 @@ def ReadPG(f, fileSize, pgidx):
     sizeLimit = expectedPGLength - (attrsStartPosition - pgStartPosition) - 4
     expectedAttrsLength = attlen  # need to read this more before reaching PGI]
 
-    if (expectedAttrsLength > sizeLimit):
+    if expectedAttrsLength > sizeLimit:
         print("ERROR: There is not enough bytes in PG to read the attributes")
         return False
 
@@ -606,7 +604,7 @@ def ReadPG(f, fileSize, pgidx):
 
     # End TAG PGI]
     tag = f.read(4)
-    if (tag != b"PGI]"):
+    if tag != b"PGI]":
         print("  Tag: " + str(tag))
         print("ERROR: PG group metadata does not end with PGI]")
         return False
diff --git a/source/utils/bp4dbg/adios2/bp4dbg/idxtable.py b/source/utils/bp4dbg/adios2/bp4dbg/idxtable.py
index 672e2510b7..10338ddaa8 100644
--- a/source/utils/bp4dbg/adios2/bp4dbg/idxtable.py
+++ b/source/utils/bp4dbg/adios2/bp4dbg/idxtable.py
@@ -1,6 +1,3 @@
-from __future__ import (absolute_import, division, print_function,
-                        unicode_literals)
-
 import numpy as np
 from os import fstat
 from .utils import *
@@ -52,7 +49,7 @@ def DumpIndexTable(fileName):
     with open(fileName, "rb") as f:
         fileSize = fstat(f.fileno()).st_size
         status = ReadHeader(f, fileSize, "Index Table")
-        if (status):
+        if status:
             status = ReadIndex(f, fileSize)
     return status
 
diff --git a/source/utils/bp4dbg/adios2/bp4dbg/metadata.py b/source/utils/bp4dbg/adios2/bp4dbg/metadata.py
index 0c7fb27ea1..3d25a2bbfc 100644
--- a/source/utils/bp4dbg/adios2/bp4dbg/metadata.py
+++ b/source/utils/bp4dbg/adios2/bp4dbg/metadata.py
@@ -1,5 +1,3 @@
-from __future__ import (absolute_import, division, print_function,
-                        unicode_literals)
 import numpy as np
 from os import fstat
 from .utils import *
@@ -12,7 +10,7 @@ def ReadEncodedStringFromBuffer(buf, pos, ID, limit, lenbytes=2):
         dt = np.dtype(np.uint16)
     namelen = np.frombuffer(buf, dtype=dt, count=1, offset=pos)[0]
     pos = pos + lenbytes
-    if (namelen > limit - lenbytes):
+    if namelen > limit - lenbytes:
         print("ERROR: " + ID + " string length ({0}) is longer than the "
               "limit to stay inside the block ({1})".format(
                   namelen, limit - lenbytes))
@@ -28,7 +26,7 @@ def ReadEncodedStringArrayFromBuffer(buf, pos, ID, limit, nStrings):
         # 2 bytes length + string without \0
         namelen = np.frombuffer(buf, dtype=np.uint16, count=1, offset=pos)[0]
         pos = pos + 2
-        if (namelen > limit - 2):
+        if namelen > limit - 2:
             print("ERROR: " + ID + " string length ({0}) is longer than the "
                   "limit to stay inside the block ({1})".format(
                       namelen, limit - 2))
@@ -174,7 +172,7 @@ def ReadCharacteristicsFromMetaData(buf, idx, pos, limit, typeID,
                 print("]")
 
             else:
-                if (isVarCharacteristics):
+                if isVarCharacteristics:
                     cData = buf[pos:pos + cLen]
                     pos = pos + cLen
                     data = bDataToNumpyArray(cData, dataTypeName, 1)
@@ -295,7 +293,7 @@ def ReadPGMD(buf, idx, pos, limit, pgStartOffset):
     print(
         "        PG length       : {0} bytes (+2 for length)".format(
             pgLength))
-    if (pgStartPosition + pgLength + 2 > limit):
+    if pgStartPosition + pgLength + 2 > limit:
         print("ERROR: There is not enough bytes {0} left in PG index block "
               "to read this single PG index ({1} bytes)").format(
                   limit - pgStartPosition, pgLength)
@@ -303,7 +301,7 @@ def ReadPGMD(buf, idx, pos, limit, pgStartOffset):
 
     pgNameLen = np.frombuffer(buf, dtype=np.uint16, count=1, offset=pos)[0]
     pos = pos + 2
-    if (pgStartPosition + pgNameLen > limit):
+    if pgStartPosition + pgNameLen > limit:
         print("ERROR: There is not enough bytes {0} left in PG index block "
               "to read the name of this single PG index ({1} bytes)").format(
                   limit - pos + 2, pgNameLen)
@@ -317,7 +315,7 @@ def ReadPGMD(buf, idx, pos, limit, pgStartOffset):
     # ColumnMajor (host language Fortran) 1 byte, 'y' or 'n'
     isColumnMajor = buf[pos]  # this is an integer value
     pos = pos + 1
-    if (isColumnMajor != ord('y') and isColumnMajor != ord('n')):
+    if isColumnMajor != ord('y') and isColumnMajor != ord('n'):
         print(
             "ERROR: Next byte for isColumnMajor must be 'y' or 'n' "
             "but it isn't = {0} (={1})".format(
@@ -331,7 +329,7 @@ def ReadPGMD(buf, idx, pos, limit, pgStartOffset):
 
     pgTimeNameLen = np.frombuffer(buf, dtype=np.uint16, count=1, offset=pos)[0]
     pos = pos + 2
-    if (pgStartPosition + pgTimeNameLen > limit):
+    if pgStartPosition + pgTimeNameLen > limit:
         print("ERROR: There is not enough bytes {0} left in PG index block "
               "to read the name of this single PG index ({1} bytes)").format(
                   limit - pos + 2, pgTimeNameLen)
@@ -364,7 +362,7 @@ def ReadVarMD(buf, idx, pos, limit, varStartOffset):
     pos = pos + 4
     print("        Var idx length  : {0} bytes (+4 for idx length)".format(
         varLength))
-    if (varStartPosition + varLength + 4 > limit):
+    if varStartPosition + varLength + 4 > limit:
         print("ERROR: There is not enough bytes in Var index block "
               "to read this single Var index")
         return False, pos
@@ -401,7 +399,7 @@ def ReadVarMD(buf, idx, pos, limit, varStartOffset):
     # 1 byte ORDER (K, C, F)
     order = buf[pos]  # this is an integer value
     pos = pos + 1
-    if (order != ord('K') and order != ord('C') and order != ord('F')):
+    if order != ord('K') and order != ord('C') and order != ord('F'):
         print(
             "ERROR: Next byte for Order must be 'K', 'C', or 'F' "
             "but it isn't = {0} (={1})".format(
@@ -425,7 +423,19 @@ def ReadVarMD(buf, idx, pos, limit, varStartOffset):
     pos = pos + 8
     print("        # of blocks     : {0}".format(cSets))
 
-    for i in range(cSets):
+#   This loop only reads the number of reported blocks
+#     for i in range(cSets):
+#         # one characteristics block
+#         newlimit = limit - (pos - varStartPosition)
+#         fileOffset = varStartOffset + (pos - varStartPosition)
+#         status, pos = ReadCharacteristicsFromMetaData(
+#             buf, i, pos, newlimit, typeID, fileOffset, True)
+#         if not status:
+#             return False
+
+#   This loop reads blocks until the reported length of variable index length
+    i = 0
+    while pos < varStartPosition + varLength:
         # one characteristics block
         newlimit = limit - (pos - varStartPosition)
         fileOffset = varStartOffset + (pos - varStartPosition)
@@ -433,6 +443,13 @@ def ReadVarMD(buf, idx, pos, limit, varStartOffset):
             buf, i, pos, newlimit, typeID, fileOffset, True)
         if not status:
             return False
+        i = i + 1
+
+    if (i != cSets):
+        print(
+            "ERROR: reported # of blocks (={0}) != # of encoded blocks "
+            " (={1})".format(
+                cSets, i))
 
     return True, pos
 
@@ -448,7 +465,7 @@ def ReadAttrMD(buf, idx, pos, limit, attrStartOffset):
     pos = pos + 4
     print("        Attr idx length : {0} bytes (+4 for idx length)".format(
         attrLength))
-    if (attrStartPosition + attrLength + 4 > limit):
+    if attrStartPosition + attrLength + 4 > limit:
         print("ERROR: There is not enough bytes in Attr index block "
               "to read this single Attr index")
         return False, pos
@@ -524,7 +541,7 @@ def ReadMetadataStep(f, fileSize, step):
         pgLength))
 
     pgStartPosition = f.tell()
-    if (pgStartPosition + pgLength > fileSize):
+    if pgStartPosition + pgLength > fileSize:
         print("ERROR: There is not enough bytes in file to read the PG index")
         return False
 
@@ -550,7 +567,7 @@ def ReadMetadataStep(f, fileSize, step):
         varLength))
 
     varsStartPosition = f.tell()
-    if (varsStartPosition + varLength > fileSize):
+    if varsStartPosition + varLength > fileSize:
         print("ERROR: There is not enough bytes in file to read the VAR index")
         return False
 
@@ -576,7 +593,7 @@ def ReadMetadataStep(f, fileSize, step):
         attrLength))
 
     attrsStartPosition = f.tell()
-    if (attrsStartPosition + attrLength > fileSize):
+    if attrsStartPosition + attrLength > fileSize:
         print("ERROR: There is not enough bytes in file "
               "to read the Attribute index")
         return False
diff --git a/source/utils/bp4dbg/adios2/bp4dbg/utils.py b/source/utils/bp4dbg/adios2/bp4dbg/utils.py
index 02ca28236a..8f1cb4849b 100644
--- a/source/utils/bp4dbg/adios2/bp4dbg/utils.py
+++ b/source/utils/bp4dbg/adios2/bp4dbg/utils.py
@@ -1,6 +1,3 @@
-from __future__ import (absolute_import, division, print_function,
-                        unicode_literals)
-
 LocalValueDim = 18446744073709551613
 
 dataTypes = {
@@ -103,7 +100,7 @@ def GetCharacteristicDataLength(cID, typeID):
 # fileType: Data, Metadata, Index Table
 def ReadHeader(f, fileSize, fileType):
     status = True
-    if (fileSize < 64):
+    if fileSize < 64:
         print("ERROR: Invalid " + fileType + ". File is smaller "
               "than the header (64 bytes)")
         return False
diff --git a/source/utils/bp4dbg/bp4dbg.py b/source/utils/bp4dbg/bp4dbg.py
index f5f0d42caf..d795f583c4 100755
--- a/source/utils/bp4dbg/bp4dbg.py
+++ b/source/utils/bp4dbg/bp4dbg.py
@@ -1,6 +1,5 @@
 #!/usr/bin/env python3
-from __future__ import (absolute_import, division, print_function,
-                        unicode_literals)
+
 import argparse
 from os.path import basename, exists, isdir
 import glob
@@ -37,31 +36,31 @@ def SetupArgs():
 
 
 def CheckFileName(args):
-    if (not exists(args.FILE)):
+    if not exists(args.FILE):
         print("ERROR: File " + args.FILE + " does not exist", flush=True)
         exit(1)
-    if (isdir(args.FILE)):
-        if (not args.no_indextable):
+    if isdir(args.FILE):
+        if not args.no_indextable:
             args.idxFileName = args.FILE + "/" + "md.idx"
             args.dumpIdx = True
-        if (not args.no_metadata):
+        if not args.no_metadata:
             args.metadataFileName = args.FILE + "/" + "md.[0-9]*"
             args.dumpMetadata = True
-        if (not args.no_data):
+        if not args.no_data:
             args.dataFileName = args.FILE + "/" + "data.[0-9]*"
             args.dumpData = True
         return
 
     name = basename(args.FILE)
-    if (name.startswith("data.")):
+    if name.startswith("data."):
         args.dataFileName = args.FILE
         args.dumpData = True
 
-    elif (name == "md.idx"):
+    elif name == "md.idx":
         args.idxFileName = args.FILE
         args.dumpIdx = True
 
-    elif (name.startswith("md.")):
+    elif name.startswith("md."):
         args.metadataFileName = args.FILE
         args.dumpMetadata = True
 
@@ -98,11 +97,11 @@ def DumpDataFiles(args):
     CheckFileName(args)
     # print(args)
 
-    if (args.dumpIdx):
+    if args.dumpIdx:
         DumpIndexTableFile(args)
 
-    if (args.dumpMetadata):
+    if args.dumpMetadata:
         DumpMetadataFiles(args)
 
-    if (args.dumpData):
+    if args.dumpData:
         DumpDataFiles(args)
diff --git a/source/utils/bpls/bpls.cmake.gen.in b/source/utils/bpls/bpls.cmake.gen.in
index 309627170a..23c2da30b4 100644
--- a/source/utils/bpls/bpls.cmake.gen.in
+++ b/source/utils/bpls/bpls.cmake.gen.in
@@ -7,15 +7,8 @@ if(ERROR_FILE)
   set(error_arg ERROR_FILE "${ERROR_FILE}")
 endif()
 
-if(@ADIOS2_HAVE_MPI@)
-  set(cmd_executor
-    @MPIEXEC_EXECUTABLE@ @MPIEXEC_EXTRA_FLAGS@ @MPIEXEC_NUMPROC_FLAG@ 1
-  )
-else()
-  set(cmd_executor)
-endif()
 execute_process(
-  COMMAND ${cmd_executor} $ ${INPUT_FILE} ${ARG1} ${ARG2} ${ARG3} ${ARG4} ${ARG5} ${ARG6} ${ARG7} ${ARG8} ${ARG9} ${ARG10} ${ARG11} ${ARG12} ${ARG13} ${ARG14} ${ARG15} ${ARG16} ${ARG17} ${ARG18} ${ARG19} ${ARG20}
+  COMMAND $ ${INPUT_FILE} ${ARG1} ${ARG2} ${ARG3} ${ARG4} ${ARG5} ${ARG6} ${ARG7} ${ARG8} ${ARG9} ${ARG10} ${ARG11} ${ARG12} ${ARG13} ${ARG14} ${ARG15} ${ARG16} ${ARG17} ${ARG18} ${ARG19} ${ARG20}
   RESULT_VARIABLE result
   ${output_arg}
   ${error_arg}
diff --git a/source/utils/bpls/bpls.cpp b/source/utils/bpls/bpls.cpp
index 6a33361f2b..2cfba743c7 100644
--- a/source/utils/bpls/bpls.cpp
+++ b/source/utils/bpls/bpls.cpp
@@ -20,10 +20,12 @@
 #include "bpls.h"
 #include "verinfo.h"
 
+#include 
 #include 
 #include 
 #include 
 #include 
+#include 
 #include 
 
 #include 
@@ -81,13 +83,13 @@ std::string format;       // format string for one data element (e.g. %6.2f)
 // Flags from arguments or defaults
 bool dump; // dump data not just list info(flag == 1)
 bool output_xml;
-bool use_regexp; // use varmasks as regular expressions
-bool sortnames;  // sort names before listing
-bool listattrs;  // do list attributes too
-bool listmeshes; // do list meshes too
-bool attrsonly;  // do list attributes only
-bool longopt;    // -l is turned on
-bool timestep;
+bool use_regexp;       // use varmasks as regular expressions
+bool sortnames;        // sort names before listing
+bool listattrs;        // do list attributes too
+bool listmeshes;       // do list meshes too
+bool attrsonly;        // do list attributes only
+bool longopt;          // -l is turned on
+bool timestep;         // read step by step
 bool noindex;          // do no print array indices with data
 bool printByteAsChar;  // print 8 bit integer arrays as string
 bool plot;             // dump histogram related information
@@ -133,7 +135,8 @@ void display_help()
         /*
         "  --sort      | -r           Sort names before listing\n"
         */
-        "  --timestep  | -t           Print values of timestep elements\n"
+        "  --timestep  | -t           Read content step by step (stream "
+        "reading)\n"
         "  --dump      | -d           Dump matched variables/attributes\n"
         "                               To match attributes too, add option "
         "-a\n"
@@ -781,6 +784,9 @@ void printSettings(void)
         printf("      -D : show decomposition of variables in the file\n");
     if (show_version)
         printf("      -V : show binary version info of file\n");
+    if (timestep)
+        printf("      -t : read step-by-step\n");
+
     if (hidden_attrs)
     {
         printf("         : show hidden attributes in the file\n");
@@ -820,7 +826,7 @@ template 
 int printAttributeValue(core::Engine *fp, core::IO *io,
                         core::Attribute *attribute)
 {
-    enum ADIOS_DATATYPES adiosvartype = type_to_enum(attribute->m_Type);
+    DataType adiosvartype = attribute->m_Type;
     if (attribute->m_IsSingleValue)
     {
         print_data((void *)&attribute->m_DataSingleValue, 0, adiosvartype,
@@ -848,7 +854,7 @@ template <>
 int printAttributeValue(core::Engine *fp, core::IO *io,
                         core::Attribute *attribute)
 {
-    enum ADIOS_DATATYPES adiosvartype = type_to_enum(attribute->m_Type);
+    DataType adiosvartype = attribute->m_Type;
     bool xmlprint = helper::EndsWith(attribute->m_Name, "xml", false);
     bool printDataAnyway = true;
 
@@ -898,8 +904,8 @@ int nEntriesMatched = 0;
 int doList_vars(core::Engine *fp, core::IO *io)
 {
 
-    const core::DataMap &variables = io->GetVariablesDataMap();
-    const core::DataMap &attributes = io->GetAttributesDataMap();
+    const core::VarMap &variables = io->GetVariables();
+    const core::AttrMap &attributes = io->GetAttributes();
 
     // make a sorted list of all variables and attributes
     EntryMap entries;
@@ -907,15 +913,25 @@ int doList_vars(core::Engine *fp, core::IO *io)
     {
         for (const auto &vpair : variables)
         {
-            Entry e(true, vpair.second.first, vpair.second.second);
-            entries.emplace(vpair.first, e);
+            Entry e(vpair.second->m_Type, vpair.second.get());
+            bool valid = true;
+            if (timestep)
+            {
+                valid = e.var->IsValidStep(fp->CurrentStep() + 1);
+                // fprintf(stdout, "Entry: ptr = %p valid = %d\n", e.var,
+                // valid);
+            }
+            if (valid)
+            {
+                entries.emplace(vpair.first, e);
+            }
         }
     }
     if (listattrs)
     {
         for (const auto &apair : attributes)
         {
-            Entry e(false, apair.second.first, apair.second.second);
+            Entry e(apair.second->m_Type, apair.second.get());
             entries.emplace(apair.first, e);
         }
     }
@@ -931,7 +947,7 @@ int doList_vars(core::Engine *fp, core::IO *io)
         int len = static_cast(entrypair.first.size());
         if (len > maxlen)
             maxlen = len;
-        len = static_cast(entrypair.second.typeName.size());
+        len = static_cast(ToString(entrypair.second.typeName).size());
         if (len > maxtypelen)
             maxtypelen = len;
     }
@@ -949,21 +965,21 @@ int doList_vars(core::Engine *fp, core::IO *io)
 
             // print definition of variable
             fprintf(outf, "%c %-*s  %-*s", commentchar, maxtypelen,
-                    entry.typeName.c_str(), maxlen, name.c_str());
+                    ToString(entry.typeName).c_str(), maxlen, name.c_str());
             if (!entry.isVar)
             {
                 // list (and print) attribute
                 if (longopt || dump)
                 {
                     fprintf(outf, "  attr   = ");
-                    if (entry.typeName == "compound")
+                    if (entry.typeName == DataType::Compound)
                     {
                         // not supported
                     }
 #define declare_template_instantiation(T)                                      \
-    else if (entry.typeName == helper::GetType())                           \
+    else if (entry.typeName == helper::GetDataType())                       \
     {                                                                          \
-        core::Attribute *a = io->InquireAttribute(name);                 \
+        core::Attribute *a = static_cast *>(entry.attr); \
         retval = printAttributeValue(fp, io, a);                               \
     }
                     ADIOS2_FOREACH_ATTRIBUTE_STDTYPE_1ARG(
@@ -979,14 +995,14 @@ int doList_vars(core::Engine *fp, core::IO *io)
             }
             else
             {
-                if (entry.typeName == "compound")
+                if (entry.typeName == DataType::Compound)
                 {
                     // not supported
                 }
 #define declare_template_instantiation(T)                                      \
-    else if (entry.typeName == helper::GetType())                           \
+    else if (entry.typeName == helper::GetDataType())                       \
     {                                                                          \
-        core::Variable *v = io->InquireVariable(name);                   \
+        core::Variable *v = static_cast *>(entry.var);    \
         retval = printVariableInfo(fp, io, v);                                 \
     }
                 ADIOS2_FOREACH_STDTYPE_1ARG(declare_template_instantiation)
@@ -1007,7 +1023,11 @@ int printVariableInfo(core::Engine *fp, core::IO *io,
                       core::Variable *variable)
 {
     size_t nsteps = variable->GetAvailableStepsCount();
-    enum ADIOS_DATATYPES adiosvartype = type_to_enum(variable->m_Type);
+    if (timestep)
+    {
+        nsteps = 1;
+    }
+    DataType adiosvartype = variable->m_Type;
     int retval = 0;
 
     bool isGlobalValue = (nsteps == 1);
@@ -1128,11 +1148,11 @@ int printVariableInfo(core::Engine *fp, core::IO *io,
                 /* End - Print the headers of statistics first */
 
                 void *min, *max, *avg, *std_dev;
-                enum ADIOS_DATATYPES vt = vartype;
+                DataType vt = vartype;
                 struct ADIOS_STAT_STEP *s = vi->statistics->steps;
-                if (vi->type == adios_complex ||
-                    vi->type == adios_double_complex)
-                    vt = adios_double;
+                if (vi->type == DataType::FloatComplex ||
+                    vi->type == DataType::DoubleComplex)
+                    vt = DataType::Double;
                 fprintf(outf, "\n%-*sglobal:", indent_len, indent_char);
                 print_data_characteristics(
                     vi->statistics->min, vi->statistics->max,
@@ -1165,14 +1185,21 @@ int printVariableInfo(core::Engine *fp, core::IO *io,
 
         if (show_decomp)
         {
-            print_decomp(fp, io, variable);
+            if (timestep)
+            {
+                print_decomp_singlestep(fp, io, variable);
+            }
+            else
+            {
+                print_decomp(fp, io, variable);
+            }
         }
     }
     else
     {
         // single GlobalValue without timesteps
         fprintf(outf, "  scalar");
-        if (longopt)
+        if (longopt && !timestep)
         {
             fprintf(outf, " = ");
             print_data(&variable->m_Value, 0, adiosvartype, false);
@@ -1181,7 +1208,14 @@ int printVariableInfo(core::Engine *fp, core::IO *io,
 
         if (show_decomp)
         {
-            print_decomp(fp, io, variable);
+            if (timestep)
+            {
+                print_decomp_singlestep(fp, io, variable);
+            }
+            else
+            {
+                print_decomp(fp, io, variable);
+            }
         }
     }
 
@@ -1190,7 +1224,14 @@ int printVariableInfo(core::Engine *fp, core::IO *io,
         // print variable content
         if (variable->m_ShapeID == ShapeID::LocalArray)
         {
-            print_decomp(fp, io, variable);
+            if (timestep)
+            {
+                print_decomp_singlestep(fp, io, variable);
+            }
+            else
+            {
+                print_decomp(fp, io, variable);
+            }
         }
         else
         {
@@ -1442,6 +1483,11 @@ int doList(const char *path)
 
     core::ADIOS adios("C++");
     core::IO &io = adios.DeclareIO("bpls");
+    if (timestep)
+    {
+        // BP4 can process metadata in chuncks to conserve memory
+        io.SetParameter("StreamReader", "true");
+    }
     core::Engine *fp = nullptr;
     std::vector engineList = getEnginesList(path);
     for (auto &engineName : engineList)
@@ -1474,13 +1520,9 @@ int doList(const char *path)
         // ntsteps = fp->tidx_stop - fp->tidx_start + 1;
         if (verbose)
         {
-            const std::map &variablesInfo =
-                io.GetAvailableVariables();
-            const std::map &attributesInfo =
-                io.GetAvailableAttributes();
             printf("File info:\n");
-            printf("  of variables:  %zu\n", variablesInfo.size());
-            printf("  of attributes: %zu\n", attributesInfo.size());
+            printf("  of variables:  %zu\n", io.GetVariables().size());
+            printf("  of attributes: %zu\n", io.GetAttributes().size());
             // printf("  of meshes:     %d\n", fp->nmeshes);
             // print_file_size(fp->file_size);
             // printf("  bp version:    %d\n", fp->version);
@@ -1497,7 +1539,37 @@ int doList(const char *path)
             printMeshes(fp);
         }
 
-        doList_vars(fp, &io);
+        if (timestep)
+        {
+            while (true)
+            {
+                adios2::StepStatus status =
+                    fp->BeginStep(adios2::StepMode::Read);
+                if (status == adios2::StepStatus::EndOfStream)
+                {
+                    break;
+                }
+                else if (status == adios2::StepStatus::NotReady)
+                {
+                    std::this_thread::sleep_for(
+                        std::chrono::milliseconds(1000));
+                    continue;
+                }
+                else if (status == adios2::StepStatus::OtherError)
+                {
+                    fprintf(stderr,
+                            "\nError: Cannot read more steps due to errors\n");
+                    break;
+                }
+                fprintf(stdout, "Step %zu:\n", fp->CurrentStep());
+                doList_vars(fp, &io);
+                fp->EndStep();
+            }
+        }
+        else
+        {
+            doList_vars(fp, &io);
+        }
 
         if (nmasks > 0 && nEntriesMatched == 0)
         {
@@ -1647,62 +1719,58 @@ void mergeLists(int nV, char **listV, int nA, char **listA, char **mlist,
     }
 }
 
-int getTypeInfo(enum ADIOS_DATATYPES adiosvartype, int *elemsize)
+int getTypeInfo(DataType adiosvartype, int *elemsize)
 {
     switch (adiosvartype)
     {
-    case adios_unsigned_byte:
+    case DataType::UInt8:
         *elemsize = 1;
         break;
-    case adios_byte:
+    case DataType::Int8:
         *elemsize = 1;
         break;
-    case adios_string:
+    case DataType::String:
         *elemsize = 1;
         break;
 
-    case adios_unsigned_short:
+    case DataType::UInt16:
         *elemsize = 2;
         break;
-    case adios_short:
+    case DataType::Int16:
         *elemsize = 2;
         break;
 
-    case adios_unsigned_integer:
+    case DataType::UInt32:
         *elemsize = 4;
         break;
-    case adios_integer:
+    case DataType::Int32:
         *elemsize = 4;
         break;
 
-    case adios_unsigned_long:
+    case DataType::UInt64:
         *elemsize = 8;
         break;
-    case adios_long:
+    case DataType::Int64:
         *elemsize = 8;
         break;
 
-    case adios_real:
+    case DataType::Float:
         *elemsize = 4;
         break;
 
-    case adios_double:
+    case DataType::Double:
         *elemsize = 8;
         break;
 
-    case adios_complex:
+    case DataType::FloatComplex:
         *elemsize = 8;
         break;
 
-    case adios_double_complex:
+    case DataType::DoubleComplex:
         *elemsize = 16;
         break;
 
-    case adios_long_double_complex:
-        *elemsize = 32;
-        break;
-
-    case adios_long_double: // do not know how to print
+    case DataType::LongDouble: // do not know how to print
     //*elemsize = 16;
     default:
         return 1;
@@ -1735,7 +1803,8 @@ int readVar(core::Engine *fp, core::IO *io, core::Variable *variable)
     int ndigits_dims[32];     // # of digits (to print) of each dimension
 
     const size_t elemsize = variable->m_ElementSize;
-    const int nsteps = static_cast(variable->GetAvailableStepsCount());
+    const int nsteps =
+        (timestep ? 1 : static_cast(variable->GetAvailableStepsCount()));
     const int ndim = static_cast(variable->m_Shape.size());
     // create the counter arrays with the appropriate lengths
     // transfer start and count arrays to format dependent arrays
@@ -1745,7 +1814,7 @@ int readVar(core::Engine *fp, core::IO *io, core::Variable *variable)
     stepStart = 0;
     stepCount = 1;
 
-    if (nsteps > 1)
+    if (!timestep && nsteps > 1)
     {
         // user selection must start with step selection
         // calculate the starting step and number of steps requested
@@ -1787,7 +1856,15 @@ int readVar(core::Engine *fp, core::IO *io, core::Variable *variable)
     size_t absstep = relative_to_absolute_step(variable, stepStart);
 
     // Get the shape of the variable for the starting step
-    auto shape = variable->Shape(absstep);
+    Dims shape;
+    if (timestep)
+    {
+        shape = variable->Shape();
+    }
+    else
+    {
+        shape = variable->Shape(absstep);
+    }
     if (verbose > 2)
     {
         printf("    starting step=%" PRIu64 " absolute step=%zu"
@@ -1879,7 +1956,7 @@ int readVar(core::Engine *fp, core::IO *io, core::Variable *variable)
         maxreadn = nelems;
 
     // special case: string. Need to use different elemsize
-    /*if (vi->type == adios_string)
+    /*if (vi->type == DataType::String)
     {
         if (vi->value)
             elemsize = strlen(vi->value) + 1;
@@ -2052,7 +2129,8 @@ int readVarBlock(core::Engine *fp, core::IO *io, core::Variable *variable,
     int ndigits_dims[32];     // # of digits (to print) of each dimension
 
     const size_t elemsize = variable->m_ElementSize;
-    const int nsteps = static_cast(variable->GetAvailableStepsCount());
+    const int nsteps =
+        (timestep ? 1 : static_cast(variable->GetAvailableStepsCount()));
     const int ndim = static_cast(variable->m_Count.size());
     // create the counter arrays with the appropriate lengths
     // transfer start and count arrays to format dependent arrays
@@ -2400,44 +2478,15 @@ void print_slice_info(core::VariableBase *variable, bool timed, uint64_t *s,
     }
 }
 
-const std::map adios_types_map = {
-    {"uint8_t", adios_unsigned_byte},
-    {"int32_t", adios_integer},
-    {"float", adios_real},
-    {"double", adios_double},
-    {"float complex", adios_complex},
-    {"double complex", adios_double_complex},
-    {"int8_t", adios_byte},
-    {"int16_t", adios_short},
-    {"int64_t", adios_long},
-    {"int64_t", adios_long},
-    {"string", adios_string},
-    {"uint8_t", adios_unsigned_byte},
-    {"uint16_t", adios_unsigned_short},
-    {"uint32_t", adios_unsigned_integer},
-    {"uint64_t", adios_unsigned_long},
-    {"uint64_t", adios_unsigned_long}};
-
-enum ADIOS_DATATYPES type_to_enum(std::string type)
-{
-    auto itType = adios_types_map.find(type);
-    if (itType == adios_types_map.end())
-    {
-        return adios_unknown;
-    }
-    return itType->second;
-}
-
-int print_data_as_string(const void *data, int maxlen,
-                         enum ADIOS_DATATYPES adiosvartype)
+int print_data_as_string(const void *data, int maxlen, DataType adiosvartype)
 {
     const char *str = (const char *)data;
     int len = maxlen;
     switch (adiosvartype)
     {
-    case adios_unsigned_byte:
-    case adios_byte:
-    case adios_string:
+    case DataType::UInt8:
+    case DataType::Int8:
+    case DataType::String:
         while (str[len - 1] == 0)
         {
             len--;
@@ -2463,15 +2512,15 @@ int print_data_as_string(const void *data, int maxlen,
         fprintf(stderr,
                 "Error in bpls code: cannot use print_data_as_string() "
                 "for type \"%d\"\n",
-                adiosvartype);
+                static_cast::type>(
+                    adiosvartype));
         return -1;
     }
     return 0;
 }
 
 int print_data_characteristics(void *min, void *max, double *avg,
-                               double *std_dev,
-                               enum ADIOS_DATATYPES adiosvartype,
+                               double *std_dev, DataType adiosvartype,
                                bool allowformat)
 {
     bool f = format.size() && allowformat;
@@ -2479,7 +2528,7 @@ int print_data_characteristics(void *min, void *max, double *avg,
 
     switch (adiosvartype)
     {
-    case adios_unsigned_byte:
+    case DataType::UInt8:
         if (min)
             fprintf(outf, (f ? fmt : "%10hhu  "), *((unsigned char *)min));
         else
@@ -2497,7 +2546,7 @@ int print_data_characteristics(void *min, void *max, double *avg,
         else
             fprintf(outf, "      null  ");
         break;
-    case adios_byte:
+    case DataType::Int8:
         if (min)
             fprintf(outf, (f ? fmt : "%10hhd  "), *((char *)min));
         else
@@ -2515,10 +2564,10 @@ int print_data_characteristics(void *min, void *max, double *avg,
         else
             fprintf(outf, "      null  ");
         break;
-    case adios_string:
+    case DataType::String:
         break;
 
-    case adios_unsigned_short:
+    case DataType::UInt16:
         if (min)
             fprintf(outf, (f ? fmt : "%10hu  "), (*(unsigned short *)min));
         else
@@ -2536,7 +2585,7 @@ int print_data_characteristics(void *min, void *max, double *avg,
         else
             fprintf(outf, "      null  ");
         break;
-    case adios_short:
+    case DataType::Int16:
         if (min)
             fprintf(outf, (f ? fmt : "%10hd  "), (*(short *)min));
         else
@@ -2555,7 +2604,7 @@ int print_data_characteristics(void *min, void *max, double *avg,
             fprintf(outf, "      null  ");
         break;
 
-    case adios_unsigned_integer:
+    case DataType::UInt32:
         if (min)
             fprintf(outf, (f ? fmt : "%10u  "), (*(unsigned int *)min));
         else
@@ -2573,7 +2622,7 @@ int print_data_characteristics(void *min, void *max, double *avg,
         else
             fprintf(outf, "      null  ");
         break;
-    case adios_integer:
+    case DataType::Int32:
         if (min)
             fprintf(outf, (f ? fmt : "%10d  "), (*(int *)min));
         else
@@ -2592,7 +2641,7 @@ int print_data_characteristics(void *min, void *max, double *avg,
             fprintf(outf, "      null  ");
         break;
 
-    case adios_unsigned_long:
+    case DataType::UInt64:
         if (min)
             fprintf(outf, (f ? fmt : "%10llu  "), (*(unsigned long long *)min));
         else
@@ -2610,7 +2659,7 @@ int print_data_characteristics(void *min, void *max, double *avg,
         else
             fprintf(outf, "      null  ");
         break;
-    case adios_long:
+    case DataType::Int64:
         if (min)
             fprintf(outf, (f ? fmt : "%10lld  "), (*(long long *)min));
         else
@@ -2629,7 +2678,7 @@ int print_data_characteristics(void *min, void *max, double *avg,
             fprintf(outf, "      null  ");
         break;
 
-    case adios_real:
+    case DataType::Float:
         if (min)
             fprintf(outf, (f ? fmt : "%10.2g  "), (*(float *)min));
         else
@@ -2647,7 +2696,7 @@ int print_data_characteristics(void *min, void *max, double *avg,
         else
             fprintf(outf, "      null  ");
         break;
-    case adios_double:
+    case DataType::Double:
         if (min)
             fprintf(outf, (f ? fmt : "%10.2g  "), (*(double *)min));
         else
@@ -2666,18 +2715,18 @@ int print_data_characteristics(void *min, void *max, double *avg,
             fprintf(outf, "      null  ");
         break;
 
-    case adios_long_double:
+    case DataType::LongDouble:
         fprintf(outf, "????????");
         break;
 
     // TO DO
     /*
-       case adios_complex:
+       case DataType::FloatComplex:
        fprintf(outf,(f ? format : "(%g,i%g) "), ((float *) data)[2*item],
        ((float *) data)[2*item+1]);
        break;
 
-       case adios_double_complex:
+       case DataType::DoubleComplex:
        fprintf(outf,(f ? format : "(%g,i%g)" ), ((double *) data)[2*item],
        ((double *) data)[2*item+1]);
        break;
@@ -2709,7 +2758,7 @@ bool print_data_xml(const char *s, const size_t length)
     return false;
 }
 
-int print_data(const void *data, int item, enum ADIOS_DATATYPES adiosvartype,
+int print_data(const void *data, int item, DataType adiosvartype,
                bool allowformat)
 {
     bool f = format.size() && allowformat;
@@ -2722,14 +2771,14 @@ int print_data(const void *data, int item, enum ADIOS_DATATYPES adiosvartype,
     // print next data item
     switch (adiosvartype)
     {
-    case adios_unsigned_byte:
+    case DataType::UInt8:
         fprintf(outf, (f ? fmt : "%hhu"), ((unsigned char *)data)[item]);
         break;
-    case adios_byte:
+    case DataType::Int8:
         fprintf(outf, (f ? fmt : "%hhd"), ((signed char *)data)[item]);
         break;
 
-    case adios_string:
+    case DataType::String:
     {
         // fprintf(outf, (f ? fmt : "\"%s\""), ((char *)data) + item);
         const std::string *dataStr =
@@ -2737,74 +2786,65 @@ int print_data(const void *data, int item, enum ADIOS_DATATYPES adiosvartype,
         fprintf(outf, (f ? fmt : "\"%s\""), dataStr[item].c_str());
         break;
     }
-    case adios_string_array:
-        // we expect one elemet of the array here
-        fprintf(outf, (f ? fmt : "\"%s\""), *((char **)data + item));
-        break;
 
-    case adios_unsigned_short:
+    case DataType::UInt16:
         fprintf(outf, (f ? fmt : "%hu"), ((unsigned short *)data)[item]);
         break;
-    case adios_short:
+    case DataType::Int16:
         fprintf(outf, (f ? fmt : "%hd"), ((signed short *)data)[item]);
         break;
 
-    case adios_unsigned_integer:
+    case DataType::UInt32:
         fprintf(outf, (f ? fmt : "%u"), ((unsigned int *)data)[item]);
         break;
-    case adios_integer:
+    case DataType::Int32:
         fprintf(outf, (f ? fmt : "%d"), ((signed int *)data)[item]);
         break;
 
-    case adios_unsigned_long:
+    case DataType::UInt64:
         fprintf(outf, (f ? fmt : "%llu"), ((unsigned long long *)data)[item]);
         break;
-    case adios_long:
+    case DataType::Int64:
         fprintf(outf, (f ? fmt : "%lld"), ((signed long long *)data)[item]);
         break;
 
-    case adios_real:
+    case DataType::Float:
         fprintf(outf, (f ? fmt : "%g"), ((float *)data)[item]);
         break;
 
-    case adios_double:
+    case DataType::Double:
         fprintf(outf, (f ? fmt : "%g"), ((double *)data)[item]);
         break;
 
-    case adios_long_double:
+    case DataType::LongDouble:
         fprintf(outf, (f ? fmt : "%Lg"), ((long double *)data)[item]);
         // fprintf(outf,(f ? fmt : "????????"));
         break;
 
-    case adios_complex:
+    case DataType::FloatComplex:
         fprintf(outf, (f ? fmt : "(%g,i%g)"), ((float *)data)[2 * item],
                 ((float *)data)[2 * item + 1]);
         break;
 
-    case adios_double_complex:
+    case DataType::DoubleComplex:
         fprintf(outf, (f ? fmt : "(%g,i%g)"), ((double *)data)[2 * item],
                 ((double *)data)[2 * item + 1]);
         break;
 
-    case adios_long_double_complex:
-        fprintf(outf, (f ? fmt : "(%Lg,i%Lg)"), ((long double *)data)[2 * item],
-                ((long double *)data)[2 * item + 1]);
-        break;
-
     default:
         break;
     } // end switch
     return 0;
 }
 
-int print_dataset(const void *data, const std::string vartype, uint64_t *s,
+int print_dataset(const void *data, const DataType vartype, uint64_t *s,
                   uint64_t *c, int tdims, int *ndigits)
 {
     int i, item, steps;
     char idxstr[128], buf[16];
     uint64_t ids[MAX_DIMS]; // current indices
     bool roll;
-    enum ADIOS_DATATYPES adiosvartype = type_to_enum(vartype);
+    DataType adiosvartype = vartype;
 
     // init current indices
     steps = 1;
@@ -2838,7 +2878,7 @@ int print_dataset(const void *data, const std::string vartype, uint64_t *s,
         // print item
         fprintf(outf, "%s", idxstr);
         if (printByteAsChar &&
-            (adiosvartype == adios_byte || adiosvartype == adios_unsigned_byte))
+            (adiosvartype == DataType::Int8 || adiosvartype == DataType::UInt8))
         {
             /* special case: k-D byte array printed as (k-1)D array of
              * strings
@@ -2927,38 +2967,45 @@ Dims get_global_array_signature(core::Engine *fp, core::IO *io,
                                 core::Variable *variable)
 {
     const size_t ndim = variable->m_Shape.size();
-    const size_t nsteps = variable->GetAvailableStepsCount();
     Dims dims(ndim, 0);
-    bool firstStep = true;
-
-    // looping over the absolute step indexes
-    // is not supported by a simple API function
-    const std::map> &indices =
-        variable->m_AvailableStepBlockIndexOffsets;
-    auto itStep = indices.begin();
-
-    for (size_t step = 0; step < nsteps; step++)
+    if (timestep)
     {
-        const size_t absstep = itStep->first;
-        Dims d = variable->Shape(absstep - 1);
-        if (d.empty())
-        {
-            continue;
-        }
+        dims = variable->Shape();
+    }
+    else
+    {
+        const size_t nsteps = variable->GetAvailableStepsCount();
+        bool firstStep = true;
 
-        for (size_t k = 0; k < ndim; k++)
+        // looping over the absolute step indexes
+        // is not supported by a simple API function
+        const std::map> &indices =
+            variable->m_AvailableStepBlockIndexOffsets;
+        auto itStep = indices.begin();
+
+        for (size_t step = 0; step < nsteps; step++)
         {
-            if (firstStep)
+            const size_t absstep = itStep->first;
+            Dims d = variable->Shape(absstep - 1);
+            if (d.empty())
             {
-                dims[k] = d[k];
+                continue;
             }
-            else if (dims[k] != d[k])
+
+            for (size_t k = 0; k < ndim; k++)
             {
-                dims[k] = 0;
+                if (firstStep)
+                {
+                    dims[k] = d[k];
+                }
+                else if (dims[k] != d[k])
+                {
+                    dims[k] = 0;
+                }
             }
+            firstStep = false;
+            ++itStep;
         }
-        firstStep = false;
-        ++itStep;
     }
     return dims;
 }
@@ -2971,27 +3018,14 @@ std::pair get_local_array_signature(core::Engine *fp,
     const size_t ndim = variable->m_Count.size();
     size_t nblocks = 0;
     Dims dims(ndim, 0);
-    std::map::Info>> allblocks =
-        fp->AllStepsBlocksInfo(*variable);
-
-    bool firstStep = true;
-    bool firstBlock = true;
 
-    for (auto &blockpair : allblocks)
+    if (timestep)
     {
-        std::vector::Info> &blocks =
-            blockpair.second;
-        const size_t blocksSize = blocks.size();
-        if (firstStep)
-        {
-            nblocks = blocksSize;
-        }
-        else if (nblocks != blocksSize)
-        {
-            nblocks = 0;
-        }
-
-        for (size_t j = 0; j < blocksSize; j++)
+        std::vector::Info> blocks =
+            fp->BlocksInfo(*variable, fp->CurrentStep());
+        nblocks = blocks.size();
+        bool firstBlock = true;
+        for (size_t j = 0; j < nblocks; j++)
         {
             for (size_t k = 0; k < ndim; k++)
             {
@@ -3006,7 +3040,46 @@ std::pair get_local_array_signature(core::Engine *fp,
             }
             firstBlock = false;
         }
-        firstStep = false;
+    }
+    else
+    {
+        std::map::Info>>
+            allblocks = fp->AllStepsBlocksInfo(*variable);
+
+        bool firstStep = true;
+        bool firstBlock = true;
+
+        for (auto &blockpair : allblocks)
+        {
+            std::vector::Info> &blocks =
+                blockpair.second;
+            const size_t blocksSize = blocks.size();
+            if (firstStep)
+            {
+                nblocks = blocksSize;
+            }
+            else if (nblocks != blocksSize)
+            {
+                nblocks = 0;
+            }
+
+            for (size_t j = 0; j < blocksSize; j++)
+            {
+                for (size_t k = 0; k < ndim; k++)
+                {
+                    if (firstBlock)
+                    {
+                        dims[k] = blocks[j].Count[k];
+                    }
+                    else if (dims[k] != blocks[j].Count[k])
+                    {
+                        dims[k] = 0;
+                    }
+                }
+                firstBlock = false;
+            }
+            firstStep = false;
+        }
     }
     return std::make_pair(nblocks, dims);
 }
@@ -3015,7 +3088,7 @@ template 
 void print_decomp(core::Engine *fp, core::IO *io, core::Variable *variable)
 {
     /* Print block info */
-    enum ADIOS_DATATYPES adiosvartype = type_to_enum(variable->m_Type);
+    DataType adiosvartype = variable->m_Type;
     std::map::Info>> allblocks =
         fp->AllStepsBlocksInfo(*variable);
     if (allblocks.empty())
@@ -3167,6 +3240,144 @@ void print_decomp(core::Engine *fp, core::IO *io, core::Variable *variable)
     }
 }
 
+template 
+void print_decomp_singlestep(core::Engine *fp, core::IO *io,
+                             core::Variable *variable)
+{
+    /* Print block info */
+    DataType adiosvartype = variable->m_Type;
+    std::vector::Info> blocks =
+        fp->BlocksInfo(*variable, fp->CurrentStep());
+
+    if (blocks.empty())
+    {
+        return;
+    }
+
+    const size_t blocksSize = blocks.size();
+    const int ndigits_nblocks = ndigits(blocksSize - 1);
+
+    if (variable->m_ShapeID == ShapeID::GlobalValue ||
+        variable->m_ShapeID == ShapeID::LocalValue)
+    {
+        // scalars
+        if (dump)
+        {
+            int col = 0;
+            int maxcols = ncols;
+            if (adiosvartype == DataType::String)
+            {
+                maxcols = 1;
+            }
+            for (size_t j = 0; j < blocksSize; j++)
+            {
+                if (col == 0 && !noindex)
+                {
+                    fprintf(outf, "    (%*zu)    ", ndigits_nblocks, j);
+                }
+                print_data(&blocks[j].Value, 0, adiosvartype, true);
+                ++col;
+                if (j < blocks.size() - 1)
+                {
+                    if (col < maxcols)
+                    {
+                        fprintf(outf, " ");
+                    }
+                    else
+                    {
+                        col = 0;
+                        fprintf(outf, "\n");
+                    }
+                }
+            }
+            fprintf(outf, "\n");
+        }
+        return;
+    }
+    else
+    {
+        // arrays
+        size_t ndim = variable->m_Count.size();
+        int ndigits_dims[32] = {
+            8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8,
+            8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8,
+        };
+        for (size_t k = 0; k < ndim; k++)
+        {
+            // get digit lengths for each dimension
+            if (variable->m_ShapeID == ShapeID::GlobalArray)
+            {
+                ndigits_dims[k] = ndigits(variable->m_Shape[k] - 1);
+            }
+            else
+            {
+                ndigits_dims[k] = ndigits(variable->m_Count[k] - 1);
+            }
+        }
+
+        size_t stepRelative = 0;
+        size_t stepAbsolute = fp->CurrentStep();
+
+        for (size_t j = 0; j < blocksSize; j++)
+        {
+            fprintf(outf, "%c         block %*zu: [", commentchar,
+                    ndigits_nblocks, j);
+
+            // just in case ndim for a block changes in LocalArrays:
+            ndim = variable->m_Count.size();
+
+            for (size_t k = 0; k < ndim; k++)
+            {
+                if (blocks[j].Count[k])
+                {
+                    if (variable->m_ShapeID == ShapeID::GlobalArray)
+                    {
+                        fprintf(outf, "%*zu:%*zu", ndigits_dims[k],
+                                blocks[j].Start[k], ndigits_dims[k],
+                                blocks[j].Start[k] + blocks[j].Count[k] - 1);
+                    }
+                    else
+                    {
+                        // blockStart is empty vector for LocalArrays
+                        fprintf(outf, "0:%*zu", ndigits_dims[k],
+                                blocks[j].Count[k] - 1);
+                    }
+                }
+                else
+                {
+                    fprintf(outf, "%-*s", 2 * ndigits_dims[k] + 1, "null");
+                }
+                if (k < ndim - 1)
+                    fprintf(outf, ", ");
+            }
+            fprintf(outf, "]");
+
+            /* Print per-block statistics if available */
+            if (longopt)
+            {
+                if (true /* TODO: variable->has_minmax */)
+                {
+                    fprintf(outf, " = ");
+                    print_data(&blocks[j].Min, 0, adiosvartype, false);
+
+                    fprintf(outf, " / ");
+                    print_data(&blocks[j].Max, 0, adiosvartype, false);
+                }
+                else
+                {
+                    fprintf(outf, "N/A / N/A");
+                }
+            }
+            fprintf(outf, "\n");
+            if (dump)
+            {
+                readVarBlock(fp, io, variable, stepRelative, j, blocks[j]);
+            }
+        }
+        ++stepRelative;
+    }
+}
+
 // parse a string "0, 3; 027" into an integer array
 // of [0,3,27]
 // exits if parsing failes
diff --git a/source/utils/bpls/bpls.h b/source/utils/bpls/bpls.h
index 9bd1d4265f..d8acdf18fd 100644
--- a/source/utils/bpls/bpls.h
+++ b/source/utils/bpls/bpls.h
@@ -7,6 +7,7 @@
 
 #include "adios2/common/ADIOSConfig.h"
 #include "adios2/common/ADIOSMacros.h"
+#include "adios2/common/ADIOSTypes.h"
 #include "adios2/core/ADIOS.h"
 #include "adios2/core/Engine.h"
 #include "adios2/core/IO.h"
@@ -34,35 +35,21 @@ namespace utils
 #define MAX_MASKS 10
 #define MAX_BUFFERSIZE (10 * 1024 * 1024)
 
-/* global defines needed for the type creation/setup functions */
-enum ADIOS_DATATYPES
-{
-    adios_unknown = -1,
-    adios_byte = 0,
-    adios_short = 1,
-    adios_integer = 2,
-    adios_long = 4,
-    adios_unsigned_byte = 50,
-    adios_unsigned_short = 51,
-    adios_unsigned_integer = 52,
-    adios_unsigned_long = 54,
-    adios_real = 5,
-    adios_double = 6,
-    adios_long_double = 7,
-    adios_string = 9,
-    adios_complex = 10,
-    adios_double_complex = 11,
-    adios_string_array = 12,
-    adios_long_double_complex = 13
-};
-
 struct Entry
 {
+    DataType typeName;
     bool isVar;
-    std::string typeName;
-    unsigned int typeIndex;
-    Entry(bool b, std::string name, unsigned idx)
-    : isVar(b), typeName(name), typeIndex(idx)
+    union
+    {
+        core::VariableBase *var;
+        core::AttributeBase *attr;
+    };
+    Entry(DataType type, core::VariableBase *v)
+    : typeName(type), isVar(true), var(v)
+    {
+    }
+    Entry(DataType type, core::AttributeBase *a)
+    : typeName(type), isVar(false), attr(a)
     {
     }
 };
@@ -80,8 +67,6 @@ int doList(const char *path);
 void mergeLists(int nV, char **listV, int nA, char **listA, char **mlist,
                 bool *isVar);
 
-enum ADIOS_DATATYPES type_to_enum(std::string type);
-
 template 
 int printVariableInfo(core::Engine *fp, core::IO *io,
                       core::Variable *variable);
@@ -110,26 +95,28 @@ bool matchesAMask(const char *name);
 int print_start(const std::string &fnamestr);
 void print_slice_info(core::VariableBase *variable, bool timed, uint64_t *s,
                       uint64_t *c, Dims count);
-int print_data(const void *data, int item, enum ADIOS_DATATYPES adiosvartypes,
+int print_data(const void *data, int item, DataType adiosvartypes,
                bool allowformat);
 
 /* s is a character array not necessarily null terminated.
  * return false on OK print, true if it not XML (not printed)*/
 bool print_data_xml(const char *s, const size_t length);
 
-int print_dataset(const void *data, const std::string vartype, uint64_t *s,
+int print_dataset(const void *data, const DataType vartype, uint64_t *s,
                   uint64_t *c, int tdims, int *ndigits);
 void print_endline(void);
 void print_stop(void);
 int print_data_hist(core::VariableBase *vi, char *varname);
 int print_data_characteristics(void *min, void *max, double *avg,
-                               double *std_dev,
-                               enum ADIOS_DATATYPES adiosvartypes,
+                               double *std_dev, DataType adiosvartypes,
                                bool allowformat);
 
 template 
 void print_decomp(core::Engine *fp, core::IO *io, core::Variable *variable);
 
+template 
+void print_decomp_singlestep(core::Engine *fp, core::IO *io,
+                             core::Variable *variable);
 // close namespace
 }
 }
diff --git a/testing/CMakeLists.txt b/testing/CMakeLists.txt
index 675147ed4d..e9650d4002 100644
--- a/testing/CMakeLists.txt
+++ b/testing/CMakeLists.txt
@@ -26,7 +26,6 @@ include(GoogleTest)
 #   mpi      - MPI_ALLOW - build MPI test, execute with mpiexec; also build Serial test
 #              MPI_NONE - build Serial test only
 #              MPI_ONLY - build MPI test only, execute with mpiexec
-#              MPI_NOEXEC - build MPI test, execute without MPI; also build Serial test
 #   src_pfx  - Source filename prefix, Test${src_pfs}${testname}.cpp
 #   tst_pfx  - Test name prefix to be added to CTest
 #   tst_sfx  - Test name suffix to be added to CTest
@@ -51,9 +50,10 @@ include(GoogleTest)
 #
 function(gtest_add_tests_helper testname mpi src_pfx tst_pfx tst_sfx)
   set(test_targets "")
-  if(NOT mpi MATCHES "^MPI_(ALLOW|NONE|ONLY|NOEXEC)$")
+  if(NOT mpi MATCHES "^MPI_(ALLOW|NONE|ONLY)$")
     message(FATAL_ERROR "Invalid mpi argument value '${mpi}'.")
   endif()
+  set(all_tests)
   if(NOT mpi STREQUAL "MPI_ONLY")
     set(tgt Test.${tst_pfx}${testname}.Serial)
     list(APPEND test_targets "${tgt}")
@@ -64,8 +64,10 @@ function(gtest_add_tests_helper testname mpi src_pfx tst_pfx tst_sfx)
     gtest_add_tests(TARGET ${tgt}
       TEST_PREFIX "${tst_pfx}"
       TEST_SUFFIX "${tst_sfx}.Serial"
+      TEST_LIST added_tests
       ${ARGN}
     )
+    list(APPEND all_tests ${added_tests})
   endif()
   if(ADIOS2_HAVE_MPI AND NOT mpi STREQUAL "MPI_NONE")
     set(tgt Test.${tst_pfx}${testname}.MPI)
@@ -74,23 +76,17 @@ function(gtest_add_tests_helper testname mpi src_pfx tst_pfx tst_sfx)
       add_executable(${tgt} Test${src_pfx}${testname}.cpp)
       target_link_libraries(${tgt} adios2::cxx11_mpi adios2::c_mpi adios2_core_mpi MPI::MPI_C adios2::thirdparty::gtest)
     endif()
-    if(NOT mpi STREQUAL "MPI_NOEXEC")
-      gtest_add_tests(TARGET ${tgt}
-        TEST_PREFIX "${tst_pfx}"
-        TEST_SUFFIX "${tst_sfx}.MPI"
-        EXEC_WRAPPER ${MPIEXEC_COMMAND}
-        TEST_LIST added_tests
-        ${ARGN}
-        )
-      set_tests_properties(${added_tests} PROPERTIES PROCESSORS "${MPIEXEC_MAX_NUMPROCS}")
-    else()
-      gtest_add_tests(TARGET ${tgt}
-        TEST_PREFIX "${tst_pfx}"
-        TEST_SUFFIX "${tst_sfx}.MPI"
-        ${ARGN}
-        )
-    endif()
+    gtest_add_tests(TARGET ${tgt}
+      TEST_PREFIX "${tst_pfx}"
+      TEST_SUFFIX "${tst_sfx}.MPI"
+      EXEC_WRAPPER ${MPIEXEC_COMMAND}
+      TEST_LIST added_tests
+      ${ARGN}
+    )
+    list(APPEND all_tests ${added_tests})
+    set_tests_properties(${added_tests} PROPERTIES PROCESSORS "${MPIEXEC_MAX_NUMPROCS}")
   endif()
+  set("Test.${tst_pfx}${testname}-TESTS" "${all_tests}" PARENT_SCOPE)
   set("Test.${tst_pfx}${testname}-TARGETS" "${test_targets}" PARENT_SCOPE)
 endfunction()
 
diff --git a/testing/adios2/CMakeLists.txt b/testing/adios2/CMakeLists.txt
index f9047a86a7..9a840fdd3d 100644
--- a/testing/adios2/CMakeLists.txt
+++ b/testing/adios2/CMakeLists.txt
@@ -11,3 +11,4 @@ add_subdirectory(xml)
 add_subdirectory(yaml)
 add_subdirectory(performance)
 add_subdirectory(helper)
+add_subdirectory(hierarchy)
diff --git a/testing/adios2/bindings/C/TestBPWriteAggregateReadLocal.cpp b/testing/adios2/bindings/C/TestBPWriteAggregateReadLocal.cpp
index 3b1cd12645..64b85be962 100644
--- a/testing/adios2/bindings/C/TestBPWriteAggregateReadLocal.cpp
+++ b/testing/adios2/bindings/C/TestBPWriteAggregateReadLocal.cpp
@@ -362,8 +362,8 @@ TEST_P(BPWriteAggregateReadLocalTest, Aggregate1DBlock0)
     LocalAggregate1DBlock0(GetParam());
 }
 
-INSTANTIATE_TEST_CASE_P(Substreams, BPWriteAggregateReadLocalTest,
-                        ::testing::Values("1", "2", "3", "4"));
+INSTANTIATE_TEST_SUITE_P(Substreams, BPWriteAggregateReadLocalTest,
+                         ::testing::Values("1", "2", "3", "4"));
 
 int main(int argc, char **argv)
 {
diff --git a/testing/adios2/bindings/C/TestBPWriteTypes.cpp b/testing/adios2/bindings/C/TestBPWriteTypes.cpp
index b0b4eaca23..a98e0338b6 100644
--- a/testing/adios2/bindings/C/TestBPWriteTypes.cpp
+++ b/testing/adios2/bindings/C/TestBPWriteTypes.cpp
@@ -8,13 +8,16 @@
  *      Author: William F Godoy godoywf@ornl.gov
  */
 
+#include 
+
+#include 
+
 #include 
 
 #if ADIOS2_USE_MPI
 #include 
 #endif
 
-#include 
 #include 
 
 #include "SmallTestData_c.h"
diff --git a/testing/adios2/bindings/fortran/TestBPWriteTypes.F90 b/testing/adios2/bindings/fortran/TestBPWriteTypes.F90
index 28f3913467..4aaa8a21bc 100644
--- a/testing/adios2/bindings/fortran/TestBPWriteTypes.F90
+++ b/testing/adios2/bindings/fortran/TestBPWriteTypes.F90
@@ -231,7 +231,7 @@ program TestBPWriteTypes
      call adios2_open(bpReader, ioRead, "ftypes.bp", adios2_mode_read, ierr)
 
      call adios2_steps(nsteps, bpReader, ierr)
-     if(nsteps /= 3)  write(*,*) 'nsteps: ', nsteps !stop 'ftypes.bp must have 3 steps'
+     if(nsteps /= 3) stop 'ftypes.bp must have 3 steps'
 
      call adios2_inquire_variable(variables(1), ioRead, "var_I8", ierr)
      if (variables(1)%name /= 'var_I8') stop 'var_I8 not recognized'
diff --git a/testing/adios2/bindings/python/CMakeLists.txt b/testing/adios2/bindings/python/CMakeLists.txt
index 8f3bf44888..d35fd4f8ad 100644
--- a/testing/adios2/bindings/python/CMakeLists.txt
+++ b/testing/adios2/bindings/python/CMakeLists.txt
@@ -20,6 +20,7 @@ if(ADIOS2_HAVE_MPI)
   add_python_mpi_test(BPWriteReadTypes)
   add_python_mpi_test(BPWriteTypesHighLevelAPI)
   add_python_mpi_test(BPWriteTypesHighLevelAPILocal)
+  add_python_mpi_test(BPWriteReadString)
   add_python_mpi_test(BPReadMultisteps)
   add_python_mpi_test(BPWriteRead2D)
   add_python_mpi_test(BPBlocksInfo)
diff --git a/testing/adios2/bindings/python/TestBPReadMultisteps.py b/testing/adios2/bindings/python/TestBPReadMultisteps.py
index 5497d007e0..606309a414 100644
--- a/testing/adios2/bindings/python/TestBPReadMultisteps.py
+++ b/testing/adios2/bindings/python/TestBPReadMultisteps.py
@@ -225,34 +225,34 @@ def check_name(name, name_list):
 
 for i in range(0, 3):
     for j in range(0, Nx):
-        if(inI8[i][j] != i):
+        if inI8[i][j] != i:
             raise ValueError('failed reading I8')
 
-        if(inI16[i][j] != i):
+        if inI16[i][j] != i:
             raise ValueError('failed reading I16')
 
-        if(inI32[i][j] != i):
+        if inI32[i][j] != i:
             raise ValueError('failed reading I32')
 
-        if(inI64[i][j] != i):
+        if inI64[i][j] != i:
             raise ValueError('failed reading I64')
 
-        if(inU8[i][j] != i):
+        if inU8[i][j] != i:
             raise ValueError('failed reading U8')
 
-        if(inU16[i][j] != i):
+        if inU16[i][j] != i:
             raise ValueError('failed reading U16')
 
-        if(inU32[i][j] != i):
+        if inU32[i][j] != i:
             raise ValueError('failed reading U32')
 
-        if(inU64[i][j] != i):
+        if inU64[i][j] != i:
             raise ValueError('failed reading U64')
 
-        if(inR32[i][j] != i):
+        if inR32[i][j] != i:
             raise ValueError('failed reading R32')
 
-        if(inR64[i][j] != i):
+        if inR64[i][j] != i:
             raise ValueError('failed reading R64')
 
 
diff --git a/testing/adios2/bindings/python/TestBPWriteRead2D.py b/testing/adios2/bindings/python/TestBPWriteRead2D.py
index 525e2db4b1..fe25dc7876 100644
--- a/testing/adios2/bindings/python/TestBPWriteRead2D.py
+++ b/testing/adios2/bindings/python/TestBPWriteRead2D.py
@@ -52,7 +52,7 @@
     ibpStream = ioRead.Open('HeatMap2D_py.bp', adios2.Mode.Read, MPI.COMM_SELF)
     var_inTemperature = ioRead.InquireVariable("temperature2D")
 
-    if(var_inTemperature is False):
+    if var_inTemperature is False:
         raise ValueError('var_inTemperature is False')
 
     assert var_inTemperature is not None
diff --git a/testing/adios2/bindings/python/TestBPWriteReadString.py b/testing/adios2/bindings/python/TestBPWriteReadString.py
new file mode 100644
index 0000000000..f17c4c7515
--- /dev/null
+++ b/testing/adios2/bindings/python/TestBPWriteReadString.py
@@ -0,0 +1,79 @@
+#!/usr/bin/env python
+#
+# Distributed under the OSI-approved Apache License, Version 2.0.  See
+# accompanying file Copyright.txt for details.
+#
+# TestBPWriteReadString.py: test writing/reading Python string type
+# in ADIOS2 File Write
+#  Created on: Oct 19, 2020
+#      Author: Dmitry Ganyushin ganyushindi@ornl.gov
+import unittest
+from mpi4py import MPI
+import adios2
+
+N_STEPS = 3
+
+
+class TestAdiosWriteReadStringfullAPI(unittest.TestCase):
+
+    def test_write_read_string_fullAPI(self):
+        comm = MPI.COMM_WORLD
+        theString = 'hello adios'
+        bpFilename = 'string_test_fullAPI.bp'
+        varname = 'mystringvar'
+        adios = adios2.ADIOS(comm)
+        ioWrite = adios.DeclareIO('ioWriter')
+        adEngine = ioWrite.Open(bpFilename, adios2.Mode.Write)
+        varMyString = ioWrite.DefineVariable(varname)
+        for step in range(N_STEPS):
+            adEngine.BeginStep()
+            adEngine.Put(varMyString, theString + str(step))
+            adEngine.EndStep()
+        adEngine.Close()
+
+        ioRead = adios.DeclareIO('ioReader')
+        adEngine = ioRead.Open(bpFilename, adios2.Mode.Read)
+        for step in range(N_STEPS):
+            adEngine.BeginStep()
+            varReadMyString = ioRead.InquireVariable(varname)
+            result = adEngine.Get(varReadMyString)
+            adEngine.EndStep()
+            self.assertEqual(result, theString + str(step))
+        adEngine.Close()
+
+    def test_write_read_string_highAPI(self):
+        comm = MPI.COMM_WORLD
+        theString = 'hello adios'
+        bpFilename = 'string_test_highAPI.bp'
+        varname = 'mystringvar'
+
+        with adios2.open(bpFilename, "w", comm) as fh:
+
+            for step in range(N_STEPS):
+                fh.write(varname, theString + str(step), end_step=True)
+
+        with adios2.open(bpFilename, "r", comm) as fh:
+            for fstep in fh:
+                step = fstep.current_step()
+                result = fstep.read(varname)
+                self.assertEqual("".join([chr(s) for s in result]),
+                                 theString + str(step))
+
+    def test_read_strings_all_steps(self):
+        comm = MPI.COMM_WORLD
+        fileName = 'string_test_all.bp'
+        with adios2.open(fileName, "w", comm) as fh:
+            for i in range(N_STEPS):
+                fh.write("string_variable", "written {}".format(i))
+                fh.end_step()
+
+        with adios2.open(fileName, "r", comm) as fh:
+            n = fh.steps()
+            name = "string_variable"
+            result = fh.read_string(name, 0, n)
+            expected_str = ["written {}".format(i) for i in range(n)]
+            self.assertEqual(result, expected_str)
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/testing/adios2/bindings/python/TestBPWriteReadTypes.py b/testing/adios2/bindings/python/TestBPWriteReadTypes.py
index b160693d96..1dbf6bbb1a 100644
--- a/testing/adios2/bindings/python/TestBPWriteReadTypes.py
+++ b/testing/adios2/bindings/python/TestBPWriteReadTypes.py
@@ -25,7 +25,7 @@ def check_name(name, name_list):
 
 
 def check_array(np1, np2, hint):
-    if((np1 == np2).all() is False):
+    if (np1 == np2).all() is False:
         print("InData: " + str(np1))
         print("Data: " + str(np2))
         raise ValueError('Array read failed ' + str(hint))
@@ -95,13 +95,11 @@ def check_array(np1, np2, hint):
 attR64 = ioWriter.DefineAttribute("attrR64", data.R64)
 
 ioWriter.SetEngine("BPFile")
-ioParams = {}
-ioParams['Threads'] = '1'
-ioParams['InitialBufferSize'] = '17Kb'
+ioParams = {'Threads': '1', 'InitialBufferSize': '17Kb'}
 ioWriter.SetParameters(ioParams)
 
 engineType = ioWriter.EngineType()
-if(engineType != "BPFile"):
+if engineType != "BPFile":
     raise ValueError(str(engineType) +
                      ' incorrect engine type, should be BPFile')
 
@@ -172,11 +170,11 @@ def check_array(np1, np2, hint):
 check_object(attrR64, "attrR64")
 
 attrStringData = attrString.DataString()
-if(attrStringData[0] != "one"):
+if attrStringData[0] != "one":
     raise ValueError('attrStringData[0] failed')
-if(attrStringData[1] != "two"):
+if attrStringData[1] != "two":
     raise ValueError('attrStringData[1] failed')
-if(attrStringData[2] != "three"):
+if attrStringData[2] != "three":
     raise ValueError('attrStringData[2] failed')
 
 attrI8Data = attrI8.Data()
@@ -245,7 +243,7 @@ def check_array(np1, np2, hint):
 
 
 result = adios.RemoveIO('writer')
-if(result is False):
+if result is False:
     raise ValueError('Could not remove IO writer')
 
 assert (reader.Steps() == nsteps)
@@ -255,7 +253,7 @@ def check_array(np1, np2, hint):
 
 ioReader.RemoveAllVariables()
 varStr = ioReader.InquireVariable("varStr")
-if(varStr is True):
+if varStr is True:
     raise ValueError('Could remove reader variables')
 
 adios.RemoveAllIOs()
diff --git a/testing/adios2/bindings/python/TestBPWriteTypesHighLevelAPI.py b/testing/adios2/bindings/python/TestBPWriteTypesHighLevelAPI.py
index b89827c344..8033b6609b 100644
--- a/testing/adios2/bindings/python/TestBPWriteTypesHighLevelAPI.py
+++ b/testing/adios2/bindings/python/TestBPWriteTypesHighLevelAPI.py
@@ -33,7 +33,7 @@
 
         data.update(rank, i, size)
 
-        if(rank == 0 and i == 0):
+        if rank == 0 and i == 0:
             fw.write("tag", "Testing ADIOS2 high-level API")
             fw.write("gvarI8", np.array(data.I8[0]))
             fw.write("gvarI16", np.array(data.I16[0]))
@@ -85,7 +85,7 @@
         fw.write("varR32", data.R32, shape, start, count)
         fw.write("varR64", data.R64, shape, start, count)
 
-        if(rank == 0 and i == 0):
+        if rank == 0 and i == 0:
             fw.write_attribute("varattrStrArray", [
                 "varattr1", "varattr2", "varattr3"], "steps")
             fw.write_attribute("varattrI8Array", data.I8, "varI8")
@@ -125,7 +125,7 @@
 #                 print("\t" + key + ": " + value)
 #             print("\n")
 
-        if(step == 0):
+        if step == 0:
             inTag = fr_step.read_string("tag")
             inI8 = fr_step.read("gvarI8")
             inI16 = fr_step.read("gvarI16")
@@ -138,38 +138,38 @@
             inR32 = fr_step.read("gvarR32")
             inR64 = fr_step.read("gvarR64")
 
-            if(inTag[0] != "Testing ADIOS2 high-level API"):
+            if inTag[0] != "Testing ADIOS2 high-level API":
                 print("InTag: " + str(inTag))
                 raise ValueError('tag variable read failed')
 
-            if(inI8 != data.I8[0]):
+            if inI8 != data.I8[0]:
                 raise ValueError('gvarI8 read failed')
 
-            if(inI16 != data.I16[0]):
+            if inI16 != data.I16[0]:
                 raise ValueError('gvarI16 read failed')
 
-            if(inI32 != data.I32[0]):
+            if inI32 != data.I32[0]:
                 raise ValueError('gvarI32 read failed')
 
-            if(inI64 != data.I64[0]):
+            if inI64 != data.I64[0]:
                 raise ValueError('gvarI64 read failed')
 
-            if(inU8 != data.U8[0]):
+            if inU8 != data.U8[0]:
                 raise ValueError('gvarU8 read failed')
 
-            if(inU16 != data.U16[0]):
+            if inU16 != data.U16[0]:
                 raise ValueError('gvarU16 read failed')
 
-            if(inU32 != data.U32[0]):
+            if inU32 != data.U32[0]:
                 raise ValueError('gvarU32 read failed')
 
-            if(inU64 != data.U64[0]):
+            if inU64 != data.U64[0]:
                 raise ValueError('gvarU64 read failed')
 
-            if(inR32 != data.R32[0]):
+            if inR32 != data.R32[0]:
                 raise ValueError('gvarR32 read failed')
 
-            if(inR64 != data.R64[0]):
+            if inR64 != data.R64[0]:
                 raise ValueError('gvarR64 read failed')
 
             # attributes
@@ -185,37 +185,37 @@
             inR32 = fr_step.read_attribute("attrR32")
             inR64 = fr_step.read_attribute("attrR64")
 
-            if(inTag[0] != "Testing single string attribute"):
+            if inTag[0] != "Testing single string attribute":
                 raise ValueError('attr string read failed')
 
-            if(inI8[0] != data.I8[0]):
+            if inI8[0] != data.I8[0]:
                 raise ValueError('attrI8 read failed')
 
-            if(inI16[0] != data.I16[0]):
+            if inI16[0] != data.I16[0]:
                 raise ValueError('attrI16 read failed')
 
-            if(inI32[0] != data.I32[0]):
+            if inI32[0] != data.I32[0]:
                 raise ValueError('attrI32 read failed')
 
-            if(inI64[0] != data.I64[0]):
+            if inI64[0] != data.I64[0]:
                 raise ValueError('attrI64 read failed')
 
-            if(inU8[0] != data.U8[0]):
+            if inU8[0] != data.U8[0]:
                 raise ValueError('attrU8 read failed')
 
-            if(inU16[0] != data.U16[0]):
+            if inU16[0] != data.U16[0]:
                 raise ValueError('attrU16 read failed')
 
-            if(inU32[0] != data.U32[0]):
+            if inU32[0] != data.U32[0]:
                 raise ValueError('attrU32 read failed')
 
-            if(inU64[0] != data.U64[0]):
+            if inU64[0] != data.U64[0]:
                 raise ValueError('attrU64 read failed')
 
-            if(inR32[0] != data.R32[0]):
+            if inR32[0] != data.R32[0]:
                 raise ValueError('attrR32 read failed')
 
-            if(inR64[0] != data.R64[0]):
+            if inR64[0] != data.R64[0]:
                 raise ValueError('attrR64 read failed')
 
             # Array attribute
@@ -231,37 +231,37 @@
             inR32 = fr_step.read_attribute("attrR32Array")
             inR64 = fr_step.read_attribute("attrR64Array")
 
-            if(inTag != ["string1", "string2", "string3"]):
+            if inTag != ["string1", "string2", "string3"]:
                 raise ValueError('attrStrArray read failed')
 
-            if((inI8 == data.I8).all() is False):
+            if (inI8 == data.I8).all() is False:
                 raise ValueError('attrI8 array read failed')
 
-            if((inI16 == data.I16).all() is False):
+            if (inI16 == data.I16).all() is False:
                 raise ValueError('attrI16 array read failed')
 
-            if((inI32 == data.I32).all() is False):
+            if (inI32 == data.I32).all() is False:
                 raise ValueError('attrI32 array read failed')
 
-            if((inI64 == data.I64).all() is False):
+            if (inI64 == data.I64).all() is False:
                 raise ValueError('attrI64 array read failed')
 
-            if((inU8 == data.U8).all() is False):
+            if (inU8 == data.U8).all() is False:
                 raise ValueError('attrU8 array read failed')
 
-            if((inU16 == data.U16).all() is False):
+            if (inU16 == data.U16).all() is False:
                 raise ValueError('attrU16 array read failed')
 
-            if((inU32 == data.U32).all() is False):
+            if (inU32 == data.U32).all() is False:
                 raise ValueError('attrU32 array read failed')
 
-            if((inU64 == data.U64).all() is False):
+            if (inU64 == data.U64).all() is False:
                 raise ValueError('attrU64 array read failed')
 
-            if((inR32 == data.R32).all() is False):
+            if (inR32 == data.R32).all() is False:
                 raise ValueError('attrR32 array read failed')
 
-            if((inR64 == data.R64).all() is False):
+            if (inR64 == data.R64).all() is False:
                 raise ValueError('attrR64 array read failed')
 
             inTags = fr_step.read_attribute_string("varattrStrArray", "steps")
@@ -276,50 +276,50 @@
             inR32 = fr_step.read_attribute("varattrR32Array", "varR32")
             inR64 = fr_step.read_attribute("varattrR64Array", "varR64")
 
-            if(inTags != ["varattr1", "varattr2", "varattr3"]):
+            if inTags != ["varattr1", "varattr2", "varattr3"]:
                 print(inTags)
                 raise ValueError('var attrStrArray read failed')
 
-            if((inI8 == data.I8).all() is False):
+            if (inI8 == data.I8).all() is False:
                 raise ValueError('var attrI8 array read failed')
 
-            if((inI16 == data.I16).all() is False):
+            if (inI16 == data.I16).all() is False:
                 raise ValueError('var attrI16 array read failed')
 
-            if((inI32 == data.I32).all() is False):
+            if (inI32 == data.I32).all() is False:
                 raise ValueError('var attrI32 array read failed')
 
-            if((inI64 == data.I64).all() is False):
+            if (inI64 == data.I64).all() is False:
                 raise ValueError('var attrI64 array read failed')
 
-            if((inU8 == data.U8).all() is False):
+            if (inU8 == data.U8).all() is False:
                 raise ValueError('var attrU8 array read failed')
 
-            if((inU16 == data.U16).all() is False):
+            if (inU16 == data.U16).all() is False:
                 raise ValueError('var attrU16 array read failed')
 
-            if((inU32 == data.U32).all() is False):
+            if (inU32 == data.U32).all() is False:
                 raise ValueError('var attrU32 array read failed')
 
-            if((inU64 == data.U64).all() is False):
+            if (inU64 == data.U64).all() is False:
                 raise ValueError('var attrU64 array read failed')
 
-            if((inR32 == data.R32).all() is False):
+            if (inR32 == data.R32).all() is False:
                 raise ValueError('var attrR32 array read failed')
 
-            if((inR64 == data.R64).all() is False):
+            if (inR64 == data.R64).all() is False:
                 raise ValueError('var attrR64 array read failed')
 
         stepStr = "Step:" + str(step)
 
         instepStr = fr_step.read_string("steps")
-        if(instepStr[0] != stepStr):
+        if instepStr[0] != stepStr:
             raise ValueError('steps variable read failed: ' +
                              instepStr + " " + stepStr)
 
         indataRanks = fr_step.read("rank", [0], [size])
         dataRanks = np.arange(0, size)
-        if((indataRanks == dataRanks).all() is False):
+        if (indataRanks == dataRanks).all() is False:
             raise ValueError('Ranks read failed')
 
         indataI8 = fr_step.read("varI8", start, count)
@@ -334,32 +334,32 @@
         indataR64 = fr_step.read("varR64", start, count)
         fr_step.end_step()
 
-        if((indataI8 == data.I8).all() is False):
+        if (indataI8 == data.I8).all() is False:
             raise ValueError('I8 array read failed')
 
-        if((indataI16 == data.I16).all() is False):
+        if (indataI16 == data.I16).all() is False:
             raise ValueError('I16 array read failed')
 
-        if((indataI32 == data.I32).all() is False):
+        if (indataI32 == data.I32).all() is False:
             raise ValueError('I32 array read failed')
 
-        if((indataI64 == data.I64).all() is False):
+        if (indataI64 == data.I64).all() is False:
             raise ValueError('I64 array read failed')
 
-        if((indataU8 == data.U8).all() is False):
+        if (indataU8 == data.U8).all() is False:
             raise ValueError('U8 array read failed')
 
-        if((indataU16 == data.U16).all() is False):
+        if (indataU16 == data.U16).all() is False:
             raise ValueError('U16 array read failed')
 
-        if((indataU32 == data.U32).all() is False):
+        if (indataU32 == data.U32).all() is False:
             raise ValueError('U32 array read failed')
 
-        if((indataU64 == data.U64).all() is False):
+        if (indataU64 == data.U64).all() is False:
             raise ValueError('U64 array read failed')
 
-        if((indataR32 == data.R32).all() is False):
+        if (indataR32 == data.R32).all() is False:
             raise ValueError('R32 array read failed')
 
-        if((indataR64 == data.R64).all() is False):
+        if (indataR64 == data.R64).all() is False:
             raise ValueError('R64 array read failed')
diff --git a/testing/adios2/bindings/python/TestBPWriteTypesHighLevelAPILocal.py b/testing/adios2/bindings/python/TestBPWriteTypesHighLevelAPILocal.py
index c2f52409ae..842d8ea5ac 100644
--- a/testing/adios2/bindings/python/TestBPWriteTypesHighLevelAPILocal.py
+++ b/testing/adios2/bindings/python/TestBPWriteTypesHighLevelAPILocal.py
@@ -14,7 +14,7 @@
 
 
 def check_array(np1, np2, hint):
-    if((np1 == np2).all() is False):
+    if (np1 == np2).all() is False:
         print("InData: " + str(np1))
         print("Data: " + str(np2))
         raise ValueError('Array read failed ' + str(hint))
diff --git a/testing/adios2/bindings/python/TestBPWriteTypesHighLevelAPI_HDF5.py b/testing/adios2/bindings/python/TestBPWriteTypesHighLevelAPI_HDF5.py
index ea71f5749f..1ac82a605d 100644
--- a/testing/adios2/bindings/python/TestBPWriteTypesHighLevelAPI_HDF5.py
+++ b/testing/adios2/bindings/python/TestBPWriteTypesHighLevelAPI_HDF5.py
@@ -31,7 +31,7 @@
     for i in range(0, 5):
         data.update(rank, i, size)
 
-        if(rank == 0 and i == 0):
+        if rank == 0 and i == 0:
             fw.write("tag", "Testing ADIOS2 high-level API")
             fw.write("gvarI8", np.array(data.I8[0]))
             fw.write("gvarI16", np.array(data.I16[0]))
@@ -82,7 +82,7 @@
         fw.write("varR32", data.R32, shape, start, count)
         fw.write("varR64", data.R64, shape, start, count)
 
-        if(rank == 0 and i == 0):
+        if rank == 0 and i == 0:
             fw.write_attribute("varattrStrArray", [
                 "varattr1", "varattr2", "varattr3"], "steps")
             fw.write_attribute("varattrI8Array", data.I8, "varI8")
@@ -119,7 +119,7 @@
 #                 print("\t" + key + ": " + value)
 #             print("\n")
 
-        if(step == 0):
+        if step == 0:
             inTag = fr_step.read_string("tag")
             inI8 = fr_step.read("gvarI8")
             inI16 = fr_step.read("gvarI16")
@@ -132,38 +132,38 @@
             inR32 = fr_step.read("gvarR32")
             inR64 = fr_step.read("gvarR64")
 
-            if(inTag[0] != "Testing ADIOS2 high-level API"):
+            if inTag[0] != "Testing ADIOS2 high-level API":
                 print("InTag: " + str(inTag))
                 raise ValueError('tag variable read failed')
 
-            if(inI8 != data.I8[0]):
+            if inI8 != data.I8[0]:
                 raise ValueError('gvarI8 read failed')
 
-            if(inI16 != data.I16[0]):
+            if inI16 != data.I16[0]:
                 raise ValueError('gvarI16 read failed')
 
-            if(inI32 != data.I32[0]):
+            if inI32 != data.I32[0]:
                 raise ValueError('gvarI32 read failed')
 
-            if(inI64 != data.I64[0]):
+            if inI64 != data.I64[0]:
                 raise ValueError('gvarI64 read failed')
 
-            if(inU8 != data.U8[0]):
+            if inU8 != data.U8[0]:
                 raise ValueError('gvarU8 read failed')
 
-            if(inU16 != data.U16[0]):
+            if inU16 != data.U16[0]:
                 raise ValueError('gvarU16 read failed')
 
-            if(inU32 != data.U32[0]):
+            if inU32 != data.U32[0]:
                 raise ValueError('gvarU32 read failed')
 
-            if(inU64 != data.U64[0]):
+            if inU64 != data.U64[0]:
                 raise ValueError('gvarU64 read failed')
 
-            if(inR32 != data.R32[0]):
+            if inR32 != data.R32[0]:
                 raise ValueError('gvarR32 read failed')
 
-            if(inR64 != data.R64[0]):
+            if inR64 != data.R64[0]:
                 raise ValueError('gvarR64 read failed')
 
             # attributes
@@ -179,37 +179,37 @@
             inR32 = fr_step.read_attribute("attrR32")
             inR64 = fr_step.read_attribute("attrR64")
 
-            if(inTag[0] != "Testing single string attribute"):
+            if inTag[0] != "Testing single string attribute":
                 raise ValueError('attr string read failed')
 
-            if(inI8[0] != data.I8[0]):
+            if inI8[0] != data.I8[0]:
                 raise ValueError('attrI8 read failed')
 
-            if(inI16[0] != data.I16[0]):
+            if inI16[0] != data.I16[0]:
                 raise ValueError('attrI16 read failed')
 
-            if(inI32[0] != data.I32[0]):
+            if inI32[0] != data.I32[0]:
                 raise ValueError('attrI32 read failed')
 
-            if(inI64[0] != data.I64[0]):
+            if inI64[0] != data.I64[0]:
                 raise ValueError('attrI64 read failed')
 
-            if(inU8[0] != data.U8[0]):
+            if inU8[0] != data.U8[0]:
                 raise ValueError('attrU8 read failed')
 
-            if(inU16[0] != data.U16[0]):
+            if inU16[0] != data.U16[0]:
                 raise ValueError('attrU16 read failed')
 
-            if(inU32[0] != data.U32[0]):
+            if inU32[0] != data.U32[0]:
                 raise ValueError('attrU32 read failed')
 
-            if(inU64[0] != data.U64[0]):
+            if inU64[0] != data.U64[0]:
                 raise ValueError('attrU64 read failed')
 
-            if(inR32[0] != data.R32[0]):
+            if inR32[0] != data.R32[0]:
                 raise ValueError('attrR32 read failed')
 
-            if(inR64[0] != data.R64[0]):
+            if inR64[0] != data.R64[0]:
                 raise ValueError('attrR64 read failed')
 
             # Array attribute
@@ -225,37 +225,37 @@
             inR32 = fr_step.read_attribute("attrR32Array")
             inR64 = fr_step.read_attribute("attrR64Array")
 
-            if(inTag != ["string1", "string2", "string3"]):
+            if inTag != ["string1", "string2", "string3"]:
                 raise ValueError('attrStrArray read failed')
 
-            if((inI8 == data.I8).all() is False):
+            if (inI8 == data.I8).all() is False:
                 raise ValueError('attrI8 array read failed')
 
-            if((inI16 == data.I16).all() is False):
+            if (inI16 == data.I16).all() is False:
                 raise ValueError('attrI16 array read failed')
 
-            if((inI32 == data.I32).all() is False):
+            if (inI32 == data.I32).all() is False:
                 raise ValueError('attrI32 array read failed')
 
-            if((inI64 == data.I64).all() is False):
+            if (inI64 == data.I64).all() is False:
                 raise ValueError('attrI64 array read failed')
 
-            if((inU8 == data.U8).all() is False):
+            if (inU8 == data.U8).all() is False:
                 raise ValueError('attrU8 array read failed')
 
-            if((inU16 == data.U16).all() is False):
+            if (inU16 == data.U16).all() is False:
                 raise ValueError('attrU16 array read failed')
 
-            if((inU32 == data.U32).all() is False):
+            if (inU32 == data.U32).all() is False:
                 raise ValueError('attrU32 array read failed')
 
-            if((inU64 == data.U64).all() is False):
+            if (inU64 == data.U64).all() is False:
                 raise ValueError('attrU64 array read failed')
 
-            if((inR32 == data.R32).all() is False):
+            if (inR32 == data.R32).all() is False:
                 raise ValueError('attrR32 array read failed')
 
-            if((inR64 == data.R64).all() is False):
+            if (inR64 == data.R64).all() is False:
                 raise ValueError('attrR64 array read failed')
 
             inTags = fr_step.read_attribute_string("varattrStrArray", "steps")
@@ -270,44 +270,44 @@
             inR32 = fr_step.read_attribute("varattrR32Array", "varR32")
             inR64 = fr_step.read_attribute("varattrR64Array", "varR64")
 
-            if(inTags != ["varattr1", "varattr2", "varattr3"]):
+            if inTags != ["varattr1", "varattr2", "varattr3"]:
                 print(inTags)
                 raise ValueError('var attrStrArray read failed')
 
-            if((inI8 == data.I8).all() is False):
+            if (inI8 == data.I8).all() is False:
                 raise ValueError('var attrI8 array read failed')
 
-            if((inI16 == data.I16).all() is False):
+            if (inI16 == data.I16).all() is False:
                 raise ValueError('var attrI16 array read failed')
 
-            if((inI32 == data.I32).all() is False):
+            if (inI32 == data.I32).all() is False:
                 raise ValueError('var attrI32 array read failed')
 
-            if((inI64 == data.I64).all() is False):
+            if (inI64 == data.I64).all() is False:
                 raise ValueError('var attrI64 array read failed')
 
-            if((inU8 == data.U8).all() is False):
+            if (inU8 == data.U8).all() is False:
                 raise ValueError('var attrU8 array read failed')
 
-            if((inU16 == data.U16).all() is False):
+            if (inU16 == data.U16).all() is False:
                 raise ValueError('var attrU16 array read failed')
 
-            if((inU32 == data.U32).all() is False):
+            if (inU32 == data.U32).all() is False:
                 raise ValueError('var attrU32 array read failed')
 
-            if((inU64 == data.U64).all() is False):
+            if (inU64 == data.U64).all() is False:
                 raise ValueError('var attrU64 array read failed')
 
-            if((inR32 == data.R32).all() is False):
+            if (inR32 == data.R32).all() is False:
                 raise ValueError('var attrR32 array read failed')
 
-            if((inR64 == data.R64).all() is False):
+            if (inR64 == data.R64).all() is False:
                 raise ValueError('var attrR64 array read failed')
 
         stepStr = "Step:" + str(step)
 
         instepStr = fr_step.read_string("steps")
-        if(instepStr[0] != stepStr):
+        if instepStr[0] != stepStr:
             raise ValueError('steps variable read failed: ' +
                              instepStr + " " + stepStr)
 
@@ -323,32 +323,32 @@
         indataR64 = fr_step.read("varR64", start, count)
         fr_step.end_step()
 
-        if((indataI8 == data.I8).all() is False):
+        if (indataI8 == data.I8).all() is False:
             raise ValueError('I8 array read failed')
 
-        if((indataI16 == data.I16).all() is False):
+        if (indataI16 == data.I16).all() is False:
             raise ValueError('I16 array read failed')
 
-        if((indataI32 == data.I32).all() is False):
+        if (indataI32 == data.I32).all() is False:
             raise ValueError('I32 array read failed')
 
-        if((indataI64 == data.I64).all() is False):
+        if (indataI64 == data.I64).all() is False:
             raise ValueError('I64 array read failed')
 
-        if((indataU8 == data.U8).all() is False):
+        if (indataU8 == data.U8).all() is False:
             raise ValueError('U8 array read failed')
 
-        if((indataU16 == data.U16).all() is False):
+        if (indataU16 == data.U16).all() is False:
             raise ValueError('U16 array read failed')
 
-        if((indataU32 == data.U32).all() is False):
+        if (indataU32 == data.U32).all() is False:
             raise ValueError('U32 array read failed')
 
-        if((indataU64 == data.U64).all() is False):
+        if (indataU64 == data.U64).all() is False:
             raise ValueError('U64 array read failed')
 
-        if((indataR32 == data.R32).all() is False):
+        if (indataR32 == data.R32).all() is False:
             raise ValueError('R32 array read failed')
 
-        if((indataR64 == data.R64).all() is False):
+        if (indataR64 == data.R64).all() is False:
             raise ValueError('R64 array read failed')
diff --git a/testing/adios2/engine/SmallTestData.h b/testing/adios2/engine/SmallTestData.h
index 929031e92a..2d309b16e7 100644
--- a/testing/adios2/engine/SmallTestData.h
+++ b/testing/adios2/engine/SmallTestData.h
@@ -7,6 +7,7 @@
 
 #include 
 
+#include 
 #include 
 #include 
 #include 
diff --git a/testing/adios2/engine/bp/CMakeLists.txt b/testing/adios2/engine/bp/CMakeLists.txt
index 5b283070f9..398fae35f8 100644
--- a/testing/adios2/engine/bp/CMakeLists.txt
+++ b/testing/adios2/engine/bp/CMakeLists.txt
@@ -5,8 +5,10 @@
 
 set(BP3_DIR ${CMAKE_CURRENT_BINARY_DIR}/bp3)
 set(BP4_DIR ${CMAKE_CURRENT_BINARY_DIR}/bp4)
+set(FS_DIR ${CMAKE_CURRENT_BINARY_DIR}/filestream)
 file(MAKE_DIRECTORY ${BP3_DIR})
 file(MAKE_DIRECTORY ${BP4_DIR})
+file(MAKE_DIRECTORY ${FS_DIR})
 
 macro(bp3_bp4_gtest_add_tests_helper testname mpi)
   gtest_add_tests_helper(${testname} ${mpi} BP Engine.BP. .BP3
@@ -41,6 +43,13 @@ bp3_bp4_gtest_add_tests_helper(WriteReadBlockInfo MPI_ALLOW)
 bp3_bp4_gtest_add_tests_helper(WriteReadVariableSpan MPI_ALLOW)
 bp3_bp4_gtest_add_tests_helper(TimeAggregation MPI_ALLOW)
 bp3_bp4_gtest_add_tests_helper(NoXMLRecovery MPI_ALLOW)
+bp3_bp4_gtest_add_tests_helper(StepsFileGlobalArray MPI_ALLOW)
+bp3_bp4_gtest_add_tests_helper(StepsFileLocalArray MPI_ALLOW)
+bp3_bp4_gtest_add_tests_helper(SelectSteps MPI_ALLOW)
+
+if(NOT MSVC)
+  bp3_bp4_gtest_add_tests_helper(BufferSize MPI_NONE)
+endif()
 
 if(ADIOS2_HAVE_MPI)
   bp3_bp4_gtest_add_tests_helper(WriteAggregateRead MPI_ONLY)
@@ -55,4 +64,18 @@ gtest_add_tests_helper(WriteNull MPI_ALLOW BP Engine.BP. .BP3
 gtest_add_tests_helper(WriteAppendReadADIOS2 MPI_ALLOW BP Engine.BP. .BP4
   WORKING_DIRECTORY ${BP4_DIR} EXTRA_ARGS "BP4"
 )
+gtest_add_tests_helper(StepsInSituGlobalArray MPI_ALLOW BP Engine.BP. .BP4
+  WORKING_DIRECTORY ${BP4_DIR} EXTRA_ARGS "BP4"
+)
+gtest_add_tests_helper(StepsInSituLocalArray MPI_ALLOW BP Engine.BP. .BP4
+  WORKING_DIRECTORY ${BP4_DIR} EXTRA_ARGS "BP4"
+)
+
+# FileStream is BP4 + StreamReader=true
+gtest_add_tests_helper(StepsInSituGlobalArray MPI_ALLOW BP Engine.BP. .FileStream
+  WORKING_DIRECTORY ${FS_DIR} EXTRA_ARGS "FileStream"
+)
+gtest_add_tests_helper(StepsInSituLocalArray MPI_ALLOW BP Engine.BP. .FileStream
+  WORKING_DIRECTORY ${FS_DIR} EXTRA_ARGS "FileStream"
+)
 
diff --git a/testing/adios2/engine/bp/TestBPBufferSize.cpp b/testing/adios2/engine/bp/TestBPBufferSize.cpp
new file mode 100644
index 0000000000..03060da0b8
--- /dev/null
+++ b/testing/adios2/engine/bp/TestBPBufferSize.cpp
@@ -0,0 +1,286 @@
+/*
+ * Distributed under the OSI-approved Apache License, Version 2.0.  See
+ * accompanying file Copyright.txt for details.
+ */
+#include 
+#include 
+
+#include 
+#include 
+#include 
+
+#include 
+#include 
+
+std::string engineName; // comes from command line
+
+class BPBufferSizeTest : public ::testing::Test
+{
+public:
+    BPBufferSizeTest() = default;
+};
+
+std::ifstream vm_input;
+void PrintVMStatus()
+{
+    std::string m_Name = "/proc/self/status";
+
+    if (!vm_input.is_open())
+    {
+        vm_input.open(m_Name.c_str());
+    }
+
+    if (vm_input.is_open())
+    {
+        vm_input.seekg(0);
+        for (std::string line; getline(vm_input, line);)
+        {
+            if (line.find("VmRSS") == 0)
+                std::cout << line << " ";
+            if (line.find("VmSize") == 0)
+                std::cout << line << " ";
+        }
+        vm_input.close();
+    }
+}
+
+void VMStatusClose()
+{
+    if (vm_input.is_open())
+    {
+        vm_input.close();
+    }
+}
+
+size_t GetAndPrintBufferSize(adios2::Engine &engine, const std::string &info,
+                             size_t step = 999999999)
+{
+
+    size_t s = engine.DebugGetDataBufferSize();
+    std::cout << std::left << std::setw(35) << info;
+    if (step < 999999999)
+    {
+        std::cout << " step " << std::setw(4) << std::to_string(step);
+    }
+    else
+    {
+        std::cout << std::setw(10) << " ";
+    }
+    std::cout << " buffer size = " << std::setw(12) << std::to_string(s) << " ";
+    PrintVMStatus();
+    std::cout << std::endl;
+    return s;
+}
+
+//******************************************************************************
+// 1D 1x8 test data
+//******************************************************************************
+
+// Put(Sync) and Put(Deferred) should have the same buffer consumption
+TEST_F(BPBufferSizeTest, SyncDeferredIdenticalUsage)
+{
+    std::string fnameSync = "ADIOS2BPBufferSizeSync.bp";
+    std::string fnameDeferred = "ADIOS2BPBufferSizeDeferred.bp";
+    std::string fnameDeferredPP = "ADIOS2BPBufferSizeDeferredPP.bp";
+    int mpiRank = 0, mpiSize = 1;
+    // Number of rows
+    const std::size_t Nx = 10485760; // 10M elements, 80MB variable
+    std::vector data;
+    data.resize(Nx);
+
+    // Number of steps
+    const std::size_t NSteps = 3;
+
+#if ADIOS2_USE_MPI
+    MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank);
+    MPI_Comm_size(MPI_COMM_WORLD, &mpiSize);
+#endif
+
+    // Write test data using BP
+    {
+#if ADIOS2_USE_MPI
+        adios2::ADIOS adios(MPI_COMM_WORLD);
+#else
+        adios2::ADIOS adios;
+#endif
+        adios2::IO io = adios.DeclareIO("TestIO");
+        adios2::Dims shape{static_cast(Nx * mpiSize)};
+        adios2::Dims start{static_cast(Nx * mpiRank)};
+        adios2::Dims count{static_cast(Nx)};
+
+        auto var1 = io.DefineVariable("r64_1", shape, start, count);
+        auto var2 = io.DefineVariable("r64_2", shape, start, count);
+        auto var3 = io.DefineVariable("r64_3", shape, start, count);
+
+        if (!engineName.empty())
+        {
+            io.SetEngine(engineName);
+        }
+        else
+        {
+            // Create the BP Engine
+            io.SetEngine("BPFile");
+        }
+        io.AddTransport("NULL");
+        io.SetParameter("Profile", "OFF");
+
+        // Write with Sync
+        size_t bufsize_sync_beginstep[NSteps];
+        size_t bufsize_sync_v1[NSteps];
+        size_t bufsize_sync_v2[NSteps];
+        size_t bufsize_sync_v3[NSteps];
+        size_t bufsize_sync_endstep[NSteps];
+        {
+            adios2::Engine engine = io.Open(fnameSync, adios2::Mode::Write);
+
+            GetAndPrintBufferSize(engine, "After Open():");
+            for (size_t step = 0; step < NSteps; ++step)
+            {
+                engine.BeginStep();
+                bufsize_sync_beginstep[step] =
+                    GetAndPrintBufferSize(engine, "After BeginStep():");
+                engine.Put(var1, data.data(), adios2::Mode::Sync);
+                bufsize_sync_v1[step] =
+                    GetAndPrintBufferSize(engine, "After Put(v1, Sync):", step);
+                engine.Put(var2, data.data(), adios2::Mode::Sync);
+                bufsize_sync_v2[step] =
+                    GetAndPrintBufferSize(engine, "After Put(v2, Sync):", step);
+                engine.Put(var3, data.data(), adios2::Mode::Sync);
+                bufsize_sync_v3[step] =
+                    GetAndPrintBufferSize(engine, "After Put(v3, Sync):", step);
+                engine.EndStep();
+                bufsize_sync_endstep[step] =
+                    GetAndPrintBufferSize(engine, "After EndStep():", step);
+            }
+
+            engine.Close();
+        }
+
+        // Write with Deferred
+        size_t bufsize_deferred_beginstep[NSteps];
+        size_t bufsize_deferred_v1[NSteps];
+        size_t bufsize_deferred_v2[NSteps];
+        size_t bufsize_deferred_v3[NSteps];
+        size_t bufsize_deferred_endstep[NSteps];
+        {
+            adios2::Engine engine = io.Open(fnameDeferred, adios2::Mode::Write);
+
+            GetAndPrintBufferSize(engine, "After Open():");
+            for (size_t step = 0; step < NSteps; ++step)
+            {
+                engine.BeginStep();
+                bufsize_deferred_beginstep[step] =
+                    GetAndPrintBufferSize(engine, "After BeginStep():");
+                engine.Put(var1, data.data(), adios2::Mode::Deferred);
+                bufsize_deferred_v1[step] = GetAndPrintBufferSize(
+                    engine, "After Put(v1, Deferred):", step);
+                engine.Put(var2, data.data(), adios2::Mode::Deferred);
+                bufsize_deferred_v2[step] = GetAndPrintBufferSize(
+                    engine, "After Put(v2, Deferred):", step);
+                engine.Put(var3, data.data(), adios2::Mode::Deferred);
+                bufsize_deferred_v3[step] = GetAndPrintBufferSize(
+                    engine, "After Put(v3, Deferred):", step);
+                engine.EndStep();
+                bufsize_deferred_endstep[step] =
+                    GetAndPrintBufferSize(engine, "After EndStep():", step);
+            }
+
+            engine.Close();
+        }
+
+        // Write with Deferred+PerformPuts
+        size_t bufsize_deferred_pp_beginstep[NSteps];
+        size_t bufsize_deferred_pp_v1[NSteps];
+        size_t bufsize_deferred_pp_v2[NSteps];
+        size_t bufsize_deferred_pp_v3[NSteps];
+        size_t bufsize_deferred_pp_endstep[NSteps];
+        {
+            adios2::Engine engine =
+                io.Open(fnameDeferredPP, adios2::Mode::Write);
+
+            GetAndPrintBufferSize(engine, "After Open():");
+            for (size_t step = 0; step < NSteps; ++step)
+            {
+                engine.BeginStep();
+                bufsize_deferred_pp_beginstep[step] =
+                    GetAndPrintBufferSize(engine, "After BeginStep():");
+                engine.Put(var1, data.data(), adios2::Mode::Deferred);
+                engine.PerformPuts();
+                bufsize_deferred_pp_v1[step] = GetAndPrintBufferSize(
+                    engine, "After Put(v1, Def)+PerformPuts():", step);
+                engine.Put(var2, data.data(), adios2::Mode::Deferred);
+                engine.PerformPuts();
+                bufsize_deferred_pp_v2[step] = GetAndPrintBufferSize(
+                    engine, "After Put(v2, Def)+PerformPuts():", step);
+                engine.Put(var3, data.data(), adios2::Mode::Deferred);
+                engine.PerformPuts();
+                bufsize_deferred_pp_v3[step] = GetAndPrintBufferSize(
+                    engine, "After Put(v3, Def)+PerformPuts():", step);
+                engine.EndStep();
+                bufsize_deferred_pp_endstep[step] =
+                    GetAndPrintBufferSize(engine, "After EndStep():", step);
+            }
+
+            engine.Close();
+        }
+
+        VMStatusClose();
+
+        /* Compare buffer size usage.
+         * Buffer size should never be substantially more than the total data
+         * size
+         * */
+        const size_t TotalDataSize = Nx * sizeof(double) * 3;
+        const size_t MaxExtra =
+            18 * 1024 * 1024; /* 18MB extra allowed in buffer */
+        for (size_t step = 0; step < NSteps; ++step)
+        {
+            EXPECT_LT(bufsize_sync_beginstep[step], TotalDataSize + MaxExtra);
+            EXPECT_LT(bufsize_sync_v1[step], TotalDataSize + MaxExtra);
+            EXPECT_LT(bufsize_sync_v2[step], TotalDataSize + MaxExtra);
+            EXPECT_LT(bufsize_sync_v3[step], TotalDataSize + MaxExtra);
+            EXPECT_LT(bufsize_sync_endstep[step], TotalDataSize + MaxExtra);
+
+            EXPECT_LT(bufsize_deferred_beginstep[step],
+                      TotalDataSize + MaxExtra);
+            EXPECT_LT(bufsize_deferred_v1[step], TotalDataSize + MaxExtra);
+            EXPECT_LT(bufsize_deferred_v2[step], TotalDataSize + MaxExtra);
+            EXPECT_LT(bufsize_deferred_v3[step], TotalDataSize + MaxExtra);
+            EXPECT_LT(bufsize_deferred_endstep[step], TotalDataSize + MaxExtra);
+
+            EXPECT_LT(bufsize_deferred_pp_beginstep[step],
+                      TotalDataSize + MaxExtra);
+            EXPECT_LT(bufsize_deferred_pp_v1[step], TotalDataSize + MaxExtra);
+            EXPECT_LT(bufsize_deferred_pp_v2[step], TotalDataSize + MaxExtra);
+            EXPECT_LT(bufsize_deferred_pp_v3[step], TotalDataSize + MaxExtra);
+            EXPECT_LT(bufsize_deferred_pp_endstep[step],
+                      TotalDataSize + MaxExtra);
+        }
+    }
+}
+
+//******************************************************************************
+// main
+//******************************************************************************
+
+int main(int argc, char **argv)
+{
+#if ADIOS2_USE_MPI
+    MPI_Init(nullptr, nullptr);
+#endif
+
+    int result;
+    ::testing::InitGoogleTest(&argc, argv);
+
+    if (argc > 1)
+    {
+        engineName = std::string(argv[1]);
+    }
+    result = RUN_ALL_TESTS();
+
+#if ADIOS2_USE_MPI
+    MPI_Finalize();
+#endif
+
+    return result;
+}
diff --git a/testing/adios2/engine/bp/TestBPChangingShape.cpp b/testing/adios2/engine/bp/TestBPChangingShape.cpp
index 91162c6f3b..76e7788777 100644
--- a/testing/adios2/engine/bp/TestBPChangingShape.cpp
+++ b/testing/adios2/engine/bp/TestBPChangingShape.cpp
@@ -47,94 +47,334 @@ TEST_F(BPChangingShape, BPWriteReadShape2D)
     adios2::ADIOS adios;
 #endif
     // Writer
-    adios2::IO outIO = adios.DeclareIO("Output");
-
-    if (!engineName.empty())
     {
-        outIO.SetEngine(engineName);
-    }
+        adios2::IO outIO = adios.DeclareIO("Output");
 
-    adios2::Engine writer = outIO.Open(fname, adios2::Mode::Write);
+        if (!engineName.empty())
+        {
+            outIO.SetEngine(engineName);
+        }
 
-    const size_t dim0 = static_cast(nproc);
-    const size_t off0 = static_cast(rank);
-    auto var = outIO.DefineVariable("v", {dim0, 1}, {off0, 0}, {1, 1});
+        adios2::Engine writer = outIO.Open(fname, adios2::Mode::Write);
 
-    std::vector buf(nsteps, 0.0);
-    for (size_t i = 0; i < buf.size(); i++)
-    {
-        buf[i] = rank + static_cast(i) / 10.0;
-    }
+        const size_t dim0 = static_cast(nproc);
+        const size_t off0 = static_cast(rank);
+        auto var =
+            outIO.DefineVariable("v", {dim0, 1}, {off0, 0}, {1, 1});
 
-    if (!rank)
-    {
-        std::cout << "Writing:" << std::endl;
+        std::vector buf(nsteps, 0.0);
+        for (size_t i = 0; i < buf.size(); i++)
+        {
+            buf[i] = rank + static_cast(i) / 10.0;
+        }
+
+        if (!rank)
+        {
+            std::cout << "Writing:" << std::endl;
+        }
+        for (size_t i = 0; i < nsteps; i++)
+        {
+            writer.BeginStep();
+
+            var.SetShape({dim0, i + 1});
+            var.SetSelection({{off0, 0}, {1, i + 1}});
+
+            if (!rank)
+            {
+                std::cout << "Step " << i << " shape (" << var.Shape()[0]
+                          << ", " << var.Shape()[1] << ")" << std::endl;
+            }
+
+            writer.Put(var, buf.data());
+
+            writer.EndStep();
+        }
+
+        writer.Close();
     }
-    for (size_t i = 1; i <= nsteps; i++)
+
+    // Reader with streaming
     {
-        writer.BeginStep();
+        adios2::IO inIO = adios.DeclareIO("Input");
 
-        var.SetShape({dim0, i});
-        var.SetSelection({{off0, 0}, {1, i}});
+        if (!engineName.empty())
+        {
+            inIO.SetEngine(engineName);
+        }
+        adios2::Engine reader = inIO.Open(fname, adios2::Mode::Read);
 
         if (!rank)
         {
-            std::cout << "Step " << i << " shape (" << var.Shape()[0] << ", "
-                      << var.Shape()[1] << ")" << std::endl;
+            std::cout << "Reading as stream with BeginStep/EndStep:"
+                      << std::endl;
         }
 
-        writer.Put(var, buf.data());
+        int i = 0;
+        while (true)
+        {
+            adios2::StepStatus status =
+                reader.BeginStep(adios2::StepMode::Read);
 
-        writer.EndStep();
-    }
+            if (status != adios2::StepStatus::OK)
+            {
+                break;
+            }
 
-    writer.Close();
+            size_t step = reader.CurrentStep();
+            size_t expected_shape = step + 1;
 
-    // Reader
+            auto var = inIO.InquireVariable("v");
+            EXPECT_TRUE(var);
 
-    adios2::IO inIO = adios.DeclareIO("Input");
+            if (!rank)
+            {
 
-    if (!engineName.empty())
-    {
-        inIO.SetEngine(engineName);
-    }
-    adios2::Engine reader = inIO.Open(fname, adios2::Mode::Read);
+                std::cout << "Step " << i << " shape (" << var.Shape()[0]
+                          << ", " << var.Shape()[1] << ")" << std::endl;
+            }
 
-    if (!rank)
-    {
-        std::cout << "Reading:" << std::endl;
+            EXPECT_EQ(var.Shape()[0], nproc);
+            EXPECT_EQ(var.Shape()[1], expected_shape);
+
+            reader.EndStep();
+            ++i;
+        }
+
+        reader.Close();
     }
 
-    int i = 0;
-    while (true)
+    // Reader with file reading
     {
-        adios2::StepStatus status = reader.BeginStep(adios2::StepMode::Read);
+        adios2::IO inIO = adios.DeclareIO("InputFile");
 
-        if (status != adios2::StepStatus::OK)
+        if (!engineName.empty())
         {
-            break;
+            inIO.SetEngine(engineName);
         }
+        adios2::Engine reader = inIO.Open(fname, adios2::Mode::Read);
 
-        size_t step = reader.CurrentStep();
-        size_t expected_shape = step + 1;
+        if (!rank)
+        {
+            std::cout << "Reading as file with SetStepSelection:" << std::endl;
+        }
 
         auto var = inIO.InquireVariable("v");
         EXPECT_TRUE(var);
 
+        for (size_t i = 0; i < nsteps; i++)
+        {
+            var.SetStepSelection({i, 1});
+            if (!rank)
+            {
+
+                std::cout << "Step " << i << " shape (" << var.Shape()[0]
+                          << ", " << var.Shape()[1] << ")" << std::endl;
+            }
+            size_t expected_shape = i + 1;
+            EXPECT_EQ(var.Shape()[0], nproc);
+            EXPECT_EQ(var.Shape()[1], expected_shape);
+        }
+
+        reader.Close();
+    }
+}
+
+TEST_F(BPChangingShape, MultiBlock)
+{
+    // Write multiple blocks and change shape in between
+    // At read, the last shape should be used not the first one
+    // This test guarantees that one can change the variable shape
+    // until EndStep()
+
+    const std::string fname("BPChangingShapeMultiblock.bp");
+    const int nsteps = 2;
+    const std::vector nblocks = {2, 3};
+    EXPECT_EQ(nsteps, nblocks.size());
+    int rank = 0, nproc = 1;
+
+#if ADIOS2_USE_MPI
+    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+    MPI_Comm_size(MPI_COMM_WORLD, &nproc);
+    adios2::ADIOS adios(MPI_COMM_WORLD);
+#else
+    adios2::ADIOS adios;
+#endif
+
+    // Writer
+    {
+        adios2::IO outIO = adios.DeclareIO("Output");
+
+        if (!engineName.empty())
+        {
+            outIO.SetEngine(engineName);
+        }
+
+        adios2::Engine writer = outIO.Open(fname, adios2::Mode::Write);
+
+        const size_t dim0 = static_cast(nproc);
+        const size_t off0 = static_cast(rank);
+        auto var =
+            outIO.DefineVariable("v", {dim0, 1}, {off0, 0}, {1, 1});
+
         if (!rank)
         {
+            std::cout << "Writing:" << std::endl;
+        }
+        for (size_t i = 0; i < nsteps; i++)
+        {
+            writer.BeginStep();
+
+            double value =
+                static_cast(rank) + static_cast(i + 1) / 10.0;
+
+            for (size_t j = 0; j < static_cast(nblocks[i]); j++)
+            {
+                var.SetShape({dim0, j + 1});
+                var.SetSelection({{off0, j}, {1, 1}});
+
+                if (!rank)
+                {
+                    std::cout << "Step " << i << " block " << j << " shape ("
+                              << var.Shape()[0] << ", " << var.Shape()[1] << ")"
+                              << " value = " << value << std::endl;
+                }
+
+                writer.Put(var, &value, adios2::Mode::Sync);
+                value += 0.01;
+            }
+            writer.EndStep();
+        }
+        writer.Close();
+    }
+
+    // Reader with streaming
+    {
+        adios2::IO inIO = adios.DeclareIO("Input");
+
+        if (!engineName.empty())
+        {
+            inIO.SetEngine(engineName);
+        }
+        adios2::Engine reader = inIO.Open(fname, adios2::Mode::Read);
 
-            std::cout << "Step " << i << " shape (" << var.Shape()[0] << ", "
-                      << var.Shape()[1] << ")" << std::endl;
+        if (!rank)
+        {
+            std::cout << "Reading as stream with BeginStep/EndStep:"
+                      << std::endl;
         }
 
-        EXPECT_EQ(var.Shape()[0], nproc);
-        EXPECT_EQ(var.Shape()[1], expected_shape);
+        int step = 0;
+        while (true)
+        {
+            adios2::StepStatus status =
+                reader.BeginStep(adios2::StepMode::Read);
+
+            if (status != adios2::StepStatus::OK)
+            {
+                break;
+            }
+
+            size_t engineStep = reader.CurrentStep();
+            size_t expected_shape = nblocks[step];
 
-        ++i;
+            auto var = inIO.InquireVariable("v");
+            EXPECT_TRUE(var);
+
+            if (!rank)
+            {
+
+                std::cout << "Step " << step << " shape (" << var.Shape()[0]
+                          << ", " << var.Shape()[1] << ")" << std::endl;
+            }
+
+            EXPECT_EQ(var.Shape()[0], nproc);
+            EXPECT_EQ(var.Shape()[1], expected_shape);
+
+            var.SetSelection(
+                {{0, 0}, {static_cast(nproc), expected_shape}});
+
+            // Check data on rank 0
+            if (!rank)
+            {
+                std::vector data(nproc * expected_shape);
+                reader.Get(var, data.data());
+
+                reader.PerformGets();
+
+                for (int i = 0; i < nproc; i++)
+                {
+                    double value = static_cast(i) +
+                                   static_cast(step + 1) / 10.0;
+
+                    for (int j = 0; j < nblocks[step]; j++)
+                    {
+                        EXPECT_EQ(data[i * nblocks[step] + j], value);
+                        value += 0.01;
+                    }
+                }
+            }
+
+            reader.EndStep();
+            ++step;
+        }
+        reader.Close();
     }
 
-    reader.Close();
+    // Reader with file reading
+    {
+        adios2::IO inIO = adios.DeclareIO("InputFile");
+
+        if (!engineName.empty())
+        {
+            inIO.SetEngine(engineName);
+        }
+        adios2::Engine reader = inIO.Open(fname, adios2::Mode::Read);
+
+        if (!rank)
+        {
+            std::cout << "Reading as file with SetStepSelection:" << std::endl;
+        }
+
+        auto var = inIO.InquireVariable("v");
+        EXPECT_TRUE(var);
+        for (int step = 0; step < nsteps; step++)
+        {
+            var.SetStepSelection({step, 1});
+            if (!rank)
+            {
+                std::cout << "Step " << step << " shape (" << var.Shape()[0]
+                          << ", " << var.Shape()[1] << ")" << std::endl;
+            }
+            size_t expected_shape = nblocks[step];
+            EXPECT_EQ(var.Shape()[0], nproc);
+            EXPECT_EQ(var.Shape()[1], expected_shape);
+
+            var.SetSelection(
+                {{0, 0}, {static_cast(nproc), expected_shape}});
+
+            // Check data on rank 0
+            if (!rank)
+            {
+                std::vector data(nproc * expected_shape);
+                reader.Get(var, data.data());
+
+                reader.EndStep();
+
+                for (int i = 0; i < nproc; i++)
+                {
+                    double value = static_cast(i) +
+                                   static_cast(step + 1) / 10.0;
+
+                    for (int j = 0; j < nblocks[step]; j++)
+                    {
+                        EXPECT_EQ(data[i * nblocks[step] + j], value);
+                        value += 0.01;
+                    }
+                }
+            }
+        }
+        reader.Close();
+    }
 }
 
 int main(int argc, char **argv)
diff --git a/testing/adios2/engine/bp/TestBPSelectSteps.cpp b/testing/adios2/engine/bp/TestBPSelectSteps.cpp
new file mode 100644
index 0000000000..521fd42a7e
--- /dev/null
+++ b/testing/adios2/engine/bp/TestBPSelectSteps.cpp
@@ -0,0 +1,328 @@
+/*
+ * Distributed under the OSI-approved Apache License, Version 2.0.  See
+ * accompanying file Copyright.txt for details.
+ */
+#include 
+#include 
+
+#include 
+#include 
+
+#include 
+#include 
+
+#include 
+
+class ADIOSReadSelectionStepsTest : public ::testing::Test
+{
+public:
+    ADIOSReadSelectionStepsTest() = default;
+};
+
+TEST_F(ADIOSReadSelectionStepsTest, Read)
+{
+    std::string filename = "ADIOSSelectSteps.bp";
+
+    // Number of steps
+    const std::size_t NSteps = 4;
+    int mpiRank = 0, mpiSize = 1;
+
+#if ADIOS2_USE_MPI
+    MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank);
+    MPI_Comm_size(MPI_COMM_WORLD, &mpiSize);
+#endif
+
+    // Write test data using BP
+    {
+#if ADIOS2_USE_MPI
+        adios2::ADIOS adios(MPI_COMM_WORLD);
+#else
+        adios2::ADIOS adios;
+#endif
+        adios2::IO ioWrite = adios.DeclareIO("TestIOWrite");
+        ioWrite.SetEngine("BPFile");
+
+        adios2::Engine engine = ioWrite.Open(filename, adios2::Mode::Write);
+        // Number of elements per process
+        const std::size_t Nx = 10;
+        adios2::Dims shape{static_cast(mpiSize * Nx)};
+        adios2::Dims start{static_cast(mpiRank * Nx)};
+        adios2::Dims count{static_cast(Nx)};
+
+        auto var0 =
+            ioWrite.DefineVariable("variable0", shape, start, count);
+        auto var1 =
+            ioWrite.DefineVariable("variable1", shape, start, count);
+        auto var2 =
+            ioWrite.DefineVariable("variable2", shape, start, count);
+        auto var3 =
+            ioWrite.DefineVariable("variable3", shape, start, count);
+
+        std::vector Ints0 = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
+        std::vector Ints1 = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1};
+        std::vector Ints2 = {2, 2, 2, 2, 2, 2, 2, 2, 2, 2};
+        std::vector Ints3 = {3, 3, 3, 3, 3, 3, 3, 3, 3, 3};
+        for (size_t step = 0; step < NSteps; ++step)
+        {
+            engine.BeginStep();
+            engine.Put(var0, Ints0.data());
+            engine.Put(var1, Ints1.data());
+            engine.Put(var2, Ints2.data());
+            engine.Put(var3, Ints3.data());
+            engine.EndStep();
+        }
+        engine.Close();
+
+#if ADIOS2_USE_MPI
+        MPI_Barrier(MPI_COMM_WORLD);
+#endif
+        adios2::IO ioRead = adios.DeclareIO("TestIORead");
+        ioRead.SetEngine("Filestream");
+        ioRead.SetParameter(filename, "1,3");
+        ioRead.SetParameter("StreamReader", "On");
+        adios2::Engine engine_s = ioRead.Open(filename, adios2::Mode::Read);
+        EXPECT_TRUE(engine_s);
+        try
+        {
+            for (int step = 0; step < NSteps; step++)
+            {
+                engine_s.BeginStep();
+                adios2::Variable var0 =
+                    ioRead.InquireVariable("variable0");
+                adios2::Variable var1 =
+                    ioRead.InquireVariable("variable1");
+                adios2::Variable var2 =
+                    ioRead.InquireVariable("variable2");
+                adios2::Variable var3 =
+                    ioRead.InquireVariable("variable3");
+
+                if (step == 0)
+                {
+                    EXPECT_FALSE(var0);
+                    EXPECT_FALSE(var1);
+                    EXPECT_FALSE(var2);
+                    EXPECT_FALSE(var3);
+                }
+                else if (step == 1)
+                {
+                    EXPECT_TRUE(var0);
+                    EXPECT_TRUE(var1);
+                    EXPECT_TRUE(var2);
+                    EXPECT_TRUE(var3);
+                }
+                else if (step == 2)
+                {
+                    EXPECT_FALSE(var0);
+                    EXPECT_FALSE(var1);
+                    EXPECT_FALSE(var2);
+                    EXPECT_FALSE(var3);
+                }
+                else if (step == 3)
+                {
+                    EXPECT_TRUE(var0);
+                    EXPECT_TRUE(var1);
+                    EXPECT_TRUE(var2);
+                    EXPECT_TRUE(var3);
+                }
+                if (var0)
+                {
+                    std::vector res;
+                    var0.SetSelection({{Nx * mpiRank}, {Nx}});
+                    engine_s.Get(var0, res, adios2::Mode::Sync);
+                    EXPECT_EQ(res, Ints0);
+                }
+                if (var1)
+                {
+                    std::vector res;
+                    var1.SetSelection({{Nx * mpiRank}, {Nx}});
+                    engine_s.Get(var1, res, adios2::Mode::Sync);
+                    EXPECT_EQ(res, Ints1);
+                }
+                if (var2)
+                {
+                    std::vector res;
+                    var2.SetSelection({{Nx * mpiRank}, {Nx}});
+                    engine_s.Get(var2, res, adios2::Mode::Sync);
+                    EXPECT_EQ(res, Ints2);
+                }
+                if (var3)
+                {
+                    std::vector res;
+                    var3.SetSelection({{Nx * mpiRank}, {Nx}});
+                    engine_s.Get(var3, res, adios2::Mode::Sync);
+                    EXPECT_EQ(res, Ints3);
+                }
+                engine_s.EndStep();
+            }
+        }
+        catch (std::exception &e)
+        {
+            std::cout << "Exception " << e.what() << std::endl;
+        }
+        engine_s.Close();
+#if ADIOS2_USE_MPI
+        MPI_Barrier(MPI_COMM_WORLD);
+#endif
+        adios2::IO ioReadBP = adios.DeclareIO("ReadBP");
+        ioReadBP.SetEngine("BPfile");
+        ioReadBP.SetParameter(filename, "1,3");
+        /** Engine derived class, spawned to start IO operations */
+        adios2::Engine engine_b = ioReadBP.Open(filename, adios2::Mode::Read);
+        EXPECT_TRUE(engine_b);
+
+        try
+        {
+            for (size_t step = 0; step < NSteps; ++step)
+            {
+                engine_b.BeginStep();
+                adios2::Variable var0 =
+                    ioReadBP.InquireVariable("variable0");
+                adios2::Variable var1 =
+                    ioReadBP.InquireVariable("variable1");
+                adios2::Variable var2 =
+                    ioReadBP.InquireVariable("variable2");
+                adios2::Variable var3 =
+                    ioReadBP.InquireVariable("variable3");
+
+                /**  Variables are not updated */
+                /**  EXPECT_EQ(var0.Steps(), 2); */
+                /**  EXPECT_EQ(engine_b.GetAbsoluteSteps(var0), {1,3}); */
+                if (step == 0)
+                {
+                    EXPECT_FALSE(var0);
+                    EXPECT_FALSE(var1);
+                    EXPECT_FALSE(var2);
+                    EXPECT_FALSE(var3);
+                }
+                else if (step == 1)
+                {
+                    EXPECT_TRUE(var0);
+                    EXPECT_TRUE(var1);
+                    EXPECT_TRUE(var2);
+                    EXPECT_TRUE(var3);
+                }
+                else if (step == 2)
+                {
+                    EXPECT_FALSE(var0);
+                    EXPECT_FALSE(var1);
+                    EXPECT_FALSE(var2);
+                    EXPECT_FALSE(var3);
+                }
+                else if (step == 3)
+                {
+                    EXPECT_TRUE(var0);
+                    EXPECT_TRUE(var1);
+                    EXPECT_TRUE(var2);
+                    EXPECT_TRUE(var3);
+                }
+                if (var0)
+                {
+                    std::vector res;
+                    var0.SetSelection({{Nx * mpiRank}, {Nx}});
+                    engine_b.Get(var0, res, adios2::Mode::Sync);
+                    EXPECT_EQ(res, Ints0);
+                }
+                if (var1)
+                {
+                    std::vector res;
+                    var1.SetSelection({{Nx * mpiRank}, {Nx}});
+                    engine_b.Get(var1, res, adios2::Mode::Sync);
+                    EXPECT_EQ(res, Ints1);
+                }
+                if (var2)
+                {
+                    std::vector res;
+                    var2.SetSelection({{Nx * mpiRank}, {Nx}});
+                    engine_b.Get(var2, res, adios2::Mode::Sync);
+                    EXPECT_EQ(res, Ints2);
+                }
+                if (var3)
+                {
+                    std::vector res;
+                    var3.SetSelection({{Nx * mpiRank}, {Nx}});
+                    engine_b.Get(var3, res, adios2::Mode::Sync);
+                    EXPECT_EQ(res, Ints3);
+                }
+                engine_b.EndStep();
+            }
+        }
+        catch (std::exception &e)
+        {
+            std::cout << "Exception " << e.what() << std::endl;
+        }
+        engine_b.Close();
+
+#if ADIOS2_USE_MPI
+        MPI_Barrier(MPI_COMM_WORLD);
+#endif
+        adios2::IO ioReadBPFull = adios.DeclareIO("ReadBPFull");
+        ioReadBPFull.SetParameter(filename, "1,3");
+        /** Engine derived class, spawned to start IO operations */
+        adios2::Engine engine_bf =
+            ioReadBPFull.Open(filename, adios2::Mode::Read);
+
+        EXPECT_TRUE(engine_bf);
+        {
+            adios2::Variable var0 =
+                ioReadBPFull.InquireVariable("variable0");
+            adios2::Variable var1 =
+                ioReadBPFull.InquireVariable("variable1");
+            adios2::Variable var2 =
+                ioReadBPFull.InquireVariable("variable2");
+            adios2::Variable var3 =
+                ioReadBPFull.InquireVariable("variable3");
+            /**  Variables are not updated */
+            /**  EXPECT_EQ(var0.Steps(), 2); */
+            /**  EXPECT_EQ(engine_b.GetAbsoluteSteps(var0), {1,3}); */
+            EXPECT_TRUE(var0);
+            EXPECT_TRUE(var1);
+            EXPECT_TRUE(var2);
+            EXPECT_TRUE(var3);
+            {
+                std::vector res;
+                var0.SetStepSelection({1, 1});
+                var0.SetSelection({{Nx * mpiRank}, {Nx}});
+                engine_bf.Get(var0, res, adios2::Mode::Sync);
+                EXPECT_EQ(res, Ints0);
+            }
+            {
+                std::vector res;
+                var1.SetStepSelection({1, 1});
+                var1.SetSelection({{Nx * mpiRank}, {Nx}});
+                engine_bf.Get(var1, res, adios2::Mode::Sync);
+                EXPECT_EQ(res, Ints1);
+            }
+
+            {
+                std::vector res;
+                var2.SetStepSelection({1, 1});
+                var2.SetSelection({{Nx * mpiRank}, {Nx}});
+                engine_bf.Get(var2, res, adios2::Mode::Sync);
+                EXPECT_EQ(res, Ints2);
+            }
+
+            {
+                std::vector res;
+                var3.SetStepSelection({1, 1});
+                var3.SetSelection({{Nx * mpiRank}, {Nx}});
+                engine_bf.Get(var3, res, adios2::Mode::Sync);
+                EXPECT_EQ(res, Ints3);
+            }
+        }
+        engine_bf.Close();
+    }
+}
+
+int main(int argc, char **argv)
+{
+#if ADIOS2_USE_MPI
+    MPI_Init(nullptr, nullptr);
+#endif
+    ::testing::InitGoogleTest(&argc, argv);
+    int result = RUN_ALL_TESTS();
+#if ADIOS2_USE_MPI
+    MPI_Finalize();
+#endif
+
+    return result;
+}
diff --git a/testing/adios2/engine/bp/TestBPStepsFileGlobalArray.cpp b/testing/adios2/engine/bp/TestBPStepsFileGlobalArray.cpp
new file mode 100644
index 0000000000..947b3b99d5
--- /dev/null
+++ b/testing/adios2/engine/bp/TestBPStepsFileGlobalArray.cpp
@@ -0,0 +1,934 @@
+/*
+ * Distributed under the OSI-approved Apache License, Version 2.0.  See
+ * accompanying file Copyright.txt for details.
+ */
+#include 
+#include 
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include 
+
+#include 
+
+std::string engineName; // comes from command line
+
+// Number of elements per process
+const std::size_t Nx = 10;
+using DataArray = std::array;
+
+class BPStepsFileGlobalArray : public ::testing::Test
+{
+protected:
+    BPStepsFileGlobalArray() = default;
+
+    const DataArray I32 = {
+        {512, 513, -510, 515, -508, 517, -506, 519, -504, 521}};
+
+    DataArray GenerateData(int step, int rank, int size)
+    {
+        DataArray d;
+        int j = rank + 1 + step * size;
+        for (size_t i = 0; i < d.size(); ++i)
+        {
+            d[i] = I32[i] + j;
+        }
+        return d;
+    }
+
+    std::string ArrayToString(int32_t *data, size_t nelems)
+    {
+        std::stringstream ss;
+        ss << "[";
+        for (size_t i = 0; i < nelems; ++i)
+        {
+            ss << data[i];
+            if (i < nelems - 1)
+            {
+                ss << " ";
+            }
+        }
+        ss << "]";
+        return ss.str();
+    }
+};
+
+enum class ReadMode
+{
+    ReadFileAll,
+    ReadFileStepByStep,
+    ReadFileStepByStepBlocks,
+    ReadStream,
+    ReadStreamBlocks
+};
+
+std::string ReadModeToString(ReadMode r)
+{
+    switch (r)
+    {
+    case ReadMode::ReadFileAll:
+        return "ReadFileAll";
+    case ReadMode::ReadFileStepByStep:
+        return "ReadFileStepByStep";
+    case ReadMode::ReadFileStepByStepBlocks:
+        return "ReadFileStepByStepBlocks";
+    case ReadMode::ReadStream:
+        return "ReadStream";
+    case ReadMode::ReadStreamBlocks:
+        return "ReadStreamBlocks";
+    }
+    return "unknown";
+}
+
+class BPStepsFileGlobalArrayReaders
+: public BPStepsFileGlobalArray,
+  public ::testing::WithParamInterface
+{
+protected:
+    ReadMode GetReadMode() { return GetParam(); };
+};
+
+// Basic case: Variable written every step
+TEST_P(BPStepsFileGlobalArrayReaders, EveryStep)
+{
+    const ReadMode readMode = GetReadMode();
+    std::string fname_prefix =
+        "BPStepsFileGlobalArray.EveryStep." + ReadModeToString(readMode);
+    int mpiRank = 0, mpiSize = 1;
+    const std::size_t NSteps = 4;
+
+#if ADIOS2_USE_MPI
+    MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank);
+    MPI_Comm_size(MPI_COMM_WORLD, &mpiSize);
+#endif
+
+    DataArray m_TestData[NSteps];
+    adios2::Dims shape{static_cast(mpiSize * Nx)};
+    adios2::Dims start{static_cast(mpiRank * Nx)};
+    adios2::Dims count{static_cast(Nx)};
+
+    std::string fname;
+#if ADIOS2_USE_MPI
+    adios2::ADIOS adios(MPI_COMM_WORLD);
+    fname = fname_prefix + ".MPI.bp";
+#else
+    adios2::ADIOS adios;
+    fname = fname_prefix + ".Serial.bp";
+#endif
+
+    // Write test data using ADIOS2
+    {
+        if (!mpiRank)
+        {
+            std::cout << "Write one variable in every step" << std::endl;
+        }
+        adios2::IO io = adios.DeclareIO("Write");
+        if (!engineName.empty())
+        {
+            io.SetEngine(engineName);
+        }
+
+        adios2::Engine engine = io.Open(fname, adios2::Mode::Write);
+
+        auto var_i32 = io.DefineVariable("i32", shape, start, count);
+
+        for (int step = 0; step < NSteps; ++step)
+        {
+            // Generate test data for each process uniquely
+            m_TestData[step] = GenerateData(step, mpiRank, mpiSize);
+            std::cout << "Rank " << mpiRank << " write step " << step << ": "
+                      << ArrayToString(m_TestData[step].data(), Nx)
+                      << std::endl;
+            engine.BeginStep();
+            engine.Put(var_i32, m_TestData[step].data());
+            engine.EndStep();
+        }
+        engine.Close();
+    }
+#if ADIOS2_USE_MPI
+    MPI_Barrier(MPI_COMM_WORLD);
+#endif
+
+    adios2::IO io = adios.DeclareIO("Read");
+    if (!engineName.empty())
+    {
+        io.SetEngine(engineName);
+    }
+    adios2::Engine engine = io.Open(fname, adios2::Mode::Read);
+    EXPECT_TRUE(engine);
+
+    if (readMode == ReadMode::ReadFileAll)
+    {
+        /// Read back data with File reading mode
+        /// Read back the whole thing and check data
+        if (!mpiRank)
+        {
+            std::cout << "Read with File reading mode, read all steps at once"
+                      << std::endl;
+        }
+        auto var_i32 = io.InquireVariable("i32");
+        EXPECT_TRUE(var_i32);
+        EXPECT_EQ(var_i32.Steps(), NSteps);
+        EXPECT_EQ(var_i32.StepsStart(), 0);
+
+        auto absSteps = engine.GetAbsoluteSteps(var_i32);
+        EXPECT_EQ(absSteps.size(), NSteps);
+        std::cout << "Absolute steps of i32 = { ";
+        for (const auto s : absSteps)
+        {
+            std::cout << s << " ";
+        }
+        std::cout << "}" << std::endl;
+        for (std::size_t i = 0; i < NSteps; ++i)
+        {
+            EXPECT_EQ(absSteps[i], i);
+        }
+
+        var_i32.SetStepSelection({0, NSteps});
+        size_t start = static_cast(mpiRank) * Nx;
+        var_i32.SetSelection({{start}, {Nx}});
+        std::array d;
+        engine.Get(var_i32, d.data(), adios2::Mode::Sync);
+        std::cout << "Rank " << mpiRank
+                  << " read all steps: " << ArrayToString(d.data(), NSteps * Nx)
+                  << std::endl;
+        for (size_t step = 0; step < NSteps; ++step)
+        {
+            for (size_t i = 0; i < Nx; ++i)
+            {
+                EXPECT_EQ(d[step * Nx + i], m_TestData[step][i]);
+            }
+        }
+        engine.Close();
+    }
+    else if (readMode == ReadMode::ReadFileStepByStep)
+    {
+        /// Read back data with File reading mode
+        /// Read back step by step and check data
+        if (!mpiRank)
+        {
+            std::cout << "Read with File reading mode, read step by step"
+                      << std::endl;
+        }
+        auto var_i32 = io.InquireVariable("i32");
+        EXPECT_TRUE(var_i32);
+        EXPECT_EQ(var_i32.Steps(), NSteps);
+        EXPECT_EQ(var_i32.StepsStart(), 0);
+        for (size_t step = 0; step < NSteps; ++step)
+        {
+            var_i32.SetStepSelection({step, 1});
+            size_t start = static_cast(mpiRank) * Nx;
+            var_i32.SetSelection({{start}, {Nx}});
+            DataArray d;
+            engine.Get(var_i32, d.data(), adios2::Mode::Sync);
+            std::cout << "Rank " << mpiRank << " read step " << step << ": "
+                      << ArrayToString(d.data(), Nx) << std::endl;
+            for (size_t i = 0; i < Nx; ++i)
+            {
+                EXPECT_EQ(d[i], m_TestData[step][i]);
+            }
+        }
+        engine.Close();
+    }
+    else if (readMode == ReadMode::ReadFileStepByStepBlocks)
+    {
+        /// Read back data with File reading mode
+        /// Read back step by step and block by block and check data
+        if (!mpiRank)
+        {
+            std::cout << "Read with File reading mode, read step by step, "
+                         "block by block"
+                      << std::endl;
+        }
+        auto var_i32 = io.InquireVariable("i32");
+        EXPECT_TRUE(var_i32);
+        EXPECT_EQ(var_i32.Steps(), NSteps);
+        EXPECT_EQ(var_i32.StepsStart(), 0);
+        for (size_t step = 0; step < NSteps; ++step)
+        {
+            var_i32.SetStepSelection({step, 1});
+            size_t blockID = static_cast(mpiRank);
+            var_i32.SetBlockSelection(blockID);
+            DataArray d;
+            engine.Get(var_i32, d.data(), adios2::Mode::Sync);
+            std::cout << "Rank " << mpiRank << " read step " << step
+                      << " block " << blockID << ": "
+                      << ArrayToString(d.data(), Nx) << std::endl;
+            auto start = var_i32.Start();
+            auto count = var_i32.Count();
+            EXPECT_EQ(start[0], mpiRank * Nx);
+            EXPECT_EQ(count[0], 1 * Nx);
+            for (size_t i = 0; i < Nx; ++i)
+            {
+                EXPECT_EQ(d[i], m_TestData[step][i]);
+            }
+        }
+        engine.Close();
+    }
+    else if (readMode == ReadMode::ReadStream)
+    {
+        /// Read back data with Stream reading mode
+        /// Read back step by step and check data
+        if (!mpiRank)
+        {
+            std::cout << "Read with Stream reading mode, read step by step"
+                      << std::endl;
+        }
+        for (size_t step = 0; step < NSteps; ++step)
+        {
+            engine.BeginStep();
+            auto var_i32 = io.InquireVariable("i32");
+            EXPECT_TRUE(var_i32);
+            // EXPECT_EQ(var_i32.Steps(), 1);
+            EXPECT_EQ(var_i32.StepsStart(), 0);
+            size_t start = static_cast(mpiRank) * Nx;
+            var_i32.SetSelection({{start}, {Nx}});
+            DataArray d;
+            engine.Get(var_i32, d.data(), adios2::Mode::Sync);
+            std::cout << "Rank " << mpiRank << " read step " << step << ": "
+                      << ArrayToString(d.data(), Nx) << std::endl;
+            for (size_t i = 0; i < Nx; ++i)
+            {
+                EXPECT_EQ(d[i], m_TestData[step][i]);
+            }
+            engine.EndStep();
+        }
+        engine.Close();
+    }
+    else if (readMode == ReadMode::ReadStreamBlocks)
+    {
+        /// Read back data with Stream reading mode
+        /// Read back step by step and check data
+        if (!mpiRank)
+        {
+            std::cout << "Read with Stream reading mode, read step by step, "
+                         "block by block"
+                      << std::endl;
+        }
+        for (size_t step = 0; step < NSteps; ++step)
+        {
+            engine.BeginStep();
+            auto var_i32 = io.InquireVariable("i32");
+            EXPECT_TRUE(var_i32);
+            // EXPECT_EQ(var_i32.Steps(), 1);
+            EXPECT_EQ(var_i32.StepsStart(), 0);
+            size_t blockID = static_cast(mpiRank);
+            var_i32.SetBlockSelection(blockID);
+            DataArray d;
+            engine.Get(var_i32, d.data(), adios2::Mode::Sync);
+            std::cout << "Rank " << mpiRank << " read step " << step
+                      << " block " << blockID << ": "
+                      << ArrayToString(d.data(), Nx) << std::endl;
+            auto start = var_i32.Start();
+            auto count = var_i32.Count();
+            EXPECT_EQ(start[0], mpiRank * Nx);
+            EXPECT_EQ(count[0], 1 * Nx);
+            for (size_t i = 0; i < Nx; ++i)
+            {
+                EXPECT_EQ(d[i], m_TestData[step][i]);
+            }
+            engine.EndStep();
+        }
+        engine.Close();
+    }
+#if ADIOS2_USE_MPI
+    MPI_Barrier(MPI_COMM_WORLD);
+#endif
+}
+
+// Variable written every other step from 2nd step
+TEST_P(BPStepsFileGlobalArrayReaders, NewVarPerStep)
+{
+    const ReadMode readMode = GetReadMode();
+    std::string fname_prefix =
+        "BPStepsFileGlobalArray.NewVarPerStep." + ReadModeToString(readMode);
+    int mpiRank = 0, mpiSize = 1;
+    const std::size_t NSteps = 4;
+
+#if ADIOS2_USE_MPI
+    MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank);
+    MPI_Comm_size(MPI_COMM_WORLD, &mpiSize);
+#endif
+
+    DataArray m_TestData[NSteps];
+    adios2::Dims shape{static_cast(mpiSize * Nx)};
+    adios2::Dims start{static_cast(mpiRank * Nx)};
+    adios2::Dims count{static_cast(Nx)};
+
+    std::string fname;
+#if ADIOS2_USE_MPI
+    adios2::ADIOS adios(MPI_COMM_WORLD);
+    fname = fname_prefix + ".MPI.bp";
+#else
+    adios2::ADIOS adios;
+    fname = fname_prefix + ".Serial.bp";
+#endif
+
+    auto lf_VarName = [](std::size_t step) -> std::string {
+        return "i32_" + std::to_string(step);
+    };
+
+    // Write test data using ADIOS2
+    {
+        if (!mpiRank)
+        {
+            std::cout << "Write a new variable in each step" << std::endl;
+        }
+        adios2::IO io = adios.DeclareIO("Write");
+        if (!engineName.empty())
+        {
+            io.SetEngine(engineName);
+        }
+
+        adios2::Engine engine = io.Open(fname, adios2::Mode::Write);
+
+        for (int step = 0; step < NSteps; ++step)
+        {
+            const std::string varName = lf_VarName(step);
+            auto var = io.DefineVariable(varName, shape, start, count);
+            // Generate test data for each process uniquely
+            m_TestData[step] = GenerateData(step, mpiRank, mpiSize);
+            std::cout << "Rank " << mpiRank << " write step " << step << " var "
+                      << varName << ": "
+                      << ArrayToString(m_TestData[step].data(), Nx)
+                      << std::endl;
+            engine.BeginStep();
+            engine.Put(var, m_TestData[step].data());
+            engine.EndStep();
+        }
+        engine.Close();
+    }
+#if ADIOS2_USE_MPI
+    MPI_Barrier(MPI_COMM_WORLD);
+#endif
+
+    adios2::IO io = adios.DeclareIO("Read");
+    if (!engineName.empty())
+    {
+        io.SetEngine(engineName);
+    }
+    adios2::Engine engine = io.Open(fname, adios2::Mode::Read);
+    EXPECT_TRUE(engine);
+
+    if (readMode == ReadMode::ReadFileAll)
+    {
+        /// Read back each variable with File reading mode
+        /// Use SetStepSelection(0,1) explicitly
+        if (!mpiRank)
+        {
+            std::cout
+                << "Read with File reading mode using explicit SetStepSelection"
+                << std::endl;
+        }
+        for (size_t step = 0; step < NSteps; ++step)
+        {
+            const std::string varName = lf_VarName(step);
+            auto var = io.InquireVariable(varName);
+            EXPECT_TRUE(var);
+            EXPECT_EQ(var.Steps(), 1);
+            EXPECT_EQ(var.StepsStart(), 0);
+            auto absSteps = engine.GetAbsoluteSteps(var);
+            /*std::cout << "Absolute steps of " << varName << " = { ";
+            for (const auto s : absSteps)
+            {
+                std::cout << s << " ";
+            }
+            std::cout << "}" << std::endl;*/
+            EXPECT_EQ(absSteps.size(), 1);
+            EXPECT_EQ(absSteps[0], step);
+
+            var.SetStepSelection({0, 1});
+            size_t start = static_cast(mpiRank) * Nx;
+            var.SetSelection({{start}, {Nx}});
+            DataArray d;
+            engine.Get(var, d.data(), adios2::Mode::Sync);
+            std::cout << "Rank " << mpiRank << " read var " << varName << ": "
+                      << ArrayToString(d.data(), Nx) << std::endl;
+
+            for (size_t i = 0; i < Nx; ++i)
+            {
+                EXPECT_EQ(d[i], m_TestData[step][i]);
+            }
+        }
+        engine.Close();
+    }
+    else if (readMode == ReadMode::ReadFileStepByStep)
+    {
+        /// Read back each variable with File reading mode
+        /// and do not use SetStepSelection() so default read after open is
+        /// tested
+        if (!mpiRank)
+        {
+            std::cout << "Read with File reading mode without SetStepSelection"
+                      << std::endl;
+        }
+        for (size_t step = 0; step < NSteps; ++step)
+        {
+            const std::string varName = lf_VarName(step);
+            auto var = io.InquireVariable(varName);
+            EXPECT_TRUE(var);
+            EXPECT_EQ(var.Steps(), 1);
+            EXPECT_EQ(var.StepsStart(), 0);
+            size_t start = static_cast(mpiRank) * Nx;
+            var.SetSelection({{start}, {Nx}});
+            DataArray d;
+            engine.Get(var, d.data(), adios2::Mode::Sync);
+            std::cout << "Rank " << mpiRank << " read var " << varName << ": "
+                      << ArrayToString(d.data(), Nx) << std::endl;
+
+            for (size_t i = 0; i < Nx; ++i)
+            {
+                EXPECT_EQ(d[i], m_TestData[step][i]);
+            }
+        }
+        engine.Close();
+    }
+    else if (readMode == ReadMode::ReadFileStepByStepBlocks)
+    {
+        /// Read back each variable with File reading mode
+        /// Read back block by block and check data
+        if (!mpiRank)
+        {
+            std::cout
+                << "Read with File reading mode using explicit SetStepSelection"
+                   ", block by block"
+                << std::endl;
+        }
+        for (size_t step = 0; step < NSteps; ++step)
+        {
+            const std::string varName = lf_VarName(step);
+            auto var = io.InquireVariable(varName);
+            EXPECT_TRUE(var);
+            EXPECT_EQ(var.Steps(), 1);
+            EXPECT_EQ(var.StepsStart(), 0);
+            var.SetStepSelection({0, 1});
+            size_t blockID = static_cast(mpiRank);
+            var.SetBlockSelection(blockID);
+            DataArray d;
+            engine.Get(var, d.data(), adios2::Mode::Sync);
+            std::cout << "Rank " << mpiRank << " read step " << step
+                      << " block " << blockID << ": "
+                      << ArrayToString(d.data(), Nx) << std::endl;
+            auto start = var.Start();
+            auto count = var.Count();
+            EXPECT_EQ(start[0], mpiRank * Nx);
+            EXPECT_EQ(count[0], 1 * Nx);
+            for (size_t i = 0; i < Nx; ++i)
+            {
+                EXPECT_EQ(d[i], m_TestData[step][i]);
+            }
+        }
+        engine.Close();
+    }
+    else if (readMode == ReadMode::ReadStream)
+    {
+        /// Read back each variable with Streaming reading mode
+        if (!mpiRank)
+        {
+            std::cout << "Read with Stream reading mode step by step"
+                      << std::endl;
+        }
+        for (size_t step = 0; step < NSteps; ++step)
+        {
+            engine.BeginStep();
+            const std::string varName = lf_VarName(step);
+            auto var = io.InquireVariable(varName);
+            EXPECT_TRUE(var);
+            EXPECT_EQ(var.Steps(), 1);
+            EXPECT_EQ(var.StepsStart(), 0);
+            size_t start = static_cast(mpiRank) * Nx;
+            var.SetSelection({{start}, {Nx}});
+            DataArray d;
+            engine.Get(var, d.data(), adios2::Mode::Sync);
+            std::cout << "Rank " << mpiRank << " read var " << varName << ": "
+                      << ArrayToString(d.data(), Nx) << std::endl;
+
+            for (size_t i = 0; i < Nx; ++i)
+            {
+                EXPECT_EQ(d[i], m_TestData[step][i]);
+            }
+            engine.EndStep();
+#if ADIOS2_USE_MPI
+            MPI_Barrier(MPI_COMM_WORLD);
+#endif
+        }
+        engine.Close();
+    }
+    else if (readMode == ReadMode::ReadStreamBlocks)
+    {
+        /// Read back each variable with Streaming reading mode
+        if (!mpiRank)
+        {
+            std::cout
+                << "Read with Stream reading mode step by step, block by block"
+                << std::endl;
+        }
+        for (size_t step = 0; step < NSteps; ++step)
+        {
+            engine.BeginStep();
+            const std::string varName = lf_VarName(step);
+            auto var = io.InquireVariable(varName);
+            EXPECT_TRUE(var);
+            EXPECT_EQ(var.Steps(), 1);
+            EXPECT_EQ(var.StepsStart(), 0);
+            size_t blockID = static_cast(mpiRank);
+            var.SetBlockSelection(blockID);
+            DataArray d;
+            engine.Get(var, d.data(), adios2::Mode::Sync);
+            std::cout << "Rank " << mpiRank << " read step " << step
+                      << " block " << blockID << ": "
+                      << ArrayToString(d.data(), Nx) << std::endl;
+            auto start = var.Start();
+            auto count = var.Count();
+            EXPECT_EQ(start[0], mpiRank * Nx);
+            EXPECT_EQ(count[0], 1 * Nx);
+            for (size_t i = 0; i < Nx; ++i)
+            {
+                EXPECT_EQ(d[i], m_TestData[step][i]);
+            }
+            engine.EndStep();
+#if ADIOS2_USE_MPI
+            MPI_Barrier(MPI_COMM_WORLD);
+#endif
+        }
+        engine.Close();
+    }
+#if ADIOS2_USE_MPI
+    MPI_Barrier(MPI_COMM_WORLD);
+#endif
+}
+
+INSTANTIATE_TEST_SUITE_P(BPStepsFileGlobalArray, BPStepsFileGlobalArrayReaders,
+                         ::testing::Values(ReadMode::ReadFileAll,
+                                           ReadMode::ReadFileStepByStep,
+                                           ReadMode::ReadFileStepByStepBlocks,
+                                           ReadMode::ReadStream,
+                                           ReadMode::ReadStreamBlocks));
+
+class BPStepsFileGlobalArrayParameters
+: public BPStepsFileGlobalArray,
+  public ::testing::WithParamInterface>
+{
+protected:
+    size_t GetNsteps() { return std::get<0>(GetParam()); };
+    size_t GetOddity() { return std::get<1>(GetParam()); };
+    ReadMode GetReadMode() { return std::get<2>(GetParam()); };
+};
+
+// Variable written every other step from 1st step
+TEST_P(BPStepsFileGlobalArrayParameters, EveryOtherStep)
+{
+    const std::size_t NSteps = GetNsteps();
+    const std::size_t Oddity = GetOddity();
+    const ReadMode readMode = GetReadMode();
+    std::string fname_prefix =
+        "BPStepsFileGlobalArray.EveryOtherStep.Steps" + std::to_string(NSteps) +
+        ".Oddity" + std::to_string(Oddity) + "." + ReadModeToString(readMode);
+    int mpiRank = 0, mpiSize = 1;
+
+#if ADIOS2_USE_MPI
+    MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank);
+    MPI_Comm_size(MPI_COMM_WORLD, &mpiSize);
+#endif
+
+    std::vector m_TestData;
+    adios2::Dims shape{static_cast(mpiSize * Nx)};
+    adios2::Dims start{static_cast(mpiRank * Nx)};
+    adios2::Dims count{static_cast(Nx)};
+
+    std::string fname;
+#if ADIOS2_USE_MPI
+    adios2::ADIOS adios(MPI_COMM_WORLD);
+    fname = fname_prefix + ".MPI.bp";
+#else
+    adios2::ADIOS adios;
+    fname = fname_prefix + ".Serial.bp";
+
+#endif
+
+    size_t stepsWritten = 0;
+
+    // Write test data using ADIOS2
+    {
+        if (!mpiRank)
+        {
+            std::cout << "Write one variable in every "
+                      << (Oddity ? "ODD" : "EVEN") << " steps, within "
+                      << std::to_string(NSteps) << " steps" << std::endl;
+        }
+        adios2::IO io = adios.DeclareIO("Write");
+        if (!engineName.empty())
+        {
+            io.SetEngine(engineName);
+        }
+
+        adios2::Engine engine = io.Open(fname, adios2::Mode::Write);
+
+        auto var_i32 = io.DefineVariable("i32", shape, start, count);
+        auto var_step = io.DefineVariable("step");
+        for (int step = 0; step < NSteps; ++step)
+        {
+            // Generate test data for each process uniquely
+            engine.BeginStep();
+            engine.Put(var_step, step);
+            if (step % 2 == Oddity)
+            {
+                m_TestData.push_back(GenerateData(step, mpiRank, mpiSize));
+                std::cout << "Rank " << mpiRank << " write step " << step
+                          << ": "
+                          << ArrayToString(m_TestData[stepsWritten].data(), Nx)
+                          << std::endl;
+                engine.Put(var_i32, m_TestData[stepsWritten].data());
+                ++stepsWritten;
+            }
+            engine.EndStep();
+        }
+        engine.Close();
+    }
+#if ADIOS2_USE_MPIstepsWritten
+    MPI_Barrier(MPI_COMM_WORLD);
+#endif
+
+    adios2::IO io = adios.DeclareIO("Read");
+    if (!engineName.empty())
+    {
+        io.SetEngine(engineName);
+    }
+    adios2::Engine engine = io.Open(fname, adios2::Mode::Read);
+    EXPECT_TRUE(engine);
+
+    if (readMode == ReadMode::ReadFileAll)
+    {
+        /// Read back data with File reading mode
+        /// Read back the whole thing and check data
+        if (!mpiRank)
+        {
+            std::cout << "Read with File reading mode, read all steps at once"
+                      << std::endl;
+        }
+
+        auto var_i32 = io.InquireVariable("i32");
+        EXPECT_TRUE(var_i32);
+        EXPECT_EQ(var_i32.Steps(), stepsWritten);
+        EXPECT_EQ(var_i32.StepsStart(), 0);
+
+        var_i32.SetStepSelection({0, stepsWritten});
+        size_t start = static_cast(mpiRank) * Nx;
+        var_i32.SetSelection({{start}, {Nx}});
+        std::vector d(stepsWritten * Nx, 0);
+        engine.Get(var_i32, d.data(), adios2::Mode::Sync);
+        std::cout << "Rank " << mpiRank
+                  << " read all steps: " << ArrayToString(d.data(), d.size())
+                  << std::endl;
+        for (size_t s = 0; s < stepsWritten; ++s)
+        {
+            for (size_t i = 0; i < Nx; ++i)
+            {
+                EXPECT_EQ(d[s * Nx + i], m_TestData[s][i]);
+            }
+        }
+        engine.Close();
+    }
+    else if (readMode == ReadMode::ReadFileStepByStep)
+    {
+        /// Read back data with File reading mode
+        /// Read back step by step and check data
+        if (!mpiRank)
+        {
+            std::cout << "Read with File reading mode, read step by step"
+                      << std::endl;
+        }
+
+        auto var_i32 = io.InquireVariable("i32");
+        EXPECT_TRUE(var_i32);
+        EXPECT_EQ(var_i32.Steps(), stepsWritten);
+        EXPECT_EQ(var_i32.StepsStart(), 0);
+
+        for (size_t s = 0; s < stepsWritten; ++s)
+        {
+            var_i32.SetStepSelection({s, 1});
+            size_t start = static_cast(mpiRank) * Nx;
+            var_i32.SetSelection({{start}, {Nx}});
+            DataArray d;
+            engine.Get(var_i32, d.data(), adios2::Mode::Sync);
+            std::cout << "Rank " << mpiRank << " read step " << s << ": "
+                      << ArrayToString(d.data(), Nx) << std::endl;
+            for (size_t i = 0; i < Nx; ++i)
+            {
+                EXPECT_EQ(d[i], m_TestData[s][i]);
+            }
+        }
+        engine.Close();
+    }
+    else if (readMode == ReadMode::ReadFileStepByStepBlocks)
+    {
+        /// Read back data with File reading mode
+        /// Read back step by step, block by block and check data
+        if (!mpiRank)
+        {
+            std::cout << "Read with File reading mode, read step by step, "
+                         "block by block"
+                      << std::endl;
+        }
+
+        auto var_i32 = io.InquireVariable("i32");
+        EXPECT_TRUE(var_i32);
+        EXPECT_EQ(var_i32.Steps(), stepsWritten);
+        EXPECT_EQ(var_i32.StepsStart(), 0);
+
+        for (size_t s = 0; s < stepsWritten; ++s)
+        {
+            var_i32.SetStepSelection({s, 1});
+            size_t blockID = static_cast(mpiRank);
+            var_i32.SetBlockSelection(blockID);
+            DataArray d;
+            engine.Get(var_i32, d.data(), adios2::Mode::Sync);
+            std::cout << "Rank " << mpiRank << " read step " << s << " block "
+                      << blockID << ": " << ArrayToString(d.data(), Nx)
+                      << std::endl;
+            auto start = var_i32.Start();
+            auto count = var_i32.Count();
+            EXPECT_EQ(start[0], mpiRank * Nx);
+            EXPECT_EQ(count[0], 1 * Nx);
+            for (size_t i = 0; i < Nx; ++i)
+            {
+                EXPECT_EQ(d[i], m_TestData[s][i]);
+            }
+        }
+        engine.Close();
+    }
+    else if (readMode == ReadMode::ReadStream)
+    {
+        /// Read back data with Stream reading mode
+        /// Read back step by step and check data
+        if (!mpiRank)
+        {
+            std::cout << "Read with Stream reading mode step by step"
+                      << std::endl;
+        }
+
+        size_t writtenStep = 0;
+        for (std::size_t step = 0; step < NSteps; ++step)
+        {
+            engine.BeginStep();
+            if (step % 2 == Oddity)
+            {
+                auto var_i32 = io.InquireVariable("i32");
+                EXPECT_TRUE(var_i32);
+                // EXPECT_EQ(var_i32.Steps(), 1);
+                EXPECT_EQ(var_i32.StepsStart(), 0);
+                size_t start = static_cast(mpiRank) * Nx;
+                var_i32.SetSelection({{start}, {Nx}});
+                DataArray d;
+                engine.Get(var_i32, d.data(), adios2::Mode::Sync);
+                std::cout << "Rank " << mpiRank << " read at step " << step
+                          << " var-step " << writtenStep << ": "
+                          << ArrayToString(d.data(), Nx) << std::endl;
+
+                for (size_t i = 0; i < Nx; ++i)
+                {
+                    EXPECT_EQ(d[i], m_TestData[writtenStep][i]);
+                }
+                ++writtenStep;
+            }
+            engine.EndStep();
+#if ADIOS2_USE_MPI
+            MPI_Barrier(MPI_COMM_WORLD);
+#endif
+        }
+        engine.Close();
+    }
+    else if (readMode == ReadMode::ReadStreamBlocks)
+    {
+        /// Read back data with Stream reading mode
+        /// Read back step by step and check data
+        if (!mpiRank)
+        {
+            std::cout << "Read with Stream reading mode, read step by step, "
+                         "block by block"
+                      << std::endl;
+        }
+
+        size_t writtenStep = 0;
+        for (size_t step = 0; step < NSteps; ++step)
+        {
+            engine.BeginStep();
+            if (step % 2 == Oddity)
+            {
+                auto var_i32 = io.InquireVariable("i32");
+                EXPECT_TRUE(var_i32);
+                // EXPECT_EQ(var_i32.Steps(), 1);
+                EXPECT_EQ(var_i32.StepsStart(), 0);
+                size_t blockID = static_cast(mpiRank);
+                var_i32.SetBlockSelection(blockID);
+                DataArray d;
+                engine.Get(var_i32, d.data(), adios2::Mode::Sync);
+                std::cout << "Rank " << mpiRank << " read at step " << step
+                          << " var-step " << writtenStep << " block " << blockID
+                          << ": " << ArrayToString(d.data(), Nx) << std::endl;
+                auto start = var_i32.Start();
+                auto count = var_i32.Count();
+                EXPECT_EQ(start[0], mpiRank * Nx);
+                EXPECT_EQ(count[0], 1 * Nx);
+                for (size_t i = 0; i < Nx; ++i)
+                {
+                    EXPECT_EQ(d[i], m_TestData[writtenStep][i]);
+                }
+                ++writtenStep;
+            }
+            engine.EndStep();
+        }
+        engine.Close();
+    }
+#if ADIOS2_USE_MPI
+    MPI_Barrier(MPI_COMM_WORLD);
+#endif
+}
+
+INSTANTIATE_TEST_SUITE_P(
+    BPStepsFileGlobalArray, BPStepsFileGlobalArrayParameters,
+    ::testing::Values(std::make_tuple(4, 0, ReadMode::ReadFileAll),
+                      std::make_tuple(4, 0, ReadMode::ReadFileStepByStep),
+                      std::make_tuple(4, 0, ReadMode::ReadFileStepByStepBlocks),
+                      std::make_tuple(4, 0, ReadMode::ReadStream),
+                      std::make_tuple(4, 0, ReadMode::ReadStreamBlocks),
+                      std::make_tuple(4, 1, ReadMode::ReadFileAll),
+                      std::make_tuple(4, 1, ReadMode::ReadFileStepByStep),
+                      std::make_tuple(4, 1, ReadMode::ReadFileStepByStepBlocks),
+                      std::make_tuple(4, 1, ReadMode::ReadStream),
+                      std::make_tuple(4, 1, ReadMode::ReadStreamBlocks),
+                      std::make_tuple(2, 1, ReadMode::ReadFileAll),
+                      std::make_tuple(2, 1, ReadMode::ReadFileStepByStep),
+                      std::make_tuple(2, 1, ReadMode::ReadFileStepByStepBlocks),
+                      std::make_tuple(2, 1, ReadMode::ReadStream),
+                      std::make_tuple(2, 1, ReadMode::ReadStreamBlocks)));
+//******************************************************************************
+// main
+//******************************************************************************
+
+int main(int argc, char **argv)
+{
+#if ADIOS2_USE_MPI
+    MPI_Init(nullptr, nullptr);
+#endif
+
+    int result;
+    ::testing::InitGoogleTest(&argc, argv);
+
+    if (argc > 1)
+    {
+        engineName = std::string(argv[1]);
+    }
+    result = RUN_ALL_TESTS();
+
+#if ADIOS2_USE_MPI
+    MPI_Finalize();
+#endif
+
+    return result;
+}
diff --git a/testing/adios2/engine/bp/TestBPStepsFileLocalArray.cpp b/testing/adios2/engine/bp/TestBPStepsFileLocalArray.cpp
new file mode 100644
index 0000000000..0402c0a8a6
--- /dev/null
+++ b/testing/adios2/engine/bp/TestBPStepsFileLocalArray.cpp
@@ -0,0 +1,609 @@
+/*
+ * Distributed under the OSI-approved Apache License, Version 2.0.  See
+ * accompanying file Copyright.txt for details.
+ */
+#include 
+#include 
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include 
+
+#include 
+
+std::string engineName; // comes from command line
+
+// Number of elements per process
+const std::size_t Nx = 10;
+using DataArray = std::array;
+
+class BPStepsFileLocalArray : public ::testing::Test
+{
+protected:
+    BPStepsFileLocalArray() = default;
+
+    const DataArray I32 = {
+        {512, 513, -510, 515, -508, 517, -506, 519, -504, 521}};
+
+    DataArray GenerateData(int step, int rank, int size)
+    {
+        DataArray d;
+        int j = rank + 1 + step * size;
+        for (size_t i = 0; i < d.size(); ++i)
+        {
+            d[i] = I32[i] + j;
+        }
+        return d;
+    }
+
+    std::string ArrayToString(int32_t *data, size_t nelems)
+    {
+        std::stringstream ss;
+        ss << "[";
+        for (size_t i = 0; i < nelems; ++i)
+        {
+            ss << data[i];
+            if (i < nelems - 1)
+            {
+                ss << " ";
+            }
+        }
+        ss << "]";
+        return ss.str();
+    }
+};
+
+enum class ReadMode
+{
+    ReadFileStepByStepBlocks,
+    ReadStreamBlocks
+};
+
+std::string ReadModeToString(const ReadMode r)
+{
+    switch (r)
+    {
+    case ReadMode::ReadFileStepByStepBlocks:
+        return "ReadFileStepByStepBlocks";
+    case ReadMode::ReadStreamBlocks:
+        return "ReadStreamBlocks";
+    }
+    return "unknown";
+}
+
+class BPStepsFileLocalArrayReaders
+: public BPStepsFileLocalArray,
+  public ::testing::WithParamInterface
+{
+protected:
+    ReadMode GetReadMode() { return GetParam(); };
+};
+
+// Basic case: Variable written every step
+TEST_P(BPStepsFileLocalArrayReaders, EveryStep)
+{
+    const ReadMode readMode = GetReadMode();
+    std::string fname_prefix =
+        "BPStepsFileLocalArray.EveryStep." + ReadModeToString(readMode);
+    int mpiRank = 0, mpiSize = 1;
+    const std::size_t NSteps = 4;
+
+#if ADIOS2_USE_MPI
+    MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank);
+    MPI_Comm_size(MPI_COMM_WORLD, &mpiSize);
+#endif
+
+    DataArray m_TestData[NSteps];
+    adios2::Dims shape{};
+    adios2::Dims start{};
+    adios2::Dims count{Nx};
+
+    std::string fname;
+#if ADIOS2_USE_MPI
+    adios2::ADIOS adios(MPI_COMM_WORLD);
+    fname = fname_prefix + ".MPI.bp";
+#else
+    adios2::ADIOS adios;
+    fname = fname_prefix + ".Serial.bp";
+#endif
+
+    // Write test data using ADIOS2
+    {
+        if (!mpiRank)
+        {
+            std::cout << "Write one variable in every step" << std::endl;
+        }
+        adios2::IO io = adios.DeclareIO("Write");
+        if (!engineName.empty())
+        {
+            io.SetEngine(engineName);
+        }
+
+        adios2::Engine engine = io.Open(fname, adios2::Mode::Write);
+
+        auto var_i32 = io.DefineVariable("i32", shape, start, count);
+
+        for (int step = 0; step < NSteps; ++step)
+        {
+            // Generate test data for each process uniquely
+            m_TestData[step] = GenerateData(step, mpiRank, mpiSize);
+            std::cout << "Rank " << mpiRank << " write step " << step << ": "
+                      << ArrayToString(m_TestData[step].data(), Nx)
+                      << std::endl;
+            engine.BeginStep();
+            engine.Put(var_i32, m_TestData[step].data());
+            engine.EndStep();
+        }
+        engine.Close();
+    }
+#if ADIOS2_USE_MPI
+    MPI_Barrier(MPI_COMM_WORLD);
+#endif
+
+    adios2::IO io = adios.DeclareIO("Read");
+    if (!engineName.empty())
+    {
+        io.SetEngine(engineName);
+    }
+    adios2::Engine engine = io.Open(fname, adios2::Mode::Read);
+    EXPECT_TRUE(engine);
+
+    if (readMode == ReadMode::ReadFileStepByStepBlocks)
+    {
+        /// Read back data with File reading mode
+        /// Read back step by step and block by block and check data
+        if (!mpiRank)
+        {
+            std::cout << "Read with File reading mode, read step by step, "
+                         "block by block"
+                      << std::endl;
+        }
+        auto var_i32 = io.InquireVariable("i32");
+        EXPECT_TRUE(var_i32);
+        EXPECT_EQ(var_i32.Steps(), NSteps);
+        EXPECT_EQ(var_i32.StepsStart(), 0);
+        for (size_t step = 0; step < NSteps; ++step)
+        {
+            var_i32.SetStepSelection({step, 1});
+            size_t blockID = static_cast(mpiRank);
+            var_i32.SetBlockSelection(blockID);
+            DataArray d;
+            engine.Get(var_i32, d.data(), adios2::Mode::Sync);
+            std::cout << "Rank " << mpiRank << " read step " << step
+                      << " block " << blockID << ": "
+                      << ArrayToString(d.data(), Nx) << std::endl;
+            auto start = var_i32.Start();
+            auto count = var_i32.Count();
+            EXPECT_EQ(start.size(), 0);
+            EXPECT_EQ(count.size(), 1);
+            EXPECT_EQ(count[0], 1 * Nx);
+            for (size_t i = 0; i < Nx; ++i)
+            {
+                EXPECT_EQ(d[i], m_TestData[step][i]);
+            }
+        }
+        engine.Close();
+    }
+    else if (readMode == ReadMode::ReadStreamBlocks)
+    {
+        /// Read back data with Stream reading mode
+        /// Read back step by step and check data
+        if (!mpiRank)
+        {
+            std::cout << "Read with Stream reading mode, read step by step, "
+                         "block by block"
+                      << std::endl;
+        }
+        for (size_t step = 0; step < NSteps; ++step)
+        {
+            engine.BeginStep();
+            auto var_i32 = io.InquireVariable("i32");
+            EXPECT_TRUE(var_i32);
+            // EXPECT_EQ(var_i32.Steps(), 1);
+            EXPECT_EQ(var_i32.StepsStart(), 0);
+            size_t blockID = static_cast(mpiRank);
+            var_i32.SetBlockSelection(blockID);
+            DataArray d;
+            engine.Get(var_i32, d.data(), adios2::Mode::Sync);
+            std::cout << "Rank " << mpiRank << " read step " << step
+                      << " block " << blockID << ": "
+                      << ArrayToString(d.data(), Nx) << std::endl;
+            auto start = var_i32.Start();
+            auto count = var_i32.Count();
+            EXPECT_EQ(start.size(), 0);
+            EXPECT_EQ(count.size(), 1);
+            EXPECT_EQ(count[0], 1 * Nx);
+            for (size_t i = 0; i < Nx; ++i)
+            {
+                EXPECT_EQ(d[i], m_TestData[step][i]);
+            }
+            engine.EndStep();
+        }
+        engine.Close();
+    }
+#if ADIOS2_USE_MPI
+    MPI_Barrier(MPI_COMM_WORLD);
+#endif
+}
+
+// Variable written every other step from 2nd step
+TEST_P(BPStepsFileLocalArrayReaders, NewVarPerStep)
+{
+    const ReadMode readMode = GetReadMode();
+    std::string fname_prefix =
+        "BPStepsFileLocalArray.NewVarPerStep." + ReadModeToString(readMode);
+    int mpiRank = 0, mpiSize = 1;
+    const std::size_t NSteps = 4;
+
+#if ADIOS2_USE_MPI
+    MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank);
+    MPI_Comm_size(MPI_COMM_WORLD, &mpiSize);
+#endif
+
+    DataArray m_TestData[NSteps];
+    adios2::Dims shape{};
+    adios2::Dims start{};
+    adios2::Dims count{Nx};
+
+    std::string fname;
+#if ADIOS2_USE_MPI
+    adios2::ADIOS adios(MPI_COMM_WORLD);
+    fname = fname_prefix + ".MPI.bp";
+#else
+    adios2::ADIOS adios;
+    fname = fname_prefix + ".Serial.bp";
+#endif
+
+    auto lf_VarName = [](int step) -> std::string {
+        return "i32_" + std::to_string(step);
+    };
+
+    // Write test data using ADIOS2
+    {
+        if (!mpiRank)
+        {
+            std::cout << "Write a new variable in each step" << std::endl;
+        }
+        adios2::IO io = adios.DeclareIO("Write");
+        if (!engineName.empty())
+        {
+            io.SetEngine(engineName);
+        }
+
+        adios2::Engine engine = io.Open(fname, adios2::Mode::Write);
+
+        for (int step = 0; step < NSteps; ++step)
+        {
+            const std::string varName = lf_VarName(step);
+            auto var = io.DefineVariable(varName, shape, start, count);
+            // Generate test data for each process uniquely
+            m_TestData[step] = GenerateData(step, mpiRank, mpiSize);
+            std::cout << "Rank " << mpiRank << " write step " << step << " var "
+                      << varName << ": "
+                      << ArrayToString(m_TestData[step].data(), Nx)
+                      << std::endl;
+            engine.BeginStep();
+            engine.Put(var, m_TestData[step].data());
+            engine.EndStep();
+        }
+        engine.Close();
+    }
+#if ADIOS2_USE_MPI
+    MPI_Barrier(MPI_COMM_WORLD);
+#endif
+
+    adios2::IO io = adios.DeclareIO("Read");
+    if (!engineName.empty())
+    {
+        io.SetEngine(engineName);
+    }
+    adios2::Engine engine = io.Open(fname, adios2::Mode::Read);
+    EXPECT_TRUE(engine);
+
+    if (readMode == ReadMode::ReadFileStepByStepBlocks)
+    {
+        /// Read back each variable with File reading mode
+        /// Read back block by block and check data
+        if (!mpiRank)
+        {
+            std::cout
+                << "Read with File reading mode using explicit SetStepSelection"
+                   ", block by block"
+                << std::endl;
+        }
+        for (int step = 0; step < NSteps; ++step)
+        {
+            const std::string varName = lf_VarName(step);
+            auto var = io.InquireVariable(varName);
+            EXPECT_TRUE(var);
+            EXPECT_EQ(var.Steps(), 1);
+            EXPECT_EQ(var.StepsStart(), 0);
+            var.SetStepSelection({0, 1});
+            size_t blockID = static_cast(mpiRank);
+            var.SetBlockSelection(blockID);
+            DataArray d;
+            engine.Get(var, d.data(), adios2::Mode::Sync);
+            std::cout << "Rank " << mpiRank << " read step " << step
+                      << " block " << blockID << ": "
+                      << ArrayToString(d.data(), Nx) << std::endl;
+            auto start = var.Start();
+            auto count = var.Count();
+            EXPECT_EQ(start.size(), 0);
+            EXPECT_EQ(count.size(), 1);
+            EXPECT_EQ(count[0], 1 * Nx);
+            for (size_t i = 0; i < Nx; ++i)
+            {
+                EXPECT_EQ(d[i], m_TestData[step][i]);
+            }
+        }
+        engine.Close();
+    }
+    else if (readMode == ReadMode::ReadStreamBlocks)
+    {
+        /// Read back each variable with Streaming reading mode
+        if (!mpiRank)
+        {
+            std::cout
+                << "Read with Stream reading mode step by step, block by block"
+                << std::endl;
+        }
+        for (int step = 0; step < NSteps; ++step)
+        {
+            engine.BeginStep();
+            const std::string varName = lf_VarName(step);
+            auto var = io.InquireVariable(varName);
+            EXPECT_TRUE(var);
+            EXPECT_EQ(var.Steps(), 1);
+            EXPECT_EQ(var.StepsStart(), 0);
+            size_t blockID = static_cast(mpiRank);
+            var.SetBlockSelection(blockID);
+            DataArray d;
+            engine.Get(var, d.data(), adios2::Mode::Sync);
+            std::cout << "Rank " << mpiRank << " read step " << step
+                      << " block " << blockID << ": "
+                      << ArrayToString(d.data(), Nx) << std::endl;
+            auto start = var.Start();
+            auto count = var.Count();
+            EXPECT_EQ(start.size(), 0);
+            EXPECT_EQ(count.size(), 1);
+            EXPECT_EQ(count[0], 1 * Nx);
+            for (size_t i = 0; i < Nx; ++i)
+            {
+                EXPECT_EQ(d[i], m_TestData[step][i]);
+            }
+            engine.EndStep();
+#if ADIOS2_USE_MPI
+            MPI_Barrier(MPI_COMM_WORLD);
+#endif
+        }
+        engine.Close();
+    }
+#if ADIOS2_USE_MPI
+    MPI_Barrier(MPI_COMM_WORLD);
+#endif
+}
+
+INSTANTIATE_TEST_SUITE_P(BPStepsFileLocalArray, BPStepsFileLocalArrayReaders,
+                         ::testing::Values(ReadMode::ReadFileStepByStepBlocks,
+                                           ReadMode::ReadStreamBlocks));
+
+class BPStepsFileLocalArrayParameters
+: public BPStepsFileLocalArray,
+  public ::testing::WithParamInterface>
+{
+protected:
+    size_t GetNsteps() { return std::get<0>(GetParam()); };
+    size_t GetOddity() { return std::get<1>(GetParam()); };
+    ReadMode GetReadMode() { return std::get<2>(GetParam()); };
+};
+
+// Variable written every other step from 1st step
+TEST_P(BPStepsFileLocalArrayParameters, EveryOtherStep)
+{
+    const std::size_t NSteps = GetNsteps();
+    const std::size_t Oddity = GetOddity();
+    const ReadMode readMode = GetReadMode();
+    std::string fname_prefix =
+        "BPStepsFileLocalArray.EveryOtherStep.Steps" + std::to_string(NSteps) +
+        ".Oddity" + std::to_string(Oddity) + "." + ReadModeToString(readMode);
+    int mpiRank = 0, mpiSize = 1;
+
+#if ADIOS2_USE_MPI
+    MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank);
+    MPI_Comm_size(MPI_COMM_WORLD, &mpiSize);
+#endif
+
+    std::vector m_TestData;
+    adios2::Dims shape{};
+    adios2::Dims start{};
+    adios2::Dims count{Nx};
+
+    std::string fname;
+#if ADIOS2_USE_MPI
+    adios2::ADIOS adios(MPI_COMM_WORLD);
+    fname = fname_prefix + ".MPI.bp";
+#else
+    adios2::ADIOS adios;
+    fname = fname_prefix + ".Serial.bp";
+
+#endif
+
+    size_t stepsWritten = 0;
+
+    // Write test data using ADIOS2
+    {
+        if (!mpiRank)
+        {
+            std::cout << "Write one variable in every "
+                      << (Oddity ? "ODD" : "EVEN") << " steps, within "
+                      << std::to_string(NSteps) << " steps" << std::endl;
+        }
+        adios2::IO io = adios.DeclareIO("Write");
+        if (!engineName.empty())
+        {
+            io.SetEngine(engineName);
+        }
+
+        adios2::Engine engine = io.Open(fname, adios2::Mode::Write);
+
+        auto var_i32 = io.DefineVariable("i32", shape, start, count);
+        auto var_step = io.DefineVariable("step");
+        for (int step = 0; step < NSteps; ++step)
+        {
+            // Generate test data for each process uniquely
+            engine.BeginStep();
+            engine.Put(var_step, step);
+            if (step % 2 == Oddity)
+            {
+                m_TestData.push_back(GenerateData(step, mpiRank, mpiSize));
+                std::cout << "Rank " << mpiRank << " write step " << step
+                          << ": "
+                          << ArrayToString(m_TestData[stepsWritten].data(), Nx)
+                          << std::endl;
+                engine.Put(var_i32, m_TestData[stepsWritten].data());
+                ++stepsWritten;
+            }
+            engine.EndStep();
+        }
+        engine.Close();
+    }
+#if ADIOS2_USE_MPI
+    MPI_Barrier(MPI_COMM_WORLD);
+#endif
+
+    adios2::IO io = adios.DeclareIO("Read");
+    if (!engineName.empty())
+    {
+        io.SetEngine(engineName);
+    }
+    adios2::Engine engine = io.Open(fname, adios2::Mode::Read);
+    EXPECT_TRUE(engine);
+
+    if (readMode == ReadMode::ReadFileStepByStepBlocks)
+    {
+        /// Read back data with File reading mode
+        /// Read back step by step, block by block and check data
+        if (!mpiRank)
+        {
+            std::cout << "Read with File reading mode, read step by step, "
+                         "block by block"
+                      << std::endl;
+        }
+
+        auto var_i32 = io.InquireVariable("i32");
+        EXPECT_TRUE(var_i32);
+        EXPECT_EQ(var_i32.Steps(), stepsWritten);
+        EXPECT_EQ(var_i32.StepsStart(), 0);
+
+        for (size_t s = 0; s < stepsWritten; ++s)
+        {
+            var_i32.SetStepSelection({s, 1});
+            size_t blockID = static_cast(mpiRank);
+            var_i32.SetBlockSelection(blockID);
+            DataArray d;
+            engine.Get(var_i32, d.data(), adios2::Mode::Sync);
+            std::cout << "Rank " << mpiRank << " read step " << s << " block "
+                      << blockID << ": " << ArrayToString(d.data(), Nx)
+                      << std::endl;
+            auto start = var_i32.Start();
+            auto count = var_i32.Count();
+            EXPECT_EQ(start.size(), 0);
+            EXPECT_EQ(count.size(), 1);
+            EXPECT_EQ(count[0], 1 * Nx);
+            for (size_t i = 0; i < Nx; ++i)
+            {
+                EXPECT_EQ(d[i], m_TestData[s][i]);
+            }
+        }
+        engine.Close();
+    }
+    else if (readMode == ReadMode::ReadStreamBlocks)
+    {
+        /// Read back data with Stream reading mode
+        /// Read back step by step and check data
+        if (!mpiRank)
+        {
+            std::cout
+                << "Read with Stream reading mode step by step, block by block"
+                << std::endl;
+        }
+
+        size_t writtenStep = 0;
+        for (int step = 0; step < NSteps; ++step)
+        {
+            engine.BeginStep();
+            if (step % 2 == Oddity)
+            {
+                auto var_i32 = io.InquireVariable("i32");
+                EXPECT_TRUE(var_i32);
+                // EXPECT_EQ(var_i32.Steps(), 1);
+                EXPECT_EQ(var_i32.StepsStart(), 0);
+                size_t blockID = static_cast(mpiRank);
+                var_i32.SetBlockSelection(blockID);
+                DataArray d;
+                engine.Get(var_i32, d.data(), adios2::Mode::Sync);
+                std::cout << "Rank " << mpiRank << " read step " << step
+                          << " block " << blockID << ": "
+                          << ArrayToString(d.data(), Nx) << std::endl;
+                auto start = var_i32.Start();
+                auto count = var_i32.Count();
+                EXPECT_EQ(start.size(), 0);
+                EXPECT_EQ(count.size(), 1);
+                EXPECT_EQ(count[0], 1 * Nx);
+                for (size_t i = 0; i < Nx; ++i)
+                {
+                    EXPECT_EQ(d[i], m_TestData[writtenStep][i]);
+                }
+                ++writtenStep;
+            }
+            engine.EndStep();
+#if ADIOS2_USE_MPI
+            MPI_Barrier(MPI_COMM_WORLD);
+#endif
+        }
+        engine.Close();
+    }
+#if ADIOS2_USE_MPI
+    MPI_Barrier(MPI_COMM_WORLD);
+#endif
+}
+
+INSTANTIATE_TEST_SUITE_P(
+    BPStepsFileLocalArray, BPStepsFileLocalArrayParameters,
+    ::testing::Values(std::make_tuple(4, 0, ReadMode::ReadFileStepByStepBlocks),
+                      std::make_tuple(4, 0, ReadMode::ReadStreamBlocks),
+                      std::make_tuple(4, 1, ReadMode::ReadFileStepByStepBlocks),
+                      std::make_tuple(4, 1, ReadMode::ReadStreamBlocks),
+                      std::make_tuple(2, 1, ReadMode::ReadFileStepByStepBlocks),
+                      std::make_tuple(2, 1, ReadMode::ReadStreamBlocks)));
+//******************************************************************************
+// main
+//******************************************************************************
+
+int main(int argc, char **argv)
+{
+#if ADIOS2_USE_MPI
+    MPI_Init(nullptr, nullptr);
+#endif
+
+    int result;
+    ::testing::InitGoogleTest(&argc, argv);
+
+    if (argc > 1)
+    {
+        engineName = std::string(argv[1]);
+    }
+    result = RUN_ALL_TESTS();
+
+#if ADIOS2_USE_MPI
+    MPI_Finalize();
+#endif
+
+    return result;
+}
diff --git a/testing/adios2/engine/bp/TestBPStepsInSituGlobalArray.cpp b/testing/adios2/engine/bp/TestBPStepsInSituGlobalArray.cpp
new file mode 100644
index 0000000000..92e2083c71
--- /dev/null
+++ b/testing/adios2/engine/bp/TestBPStepsInSituGlobalArray.cpp
@@ -0,0 +1,698 @@
+/*
+ * Distributed under the OSI-approved Apache License, Version 2.0.  See
+ * accompanying file Copyright.txt for details.
+ */
+#include 
+#include 
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include 
+
+#include 
+
+std::string engineName; // comes from command line
+
+// Number of elements per process
+const std::size_t Nx = 10;
+using DataArray = std::array;
+
+class BPStepsInSituGlobalArray : public ::testing::Test
+{
+protected:
+    BPStepsInSituGlobalArray() = default;
+
+    const DataArray I32 = {
+        {512, 513, -510, 515, -508, 517, -506, 519, -504, 521}};
+
+    DataArray GenerateData(const int step, const int rank, const int size)
+    {
+        DataArray d;
+        int j = rank + 1 + step * size;
+        for (size_t i = 0; i < d.size(); ++i)
+        {
+            d[i] = I32[i] + j;
+        }
+        return d;
+    }
+
+    std::string ArrayToString(const int32_t *data, const size_t nelems)
+    {
+        std::stringstream ss;
+        ss << "[";
+        for (size_t i = 0; i < nelems; ++i)
+        {
+            ss << data[i];
+            if (i < nelems - 1)
+            {
+                ss << " ";
+            }
+        }
+        ss << "]";
+        return ss.str();
+    }
+};
+
+enum class ReadMode
+{
+    ReadGlobal,
+    ReadBlocks
+};
+
+std::string ReadModeToString(const ReadMode r)
+{
+    switch (r)
+    {
+    case ReadMode::ReadGlobal:
+        return "ReadGlobal";
+    case ReadMode::ReadBlocks:
+        return "ReadBlocks";
+    }
+    return "unknown";
+}
+
+enum class Act
+{
+    Write,
+    Read
+};
+
+const std::vector> Schedules = {
+    {Act::Write, Act::Write, Act::Write, Act::Read, Act::Read, Act::Read},
+    {Act::Write, Act::Write, Act::Read, Act::Write, Act::Read, Act::Read},
+    {Act::Write, Act::Write, Act::Read, Act::Read, Act::Write, Act::Read},
+    {Act::Write, Act::Read, Act::Write, Act::Write, Act::Read, Act::Read},
+    {Act::Write, Act::Read, Act::Write, Act::Read, Act::Write, Act::Read}};
+
+std::string ScheduleToString(const std::vector &schedule)
+{
+    std::stringstream ss;
+    ss << "[";
+    for (int i = 0; i < schedule.size(); ++i)
+    {
+        if (schedule[i] == Act::Write)
+        {
+            ss << "Write";
+        }
+        else if (schedule[i] == Act::Read)
+        {
+            ss << "Read";
+        }
+        if (i < schedule.size() - 1)
+        {
+            ss << " ";
+        }
+    }
+    ss << "]";
+    return ss.str();
+}
+
+class BPStepsInSituGlobalArrayReaders
+: public BPStepsInSituGlobalArray,
+  public ::testing::WithParamInterface>
+{
+protected:
+    const std::vector &GetSchedule()
+    {
+        return Schedules[std::get<0>(GetParam())];
+    };
+    ReadMode GetReadMode() { return std::get<1>(GetParam()); };
+    size_t GetScheduleID() { return std::get<0>(GetParam()); };
+};
+
+// Basic case: Variable written every step
+TEST_P(BPStepsInSituGlobalArrayReaders, EveryStep)
+{
+    const std::vector &schedule = GetSchedule();
+    const ReadMode readMode = GetReadMode();
+    const std::string fname_prefix = "BPStepsInSituGlobalArray.EveryStep." +
+                                     std::to_string(GetScheduleID()) + "." +
+                                     ReadModeToString(readMode);
+    int mpiRank = 0, mpiSize = 1;
+
+#if ADIOS2_USE_MPI
+    MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank);
+    MPI_Comm_size(MPI_COMM_WORLD, &mpiSize);
+#endif
+
+    std::vector m_TestData;
+    adios2::Dims shape{static_cast(mpiSize * Nx)};
+    adios2::Dims start{static_cast(mpiRank * Nx)};
+    adios2::Dims count{static_cast(Nx)};
+
+    std::string fname;
+#if ADIOS2_USE_MPI
+    adios2::ADIOS adios(MPI_COMM_WORLD);
+    fname = fname_prefix + ".MPI.bp";
+#else
+    adios2::ADIOS adios;
+    fname = fname_prefix + ".Serial.bp";
+#endif
+
+    if (!mpiRank)
+    {
+        std::cout << "Test with Schedule " << GetScheduleID() << " "
+                  << ScheduleToString(schedule) << " Read Mode "
+                  << ReadModeToString(readMode) << std::endl;
+    }
+
+    // Start writer
+    adios2::IO iow = adios.DeclareIO("Write");
+    if (!engineName.empty())
+    {
+        iow.SetEngine(engineName);
+    }
+    adios2::Engine writer = iow.Open(fname, adios2::Mode::Write);
+    EXPECT_TRUE(writer);
+    auto var_i32 = iow.DefineVariable("i32", shape, start, count);
+    EXPECT_TRUE(var_i32);
+
+#if ADIOS2_USE_MPI
+    MPI_Barrier(MPI_COMM_WORLD);
+#endif
+
+    // Start reader
+    adios2::IO ior = adios.DeclareIO("Read");
+    if (!engineName.empty())
+    {
+        ior.SetEngine(engineName);
+    }
+    ior.SetParameter("OpenTimeoutSecs", "10.0");
+    adios2::Engine reader = ior.Open(fname, adios2::Mode::Read);
+    EXPECT_TRUE(reader);
+
+    int stepsWritten = 0;
+    int stepsRead = 0;
+
+    for (const auto act : schedule)
+    {
+        if (act == Act::Write)
+        {
+            // Write test data using ADIOS2
+            if (!mpiRank)
+            {
+                std::cout << "Write step " << stepsWritten << std::endl;
+            }
+
+            // Generate test data for each process uniquely
+            m_TestData.push_back(GenerateData(stepsWritten, mpiRank, mpiSize));
+            std::cout << "Rank " << mpiRank << " write step " << stepsWritten
+                      << ": "
+                      << ArrayToString(m_TestData[stepsWritten].data(), Nx)
+                      << std::endl;
+            writer.BeginStep();
+            writer.Put(var_i32, m_TestData[stepsWritten].data());
+            writer.EndStep();
+            ++stepsWritten;
+        }
+        else if (act == Act::Read)
+        {
+            if (readMode == ReadMode::ReadGlobal)
+            {
+                /// Read back data with global selection
+                if (!mpiRank)
+                {
+                    std::cout << "Read step " << stepsRead
+                              << " with Global selection" << std::endl;
+                }
+
+                reader.BeginStep();
+                auto var_i32 = ior.InquireVariable("i32");
+                EXPECT_TRUE(var_i32);
+                // EXPECT_EQ(var_i32.Steps(), 1);
+                EXPECT_EQ(var_i32.StepsStart(), 0);
+                size_t start = static_cast(mpiRank) * Nx;
+                var_i32.SetSelection({{start}, {Nx}});
+                DataArray d;
+                reader.Get(var_i32, d.data(), adios2::Mode::Sync);
+                std::cout << "Rank " << mpiRank << " read step " << stepsRead
+                          << ": " << ArrayToString(d.data(), Nx) << std::endl;
+                for (size_t i = 0; i < Nx; ++i)
+                {
+                    EXPECT_EQ(d[i], m_TestData[stepsRead][i]);
+                }
+                reader.EndStep();
+            }
+            else if (readMode == ReadMode::ReadBlocks)
+            {
+                /// Read back data with block selection
+                if (!mpiRank)
+                {
+                    std::cout << "Read step " << stepsRead
+                              << " with Block selection" << std::endl;
+                }
+
+                reader.BeginStep();
+                auto var_i32 = ior.InquireVariable("i32");
+                EXPECT_TRUE(var_i32);
+                // EXPECT_EQ(var_i32.Steps(), 1);
+                EXPECT_EQ(var_i32.StepsStart(), 0);
+                size_t blockID = static_cast(mpiRank);
+                var_i32.SetBlockSelection(blockID);
+                DataArray d;
+                reader.Get(var_i32, d.data(), adios2::Mode::Sync);
+                std::cout << "Rank " << mpiRank << " read step " << stepsRead
+                          << " block " << blockID << ": "
+                          << ArrayToString(d.data(), Nx) << std::endl;
+                auto start = var_i32.Start();
+                auto count = var_i32.Count();
+                EXPECT_EQ(start[0], mpiRank * Nx);
+                EXPECT_EQ(count[0], 1 * Nx);
+                for (size_t i = 0; i < Nx; ++i)
+                {
+                    EXPECT_EQ(d[i], m_TestData[stepsRead][i]);
+                }
+                reader.EndStep();
+            }
+            ++stepsRead;
+        }
+#if ADIOS2_USE_MPI
+        std::flush(std::cout);
+        MPI_Barrier(MPI_COMM_WORLD);
+#endif
+    }
+    writer.Close();
+    reader.Close();
+}
+
+// A new variable is created and written every step
+TEST_P(BPStepsInSituGlobalArrayReaders, NewVarPerStep)
+{
+    const std::vector &schedule = GetSchedule();
+    const ReadMode readMode = GetReadMode();
+    const std::string fname_prefix = "BPStepsInSituGlobalArray.NewVarPerStep." +
+                                     std::to_string(GetScheduleID()) + "." +
+                                     ReadModeToString(readMode);
+    int mpiRank = 0, mpiSize = 1;
+
+#if ADIOS2_USE_MPI
+    MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank);
+    MPI_Comm_size(MPI_COMM_WORLD, &mpiSize);
+#endif
+
+    std::vector m_TestData;
+    adios2::Dims shape{static_cast(mpiSize * Nx)};
+    adios2::Dims start{static_cast(mpiRank * Nx)};
+    adios2::Dims count{static_cast(Nx)};
+
+    std::string fname;
+#if ADIOS2_USE_MPI
+    adios2::ADIOS adios(MPI_COMM_WORLD);
+    fname = fname_prefix + ".MPI.bp";
+#else
+    adios2::ADIOS adios;
+    fname = fname_prefix + ".Serial.bp";
+#endif
+
+    auto lf_VarName = [](int step) -> std::string {
+        return "i32_" + std::to_string(step);
+    };
+
+    std::cout << "Test with Schedule " << GetScheduleID() << " "
+              << ScheduleToString(schedule) << " Read Mode "
+              << ReadModeToString(readMode) << std::endl;
+
+    // Start writer
+    adios2::IO iow = adios.DeclareIO("Write");
+    if (!engineName.empty())
+    {
+        iow.SetEngine(engineName);
+    }
+    adios2::Engine writer = iow.Open(fname, adios2::Mode::Write);
+    EXPECT_TRUE(writer);
+    auto var_i32 = iow.DefineVariable("i32", shape, start, count);
+    EXPECT_TRUE(var_i32);
+
+#if ADIOS2_USE_MPI
+    MPI_Barrier(MPI_COMM_WORLD);
+#endif
+
+    // Start reader
+    adios2::IO ior = adios.DeclareIO("Read");
+    if (!engineName.empty())
+    {
+        ior.SetEngine(engineName);
+    }
+    ior.SetParameter("OpenTimeoutSecs", "10.0");
+    adios2::Engine reader = ior.Open(fname, adios2::Mode::Read);
+    EXPECT_TRUE(reader);
+
+    int stepsWritten = 0;
+    int stepsRead = 0;
+
+    for (const auto act : schedule)
+    {
+        if (act == Act::Write)
+        {
+            if (!mpiRank)
+            {
+                std::cout << "Write step " << stepsWritten << std::endl;
+            }
+
+            const std::string varName = lf_VarName(stepsWritten);
+            auto var =
+                iow.DefineVariable(varName, shape, start, count);
+            // Generate test data for each process uniquely
+            m_TestData.push_back(GenerateData(stepsWritten, mpiRank, mpiSize));
+            std::cout << "Rank " << mpiRank << " write step " << stepsWritten
+                      << " var " << varName << ": "
+                      << ArrayToString(m_TestData[stepsWritten].data(), Nx)
+                      << std::endl;
+            writer.BeginStep();
+            writer.Put(var, m_TestData[stepsWritten].data());
+            writer.EndStep();
+            ++stepsWritten;
+        }
+        else if (act == Act::Read)
+        {
+            if (readMode == ReadMode::ReadGlobal)
+            {
+                /// Read back data with global selection
+                if (!mpiRank)
+                {
+                    std::cout << "Read step " << stepsRead
+                              << " with Global selection" << std::endl;
+                }
+
+                reader.BeginStep();
+                const std::string varName = lf_VarName(stepsRead);
+                auto var = ior.InquireVariable(varName);
+                EXPECT_TRUE(var);
+                EXPECT_EQ(var.Steps(), 1);
+                EXPECT_EQ(var.StepsStart(), 0);
+                size_t start = static_cast(mpiRank) * Nx;
+                var.SetSelection({{start}, {Nx}});
+                DataArray d;
+                reader.Get(var, d.data(), adios2::Mode::Sync);
+                std::cout << "Rank " << mpiRank << " read var " << varName
+                          << ": " << ArrayToString(d.data(), Nx) << std::endl;
+
+                for (size_t i = 0; i < Nx; ++i)
+                {
+                    EXPECT_EQ(d[i], m_TestData[stepsRead][i]);
+                }
+                reader.EndStep();
+            }
+            else if (readMode == ReadMode::ReadBlocks)
+            {
+                /// Read back data with block selection
+                if (!mpiRank)
+                {
+                    std::cout << "Read step " << stepsRead
+                              << " with Block selection" << std::endl;
+                }
+
+                reader.BeginStep();
+                const std::string varName = lf_VarName(stepsRead);
+                auto var = ior.InquireVariable(varName);
+                EXPECT_TRUE(var);
+                EXPECT_EQ(var.Steps(), 1);
+                EXPECT_EQ(var.StepsStart(), 0);
+                size_t blockID = static_cast(mpiRank);
+                var.SetBlockSelection(blockID);
+                DataArray d;
+                reader.Get(var, d.data(), adios2::Mode::Sync);
+                std::cout << "Rank " << mpiRank << " read step " << stepsRead
+                          << " block " << blockID << ": "
+                          << ArrayToString(d.data(), Nx) << std::endl;
+                auto start = var.Start();
+                auto count = var.Count();
+                EXPECT_EQ(start[0], mpiRank * Nx);
+                EXPECT_EQ(count[0], 1 * Nx);
+                for (size_t i = 0; i < Nx; ++i)
+                {
+                    EXPECT_EQ(d[i], m_TestData[stepsRead][i]);
+                }
+                reader.EndStep();
+            }
+            ++stepsRead;
+        }
+#if ADIOS2_USE_MPI
+        std::flush(std::cout);
+        MPI_Barrier(MPI_COMM_WORLD);
+#endif
+    }
+    writer.Close();
+    reader.Close();
+}
+
+INSTANTIATE_TEST_SUITE_P(
+    BPStepsInSituGlobalArray, BPStepsInSituGlobalArrayReaders,
+    ::testing::Values(std::make_tuple(0, ReadMode::ReadGlobal),
+                      std::make_tuple(1, ReadMode::ReadGlobal),
+                      std::make_tuple(2, ReadMode::ReadGlobal),
+                      std::make_tuple(3, ReadMode::ReadGlobal),
+                      std::make_tuple(4, ReadMode::ReadGlobal),
+                      std::make_tuple(0, ReadMode::ReadBlocks),
+                      std::make_tuple(1, ReadMode::ReadBlocks),
+                      std::make_tuple(2, ReadMode::ReadBlocks),
+                      std::make_tuple(3, ReadMode::ReadBlocks),
+                      std::make_tuple(4, ReadMode::ReadBlocks)));
+
+class BPStepsInSituGlobalArrayParameters
+: public BPStepsInSituGlobalArray,
+  public ::testing::WithParamInterface>
+{
+protected:
+    const std::vector &GetSchedule()
+    {
+        return Schedules[std::get<0>(GetParam())];
+    };
+    size_t GetOddity() { return std::get<1>(GetParam()); };
+    ReadMode GetReadMode() { return std::get<2>(GetParam()); };
+    size_t GetScheduleID() { return std::get<0>(GetParam()); };
+};
+
+// A variable written every other step either from step 0 (EVEN) or from
+// step 1 (ODD)
+TEST_P(BPStepsInSituGlobalArrayParameters, EveryOtherStep)
+{
+    const std::vector &schedule = GetSchedule();
+    const std::size_t Oddity = GetOddity();
+    const ReadMode readMode = GetReadMode();
+    const std::string fname_prefix =
+        "BPStepsInSituGlobalArray.EveryOtherStep.Schedule" +
+        std::to_string(GetScheduleID()) + ".Oddity" + std::to_string(Oddity) +
+        "." + ReadModeToString(readMode);
+    int mpiRank = 0, mpiSize = 1;
+
+#if ADIOS2_USE_MPI
+    MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank);
+    MPI_Comm_size(MPI_COMM_WORLD, &mpiSize);
+#endif
+
+    std::vector m_TestData;
+    adios2::Dims shape{static_cast(mpiSize * Nx)};
+    adios2::Dims start{static_cast(mpiRank * Nx)};
+    adios2::Dims count{static_cast(Nx)};
+
+    std::string fname;
+#if ADIOS2_USE_MPI
+    adios2::ADIOS adios(MPI_COMM_WORLD);
+    fname = fname_prefix + ".MPI.bp";
+#else
+    adios2::ADIOS adios;
+    fname = fname_prefix + ".Serial.bp";
+
+#endif
+
+    if (!mpiRank)
+    {
+        std::cout << "Test with Schedule " << GetScheduleID() << " "
+                  << ScheduleToString(schedule) << " Oddity " << Oddity
+                  << " Read Mode " << ReadModeToString(readMode) << std::endl;
+    }
+
+    // Start writer
+    adios2::IO iow = adios.DeclareIO("Write");
+    if (!engineName.empty())
+    {
+        iow.SetEngine(engineName);
+    }
+    adios2::Engine writer = iow.Open(fname, adios2::Mode::Write);
+    EXPECT_TRUE(writer);
+    auto var_i32 = iow.DefineVariable("i32", shape, start, count);
+    EXPECT_TRUE(var_i32);
+    auto var_step = iow.DefineVariable("step");
+    EXPECT_TRUE(var_step);
+
+#if ADIOS2_USE_MPI
+    MPI_Barrier(MPI_COMM_WORLD);
+#endif
+
+    // Start reader
+    adios2::IO ior = adios.DeclareIO("Read");
+    if (!engineName.empty())
+    {
+        ior.SetEngine(engineName);
+    }
+    ior.SetParameter("OpenTimeoutSecs", "10.0");
+    adios2::Engine reader = ior.Open(fname, adios2::Mode::Read);
+    EXPECT_TRUE(reader);
+
+    int stepsWritten = 0;
+    int stepsRead = 0;
+    int varStepsWritten = 0;
+    int varStepsRead = 0;
+    for (const auto act : schedule)
+    {
+        if (act == Act::Write)
+        {
+            if (!mpiRank)
+            {
+                std::cout << "Write step " << stepsWritten << std::endl;
+            }
+
+            // Generate test data for each process uniquely
+            writer.BeginStep();
+            writer.Put(var_step, stepsWritten);
+            if (stepsWritten % 2 == Oddity)
+            {
+                m_TestData.push_back(
+                    GenerateData(stepsWritten, mpiRank, mpiSize));
+                std::cout << "Rank " << mpiRank << " at step " << stepsWritten
+                          << " write var-step " << varStepsWritten << ": "
+                          << ArrayToString(m_TestData[varStepsWritten].data(),
+                                           Nx)
+                          << std::endl;
+                writer.Put(var_i32, m_TestData[varStepsWritten].data());
+                ++varStepsWritten;
+            }
+            writer.EndStep();
+            ++stepsWritten;
+        }
+        else if (act == Act::Read)
+        {
+
+            if (readMode == ReadMode::ReadGlobal)
+            {
+                /// Read back data with global selection
+                if (!mpiRank)
+                {
+                    std::cout << "Read step " << stepsRead
+                              << " with Global selection" << std::endl;
+                }
+
+                reader.BeginStep();
+                if (stepsRead % 2 == Oddity)
+                {
+                    auto var_i32 = ior.InquireVariable("i32");
+                    EXPECT_TRUE(var_i32);
+                    // EXPECT_EQ(var_i32.Steps(), 1);
+                    EXPECT_EQ(var_i32.StepsStart(), 0);
+                    size_t start = static_cast(mpiRank) * Nx;
+                    var_i32.SetSelection({{start}, {Nx}});
+                    DataArray d;
+                    reader.Get(var_i32, d.data(), adios2::Mode::Sync);
+                    std::cout << "Rank " << mpiRank << " read at step "
+                              << stepsRead << " var-step " << varStepsRead
+                              << ": " << ArrayToString(d.data(), Nx)
+                              << std::endl;
+
+                    for (size_t i = 0; i < Nx; ++i)
+                    {
+                        EXPECT_EQ(d[i], m_TestData[varStepsRead][i]);
+                    }
+                    ++varStepsRead;
+                }
+                reader.EndStep();
+            }
+            else if (readMode == ReadMode::ReadBlocks)
+            {
+                /// Read back data with block selection
+                if (!mpiRank)
+                {
+                    std::cout << "Read step " << stepsRead
+                              << " with Block selection" << std::endl;
+                }
+                reader.BeginStep();
+                if (stepsRead % 2 == Oddity)
+                {
+                    auto var_i32 = ior.InquireVariable("i32");
+                    EXPECT_TRUE(var_i32);
+                    // EXPECT_EQ(var_i32.Steps(), 1);
+                    EXPECT_EQ(var_i32.StepsStart(), 0);
+                    size_t blockID = static_cast(mpiRank);
+                    var_i32.SetBlockSelection(blockID);
+                    DataArray d;
+                    reader.Get(var_i32, d.data(), adios2::Mode::Sync);
+                    std::cout << "Rank " << mpiRank << " read at step "
+                              << stepsRead << " var-step " << varStepsRead
+                              << " block " << blockID << ": "
+                              << ArrayToString(d.data(), Nx) << std::endl;
+                    auto start = var_i32.Start();
+                    auto count = var_i32.Count();
+                    EXPECT_EQ(start[0], mpiRank * Nx);
+                    EXPECT_EQ(count[0], 1 * Nx);
+                    for (size_t i = 0; i < Nx; ++i)
+                    {
+                        EXPECT_EQ(d[i], m_TestData[varStepsRead][i]);
+                    }
+                    ++varStepsRead;
+                }
+                reader.EndStep();
+            }
+            ++stepsRead;
+        }
+
+#if ADIOS2_USE_MPI
+        std::flush(std::cout);
+        MPI_Barrier(MPI_COMM_WORLD);
+#endif
+    }
+    writer.Close();
+    reader.Close();
+}
+
+INSTANTIATE_TEST_SUITE_P(
+    BPStepsInSituGlobalArray, BPStepsInSituGlobalArrayParameters,
+    ::testing::Values(std::make_tuple(0, 0, ReadMode::ReadGlobal),
+                      std::make_tuple(0, 0, ReadMode::ReadBlocks),
+                      std::make_tuple(0, 1, ReadMode::ReadGlobal),
+                      std::make_tuple(0, 1, ReadMode::ReadBlocks),
+                      std::make_tuple(1, 0, ReadMode::ReadGlobal),
+                      std::make_tuple(1, 0, ReadMode::ReadBlocks),
+                      std::make_tuple(1, 1, ReadMode::ReadGlobal),
+                      std::make_tuple(1, 1, ReadMode::ReadBlocks),
+                      std::make_tuple(2, 0, ReadMode::ReadGlobal),
+                      std::make_tuple(2, 0, ReadMode::ReadBlocks),
+                      std::make_tuple(2, 1, ReadMode::ReadGlobal),
+                      std::make_tuple(2, 1, ReadMode::ReadBlocks),
+                      std::make_tuple(3, 0, ReadMode::ReadGlobal),
+                      std::make_tuple(3, 0, ReadMode::ReadBlocks),
+                      std::make_tuple(3, 1, ReadMode::ReadGlobal),
+                      std::make_tuple(3, 1, ReadMode::ReadBlocks),
+                      std::make_tuple(4, 0, ReadMode::ReadGlobal),
+                      std::make_tuple(4, 0, ReadMode::ReadBlocks),
+                      std::make_tuple(4, 1, ReadMode::ReadGlobal),
+                      std::make_tuple(4, 1, ReadMode::ReadBlocks)));
+//******************************************************************************
+// main
+//******************************************************************************
+
+int main(int argc, char **argv)
+{
+#if ADIOS2_USE_MPI
+    MPI_Init(nullptr, nullptr);
+#endif
+
+    int result;
+    ::testing::InitGoogleTest(&argc, argv);
+
+    if (argc > 1)
+    {
+        engineName = std::string(argv[1]);
+    }
+    result = RUN_ALL_TESTS();
+
+#if ADIOS2_USE_MPI
+    MPI_Finalize();
+#endif
+
+    return result;
+}
diff --git a/testing/adios2/engine/bp/TestBPStepsInSituLocalArray.cpp b/testing/adios2/engine/bp/TestBPStepsInSituLocalArray.cpp
new file mode 100644
index 0000000000..a8778eee20
--- /dev/null
+++ b/testing/adios2/engine/bp/TestBPStepsInSituLocalArray.cpp
@@ -0,0 +1,551 @@
+/*
+ * Distributed under the OSI-approved Apache License, Version 2.0.  See
+ * accompanying file Copyright.txt for details.
+ */
+#include 
+#include 
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include 
+
+#include 
+
+std::string engineName; // comes from command line
+
+// Number of elements per process
+const std::size_t Nx = 10;
+using DataArray = std::array;
+
+class BPStepsInSituLocalArray : public ::testing::Test
+{
+protected:
+    BPStepsInSituLocalArray() = default;
+
+    const DataArray I32 = {
+        {512, 513, -510, 515, -508, 517, -506, 519, -504, 521}};
+
+    DataArray GenerateData(const int step, const int rank, const int size)
+    {
+        DataArray d;
+        int j = rank + 1 + step * size;
+        for (size_t i = 0; i < d.size(); ++i)
+        {
+            d[i] = I32[i] + j;
+        }
+        return d;
+    }
+
+    std::string ArrayToString(const int32_t *data, const size_t nelems)
+    {
+        std::stringstream ss;
+        ss << "[";
+        for (size_t i = 0; i < nelems; ++i)
+        {
+            ss << data[i];
+            if (i < nelems - 1)
+            {
+                ss << " ";
+            }
+        }
+        ss << "]";
+        return ss.str();
+    }
+};
+
+enum class Act
+{
+    Write,
+    Read
+};
+
+const std::vector> Schedules = {
+    {Act::Write, Act::Write, Act::Write, Act::Read, Act::Read, Act::Read},
+    {Act::Write, Act::Write, Act::Read, Act::Write, Act::Read, Act::Read},
+    {Act::Write, Act::Write, Act::Read, Act::Read, Act::Write, Act::Read},
+    {Act::Write, Act::Read, Act::Write, Act::Write, Act::Read, Act::Read},
+    {Act::Write, Act::Read, Act::Write, Act::Read, Act::Write, Act::Read}};
+
+std::string ScheduleToString(const std::vector &schedule)
+{
+    std::stringstream ss;
+    ss << "[";
+    for (int i = 0; i < schedule.size(); ++i)
+    {
+        if (schedule[i] == Act::Write)
+        {
+            ss << "Write";
+        }
+        else if (schedule[i] == Act::Read)
+        {
+            ss << "Read";
+        }
+        if (i < schedule.size() - 1)
+        {
+            ss << " ";
+        }
+    }
+    ss << "]";
+    return ss.str();
+}
+
+class BPStepsInSituLocalArrayReaders
+: public BPStepsInSituLocalArray,
+  public ::testing::WithParamInterface
+{
+protected:
+    const std::vector &GetSchedule() { return Schedules[GetParam()]; };
+    size_t GetScheduleID() { return GetParam(); };
+};
+
+// Basic case: Variable written every step
+TEST_P(BPStepsInSituLocalArrayReaders, EveryStep)
+{
+    const std::vector &schedule = GetSchedule();
+    const std::string fname_prefix =
+        "BPStepsInSituLocalArray.EveryStep." + std::to_string(GetScheduleID());
+    int mpiRank = 0, mpiSize = 1;
+
+#if ADIOS2_USE_MPI
+    MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank);
+    MPI_Comm_size(MPI_COMM_WORLD, &mpiSize);
+#endif
+
+    std::vector m_TestData;
+    adios2::Dims shape{};
+    adios2::Dims start{};
+    adios2::Dims count{Nx};
+
+    std::string fname;
+#if ADIOS2_USE_MPI
+    adios2::ADIOS adios(MPI_COMM_WORLD);
+    fname = fname_prefix + ".MPI.bp";
+#else
+    adios2::ADIOS adios;
+    fname = fname_prefix + ".Serial.bp";
+#endif
+    if (!mpiRank)
+    {
+        std::cout << "Test with Schedule " << GetScheduleID() << " "
+                  << ScheduleToString(schedule) << std::endl;
+    }
+
+    // Start writer
+    adios2::IO iow = adios.DeclareIO("Write");
+    if (!engineName.empty())
+    {
+        iow.SetEngine(engineName);
+    }
+    adios2::Engine writer = iow.Open(fname, adios2::Mode::Write);
+    EXPECT_TRUE(writer);
+    auto var_i32 = iow.DefineVariable("i32", shape, start, count);
+    EXPECT_TRUE(var_i32);
+
+#if ADIOS2_USE_MPI
+    MPI_Barrier(MPI_COMM_WORLD);
+#endif
+
+    // Start reader
+    adios2::IO ior = adios.DeclareIO("Read");
+    if (!engineName.empty())
+    {
+        ior.SetEngine(engineName);
+    }
+    ior.SetParameter("OpenTimeoutSecs", "10.0");
+    adios2::Engine reader = ior.Open(fname, adios2::Mode::Read);
+    EXPECT_TRUE(reader);
+
+    int stepsWritten = 0;
+    int stepsRead = 0;
+
+    for (const auto act : schedule)
+    {
+        if (act == Act::Write)
+        {
+            // Write test data using ADIOS2
+            if (!mpiRank)
+            {
+                std::cout << "Write step " << stepsWritten << std::endl;
+            }
+
+            // Generate test data for each process uniquely
+            m_TestData.push_back(GenerateData(stepsWritten, mpiRank, mpiSize));
+            std::cout << "Rank " << mpiRank << " write step " << stepsWritten
+                      << ": "
+                      << ArrayToString(m_TestData[stepsWritten].data(), Nx)
+                      << std::endl;
+            writer.BeginStep();
+            writer.Put(var_i32, m_TestData[stepsWritten].data());
+            writer.EndStep();
+            ++stepsWritten;
+        }
+        else if (act == Act::Read)
+        {
+            /// Read back data with block selection
+            if (!mpiRank)
+            {
+                std::cout << "Read step " << stepsRead
+                          << " with Block selection" << std::endl;
+            }
+
+            reader.BeginStep();
+            auto var_i32 = ior.InquireVariable("i32");
+            EXPECT_TRUE(var_i32);
+            // EXPECT_EQ(var_i32.Steps(), 1);
+            EXPECT_EQ(var_i32.StepsStart(), 0);
+            size_t blockID = static_cast(mpiRank);
+            var_i32.SetBlockSelection(blockID);
+            DataArray d;
+            reader.Get(var_i32, d.data(), adios2::Mode::Sync);
+            std::cout << "Rank " << mpiRank << " read step " << stepsRead
+                      << " block " << blockID << ": "
+                      << ArrayToString(d.data(), Nx) << std::endl;
+            auto start = var_i32.Start();
+            auto count = var_i32.Count();
+            EXPECT_EQ(start.size(), 0);
+            EXPECT_EQ(count.size(), 1);
+            EXPECT_EQ(count[0], 1 * Nx);
+            for (size_t i = 0; i < Nx; ++i)
+            {
+                EXPECT_EQ(d[i], m_TestData[stepsRead][i]);
+            }
+            reader.EndStep();
+            ++stepsRead;
+        }
+#if ADIOS2_USE_MPI
+        std::flush(std::cout);
+        MPI_Barrier(MPI_COMM_WORLD);
+#endif
+    }
+    writer.Close();
+    reader.Close();
+}
+
+// A new variable is created and written every step
+TEST_P(BPStepsInSituLocalArrayReaders, NewVarPerStep)
+{
+    const std::vector &schedule = GetSchedule();
+    const std::string fname_prefix = "BPStepsInSituLocalArray.NewVarPerStep." +
+                                     std::to_string(GetScheduleID());
+    int mpiRank = 0, mpiSize = 1;
+
+#if ADIOS2_USE_MPI
+    MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank);
+    MPI_Comm_size(MPI_COMM_WORLD, &mpiSize);
+#endif
+
+    std::vector m_TestData;
+    adios2::Dims shape{};
+    adios2::Dims start{};
+    adios2::Dims count{Nx};
+
+    std::string fname;
+#if ADIOS2_USE_MPI
+    adios2::ADIOS adios(MPI_COMM_WORLD);
+    fname = fname_prefix + ".MPI.bp";
+#else
+    adios2::ADIOS adios;
+    fname = fname_prefix + ".Serial.bp";
+#endif
+
+    auto lf_VarName = [](int step) -> std::string {
+        return "i32_" + std::to_string(step);
+    };
+
+    if (!mpiRank)
+    {
+        std::cout << "Test with Schedule " << GetScheduleID() << " "
+                  << ScheduleToString(schedule) << std::endl;
+    }
+
+    // Start writer
+    adios2::IO iow = adios.DeclareIO("Write");
+    if (!engineName.empty())
+    {
+        iow.SetEngine(engineName);
+    }
+    adios2::Engine writer = iow.Open(fname, adios2::Mode::Write);
+    EXPECT_TRUE(writer);
+    auto var_i32 = iow.DefineVariable("i32", shape, start, count);
+    EXPECT_TRUE(var_i32);
+
+#if ADIOS2_USE_MPI
+    MPI_Barrier(MPI_COMM_WORLD);
+#endif
+
+    // Start reader
+    adios2::IO ior = adios.DeclareIO("Read");
+    if (!engineName.empty())
+    {
+        ior.SetEngine(engineName);
+    }
+    ior.SetParameter("OpenTimeoutSecs", "10.0");
+    adios2::Engine reader = ior.Open(fname, adios2::Mode::Read);
+    EXPECT_TRUE(reader);
+
+    int stepsWritten = 0;
+    int stepsRead = 0;
+
+    for (const auto act : schedule)
+    {
+        if (act == Act::Write)
+        {
+            if (!mpiRank)
+            {
+                std::cout << "Write step " << stepsWritten << std::endl;
+            }
+
+            const std::string varName = lf_VarName(stepsWritten);
+            auto var =
+                iow.DefineVariable(varName, shape, start, count);
+            // Generate test data for each process uniquely
+            m_TestData.push_back(GenerateData(stepsWritten, mpiRank, mpiSize));
+            std::cout << "Rank " << mpiRank << " write step " << stepsWritten
+                      << " var " << varName << ": "
+                      << ArrayToString(m_TestData[stepsWritten].data(), Nx)
+                      << std::endl;
+            writer.BeginStep();
+            writer.Put(var, m_TestData[stepsWritten].data());
+            writer.EndStep();
+            ++stepsWritten;
+        }
+        else if (act == Act::Read)
+        {
+            /// Read back data with block selection
+            if (!mpiRank)
+            {
+                std::cout << "Read step " << stepsRead
+                          << " with Block selection" << std::endl;
+            }
+
+            reader.BeginStep();
+            const std::string varName = lf_VarName(stepsRead);
+            auto var = ior.InquireVariable(varName);
+            EXPECT_TRUE(var);
+            EXPECT_EQ(var.Steps(), 1);
+            EXPECT_EQ(var.StepsStart(), 0);
+            size_t blockID = static_cast(mpiRank);
+            var.SetBlockSelection(blockID);
+            DataArray d;
+            reader.Get(var, d.data(), adios2::Mode::Sync);
+            std::cout << "Rank " << mpiRank << " read step " << stepsRead
+                      << " block " << blockID << ": "
+                      << ArrayToString(d.data(), Nx) << std::endl;
+            auto start = var.Start();
+            auto count = var.Count();
+            EXPECT_EQ(start.size(), 0);
+            EXPECT_EQ(count.size(), 1);
+            EXPECT_EQ(count[0], 1 * Nx);
+            for (size_t i = 0; i < Nx; ++i)
+            {
+                EXPECT_EQ(d[i], m_TestData[stepsRead][i]);
+            }
+            reader.EndStep();
+            ++stepsRead;
+        }
+#if ADIOS2_USE_MPI
+        std::flush(std::cout);
+        MPI_Barrier(MPI_COMM_WORLD);
+#endif
+    }
+    writer.Close();
+    reader.Close();
+}
+
+INSTANTIATE_TEST_SUITE_P(BPStepsInSituLocalArray,
+                         BPStepsInSituLocalArrayReaders,
+                         ::testing::Values(0, 1, 2, 3, 4));
+
+class BPStepsInSituLocalArrayParameters
+: public BPStepsInSituLocalArray,
+  public ::testing::WithParamInterface>
+{
+protected:
+    const std::vector &GetSchedule()
+    {
+        return Schedules[std::get<0>(GetParam())];
+    };
+    size_t GetOddity() { return std::get<1>(GetParam()); };
+    size_t GetScheduleID() { return std::get<0>(GetParam()); };
+};
+
+// A variable written every other step either from step 0 (EVEN) or from
+// step 1 (ODD)
+TEST_P(BPStepsInSituLocalArrayParameters, EveryOtherStep)
+{
+    const std::vector &schedule = GetSchedule();
+    const std::size_t Oddity = GetOddity();
+    const std::string fname_prefix =
+        "BPStepsInSituLocalArray.EveryOtherStep.Schedule" +
+        std::to_string(GetScheduleID()) + ".Oddity" + std::to_string(Oddity);
+    int mpiRank = 0, mpiSize = 1;
+
+#if ADIOS2_USE_MPI
+    MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank);
+    MPI_Comm_size(MPI_COMM_WORLD, &mpiSize);
+#endif
+
+    std::vector m_TestData;
+    adios2::Dims shape{};
+    adios2::Dims start{};
+    adios2::Dims count{Nx};
+
+    std::string fname;
+#if ADIOS2_USE_MPI
+    adios2::ADIOS adios(MPI_COMM_WORLD);
+    fname = fname_prefix + ".MPI.bp";
+#else
+    adios2::ADIOS adios;
+    fname = fname_prefix + ".Serial.bp";
+
+#endif
+
+    if (!mpiRank)
+    {
+        std::cout << "Test with Schedule " << GetScheduleID() << " "
+                  << ScheduleToString(schedule) << " Oddity " << Oddity
+                  << std::endl;
+    }
+
+    // Start writer
+    adios2::IO iow = adios.DeclareIO("Write");
+    if (!engineName.empty())
+    {
+        iow.SetEngine(engineName);
+    }
+    adios2::Engine writer = iow.Open(fname, adios2::Mode::Write);
+    EXPECT_TRUE(writer);
+    auto var_i32 = iow.DefineVariable("i32", shape, start, count);
+    EXPECT_TRUE(var_i32);
+    auto var_step = iow.DefineVariable("step");
+    EXPECT_TRUE(var_step);
+
+#if ADIOS2_USE_MPI
+    MPI_Barrier(MPI_COMM_WORLD);
+#endif
+
+    // Start reader
+    adios2::IO ior = adios.DeclareIO("Read");
+    if (!engineName.empty())
+    {
+        ior.SetEngine(engineName);
+    }
+    ior.SetParameter("OpenTimeoutSecs", "10.0");
+    adios2::Engine reader = ior.Open(fname, adios2::Mode::Read);
+    EXPECT_TRUE(reader);
+
+    int stepsWritten = 0;
+    int stepsRead = 0;
+    int varStepsWritten = 0;
+    int varStepsRead = 0;
+    for (const auto act : schedule)
+    {
+        if (act == Act::Write)
+        {
+            if (!mpiRank)
+            {
+                std::cout << "Write step " << stepsWritten << std::endl;
+            }
+
+            // Generate test data for each process uniquely
+            writer.BeginStep();
+            writer.Put(var_step, stepsWritten);
+            if (stepsWritten % 2 == Oddity)
+            {
+                m_TestData.push_back(
+                    GenerateData(stepsWritten, mpiRank, mpiSize));
+                std::cout << "Rank " << mpiRank << " at step " << stepsWritten
+                          << " write var-step " << varStepsWritten << ": "
+                          << ArrayToString(m_TestData[varStepsWritten].data(),
+                                           Nx)
+                          << std::endl;
+                writer.Put(var_i32, m_TestData[varStepsWritten].data());
+                ++varStepsWritten;
+            }
+            writer.EndStep();
+            ++stepsWritten;
+        }
+        else if (act == Act::Read)
+        {
+            /// Read back data with block selection
+            if (!mpiRank)
+            {
+                std::cout << "Read step " << stepsRead
+                          << " with Block selection" << std::endl;
+            }
+            reader.BeginStep();
+            if (stepsRead % 2 == Oddity)
+            {
+                auto var_i32 = ior.InquireVariable("i32");
+                EXPECT_TRUE(var_i32);
+                // EXPECT_EQ(var_i32.Steps(), 1);
+                EXPECT_EQ(var_i32.StepsStart(), 0);
+                size_t blockID = static_cast(mpiRank);
+                var_i32.SetBlockSelection(blockID);
+                DataArray d;
+                reader.Get(var_i32, d.data(), adios2::Mode::Sync);
+                std::cout << "Rank " << mpiRank << " read at step " << stepsRead
+                          << " var-step " << varStepsRead << " block "
+                          << blockID << ": " << ArrayToString(d.data(), Nx)
+                          << std::endl;
+                auto start = var_i32.Start();
+                auto count = var_i32.Count();
+                EXPECT_EQ(start.size(), 0);
+                EXPECT_EQ(count.size(), 1);
+                EXPECT_EQ(count[0], 1 * Nx);
+                for (size_t i = 0; i < Nx; ++i)
+                {
+                    EXPECT_EQ(d[i], m_TestData[varStepsRead][i]);
+                }
+                ++varStepsRead;
+            }
+            reader.EndStep();
+            ++stepsRead;
+        }
+
+#if ADIOS2_USE_MPI
+        std::flush(std::cout);
+        MPI_Barrier(MPI_COMM_WORLD);
+#endif
+    }
+    writer.Close();
+    reader.Close();
+}
+
+INSTANTIATE_TEST_SUITE_P(
+    BPStepsInSituLocalArray, BPStepsInSituLocalArrayParameters,
+    ::testing::Values(std::make_tuple(0, 0), std::make_tuple(0, 1),
+                      std::make_tuple(1, 0), std::make_tuple(1, 1),
+                      std::make_tuple(2, 0), std::make_tuple(2, 1),
+                      std::make_tuple(3, 0), std::make_tuple(3, 1),
+                      std::make_tuple(4, 0), std::make_tuple(4, 1)));
+//******************************************************************************
+// main
+//******************************************************************************
+
+int main(int argc, char **argv)
+{
+#if ADIOS2_USE_MPI
+    MPI_Init(nullptr, nullptr);
+#endif
+
+    int result;
+    ::testing::InitGoogleTest(&argc, argv);
+
+    if (argc > 1)
+    {
+        engineName = std::string(argv[1]);
+    }
+    result = RUN_ALL_TESTS();
+
+#if ADIOS2_USE_MPI
+    MPI_Finalize();
+#endif
+
+    return result;
+}
diff --git a/testing/adios2/engine/bp/TestBPTimeAggregation.cpp b/testing/adios2/engine/bp/TestBPTimeAggregation.cpp
index c5bed38fe1..b0ebe23b36 100644
--- a/testing/adios2/engine/bp/TestBPTimeAggregation.cpp
+++ b/testing/adios2/engine/bp/TestBPTimeAggregation.cpp
@@ -42,6 +42,8 @@ void TimeAggregation1D8(const std::string flushstepscount)
 #else
     adios2::ADIOS adios;
 #endif
+    const std::string TestName =
+        "TimeAggregation1D8 flush every " + flushstepscount + " steps";
     {
         adios2::IO io = adios.DeclareIO("TestIO");
 
@@ -73,6 +75,8 @@ void TimeAggregation1D8(const std::string flushstepscount)
             auto var_r32 = io.DefineVariable("r32", shape, start, count);
             auto var_r64 =
                 io.DefineVariable("r64", shape, start, count);
+
+            io.DefineAttribute("TestName", TestName);
         }
 
         if (!engineName.empty())
@@ -86,6 +90,7 @@ void TimeAggregation1D8(const std::string flushstepscount)
         }
 
         io.AddTransport("file");
+        io.SetParameter("AggregatorRatio", "1");
 
         SmallTestData m_TestData;
 
@@ -230,6 +235,12 @@ void TimeAggregation1D8(const std::string flushstepscount)
         ASSERT_EQ(var_r64.Steps(), NSteps);
         ASSERT_EQ(var_r64.Shape()[0], mpiSize * Nx);
 
+        auto attr = io.InquireAttribute("TestName");
+        EXPECT_TRUE(attr);
+        ASSERT_EQ(attr.Data().size() == 1, true);
+        ASSERT_EQ(attr.Type(), adios2::GetType());
+        ASSERT_EQ(attr.Data().front(), TestName);
+
         // TODO: other types
 
         SmallTestData testData;
@@ -355,6 +366,9 @@ void TimeAggregation2D4x2(const std::string flushstepscount)
 #else
     adios2::ADIOS adios;
 #endif
+    const std::string TestName =
+        "TimeAggregation2D4x2 flush every " + flushstepscount + " steps";
+
     {
         adios2::IO io = adios.DeclareIO("TestIO");
 
@@ -387,6 +401,8 @@ void TimeAggregation2D4x2(const std::string flushstepscount)
             auto var_r32 = io.DefineVariable("r32", shape, start, count);
             auto var_r64 =
                 io.DefineVariable("r64", shape, start, count);
+
+            io.DefineAttribute("TestName", TestName);
         }
 
         if (!engineName.empty())
@@ -400,6 +416,7 @@ void TimeAggregation2D4x2(const std::string flushstepscount)
         }
 
         io.AddTransport("file");
+        io.SetParameter("AggregatorRatio", "1");
 
         SmallTestData m_TestData;
 
@@ -552,6 +569,12 @@ void TimeAggregation2D4x2(const std::string flushstepscount)
         ASSERT_EQ(var_r64.Shape()[0], Ny);
         ASSERT_EQ(var_r64.Shape()[1], static_cast(mpiSize * Nx));
 
+        auto attr = io.InquireAttribute("TestName");
+        EXPECT_TRUE(attr);
+        ASSERT_EQ(attr.Data().size() == 1, true);
+        ASSERT_EQ(attr.Type(), adios2::GetType());
+        ASSERT_EQ(attr.Data().front(), TestName);
+
         std::string IString;
         std::array I8;
         std::array I16;
@@ -665,8 +688,8 @@ TEST_P(BPTestTimeAggregation, BPTimeAggregation2D4x2)
     TimeAggregation2D4x2(GetParam());
 }
 
-INSTANTIATE_TEST_CASE_P(FlushStepsCount, BPTestTimeAggregation,
-                        ::testing::Values("1", "2", "3", "6", "8", "10"));
+INSTANTIATE_TEST_SUITE_P(FlushStepsCount, BPTestTimeAggregation,
+                         ::testing::Values("1", "2", "3", "6", "8", "10"));
 
 int main(int argc, char **argv)
 {
diff --git a/testing/adios2/engine/bp/TestBPWriteAggregateRead.cpp b/testing/adios2/engine/bp/TestBPWriteAggregateRead.cpp
index b9acd3762d..eb9ba1f9d6 100644
--- a/testing/adios2/engine/bp/TestBPWriteAggregateRead.cpp
+++ b/testing/adios2/engine/bp/TestBPWriteAggregateRead.cpp
@@ -40,7 +40,7 @@ void WriteAggRead1D8(const std::string substreams)
 
         if (mpiSize > 1)
         {
-            io.SetParameter("Substreams", substreams);
+            io.SetParameter("NumAggregators", substreams);
         }
 
         // Declare 1D variables (NumOfProcesses * Nx)
@@ -69,6 +69,14 @@ void WriteAggRead1D8(const std::string substreams)
             auto var_r32 = io.DefineVariable("r32", shape, start, count);
             auto var_r64 =
                 io.DefineVariable("r64", shape, start, count);
+
+            /* add operations
+            adios2::Operator ZFPOp =
+                adios.DefineOperator("ZFPCompressor", adios2::ops::LossyZFP);
+
+            var_r32.AddOperation(ZFPOp, {{adios2::ops::zfp::key::rate, "32"}});
+            var_r64.AddOperation(ZFPOp, {{adios2::ops::zfp::key::rate, "64"}});
+            */
         }
 
         if (!engineName.empty())
@@ -953,8 +961,8 @@ TEST_P(BPWriteAggregateReadTest, ADIOS2BPWriteAggregateRead2D4x2)
     WriteAggRead2D4x2(GetParam());
 }
 
-INSTANTIATE_TEST_CASE_P(Substreams, BPWriteAggregateReadTest,
-                        ::testing::Values("1", "2", "3", "4", "5"));
+INSTANTIATE_TEST_SUITE_P(Substreams, BPWriteAggregateReadTest,
+                         ::testing::Values("1", "2", "3", "4", "5", "0"));
 
 int main(int argc, char **argv)
 {
diff --git a/testing/adios2/engine/bp/TestBPWriteAppendReadADIOS2.cpp b/testing/adios2/engine/bp/TestBPWriteAppendReadADIOS2.cpp
index 32731b4695..ef7288f05a 100644
--- a/testing/adios2/engine/bp/TestBPWriteAppendReadADIOS2.cpp
+++ b/testing/adios2/engine/bp/TestBPWriteAppendReadADIOS2.cpp
@@ -171,6 +171,7 @@ TEST_F(BPWriteAppendReadTestADIOS2, ADIOS2BPWriteAppendRead2D2x4)
             io.SetEngine("BP4");
         }
         io.AddTransport("file");
+        io.SetParameter("AggregatorRatio", "1");
 
         adios2::Engine bpWriter = io.Open(fname, adios2::Mode::Write);
 
@@ -322,6 +323,7 @@ TEST_F(BPWriteAppendReadTestADIOS2, ADIOS2BPWriteAppendRead2D2x4)
             io.SetEngine("BP4");
         }
         io.AddTransport("file");
+        io.SetParameter("AggregatorRatio", "1");
 
         adios2::Engine bpAppender = io.Open(fname, adios2::Mode::Append);
 
diff --git a/testing/adios2/engine/bp/TestBPWriteMemorySelectionRead.cpp b/testing/adios2/engine/bp/TestBPWriteMemorySelectionRead.cpp
index f71084900e..a5a353904b 100644
--- a/testing/adios2/engine/bp/TestBPWriteMemorySelectionRead.cpp
+++ b/testing/adios2/engine/bp/TestBPWriteMemorySelectionRead.cpp
@@ -968,8 +968,8 @@ TEST_P(BPWriteMemSelReadVector, BPMemorySelectionSteps3D4x2x8)
     BPSteps3D8x2x4(GetParam());
 }
 
-INSTANTIATE_TEST_CASE_P(ghostCells, BPWriteMemSelReadVector,
-                        ::testing::Values(1));
+INSTANTIATE_TEST_SUITE_P(ghostCells, BPWriteMemSelReadVector,
+                         ::testing::Values(1));
 
 int main(int argc, char **argv)
 {
diff --git a/testing/adios2/engine/bp/TestBPWriteReadAttributes.cpp b/testing/adios2/engine/bp/TestBPWriteReadAttributes.cpp
index c1fe0bcf5a..59db39eb47 100644
--- a/testing/adios2/engine/bp/TestBPWriteReadAttributes.cpp
+++ b/testing/adios2/engine/bp/TestBPWriteReadAttributes.cpp
@@ -949,9 +949,9 @@ TEST_F(BPWriteReadAttributes, WriteReadStreamVar)
         io.DefineAttribute("smile", "\u263A", var2.Name(),
                                         separator);
 
-        io.DefineAttribute("utf8", std::string(u8"महसुस"),
+        io.DefineAttribute("utf8", std::string("महसुस"),
                                         var1.Name(), separator);
-        io.DefineAttribute("utf8", std::string(u8"महसुस"),
+        io.DefineAttribute("utf8", std::string("महसुस"),
                                         var2.Name(), separator);
 #endif
         adios2::Engine bpWriter = io.Open(fName, adios2::Mode::Write);
@@ -1016,7 +1016,7 @@ TEST_F(BPWriteReadAttributes, WriteReadStreamVar)
             EXPECT_EQ(itUTF8->second.at("Type"), "string");
             EXPECT_EQ(itUTF8->second.at("Elements"), "1");
             EXPECT_EQ(itUTF8->second.at("Value"),
-                      "\"" + std::string(u8"महसुस") + "\"");
+                      "\"" + std::string("महसुस") + "\"");
 #endif
         };
 
diff --git a/testing/adios2/engine/bp/TestBPWriteReadBlockInfo.cpp b/testing/adios2/engine/bp/TestBPWriteReadBlockInfo.cpp
index 81db013ac2..d957915a60 100644
--- a/testing/adios2/engine/bp/TestBPWriteReadBlockInfo.cpp
+++ b/testing/adios2/engine/bp/TestBPWriteReadBlockInfo.cpp
@@ -137,6 +137,7 @@ TEST_F(BPWriteReadBlockInfo, BPWriteReadBlockInfo1D8)
             // Create the BP Engine
             io.SetEngine("BPFile");
         }
+        io.SetParameter("AggregatorRatio", "1");
 
         adios2::Engine bpWriter = io.Open(fname, adios2::Mode::Write);
 
@@ -459,6 +460,7 @@ TEST_F(BPWriteReadBlockInfo, BPWriteReadBlockInfo2D2x4)
             // Create the BP Engine
             io.SetEngine("BPFile");
         }
+        io.SetParameter("AggregatorRatio", "1");
 
         adios2::Engine bpWriter = io.Open(fname, adios2::Mode::Write);
 
diff --git a/testing/adios2/engine/bp/TestBPWriteReadLocalVariables.cpp b/testing/adios2/engine/bp/TestBPWriteReadLocalVariables.cpp
index af211fb376..f50e97a15a 100644
--- a/testing/adios2/engine/bp/TestBPWriteReadLocalVariables.cpp
+++ b/testing/adios2/engine/bp/TestBPWriteReadLocalVariables.cpp
@@ -1770,6 +1770,12 @@ TEST_F(BPWriteReadLocalVariables, ADIOS2BPWriteReadLocal1DSubFile)
     std::iota(data[0].begin(), data[0].end(), startBlock0);
     std::iota(data[1].begin(), data[1].end(), startBlock1);
 
+    /* This is a test for only BP3 */
+    if (engineName != "BP3")
+    {
+        return;
+    }
+
 #if ADIOS2_USE_MPI
     adios2::ADIOS adios(MPI_COMM_WORLD);
 #else
@@ -1778,6 +1784,7 @@ TEST_F(BPWriteReadLocalVariables, ADIOS2BPWriteReadLocal1DSubFile)
     {
         adios2::IO io = adios.DeclareIO("TestIO");
         io.SetEngine("BP3");
+        io.SetParameter("AggregatorRatio", "1");
         const adios2::Dims shape{};
         const adios2::Dims start{};
         const adios2::Dims count{Nx0};
diff --git a/testing/adios2/engine/bp/TestBPWriteReadMultiblock.cpp b/testing/adios2/engine/bp/TestBPWriteReadMultiblock.cpp
index f16803019a..e8f679ba7c 100644
--- a/testing/adios2/engine/bp/TestBPWriteReadMultiblock.cpp
+++ b/testing/adios2/engine/bp/TestBPWriteReadMultiblock.cpp
@@ -1286,6 +1286,7 @@ TEST_F(BPWriteReadMultiblockTest, ADIOS2BPWriteReadMultiblock2D4x2)
         }
 
         io.AddTransport("file");
+        io.SetParameter("AggregatorRatio", "1");
 
         adios2::Engine bpWriter = io.Open(fname, adios2::Mode::Write);
 
diff --git a/testing/adios2/engine/bp/operations/TestBPWriteReadBZIP2.cpp b/testing/adios2/engine/bp/operations/TestBPWriteReadBZIP2.cpp
index ce5898c9d0..afd205e6e4 100644
--- a/testing/adios2/engine/bp/operations/TestBPWriteReadBZIP2.cpp
+++ b/testing/adios2/engine/bp/operations/TestBPWriteReadBZIP2.cpp
@@ -878,7 +878,7 @@ TEST_P(BPWriteReadBZIP2, ADIOS2BPWriteReadBZIP23DSel)
     BZIP2Accuracy3DSel(GetParam());
 }
 
-INSTANTIATE_TEST_CASE_P(
+INSTANTIATE_TEST_SUITE_P(
     BZIP2Accuracy, BPWriteReadBZIP2,
     ::testing::Values(adios2::ops::bzip2::value::blockSize100k_1,
                       adios2::ops::bzip2::value::blockSize100k_2,
diff --git a/testing/adios2/engine/bp/operations/TestBPWriteReadBlosc.cpp b/testing/adios2/engine/bp/operations/TestBPWriteReadBlosc.cpp
index 01a0acbf69..91804d859e 100644
--- a/testing/adios2/engine/bp/operations/TestBPWriteReadBlosc.cpp
+++ b/testing/adios2/engine/bp/operations/TestBPWriteReadBlosc.cpp
@@ -878,16 +878,17 @@ TEST_P(BPWriteReadBlosc, ADIOS2BPWriteReadBlosc3DSel)
     BloscAccuracy3DSel(GetParam());
 }
 
-INSTANTIATE_TEST_CASE_P(BloscAccuracy, BPWriteReadBlosc,
-                        ::testing::Values(adios2::ops::blosc::value::clevel_1,
-                                          adios2::ops::blosc::value::clevel_2,
-                                          adios2::ops::blosc::value::clevel_3,
-                                          adios2::ops::blosc::value::clevel_4,
-                                          adios2::ops::blosc::value::clevel_5,
-                                          adios2::ops::blosc::value::clevel_6,
-                                          adios2::ops::blosc::value::clevel_7,
-                                          adios2::ops::blosc::value::clevel_8,
-                                          adios2::ops::blosc::value::clevel_9));
+INSTANTIATE_TEST_SUITE_P(
+    BloscAccuracy, BPWriteReadBlosc,
+    ::testing::Values(adios2::ops::blosc::value::clevel_1,
+                      adios2::ops::blosc::value::clevel_2,
+                      adios2::ops::blosc::value::clevel_3,
+                      adios2::ops::blosc::value::clevel_4,
+                      adios2::ops::blosc::value::clevel_5,
+                      adios2::ops::blosc::value::clevel_6,
+                      adios2::ops::blosc::value::clevel_7,
+                      adios2::ops::blosc::value::clevel_8,
+                      adios2::ops::blosc::value::clevel_9));
 
 int main(int argc, char **argv)
 {
diff --git a/testing/adios2/engine/bp/operations/TestBPWriteReadMGARD.cpp b/testing/adios2/engine/bp/operations/TestBPWriteReadMGARD.cpp
index 1481a67c56..40f73cf3c3 100644
--- a/testing/adios2/engine/bp/operations/TestBPWriteReadMGARD.cpp
+++ b/testing/adios2/engine/bp/operations/TestBPWriteReadMGARD.cpp
@@ -5,6 +5,7 @@
 #include 
 #include 
 
+#include 
 #include  //std::cout
 #include   //std::iota
 #include 
@@ -997,9 +998,9 @@ TEST_P(BPWriteReadMGARD, BPWRMGARD2D) { MGARDAccuracy2D(GetParam()); }
 
 TEST_P(BPWriteReadMGARD, BPWRMGARD3D) { MGARDAccuracy3D(GetParam()); }
 
-INSTANTIATE_TEST_CASE_P(MGARDAccuracy, BPWriteReadMGARD,
-                        ::testing::Values("0.01", "0.001", "0.0001",
-                                          "0.00001"));
+INSTANTIATE_TEST_SUITE_P(MGARDAccuracy, BPWriteReadMGARD,
+                         ::testing::Values("0.01", "0.001", "0.0001",
+                                           "0.00001"));
 
 int main(int argc, char **argv)
 {
diff --git a/testing/adios2/engine/bp/operations/TestBPWriteReadPNG.cpp b/testing/adios2/engine/bp/operations/TestBPWriteReadPNG.cpp
index 6cc1e49757..63443cbd39 100644
--- a/testing/adios2/engine/bp/operations/TestBPWriteReadPNG.cpp
+++ b/testing/adios2/engine/bp/operations/TestBPWriteReadPNG.cpp
@@ -438,7 +438,7 @@ class BPWRPNG : public ::testing::TestWithParam
 
 TEST_P(BPWRPNG, BPWRPNG2D) { PNGAccuracy2D(GetParam()); }
 
-INSTANTIATE_TEST_CASE_P(
+INSTANTIATE_TEST_SUITE_P(
     PNGAccuracy, BPWRPNG,
     ::testing::Values(adios2::ops::png::value::compression_level_1,
                       adios2::ops::png::value::compression_level_2,
diff --git a/testing/adios2/engine/bp/operations/TestBPWriteReadSZ.cpp b/testing/adios2/engine/bp/operations/TestBPWriteReadSZ.cpp
index d93031ea76..14e0630fe4 100644
--- a/testing/adios2/engine/bp/operations/TestBPWriteReadSZ.cpp
+++ b/testing/adios2/engine/bp/operations/TestBPWriteReadSZ.cpp
@@ -1035,9 +1035,9 @@ TEST_P(BPWriteReadSZ, BPWRSZ2DSel) { SZAccuracy2DSel(GetParam()); }
 TEST_P(BPWriteReadSZ, BPWRSZ3DSel) { SZAccuracy3DSel(GetParam()); }
 TEST_F(BPWriteReadSZ, BPWRSZ2DSmallSel) { SZAccuracy2DSmallSel("0.01"); }
 
-INSTANTIATE_TEST_CASE_P(SZAccuracy, BPWriteReadSZ,
-                        ::testing::Values("0.01", "0.001", "0.0001",
-                                          "0.00001"));
+INSTANTIATE_TEST_SUITE_P(SZAccuracy, BPWriteReadSZ,
+                         ::testing::Values("0.01", "0.001", "0.0001",
+                                           "0.00001"));
 
 int main(int argc, char **argv)
 {
diff --git a/testing/adios2/engine/bp/operations/TestBPWriteReadZfp.cpp b/testing/adios2/engine/bp/operations/TestBPWriteReadZfp.cpp
index ab7ce3ea36..9896a202e9 100644
--- a/testing/adios2/engine/bp/operations/TestBPWriteReadZfp.cpp
+++ b/testing/adios2/engine/bp/operations/TestBPWriteReadZfp.cpp
@@ -5,6 +5,7 @@
 #include 
 #include 
 
+#include 
 #include 
 #include  //std::iota
 #include 
@@ -960,7 +961,7 @@ TEST_P(BPWRZFP, ADIOS2BPWRZFP2DSel) { ZFPRate2DSel(GetParam()); }
 TEST_P(BPWRZFP, ADIOS2BPWRZFP3DSel) { ZFPRate3DSel(GetParam()); }
 TEST_P(BPWRZFP, ADIOS2BPWRZFP2DSmallSel) { ZFPRate2DSmallSel(GetParam()); }
 
-INSTANTIATE_TEST_CASE_P(ZFPRate, BPWRZFP, ::testing::Values("8", "9", "10"));
+INSTANTIATE_TEST_SUITE_P(ZFPRate, BPWRZFP, ::testing::Values("8", "9", "10"));
 
 int main(int argc, char **argv)
 {
diff --git a/testing/adios2/engine/bp/operations/TestBPWriteReadZfpConfig.cpp b/testing/adios2/engine/bp/operations/TestBPWriteReadZfpConfig.cpp
index 64adbd0480..5adcc94a25 100644
--- a/testing/adios2/engine/bp/operations/TestBPWriteReadZfpConfig.cpp
+++ b/testing/adios2/engine/bp/operations/TestBPWriteReadZfpConfig.cpp
@@ -876,7 +876,7 @@ TEST_P(BPWriteReadZfpConfig, ADIOS2BPWriteReadZfp2DSmallSel)
     ZfpRate2DSmallSel(GetParam());
 }
 
-INSTANTIATE_TEST_CASE_P(
+INSTANTIATE_TEST_SUITE_P(
     ZfpConfigFile, BPWriteReadZfpConfig,
     ::testing::Values("configZfp_rate8.xml", "configZfp_rate8Simple.xml",
                       "configZfp_rate9.xml", "configZfp_rate9Simple.xml",
diff --git a/testing/adios2/engine/bp/operations/TestBPWriteReadZfpHighLevelAPI.cpp b/testing/adios2/engine/bp/operations/TestBPWriteReadZfpHighLevelAPI.cpp
index 21ab379dcc..1595e8e3bd 100644
--- a/testing/adios2/engine/bp/operations/TestBPWriteReadZfpHighLevelAPI.cpp
+++ b/testing/adios2/engine/bp/operations/TestBPWriteReadZfpHighLevelAPI.cpp
@@ -722,8 +722,8 @@ TEST_P(BPWriteReadZfpHighLevelAPI, ADIOS2BPWriteReadZfp2DSmallSel)
     ZfpRate2DSmallSel(GetParam());
 }
 
-INSTANTIATE_TEST_CASE_P(ZfpRate, BPWriteReadZfpHighLevelAPI,
-                        ::testing::Values(8., 9., 10));
+INSTANTIATE_TEST_SUITE_P(ZfpRate, BPWriteReadZfpHighLevelAPI,
+                         ::testing::Values(8., 9., 10));
 
 int main(int argc, char **argv)
 {
diff --git a/testing/adios2/engine/dataman/CMakeLists.txt b/testing/adios2/engine/dataman/CMakeLists.txt
index 5b5878069a..74379caa1e 100644
--- a/testing/adios2/engine/dataman/CMakeLists.txt
+++ b/testing/adios2/engine/dataman/CMakeLists.txt
@@ -3,14 +3,29 @@
 # accompanying file Copyright.txt for details.
 #------------------------------------------------------------------------------#
 
-if(ADIOS2_HAVE_MPI)
-  gtest_add_tests_helper(1D MPI_NOEXEC DataMan Engine.DataMan. "")
-  gtest_add_tests_helper(2DMemSelect MPI_NOEXEC DataMan Engine.DataMan. "")
-  gtest_add_tests_helper(3DMemSelect MPI_NOEXEC DataMan Engine.DataMan. "")
-  gtest_add_tests_helper(WriterDoubleBuffer MPI_NOEXEC DataMan Engine.DataMan. "")
-  gtest_add_tests_helper(WriterSingleBuffer MPI_NOEXEC DataMan Engine.DataMan. "")
-  gtest_add_tests_helper(ReaderDoubleBuffer MPI_NOEXEC DataMan Engine.DataMan. "")
-  gtest_add_tests_helper(ReaderSingleBuffer MPI_NOEXEC DataMan Engine.DataMan. "")
-# Turning off DataMan.Reliable test as unstable - May 27, 2020
-#  gtest_add_tests_helper(Reliable MPI_NOEXEC DataMan Engine.DataMan. "")
+foreach(tst IN ITEMS
+  1D
+  2DMemSelect 3DMemSelect
+  WriterDoubleBuffer WriterSingleBuffer
+  ReaderDoubleBuffer ReaderSingleBuffer
+  Reliable
+  )
+  gtest_add_tests_helper(${tst} MPI_NONE DataMan Engine.DataMan. "")
+  set_tests_properties(${Test.Engine.DataMan.${tst}-TESTS}
+    PROPERTIES RUN_SERIAL TRUE
+  )
+endforeach()
+
+if(ADIOS2_HAVE_ZFP)
+    gtest_add_tests_helper(2DZfp MPI_NONE DataMan Engine.DataMan. "")
+    set_tests_properties(${Test.Engine.DataMan.2DZfp-TESTS}
+        PROPERTIES RUN_SERIAL TRUE
+        )
+endif()
+
+if(ADIOS2_HAVE_BZip2)
+    gtest_add_tests_helper(2DBzip2 MPI_NONE DataMan Engine.DataMan. "")
+    set_tests_properties(${Test.Engine.DataMan.2DBzip2-TESTS}
+        PROPERTIES RUN_SERIAL TRUE
+        )
 endif()
diff --git a/testing/adios2/engine/dataman/TestDataMan1D.cpp b/testing/adios2/engine/dataman/TestDataMan1D.cpp
index fcb3591faf..d688e75603 100644
--- a/testing/adios2/engine/dataman/TestDataMan1D.cpp
+++ b/testing/adios2/engine/dataman/TestDataMan1D.cpp
@@ -18,9 +18,6 @@
 
 using namespace adios2;
 
-int mpiRank = 0;
-int mpiSize = 1;
-
 size_t print_lines = 0;
 size_t to_print_lines = 10;
 
@@ -29,8 +26,8 @@ void GenData(std::vector> &data, const size_t step)
 {
     for (size_t i = 0; i < data.size(); ++i)
     {
-        data[i] = {static_cast(i + mpiRank * 10000 + step * 100),
-                   static_cast(i + mpiRank * 10000)};
+        data[i] = {static_cast(i + 10000 + step * 100),
+                   static_cast(i + 10000)};
     }
 }
 
@@ -39,14 +36,14 @@ void GenData(std::vector &data, const size_t step)
 {
     for (size_t i = 0; i < data.size(); ++i)
     {
-        data[i] = i + mpiRank * 10000 + step * 100;
+        data[i] = i + 10000 + step * 100;
     }
 }
 
 template 
 void PrintData(const T *data, const size_t size, const size_t step)
 {
-    std::cout << "Rank: " << mpiRank << " Step: " << step << " [";
+    std::cout << "Step: " << step << " Size:" << size << "\n";
     size_t printsize = 32;
     if (size < printsize)
     {
@@ -68,11 +65,6 @@ void VerifyData(const std::complex *data, const size_t size, size_t step)
     {
         ASSERT_EQ(data[i], tmpdata[i]);
     }
-    if (print_lines < to_print_lines)
-    {
-        PrintData(data, size, step);
-        ++print_lines;
-    }
 }
 
 template 
@@ -84,11 +76,6 @@ void VerifyData(const T *data, const size_t size, size_t step)
     {
         ASSERT_EQ(data[i], tmpdata[i]);
     }
-    if (print_lines < to_print_lines)
-    {
-        PrintData(data, size, step);
-        ++print_lines;
-    }
 }
 
 template 
@@ -102,11 +89,7 @@ void DataManWriter(const Dims &shape, const Dims &start, const Dims &count,
 {
     size_t datasize = std::accumulate(count.begin(), count.end(), 1,
                                       std::multiplies());
-#if ADIOS2_USE_MPI
-    adios2::ADIOS adios(MPI_COMM_SELF);
-#else
     adios2::ADIOS adios;
-#endif
     adios2::IO dataManIO = adios.DeclareIO("WAN");
     dataManIO.SetEngine("DataMan");
     dataManIO.SetParameters(engineParams);
@@ -178,11 +161,7 @@ void DataManReader(const Dims &shape, const Dims &start, const Dims &count,
 {
     size_t datasize = std::accumulate(count.begin(), count.end(), 1,
                                       std::multiplies());
-#if ADIOS2_USE_MPI
-    adios2::ADIOS adios(MPI_COMM_SELF);
-#else
     adios2::ADIOS adios;
-#endif
     adios2::IO dataManIO = adios.DeclareIO("WAN");
     dataManIO.SetEngine("DataMan");
     dataManIO.SetParameters(engineParams);
@@ -208,15 +187,6 @@ void DataManReader(const Dims &shape, const Dims &start, const Dims &count,
             received_steps = true;
             const auto &vars = dataManIO.AvailableVariables();
             ASSERT_EQ(vars.size(), 11);
-            if (print_lines < 10)
-            {
-                std::cout << "All available variables : ";
-                for (const auto &var : vars)
-                {
-                    std::cout << var.first << ", ";
-                }
-                std::cout << std::endl;
-            }
             currentStep = dataManReader.CurrentStep();
             adios2::Variable bpChars =
                 dataManIO.InquireVariable("bpChars");
@@ -280,8 +250,6 @@ void DataManReader(const Dims &shape, const Dims &start, const Dims &count,
         }
         else if (status == adios2::StepStatus::EndOfStream)
         {
-            std::cout << "DataManReader end of stream at Step " << currentStep
-                      << std::endl;
             break;
         }
         else if (status == adios2::StepStatus::NotReady)
@@ -293,11 +261,6 @@ void DataManReader(const Dims &shape, const Dims &start, const Dims &count,
     {
         auto attInt = dataManIO.InquireAttribute("AttInt");
         ASSERT_EQ(110, attInt.Data()[0]);
-        ASSERT_NE(111, attInt.Data()[0]);
-    }
-    else
-    {
-        std::cout << "no steps received " << std::endl;
     }
     dataManReader.Close();
 }
@@ -322,40 +285,18 @@ TEST_F(DataManEngineTest, 1D)
     // run workflow
     auto r =
         std::thread(DataManReader, shape, start, count, steps, engineParams);
-    std::cout << "Reader thread started" << std::endl;
     auto w =
         std::thread(DataManWriter, shape, start, count, steps, engineParams);
-    std::cout << "Writer thread started" << std::endl;
     w.join();
-    std::cout << "Writer thread ended" << std::endl;
     r.join();
-    std::cout << "Reader thread ended" << std::endl;
 }
 #endif // ZEROMQ
 
 int main(int argc, char **argv)
 {
-#if ADIOS2_USE_MPI
-    int mpi_provided;
-    MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &mpi_provided);
-    std::cout << "MPI_Init_thread required Mode " << MPI_THREAD_MULTIPLE
-              << " and provided Mode " << mpi_provided << std::endl;
-    if (mpi_provided != MPI_THREAD_MULTIPLE)
-    {
-        MPI_Finalize();
-        return 0;
-    }
-    MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank);
-    MPI_Comm_size(MPI_COMM_WORLD, &mpiSize);
-#endif
-
     int result;
     ::testing::InitGoogleTest(&argc, argv);
     result = RUN_ALL_TESTS();
 
-#if ADIOS2_USE_MPI
-    MPI_Finalize();
-#endif
-
     return result;
 }
diff --git a/testing/adios2/engine/dataman/TestDataMan2DBzip2.cpp b/testing/adios2/engine/dataman/TestDataMan2DBzip2.cpp
new file mode 100644
index 0000000000..ab90fe4033
--- /dev/null
+++ b/testing/adios2/engine/dataman/TestDataMan2DBzip2.cpp
@@ -0,0 +1,343 @@
+/*
+ * Distributed under the OSI-approved Apache License, Version 2.0.  See
+ * accompanying file Copyright.txt for details.
+ *
+ * TestDataMan2DZfp.cpp
+ *
+ *  Created on: Nov 24, 2020
+ *      Author: Jason Wang
+ */
+
+#include 
+#include 
+#include 
+#include 
+#include 
+
+using namespace adios2;
+size_t print_lines = 0;
+
+class DataManEngineTest : public ::testing::Test
+{
+public:
+    DataManEngineTest() = default;
+};
+
+template 
+void PrintData(const T *data, const size_t step, const Dims &start,
+               const Dims &count)
+{
+    size_t size = std::accumulate(count.begin(), count.end(), 1,
+                                  std::multiplies());
+    std::cout << "Step: " << step << " Size:" << size << "\n";
+    size_t printsize = 128;
+
+    if (size < printsize)
+    {
+        printsize = size;
+    }
+    int s = 0;
+    for (size_t i = 0; i < printsize; ++i)
+    {
+        ++s;
+        std::cout << data[i] << " ";
+        if (s == count[1])
+        {
+            std::cout << std::endl;
+            s = 0;
+        }
+    }
+
+    std::cout << "]" << std::endl;
+}
+
+template 
+void GenData(std::vector &data, const size_t step, const Dims &start,
+             const Dims &count, const Dims &shape)
+{
+    if (start.size() == 2)
+    {
+        for (size_t i = 0; i < count[0]; ++i)
+        {
+            for (size_t j = 0; j < count[1]; ++j)
+            {
+                data[i * count[1] + j] =
+                    (i + start[1]) * shape[1] + j + start[0] + 0.01;
+            }
+        }
+    }
+}
+
+template 
+void VerifyData(const std::complex *data, size_t step, const Dims &start,
+                const Dims &count, const Dims &shape)
+{
+    size_t size = std::accumulate(count.begin(), count.end(), 1,
+                                  std::multiplies());
+    std::vector> tmpdata(size);
+    GenData(tmpdata, step, start, count, shape);
+    for (size_t i = 0; i < size; ++i)
+    {
+        ASSERT_EQ(data[i], tmpdata[i]);
+    }
+}
+
+template 
+void VerifyData(const T *data, size_t step, const Dims &start,
+                const Dims &count, const Dims &shape)
+{
+    size_t size = std::accumulate(count.begin(), count.end(), 1,
+                                  std::multiplies());
+    std::vector tmpdata(size);
+    GenData(tmpdata, step, start, count, shape);
+    for (size_t i = 0; i < size; ++i)
+    {
+        ASSERT_EQ(data[i], tmpdata[i]);
+    }
+}
+
+void DataManWriterP2PMemSelect(const Dims &shape, const Dims &start,
+                               const Dims &count, const size_t steps,
+                               const adios2::Params &engineParams)
+{
+    size_t datasize = std::accumulate(count.begin(), count.end(), 1,
+                                      std::multiplies());
+    adios2::ADIOS adios;
+    adios2::IO dataManIO = adios.DeclareIO("WAN");
+    dataManIO.SetEngine("DataMan");
+    dataManIO.SetParameters(engineParams);
+    std::vector myChars(datasize);
+    std::vector myUChars(datasize);
+    std::vector myShorts(datasize);
+    std::vector myUShorts(datasize);
+    std::vector myInts(datasize);
+    std::vector myUInts(datasize);
+    std::vector myFloats(datasize);
+    std::vector myDoubles(datasize);
+    std::vector> myComplexes(datasize);
+    std::vector> myDComplexes(datasize);
+    auto bpChars =
+        dataManIO.DefineVariable("bpChars", shape, start, count);
+    auto bpUChars = dataManIO.DefineVariable("bpUChars", shape,
+                                                            start, count);
+    auto bpShorts =
+        dataManIO.DefineVariable("bpShorts", shape, start, count);
+    auto bpUShorts = dataManIO.DefineVariable(
+        "bpUShorts", shape, start, count);
+    auto bpInts = dataManIO.DefineVariable("bpInts", shape, start, count);
+    auto bpUInts =
+        dataManIO.DefineVariable("bpUInts", shape, start, count);
+    auto bpFloats =
+        dataManIO.DefineVariable("bpFloats", shape, start, count);
+    adios2::Operator bzip2Op =
+        adios.DefineOperator("Compressor", adios2::ops::LosslessBZIP2);
+    bpFloats.AddOperation(bzip2Op, {});
+    auto bpDoubles =
+        dataManIO.DefineVariable("bpDoubles", shape, start, count);
+    auto bpComplexes = dataManIO.DefineVariable>(
+        "bpComplexes", shape, start, count);
+    auto bpDComplexes = dataManIO.DefineVariable>(
+        "bpDComplexes", shape, start, count);
+    dataManIO.DefineAttribute("AttInt", 110);
+    adios2::Engine dataManWriter =
+        dataManIO.Open("stream", adios2::Mode::Write);
+    for (int i = 0; i < steps; ++i)
+    {
+        dataManWriter.BeginStep();
+        GenData(myChars, i, start, count, shape);
+        GenData(myUChars, i, start, count, shape);
+        GenData(myShorts, i, start, count, shape);
+        GenData(myUShorts, i, start, count, shape);
+        GenData(myInts, i, start, count, shape);
+        GenData(myUInts, i, start, count, shape);
+        GenData(myFloats, i, start, count, shape);
+        GenData(myDoubles, i, start, count, shape);
+        GenData(myComplexes, i, start, count, shape);
+        GenData(myDComplexes, i, start, count, shape);
+        dataManWriter.Put(bpChars, myChars.data(), adios2::Mode::Sync);
+        dataManWriter.Put(bpUChars, myUChars.data(), adios2::Mode::Sync);
+        dataManWriter.Put(bpShorts, myShorts.data(), adios2::Mode::Sync);
+        dataManWriter.Put(bpUShorts, myUShorts.data(), adios2::Mode::Sync);
+        dataManWriter.Put(bpInts, myInts.data(), adios2::Mode::Sync);
+        dataManWriter.Put(bpUInts, myUInts.data(), adios2::Mode::Sync);
+        dataManWriter.Put(bpFloats, myFloats.data(), adios2::Mode::Sync);
+        dataManWriter.Put(bpDoubles, myDoubles.data(), adios2::Mode::Sync);
+        dataManWriter.Put(bpComplexes, myComplexes.data(), adios2::Mode::Sync);
+        dataManWriter.Put(bpDComplexes, myDComplexes.data(),
+                          adios2::Mode::Sync);
+        dataManWriter.EndStep();
+    }
+    dataManWriter.Close();
+}
+
+void DataManReaderP2PMemSelect(const Dims &shape, const Dims &start,
+                               const Dims &count, const Dims &memStart,
+                               const Dims &memCount, const size_t steps,
+                               const adios2::Params &engineParams)
+{
+    adios2::ADIOS adios;
+    adios2::IO dataManIO = adios.DeclareIO("WAN");
+    dataManIO.SetEngine("DataMan");
+    dataManIO.SetParameters(engineParams);
+    adios2::Engine dataManReader = dataManIO.Open("stream", adios2::Mode::Read);
+
+    size_t datasize = std::accumulate(memCount.begin(), memCount.end(), 1,
+                                      std::multiplies());
+    std::vector myChars(datasize);
+    std::vector myUChars(datasize);
+    std::vector myShorts(datasize);
+    std::vector myUShorts(datasize);
+    std::vector myInts(datasize);
+    std::vector myUInts(datasize);
+    std::vector myFloats(datasize);
+    std::vector myDoubles(datasize);
+    std::vector> myComplexes(datasize);
+    std::vector> myDComplexes(datasize);
+    bool received_steps = false;
+    size_t currentStep;
+    while (true)
+    {
+        adios2::StepStatus status = dataManReader.BeginStep();
+        if (status == adios2::StepStatus::OK)
+        {
+            received_steps = true;
+            const auto &vars = dataManIO.AvailableVariables();
+            ASSERT_EQ(vars.size(), 10);
+            currentStep = dataManReader.CurrentStep();
+            GenData(myChars, currentStep, memStart, memCount, shape);
+            GenData(myUChars, currentStep, memStart, memCount, shape);
+            GenData(myShorts, currentStep, memStart, memCount, shape);
+            GenData(myUShorts, currentStep, memStart, memCount, shape);
+            GenData(myInts, currentStep, memStart, memCount, shape);
+            GenData(myUInts, currentStep, memStart, memCount, shape);
+            GenData(myFloats, currentStep, memStart, memCount, shape);
+            GenData(myDoubles, currentStep, memStart, memCount, shape);
+            GenData(myComplexes, currentStep, memStart, memCount, shape);
+            GenData(myDComplexes, currentStep, memStart, memCount, shape);
+            adios2::Variable bpChars =
+                dataManIO.InquireVariable("bpChars");
+            adios2::Variable bpUChars =
+                dataManIO.InquireVariable("bpUChars");
+            adios2::Variable bpShorts =
+                dataManIO.InquireVariable("bpShorts");
+            adios2::Variable bpUShorts =
+                dataManIO.InquireVariable("bpUShorts");
+            adios2::Variable bpInts =
+                dataManIO.InquireVariable("bpInts");
+            adios2::Variable bpUInts =
+                dataManIO.InquireVariable("bpUInts");
+            adios2::Variable bpFloats =
+                dataManIO.InquireVariable("bpFloats");
+            adios2::Variable bpDoubles =
+                dataManIO.InquireVariable("bpDoubles");
+            adios2::Variable> bpComplexes =
+                dataManIO.InquireVariable>("bpComplexes");
+            adios2::Variable> bpDComplexes =
+                dataManIO.InquireVariable>("bpDComplexes");
+            auto charsBlocksInfo = dataManReader.AllStepsBlocksInfo(bpChars);
+
+            bpChars.SetSelection({start, count});
+            bpUChars.SetSelection({start, count});
+            bpShorts.SetSelection({start, count});
+            bpUShorts.SetSelection({start, count});
+            bpInts.SetSelection({start, count});
+            bpUInts.SetSelection({start, count});
+            bpFloats.SetSelection({start, count});
+            bpDoubles.SetSelection({start, count});
+            bpComplexes.SetSelection({start, count});
+            bpDComplexes.SetSelection({start, count});
+
+            bpChars.SetMemorySelection({memStart, memCount});
+            bpUChars.SetMemorySelection({memStart, memCount});
+            bpShorts.SetMemorySelection({memStart, memCount});
+            bpUShorts.SetMemorySelection({memStart, memCount});
+            bpInts.SetMemorySelection({memStart, memCount});
+            bpUInts.SetMemorySelection({memStart, memCount});
+            bpFloats.SetMemorySelection({memStart, memCount});
+            bpDoubles.SetMemorySelection({memStart, memCount});
+            bpComplexes.SetMemorySelection({memStart, memCount});
+            bpDComplexes.SetMemorySelection({memStart, memCount});
+
+            dataManReader.Get(bpChars, myChars.data(), adios2::Mode::Sync);
+            dataManReader.Get(bpUChars, myUChars.data(), adios2::Mode::Sync);
+            dataManReader.Get(bpShorts, myShorts.data(), adios2::Mode::Sync);
+            dataManReader.Get(bpUShorts, myUShorts.data(), adios2::Mode::Sync);
+            dataManReader.Get(bpInts, myInts.data(), adios2::Mode::Sync);
+            dataManReader.Get(bpUInts, myUInts.data(), adios2::Mode::Sync);
+            dataManReader.Get(bpFloats, myFloats.data(), adios2::Mode::Sync);
+            dataManReader.Get(bpDoubles, myDoubles.data(), adios2::Mode::Sync);
+            dataManReader.Get(bpComplexes, myComplexes.data(),
+                              adios2::Mode::Sync);
+            dataManReader.Get(bpDComplexes, myDComplexes.data(),
+                              adios2::Mode::Sync);
+            VerifyData(myChars.data(), currentStep, memStart, memCount, shape);
+            VerifyData(myUChars.data(), currentStep, memStart, memCount, shape);
+            VerifyData(myShorts.data(), currentStep, memStart, memCount, shape);
+            VerifyData(myUShorts.data(), currentStep, memStart, memCount,
+                       shape);
+            VerifyData(myInts.data(), currentStep, memStart, memCount, shape);
+            VerifyData(myUInts.data(), currentStep, memStart, memCount, shape);
+            VerifyData(myFloats.data(), currentStep, memStart, memCount, shape);
+            VerifyData(myDoubles.data(), currentStep, memStart, memCount,
+                       shape);
+            VerifyData(myComplexes.data(), currentStep, memStart, memCount,
+                       shape);
+            VerifyData(myDComplexes.data(), currentStep, memStart, memCount,
+                       shape);
+            dataManReader.EndStep();
+        }
+        else if (status == adios2::StepStatus::EndOfStream)
+        {
+            break;
+        }
+        else if (status == adios2::StepStatus::NotReady)
+        {
+            continue;
+        }
+    }
+    if (received_steps)
+    {
+        auto attInt = dataManIO.InquireAttribute("AttInt");
+        ASSERT_EQ(110, attInt.Data()[0]);
+    }
+    dataManReader.Close();
+    print_lines = 0;
+}
+
+#ifdef ADIOS2_HAVE_ZEROMQ
+TEST_F(DataManEngineTest, 2D_Bzip2)
+{
+    // set parameters
+    Dims shape = {10, 10};
+    Dims start = {2, 2};
+    Dims count = {5, 5};
+    Dims memstart = start;
+    Dims memcount = count;
+    memstart = {1, 1};
+    memcount = {7, 9};
+
+    size_t steps = 5000;
+    adios2::Params engineParams = {
+        {"IPAddress", "127.0.0.1"}, {"Port", "12320"}, {"Verbose", "0"}};
+
+    auto r = std::thread(DataManReaderP2PMemSelect, shape, start, count,
+                         memstart, memcount, steps, engineParams);
+
+    auto w = std::thread(DataManWriterP2PMemSelect, shape, start, count, steps,
+                         engineParams);
+
+    w.join();
+
+    r.join();
+}
+
+#endif // ZEROMQ
+
+int main(int argc, char **argv)
+{
+    int result;
+    ::testing::InitGoogleTest(&argc, argv);
+    result = RUN_ALL_TESTS();
+
+    return result;
+}
diff --git a/testing/adios2/engine/dataman/TestDataMan2DMemSelect.cpp b/testing/adios2/engine/dataman/TestDataMan2DMemSelect.cpp
index 46babffe33..1f65efe1a9 100644
--- a/testing/adios2/engine/dataman/TestDataMan2DMemSelect.cpp
+++ b/testing/adios2/engine/dataman/TestDataMan2DMemSelect.cpp
@@ -10,15 +10,10 @@
 
 #include 
 #include 
-#if ADIOS2_USE_MPI
-#include 
-#endif
 #include 
 #include 
 
 using namespace adios2;
-int mpiRank = 0;
-int mpiSize = 1;
 size_t print_lines = 0;
 
 class DataManEngineTest : public ::testing::Test
@@ -33,8 +28,7 @@ void PrintData(const T *data, const size_t step, const Dims &start,
 {
     size_t size = std::accumulate(count.begin(), count.end(), 1,
                                   std::multiplies());
-    std::cout << "Rank: " << mpiRank << " Step: " << step << " Size:" << size
-              << "\n";
+    std::cout << "Step: " << step << " Size:" << size << "\n";
     size_t printsize = 128;
 
     if (size < printsize)
@@ -85,11 +79,6 @@ void VerifyData(const std::complex *data, size_t step, const Dims &start,
     {
         ASSERT_EQ(data[i], tmpdata[i]);
     }
-    if (print_lines < 32)
-    {
-        PrintData(data, step, start, count);
-        ++print_lines;
-    }
 }
 
 template 
@@ -100,11 +89,6 @@ void VerifyData(const T *data, size_t step, const Dims &start,
                                   std::multiplies());
     bool compressed = false;
     std::vector tmpdata(size);
-    if (print_lines < 32)
-    {
-        PrintData(data, step, start, count);
-        ++print_lines;
-    }
     GenData(tmpdata, step, start, count, shape);
     for (size_t i = 0; i < size; ++i)
     {
@@ -121,11 +105,7 @@ void DataManWriterP2PMemSelect(const Dims &shape, const Dims &start,
 {
     size_t datasize = std::accumulate(count.begin(), count.end(), 1,
                                       std::multiplies());
-#if ADIOS2_USE_MPI
-    adios2::ADIOS adios(MPI_COMM_SELF);
-#else
     adios2::ADIOS adios;
-#endif
     adios2::IO dataManIO = adios.DeclareIO("WAN");
     dataManIO.SetEngine("DataMan");
     dataManIO.SetParameters(engineParams);
@@ -195,11 +175,7 @@ void DataManReaderP2PMemSelect(const Dims &shape, const Dims &start,
                                const Dims &memCount, const size_t steps,
                                const adios2::Params &engineParams)
 {
-#if ADIOS2_USE_MPI
-    adios2::ADIOS adios(MPI_COMM_SELF);
-#else
     adios2::ADIOS adios;
-#endif
     adios2::IO dataManIO = adios.DeclareIO("WAN");
     dataManIO.SetEngine("DataMan");
     dataManIO.SetParameters(engineParams);
@@ -227,15 +203,6 @@ void DataManReaderP2PMemSelect(const Dims &shape, const Dims &start,
             received_steps = true;
             const auto &vars = dataManIO.AvailableVariables();
             ASSERT_EQ(vars.size(), 10);
-            if (print_lines == 0)
-            {
-                std::cout << "All available variables : ";
-                for (const auto &var : vars)
-                {
-                    std::cout << var.first << ", ";
-                }
-                std::cout << std::endl;
-            }
             currentStep = dataManReader.CurrentStep();
             GenData(myChars, currentStep, memStart, memCount, shape);
             GenData(myUChars, currentStep, memStart, memCount, shape);
@@ -331,14 +298,7 @@ void DataManReaderP2PMemSelect(const Dims &shape, const Dims &start,
     if (received_steps)
     {
         auto attInt = dataManIO.InquireAttribute("AttInt");
-        std::cout << "Attribute received " << attInt.Data()[0]
-                  << ", expected 110" << std::endl;
         ASSERT_EQ(110, attInt.Data()[0]);
-        ASSERT_NE(111, attInt.Data()[0]);
-    }
-    else
-    {
-        std::cout << "no steps received " << std::endl;
     }
     dataManReader.Close();
     print_lines = 0;
@@ -362,42 +322,22 @@ TEST_F(DataManEngineTest, 2D_MemSelect)
 
     auto r = std::thread(DataManReaderP2PMemSelect, shape, start, count,
                          memstart, memcount, steps, engineParams);
-    std::cout << "Reader thread started" << std::endl;
 
     auto w = std::thread(DataManWriterP2PMemSelect, shape, start, count, steps,
                          engineParams);
-    std::cout << "Writer thread started" << std::endl;
 
     w.join();
-    std::cout << "Writer thread ended" << std::endl;
 
     r.join();
-    std::cout << "Reader thread ended" << std::endl;
 }
 
 #endif // ZEROMQ
 
 int main(int argc, char **argv)
 {
-#if ADIOS2_USE_MPI
-    int mpi_provided;
-    MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &mpi_provided);
-    std::cout << "MPI_Init_thread required Mode " << MPI_THREAD_MULTIPLE
-              << " and provided Mode " << mpi_provided << std::endl;
-    if (mpi_provided != MPI_THREAD_MULTIPLE)
-    {
-        MPI_Finalize();
-        return 0;
-    }
-    MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank);
-    MPI_Comm_size(MPI_COMM_WORLD, &mpiSize);
-#endif
     int result;
     ::testing::InitGoogleTest(&argc, argv);
     result = RUN_ALL_TESTS();
-#if ADIOS2_USE_MPI
-    MPI_Finalize();
-#endif
 
     return result;
 }
diff --git a/testing/adios2/engine/dataman/TestDataMan2DZfp.cpp b/testing/adios2/engine/dataman/TestDataMan2DZfp.cpp
new file mode 100644
index 0000000000..a0b209f0ea
--- /dev/null
+++ b/testing/adios2/engine/dataman/TestDataMan2DZfp.cpp
@@ -0,0 +1,345 @@
+/*
+ * Distributed under the OSI-approved Apache License, Version 2.0.  See
+ * accompanying file Copyright.txt for details.
+ *
+ * TestDataMan2DZfp.cpp
+ *
+ *  Created on: Nov 24, 2020
+ *      Author: Jason Wang
+ */
+
+#include 
+#include 
+#include 
+#include 
+#include 
+
+using namespace adios2;
+size_t print_lines = 0;
+
+class DataManEngineTest : public ::testing::Test
+{
+public:
+    DataManEngineTest() = default;
+};
+
+template 
+void PrintData(const T *data, const size_t step, const Dims &start,
+               const Dims &count)
+{
+    size_t size = std::accumulate(count.begin(), count.end(), 1,
+                                  std::multiplies());
+    std::cout << "Step: " << step << " Size:" << size << "\n";
+    size_t printsize = 128;
+
+    if (size < printsize)
+    {
+        printsize = size;
+    }
+    int s = 0;
+    for (size_t i = 0; i < printsize; ++i)
+    {
+        ++s;
+        std::cout << data[i] << " ";
+        if (s == count[1])
+        {
+            std::cout << std::endl;
+            s = 0;
+        }
+    }
+
+    std::cout << "]" << std::endl;
+}
+
+template 
+void GenData(std::vector &data, const size_t step, const Dims &start,
+             const Dims &count, const Dims &shape)
+{
+    if (start.size() == 2)
+    {
+        for (size_t i = 0; i < count[0]; ++i)
+        {
+            for (size_t j = 0; j < count[1]; ++j)
+            {
+                data[i * count[1] + j] =
+                    (i + start[1]) * shape[1] + j + start[0] + 0.01;
+            }
+        }
+    }
+}
+
+template 
+void VerifyData(const std::complex *data, size_t step, const Dims &start,
+                const Dims &count, const Dims &shape)
+{
+    size_t size = std::accumulate(count.begin(), count.end(), 1,
+                                  std::multiplies());
+    std::vector> tmpdata(size);
+    GenData(tmpdata, step, start, count, shape);
+    for (size_t i = 0; i < size; ++i)
+    {
+        ASSERT_EQ(abs(data[i].real() - tmpdata[i].real()) < 0.01, true);
+        ASSERT_EQ(abs(data[i].imag() - tmpdata[i].imag()) < 0.01, true);
+    }
+}
+
+template 
+void VerifyData(const T *data, size_t step, const Dims &start,
+                const Dims &count, const Dims &shape)
+{
+    size_t size = std::accumulate(count.begin(), count.end(), 1,
+                                  std::multiplies());
+    std::vector tmpdata(size);
+    GenData(tmpdata, step, start, count, shape);
+    for (size_t i = 0; i < size; ++i)
+    {
+        ASSERT_EQ(abs((double)(data[i] - tmpdata[i])) < 0.01, true);
+    }
+}
+
+void DataManWriterP2PMemSelect(const Dims &shape, const Dims &start,
+                               const Dims &count, const size_t steps,
+                               const adios2::Params &engineParams)
+{
+    size_t datasize = std::accumulate(count.begin(), count.end(), 1,
+                                      std::multiplies());
+    adios2::ADIOS adios;
+    adios2::IO dataManIO = adios.DeclareIO("WAN");
+    dataManIO.SetEngine("DataMan");
+    dataManIO.SetParameters(engineParams);
+    std::vector myChars(datasize);
+    std::vector myUChars(datasize);
+    std::vector myShorts(datasize);
+    std::vector myUShorts(datasize);
+    std::vector myInts(datasize);
+    std::vector myUInts(datasize);
+    std::vector myFloats(datasize);
+    std::vector myDoubles(datasize);
+    std::vector> myComplexes(datasize);
+    std::vector> myDComplexes(datasize);
+    auto bpChars =
+        dataManIO.DefineVariable("bpChars", shape, start, count);
+    auto bpUChars = dataManIO.DefineVariable("bpUChars", shape,
+                                                            start, count);
+    auto bpShorts =
+        dataManIO.DefineVariable("bpShorts", shape, start, count);
+    auto bpUShorts = dataManIO.DefineVariable(
+        "bpUShorts", shape, start, count);
+    auto bpInts = dataManIO.DefineVariable("bpInts", shape, start, count);
+    auto bpUInts =
+        dataManIO.DefineVariable("bpUInts", shape, start, count);
+    auto bpFloats =
+        dataManIO.DefineVariable("bpFloats", shape, start, count);
+    adios2::Operator zfpOp =
+        adios.DefineOperator("zfpCompressor", adios2::ops::LossyZFP);
+    bpFloats.AddOperation(zfpOp, {{adios2::ops::zfp::key::accuracy, "0.1"}});
+    auto bpDoubles =
+        dataManIO.DefineVariable("bpDoubles", shape, start, count);
+    auto bpComplexes = dataManIO.DefineVariable>(
+        "bpComplexes", shape, start, count);
+    auto bpDComplexes = dataManIO.DefineVariable>(
+        "bpDComplexes", shape, start, count);
+    dataManIO.DefineAttribute("AttInt", 110);
+    adios2::Engine dataManWriter =
+        dataManIO.Open("stream", adios2::Mode::Write);
+    for (int i = 0; i < steps; ++i)
+    {
+        dataManWriter.BeginStep();
+        GenData(myChars, i, start, count, shape);
+        GenData(myUChars, i, start, count, shape);
+        GenData(myShorts, i, start, count, shape);
+        GenData(myUShorts, i, start, count, shape);
+        GenData(myInts, i, start, count, shape);
+        GenData(myUInts, i, start, count, shape);
+        GenData(myFloats, i, start, count, shape);
+        GenData(myDoubles, i, start, count, shape);
+        GenData(myComplexes, i, start, count, shape);
+        GenData(myDComplexes, i, start, count, shape);
+        dataManWriter.Put(bpChars, myChars.data(), adios2::Mode::Sync);
+        dataManWriter.Put(bpUChars, myUChars.data(), adios2::Mode::Sync);
+        dataManWriter.Put(bpShorts, myShorts.data(), adios2::Mode::Sync);
+        dataManWriter.Put(bpUShorts, myUShorts.data(), adios2::Mode::Sync);
+        dataManWriter.Put(bpInts, myInts.data(), adios2::Mode::Sync);
+        dataManWriter.Put(bpUInts, myUInts.data(), adios2::Mode::Sync);
+        dataManWriter.Put(bpFloats, myFloats.data(), adios2::Mode::Sync);
+        dataManWriter.Put(bpDoubles, myDoubles.data(), adios2::Mode::Sync);
+        dataManWriter.Put(bpComplexes, myComplexes.data(), adios2::Mode::Sync);
+        dataManWriter.Put(bpDComplexes, myDComplexes.data(),
+                          adios2::Mode::Sync);
+        dataManWriter.EndStep();
+    }
+    dataManWriter.Close();
+}
+
+void DataManReaderP2PMemSelect(const Dims &shape, const Dims &start,
+                               const Dims &count, const Dims &memStart,
+                               const Dims &memCount, const size_t steps,
+                               const adios2::Params &engineParams)
+{
+    adios2::ADIOS adios;
+    adios2::IO dataManIO = adios.DeclareIO("WAN");
+    dataManIO.SetEngine("DataMan");
+    dataManIO.SetParameters(engineParams);
+    adios2::Engine dataManReader = dataManIO.Open("stream", adios2::Mode::Read);
+
+    size_t datasize = std::accumulate(memCount.begin(), memCount.end(), 1,
+                                      std::multiplies());
+    std::vector myChars(datasize);
+    std::vector myUChars(datasize);
+    std::vector myShorts(datasize);
+    std::vector myUShorts(datasize);
+    std::vector myInts(datasize);
+    std::vector myUInts(datasize);
+    std::vector myFloats(datasize);
+    std::vector myDoubles(datasize);
+    std::vector> myComplexes(datasize);
+    std::vector> myDComplexes(datasize);
+    bool received_steps = false;
+    size_t currentStep;
+    while (true)
+    {
+        adios2::StepStatus status = dataManReader.BeginStep();
+        if (status == adios2::StepStatus::OK)
+        {
+            received_steps = true;
+            const auto &vars = dataManIO.AvailableVariables();
+            ASSERT_EQ(vars.size(), 10);
+            currentStep = dataManReader.CurrentStep();
+            GenData(myChars, currentStep, memStart, memCount, shape);
+            GenData(myUChars, currentStep, memStart, memCount, shape);
+            GenData(myShorts, currentStep, memStart, memCount, shape);
+            GenData(myUShorts, currentStep, memStart, memCount, shape);
+            GenData(myInts, currentStep, memStart, memCount, shape);
+            GenData(myUInts, currentStep, memStart, memCount, shape);
+            GenData(myFloats, currentStep, memStart, memCount, shape);
+            GenData(myDoubles, currentStep, memStart, memCount, shape);
+            GenData(myComplexes, currentStep, memStart, memCount, shape);
+            GenData(myDComplexes, currentStep, memStart, memCount, shape);
+            adios2::Variable bpChars =
+                dataManIO.InquireVariable("bpChars");
+            adios2::Variable bpUChars =
+                dataManIO.InquireVariable("bpUChars");
+            adios2::Variable bpShorts =
+                dataManIO.InquireVariable("bpShorts");
+            adios2::Variable bpUShorts =
+                dataManIO.InquireVariable("bpUShorts");
+            adios2::Variable bpInts =
+                dataManIO.InquireVariable("bpInts");
+            adios2::Variable bpUInts =
+                dataManIO.InquireVariable("bpUInts");
+            adios2::Variable bpFloats =
+                dataManIO.InquireVariable("bpFloats");
+            adios2::Variable bpDoubles =
+                dataManIO.InquireVariable("bpDoubles");
+            adios2::Variable> bpComplexes =
+                dataManIO.InquireVariable>("bpComplexes");
+            adios2::Variable> bpDComplexes =
+                dataManIO.InquireVariable>("bpDComplexes");
+            auto charsBlocksInfo = dataManReader.AllStepsBlocksInfo(bpChars);
+
+            bpChars.SetSelection({start, count});
+            bpUChars.SetSelection({start, count});
+            bpShorts.SetSelection({start, count});
+            bpUShorts.SetSelection({start, count});
+            bpInts.SetSelection({start, count});
+            bpUInts.SetSelection({start, count});
+            bpFloats.SetSelection({start, count});
+            bpDoubles.SetSelection({start, count});
+            bpComplexes.SetSelection({start, count});
+            bpDComplexes.SetSelection({start, count});
+
+            bpChars.SetMemorySelection({memStart, memCount});
+            bpUChars.SetMemorySelection({memStart, memCount});
+            bpShorts.SetMemorySelection({memStart, memCount});
+            bpUShorts.SetMemorySelection({memStart, memCount});
+            bpInts.SetMemorySelection({memStart, memCount});
+            bpUInts.SetMemorySelection({memStart, memCount});
+            bpFloats.SetMemorySelection({memStart, memCount});
+            bpDoubles.SetMemorySelection({memStart, memCount});
+            bpComplexes.SetMemorySelection({memStart, memCount});
+            bpDComplexes.SetMemorySelection({memStart, memCount});
+
+            dataManReader.Get(bpChars, myChars.data(), adios2::Mode::Sync);
+            dataManReader.Get(bpUChars, myUChars.data(), adios2::Mode::Sync);
+            dataManReader.Get(bpShorts, myShorts.data(), adios2::Mode::Sync);
+            dataManReader.Get(bpUShorts, myUShorts.data(), adios2::Mode::Sync);
+            dataManReader.Get(bpInts, myInts.data(), adios2::Mode::Sync);
+            dataManReader.Get(bpUInts, myUInts.data(), adios2::Mode::Sync);
+            dataManReader.Get(bpFloats, myFloats.data(), adios2::Mode::Sync);
+            dataManReader.Get(bpDoubles, myDoubles.data(), adios2::Mode::Sync);
+            dataManReader.Get(bpComplexes, myComplexes.data(),
+                              adios2::Mode::Sync);
+            dataManReader.Get(bpDComplexes, myDComplexes.data(),
+                              adios2::Mode::Sync);
+            VerifyData(myChars.data(), currentStep, memStart, memCount, shape);
+            VerifyData(myUChars.data(), currentStep, memStart, memCount, shape);
+            VerifyData(myShorts.data(), currentStep, memStart, memCount, shape);
+            VerifyData(myUShorts.data(), currentStep, memStart, memCount,
+                       shape);
+            VerifyData(myInts.data(), currentStep, memStart, memCount, shape);
+            VerifyData(myUInts.data(), currentStep, memStart, memCount, shape);
+            VerifyData(myFloats.data(), currentStep, memStart, memCount, shape);
+            VerifyData(myDoubles.data(), currentStep, memStart, memCount,
+                       shape);
+            VerifyData(myComplexes.data(), currentStep, memStart, memCount,
+                       shape);
+            VerifyData(myDComplexes.data(), currentStep, memStart, memCount,
+                       shape);
+            dataManReader.EndStep();
+        }
+        else if (status == adios2::StepStatus::EndOfStream)
+        {
+            break;
+        }
+        else if (status == adios2::StepStatus::NotReady)
+        {
+            continue;
+        }
+    }
+    if (received_steps)
+    {
+        auto attInt = dataManIO.InquireAttribute("AttInt");
+        ASSERT_EQ(110, attInt.Data()[0]);
+        ASSERT_NE(111, attInt.Data()[0]);
+    }
+    dataManReader.Close();
+    print_lines = 0;
+}
+
+#ifdef ADIOS2_HAVE_ZEROMQ
+TEST_F(DataManEngineTest, 2D_Zfp)
+{
+    // set parameters
+    Dims shape = {10, 10};
+    Dims start = {2, 2};
+    Dims count = {5, 5};
+    Dims memstart = start;
+    Dims memcount = count;
+    memstart = {1, 1};
+    memcount = {7, 9};
+
+    size_t steps = 5000;
+    adios2::Params engineParams = {
+        {"IPAddress", "127.0.0.1"}, {"Port", "12320"}, {"Verbose", "0"}};
+
+    auto r = std::thread(DataManReaderP2PMemSelect, shape, start, count,
+                         memstart, memcount, steps, engineParams);
+
+    auto w = std::thread(DataManWriterP2PMemSelect, shape, start, count, steps,
+                         engineParams);
+
+    w.join();
+
+    r.join();
+}
+
+#endif // ZEROMQ
+
+int main(int argc, char **argv)
+{
+    int result;
+    ::testing::InitGoogleTest(&argc, argv);
+    result = RUN_ALL_TESTS();
+
+    return result;
+}
diff --git a/testing/adios2/engine/dataman/TestDataMan3DMemSelect.cpp b/testing/adios2/engine/dataman/TestDataMan3DMemSelect.cpp
index b44c70215f..0f9ddda7a3 100644
--- a/testing/adios2/engine/dataman/TestDataMan3DMemSelect.cpp
+++ b/testing/adios2/engine/dataman/TestDataMan3DMemSelect.cpp
@@ -10,15 +10,10 @@
 
 #include 
 #include 
-#if ADIOS2_USE_MPI
-#include 
-#endif
 #include 
 #include 
 
 using namespace adios2;
-int mpiRank = 0;
-int mpiSize = 1;
 size_t print_lines = 0;
 
 Dims shape = {4, 4, 4};
@@ -58,8 +53,7 @@ void PrintData(const T *data, const size_t step, const Dims &start,
 {
     size_t size = std::accumulate(count.begin(), count.end(), 1,
                                   std::multiplies());
-    std::cout << "Rank: " << mpiRank << " Step: " << step << " Size:" << size
-              << "\n";
+    std::cout << "Step: " << step << " Size:" << size << "\n";
     size_t printsize = 128;
 
     if (size < printsize)
@@ -87,11 +81,6 @@ void VerifyData(const int *data, size_t step, const Dims &start,
     size_t size = std::accumulate(count.begin(), count.end(), 1,
                                   std::multiplies());
     bool compressed = false;
-    if (print_lines < 32)
-    {
-        PrintData(data, step, start, count);
-        ++print_lines;
-    }
     for (size_t i = 0; i < size; ++i)
     {
         if (!compressed)
@@ -105,11 +94,7 @@ void DataManWriterP2PMemSelect(const Dims &shape, const Dims &start,
                                const Dims &count, const size_t steps,
                                const adios2::Params &engineParams)
 {
-#if ADIOS2_USE_MPI
-    adios2::ADIOS adios(MPI_COMM_SELF);
-#else
     adios2::ADIOS adios;
-#endif
     adios2::IO dataManIO = adios.DeclareIO("WAN");
     dataManIO.SetEngine("DataMan");
     dataManIO.SetParameters(engineParams);
@@ -130,17 +115,12 @@ void DataManReaderP2PMemSelect(const Dims &shape, const Dims &start,
                                const Dims &memCount, const size_t steps,
                                const adios2::Params &engineParams)
 {
-#if ADIOS2_USE_MPI
-    adios2::ADIOS adios(MPI_COMM_SELF);
-#else
     adios2::ADIOS adios;
-#endif
     adios2::IO dataManIO = adios.DeclareIO("WAN");
     dataManIO.SetEngine("DataMan");
     dataManIO.SetParameters(engineParams);
     adios2::Engine dataManReader = dataManIO.Open("stream", adios2::Mode::Read);
     std::vector myInts = reader_data;
-    size_t i;
     while (true)
     {
         adios2::StepStatus status = dataManReader.BeginStep(StepMode::Read, 5);
@@ -148,15 +128,6 @@ void DataManReaderP2PMemSelect(const Dims &shape, const Dims &start,
         {
             const auto &vars = dataManIO.AvailableVariables();
             ASSERT_EQ(vars.size(), 1);
-            if (print_lines == 0)
-            {
-                std::cout << "All available variables : ";
-                for (const auto &var : vars)
-                {
-                    std::cout << var.first << ", ";
-                }
-                std::cout << std::endl;
-            }
             size_t currentStep = dataManReader.CurrentStep();
             adios2::Variable bpInts =
                 dataManIO.InquireVariable("bpInts");
@@ -168,8 +139,6 @@ void DataManReaderP2PMemSelect(const Dims &shape, const Dims &start,
         }
         else if (status == adios2::StepStatus::EndOfStream)
         {
-            std::cout << "DataManReader end of stream at Step " << i
-                      << std::endl;
             break;
         }
         else if (status == adios2::StepStatus::NotReady)
@@ -185,49 +154,29 @@ void DataManReaderP2PMemSelect(const Dims &shape, const Dims &start,
 TEST_F(DataManEngineTest, 3D_MemSelect)
 {
 
-    size_t steps = 5000;
+    size_t steps = 3500;
     adios2::Params engineParams = {{"IPAddress", "127.0.0.1"},
                                    {"Port", "12340"}};
     // run workflow
 
     auto r = std::thread(DataManReaderP2PMemSelect, shape, start, count,
                          memstart, memcount, steps, engineParams);
-    std::cout << "Reader thread started" << std::endl;
 
     auto w = std::thread(DataManWriterP2PMemSelect, shape, start, count, steps,
                          engineParams);
-    std::cout << "Writer thread started" << std::endl;
 
     w.join();
-    std::cout << "Writer thread ended" << std::endl;
 
     r.join();
-    std::cout << "Reader thread ended" << std::endl;
 }
 
 #endif // ZEROMQ
 
 int main(int argc, char **argv)
 {
-#if ADIOS2_USE_MPI
-    int mpi_provided;
-    MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &mpi_provided);
-    std::cout << "MPI_Init_thread required Mode " << MPI_THREAD_MULTIPLE
-              << " and provided Mode " << mpi_provided << std::endl;
-    if (mpi_provided != MPI_THREAD_MULTIPLE)
-    {
-        MPI_Finalize();
-        return 0;
-    }
-    MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank);
-    MPI_Comm_size(MPI_COMM_WORLD, &mpiSize);
-#endif
     int result;
     ::testing::InitGoogleTest(&argc, argv);
     result = RUN_ALL_TESTS();
-#if ADIOS2_USE_MPI
-    MPI_Finalize();
-#endif
 
     return result;
 }
diff --git a/testing/adios2/engine/dataman/TestDataManReaderDoubleBuffer.cpp b/testing/adios2/engine/dataman/TestDataManReaderDoubleBuffer.cpp
index ecd7369c96..b70d15ba8c 100644
--- a/testing/adios2/engine/dataman/TestDataManReaderDoubleBuffer.cpp
+++ b/testing/adios2/engine/dataman/TestDataManReaderDoubleBuffer.cpp
@@ -13,9 +13,6 @@
 
 using namespace adios2;
 
-int mpiRank = 0;
-int mpiSize = 1;
-
 size_t print_lines = 0;
 size_t to_print_lines = 10;
 
@@ -24,8 +21,8 @@ void GenData(std::vector> &data, const size_t step)
 {
     for (size_t i = 0; i < data.size(); ++i)
     {
-        data[i] = {static_cast(i + mpiRank * 10000 + step * 100),
-                   static_cast(i + mpiRank * 10000)};
+        data[i] = {static_cast(i + 10000 + step * 100),
+                   static_cast(i + 10000)};
     }
 }
 
@@ -34,14 +31,14 @@ void GenData(std::vector &data, const size_t step)
 {
     for (size_t i = 0; i < data.size(); ++i)
     {
-        data[i] = i + mpiRank * 10000 + step * 100;
+        data[i] = i + 10000 + step * 100;
     }
 }
 
 template 
 void PrintData(const T *data, const size_t size, const size_t step)
 {
-    std::cout << "Rank: " << mpiRank << " Step: " << step << " [";
+    std::cout << "Step: " << step << " [";
     size_t printsize = 32;
     if (size < printsize)
     {
@@ -63,11 +60,6 @@ void VerifyData(const std::complex *data, const size_t size, size_t step)
     {
         ASSERT_EQ(data[i], tmpdata[i]);
     }
-    if (print_lines < to_print_lines)
-    {
-        PrintData(data, size, step);
-        ++print_lines;
-    }
 }
 
 template 
@@ -79,11 +71,6 @@ void VerifyData(const T *data, const size_t size, size_t step)
     {
         ASSERT_EQ(data[i], tmpdata[i]);
     }
-    if (print_lines < to_print_lines)
-    {
-        PrintData(data, size, step);
-        ++print_lines;
-    }
 }
 
 template 
@@ -97,11 +84,7 @@ void DataManWriter(const Dims &shape, const Dims &start, const Dims &count,
 {
     size_t datasize = std::accumulate(count.begin(), count.end(), 1,
                                       std::multiplies());
-#if ADIOS2_USE_MPI
-    adios2::ADIOS adios(MPI_COMM_SELF);
-#else
     adios2::ADIOS adios;
-#endif
     adios2::IO dataManIO = adios.DeclareIO("WAN");
     dataManIO.SetEngine("DataMan");
     dataManIO.SetParameters(engineParams);
@@ -173,11 +156,7 @@ void DataManReader(const Dims &shape, const Dims &start, const Dims &count,
 {
     size_t datasize = std::accumulate(count.begin(), count.end(), 1,
                                       std::multiplies());
-#if ADIOS2_USE_MPI
-    adios2::ADIOS adios(MPI_COMM_SELF);
-#else
     adios2::ADIOS adios;
-#endif
     adios2::IO dataManIO = adios.DeclareIO("WAN");
     dataManIO.SetEngine("DataMan");
     dataManIO.SetParameters(engineParams);
@@ -203,15 +182,6 @@ void DataManReader(const Dims &shape, const Dims &start, const Dims &count,
             received_steps = true;
             const auto &vars = dataManIO.AvailableVariables();
             ASSERT_EQ(vars.size(), 11);
-            if (print_lines < 10)
-            {
-                std::cout << "All available variables : ";
-                for (const auto &var : vars)
-                {
-                    std::cout << var.first << ", ";
-                }
-                std::cout << std::endl;
-            }
             currentStep = dataManReader.CurrentStep();
             adios2::Variable bpChars =
                 dataManIO.InquireVariable("bpChars");
@@ -275,8 +245,6 @@ void DataManReader(const Dims &shape, const Dims &start, const Dims &count,
         }
         else if (status == adios2::StepStatus::EndOfStream)
         {
-            std::cout << "DataManReader end of stream at Step " << currentStep
-                      << std::endl;
             break;
         }
         else if (status == adios2::StepStatus::NotReady)
@@ -290,10 +258,6 @@ void DataManReader(const Dims &shape, const Dims &start, const Dims &count,
         ASSERT_EQ(110, attInt.Data()[0]);
         ASSERT_NE(111, attInt.Data()[0]);
     }
-    else
-    {
-        std::cout << "no steps received " << std::endl;
-    }
     dataManReader.Close();
 }
 
@@ -318,42 +282,20 @@ TEST_F(DataManEngineTest, ReaderDoubleBuffer)
                                          {"DoubleBuffer", "true"}};
     auto r = std::thread(DataManReader, shape, start, count, steps,
                          readerEngineParams);
-    std::cout << "Reader thread started" << std::endl;
     adios2::Params writerEngineParams = {{"IPAddress", "127.0.0.1"},
                                          {"Port", "12360"}};
     auto w = std::thread(DataManWriter, shape, start, count, steps,
                          writerEngineParams);
-    std::cout << "Writer thread started" << std::endl;
     w.join();
-    std::cout << "Writer thread ended" << std::endl;
     r.join();
-    std::cout << "Reader thread ended" << std::endl;
 }
 #endif // ZEROMQ
 
 int main(int argc, char **argv)
 {
-#if ADIOS2_USE_MPI
-    int mpi_provided;
-    MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &mpi_provided);
-    std::cout << "MPI_Init_thread required Mode " << MPI_THREAD_MULTIPLE
-              << " and provided Mode " << mpi_provided << std::endl;
-    if (mpi_provided != MPI_THREAD_MULTIPLE)
-    {
-        MPI_Finalize();
-        return 0;
-    }
-    MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank);
-    MPI_Comm_size(MPI_COMM_WORLD, &mpiSize);
-#endif
-
     int result;
     ::testing::InitGoogleTest(&argc, argv);
     result = RUN_ALL_TESTS();
 
-#if ADIOS2_USE_MPI
-    MPI_Finalize();
-#endif
-
     return result;
 }
diff --git a/testing/adios2/engine/dataman/TestDataManReaderSingleBuffer.cpp b/testing/adios2/engine/dataman/TestDataManReaderSingleBuffer.cpp
index 6b5e30926b..3bd18d82e9 100644
--- a/testing/adios2/engine/dataman/TestDataManReaderSingleBuffer.cpp
+++ b/testing/adios2/engine/dataman/TestDataManReaderSingleBuffer.cpp
@@ -13,9 +13,6 @@
 
 using namespace adios2;
 
-int mpiRank = 0;
-int mpiSize = 1;
-
 size_t print_lines = 0;
 size_t to_print_lines = 10;
 
@@ -24,8 +21,8 @@ void GenData(std::vector> &data, const size_t step)
 {
     for (size_t i = 0; i < data.size(); ++i)
     {
-        data[i] = {static_cast(i + mpiRank * 10000 + step * 100),
-                   static_cast(i + mpiRank * 10000)};
+        data[i] = {static_cast(i + 10000 + step * 100),
+                   static_cast(i + 10000)};
     }
 }
 
@@ -34,14 +31,14 @@ void GenData(std::vector &data, const size_t step)
 {
     for (size_t i = 0; i < data.size(); ++i)
     {
-        data[i] = i + mpiRank * 10000 + step * 100;
+        data[i] = i + 10000 + step * 100;
     }
 }
 
 template 
 void PrintData(const T *data, const size_t size, const size_t step)
 {
-    std::cout << "Rank: " << mpiRank << " Step: " << step << " [";
+    std::cout << "Step: " << step << " [";
     size_t printsize = 32;
     if (size < printsize)
     {
@@ -63,11 +60,6 @@ void VerifyData(const std::complex *data, const size_t size, size_t step)
     {
         ASSERT_EQ(data[i], tmpdata[i]);
     }
-    if (print_lines < to_print_lines)
-    {
-        PrintData(data, size, step);
-        ++print_lines;
-    }
 }
 
 template 
@@ -79,11 +71,6 @@ void VerifyData(const T *data, const size_t size, size_t step)
     {
         ASSERT_EQ(data[i], tmpdata[i]);
     }
-    if (print_lines < to_print_lines)
-    {
-        PrintData(data, size, step);
-        ++print_lines;
-    }
 }
 
 template 
@@ -203,15 +190,6 @@ void DataManReader(const Dims &shape, const Dims &start, const Dims &count,
             received_steps = true;
             const auto &vars = dataManIO.AvailableVariables();
             ASSERT_EQ(vars.size(), 11);
-            if (print_lines < 10)
-            {
-                std::cout << "All available variables : ";
-                for (const auto &var : vars)
-                {
-                    std::cout << var.first << ", ";
-                }
-                std::cout << std::endl;
-            }
             currentStep = dataManReader.CurrentStep();
             adios2::Variable bpChars =
                 dataManIO.InquireVariable("bpChars");
@@ -275,8 +253,6 @@ void DataManReader(const Dims &shape, const Dims &start, const Dims &count,
         }
         else if (status == adios2::StepStatus::EndOfStream)
         {
-            std::cout << "DataManReader end of stream at Step " << currentStep
-                      << std::endl;
             break;
         }
         else if (status == adios2::StepStatus::NotReady)
@@ -290,10 +266,6 @@ void DataManReader(const Dims &shape, const Dims &start, const Dims &count,
         ASSERT_EQ(110, attInt.Data()[0]);
         ASSERT_NE(111, attInt.Data()[0]);
     }
-    else
-    {
-        std::cout << "no steps received " << std::endl;
-    }
     dataManReader.Close();
 }
 
@@ -318,16 +290,12 @@ TEST_F(DataManEngineTest, ReaderSingleBuffer)
                                          {"DoubleBuffer", "false"}};
     auto r = std::thread(DataManReader, shape, start, count, steps,
                          readerEngineParams);
-    std::cout << "Reader thread started" << std::endl;
     adios2::Params writerEngineParams = {{"IPAddress", "127.0.0.1"},
                                          {"Port", "12360"}};
     auto w = std::thread(DataManWriter, shape, start, count, steps,
                          writerEngineParams);
-    std::cout << "Writer thread started" << std::endl;
     w.join();
-    std::cout << "Writer thread ended" << std::endl;
     r.join();
-    std::cout << "Reader thread ended" << std::endl;
 }
 #endif // ZEROMQ
 
@@ -336,8 +304,6 @@ int main(int argc, char **argv)
 #if ADIOS2_USE_MPI
     int mpi_provided;
     MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &mpi_provided);
-    std::cout << "MPI_Init_thread required Mode " << MPI_THREAD_MULTIPLE
-              << " and provided Mode " << mpi_provided << std::endl;
     if (mpi_provided != MPI_THREAD_MULTIPLE)
     {
         MPI_Finalize();
diff --git a/testing/adios2/engine/dataman/TestDataManReliable.cpp b/testing/adios2/engine/dataman/TestDataManReliable.cpp
index cca20b8e69..6321867861 100644
--- a/testing/adios2/engine/dataman/TestDataManReliable.cpp
+++ b/testing/adios2/engine/dataman/TestDataManReliable.cpp
@@ -13,9 +13,6 @@
 
 using namespace adios2;
 
-int mpiRank = 0;
-int mpiSize = 1;
-
 size_t print_lines = 0;
 size_t to_print_lines = 10;
 
@@ -24,8 +21,8 @@ void GenData(std::vector> &data, const size_t step)
 {
     for (size_t i = 0; i < data.size(); ++i)
     {
-        data[i] = {static_cast(i + mpiRank * 10000 + step * 100),
-                   static_cast(i + mpiRank * 10000)};
+        data[i] = {static_cast(i + 10000 + step * 100),
+                   static_cast(i + 10000)};
     }
 }
 
@@ -34,14 +31,14 @@ void GenData(std::vector &data, const size_t step)
 {
     for (size_t i = 0; i < data.size(); ++i)
     {
-        data[i] = i + mpiRank * 10000 + step * 100;
+        data[i] = i + 10000 + step * 100;
     }
 }
 
 template 
 void PrintData(const T *data, const size_t size, const size_t step)
 {
-    std::cout << "Rank: " << mpiRank << " Step: " << step << " [";
+    std::cout << "Step: " << step << " [";
     size_t printsize = 32;
     if (size < printsize)
     {
@@ -63,11 +60,6 @@ void VerifyData(const std::complex *data, const size_t size, size_t step)
     {
         ASSERT_EQ(data[i], tmpdata[i]);
     }
-    if (print_lines < to_print_lines)
-    {
-        PrintData(data, size, step);
-        ++print_lines;
-    }
 }
 
 template 
@@ -79,11 +71,6 @@ void VerifyData(const T *data, const size_t size, size_t step)
     {
         ASSERT_EQ(data[i], tmpdata[i]);
     }
-    if (print_lines < to_print_lines)
-    {
-        PrintData(data, size, step);
-        ++print_lines;
-    }
 }
 
 template 
@@ -97,11 +84,7 @@ void DataManWriter(const Dims &shape, const Dims &start, const Dims &count,
 {
     size_t datasize = std::accumulate(count.begin(), count.end(), 1,
                                       std::multiplies());
-#if ADIOS2_USE_MPI
-    adios2::ADIOS adios(MPI_COMM_SELF);
-#else
     adios2::ADIOS adios;
-#endif
     adios2::IO dataManIO = adios.DeclareIO("WAN");
     dataManIO.SetEngine("DataMan");
     dataManIO.SetParameters(engineParams);
@@ -173,11 +156,7 @@ void DataManReader(const Dims &shape, const Dims &start, const Dims &count,
 {
     size_t datasize = std::accumulate(count.begin(), count.end(), 1,
                                       std::multiplies());
-#if ADIOS2_USE_MPI
-    adios2::ADIOS adios(MPI_COMM_SELF);
-#else
     adios2::ADIOS adios;
-#endif
     adios2::IO dataManIO = adios.DeclareIO("WAN");
     dataManIO.SetEngine("DataMan");
     dataManIO.SetParameters(engineParams);
@@ -203,15 +182,6 @@ void DataManReader(const Dims &shape, const Dims &start, const Dims &count,
             received_steps = true;
             const auto &vars = dataManIO.AvailableVariables();
             ASSERT_EQ(vars.size(), 11);
-            if (print_lines < 10)
-            {
-                std::cout << "All available variables : ";
-                for (const auto &var : vars)
-                {
-                    std::cout << var.first << ", ";
-                }
-                std::cout << std::endl;
-            }
             currentStep = dataManReader.CurrentStep();
             adios2::Variable bpChars =
                 dataManIO.InquireVariable("bpChars");
@@ -275,8 +245,6 @@ void DataManReader(const Dims &shape, const Dims &start, const Dims &count,
         }
         else if (status == adios2::StepStatus::EndOfStream)
         {
-            std::cout << "DataManReader end of stream at Step " << currentStep
-                      << std::endl;
             break;
         }
         else if (status == adios2::StepStatus::NotReady)
@@ -290,10 +258,6 @@ void DataManReader(const Dims &shape, const Dims &start, const Dims &count,
         ASSERT_EQ(110, attInt.Data()[0]);
         ASSERT_NE(111, attInt.Data()[0]);
     }
-    else
-    {
-        std::cout << "no steps received " << std::endl;
-    }
     dataManReader.Close();
 }
 
@@ -318,43 +282,21 @@ TEST_F(DataManEngineTest, Reliable)
                                          {"TransportMode", "reliable"}};
     auto r = std::thread(DataManReader, shape, start, count, steps,
                          readerEngineParams);
-    std::cout << "Reader thread started" << std::endl;
     adios2::Params writerEngineParams = {{"IPAddress", "127.0.0.1"},
                                          {"Port", "12360"},
                                          {"TransportMode", "reliable"}};
     auto w = std::thread(DataManWriter, shape, start, count, steps,
                          writerEngineParams);
-    std::cout << "Writer thread started" << std::endl;
     w.join();
-    std::cout << "Writer thread ended" << std::endl;
     r.join();
-    std::cout << "Reader thread ended" << std::endl;
 }
 #endif // ZEROMQ
 
 int main(int argc, char **argv)
 {
-#if ADIOS2_USE_MPI
-    int mpi_provided;
-    MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &mpi_provided);
-    std::cout << "MPI_Init_thread required Mode " << MPI_THREAD_MULTIPLE
-              << " and provided Mode " << mpi_provided << std::endl;
-    if (mpi_provided != MPI_THREAD_MULTIPLE)
-    {
-        MPI_Finalize();
-        return 0;
-    }
-    MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank);
-    MPI_Comm_size(MPI_COMM_WORLD, &mpiSize);
-#endif
-
     int result;
     ::testing::InitGoogleTest(&argc, argv);
     result = RUN_ALL_TESTS();
 
-#if ADIOS2_USE_MPI
-    MPI_Finalize();
-#endif
-
     return result;
 }
diff --git a/testing/adios2/engine/dataman/TestDataManWriterDoubleBuffer.cpp b/testing/adios2/engine/dataman/TestDataManWriterDoubleBuffer.cpp
index 97c05b9035..f3ab6ac484 100644
--- a/testing/adios2/engine/dataman/TestDataManWriterDoubleBuffer.cpp
+++ b/testing/adios2/engine/dataman/TestDataManWriterDoubleBuffer.cpp
@@ -18,9 +18,6 @@
 
 using namespace adios2;
 
-int mpiRank = 0;
-int mpiSize = 1;
-
 size_t print_lines = 0;
 size_t to_print_lines = 10;
 
@@ -29,8 +26,8 @@ void GenData(std::vector> &data, const size_t step)
 {
     for (size_t i = 0; i < data.size(); ++i)
     {
-        data[i] = {static_cast(i + mpiRank * 10000 + step * 100),
-                   static_cast(i + mpiRank * 10000)};
+        data[i] = {static_cast(i + 10000 + step * 100),
+                   static_cast(i + 10000)};
     }
 }
 
@@ -39,14 +36,14 @@ void GenData(std::vector &data, const size_t step)
 {
     for (size_t i = 0; i < data.size(); ++i)
     {
-        data[i] = i + mpiRank * 10000 + step * 100;
+        data[i] = i + 10000 + step * 100;
     }
 }
 
 template 
 void PrintData(const T *data, const size_t size, const size_t step)
 {
-    std::cout << "Rank: " << mpiRank << " Step: " << step << " [";
+    std::cout << "Step: " << step << " [";
     size_t printsize = 32;
     if (size < printsize)
     {
@@ -68,11 +65,6 @@ void VerifyData(const std::complex *data, const size_t size, size_t step)
     {
         ASSERT_EQ(data[i], tmpdata[i]);
     }
-    if (print_lines < to_print_lines)
-    {
-        PrintData(data, size, step);
-        ++print_lines;
-    }
 }
 
 template 
@@ -84,11 +76,6 @@ void VerifyData(const T *data, const size_t size, size_t step)
     {
         ASSERT_EQ(data[i], tmpdata[i]);
     }
-    if (print_lines < to_print_lines)
-    {
-        PrintData(data, size, step);
-        ++print_lines;
-    }
 }
 
 template 
@@ -208,15 +195,6 @@ void DataManReader(const Dims &shape, const Dims &start, const Dims &count,
             received_steps = true;
             const auto &vars = dataManIO.AvailableVariables();
             ASSERT_EQ(vars.size(), 11);
-            if (print_lines < 10)
-            {
-                std::cout << "All available variables : ";
-                for (const auto &var : vars)
-                {
-                    std::cout << var.first << ", ";
-                }
-                std::cout << std::endl;
-            }
             currentStep = dataManReader.CurrentStep();
             adios2::Variable bpChars =
                 dataManIO.InquireVariable("bpChars");
@@ -280,8 +258,6 @@ void DataManReader(const Dims &shape, const Dims &start, const Dims &count,
         }
         else if (status == adios2::StepStatus::EndOfStream)
         {
-            std::cout << "DataManReader end of stream at Step " << currentStep
-                      << std::endl;
             break;
         }
         else if (status == adios2::StepStatus::NotReady)
@@ -293,11 +269,6 @@ void DataManReader(const Dims &shape, const Dims &start, const Dims &count,
     {
         auto attInt = dataManIO.InquireAttribute("AttInt");
         ASSERT_EQ(110, attInt.Data()[0]);
-        ASSERT_NE(111, attInt.Data()[0]);
-    }
-    else
-    {
-        std::cout << "no steps received " << std::endl;
     }
     dataManReader.Close();
 }
@@ -322,17 +293,13 @@ TEST_F(DataManEngineTest, WriterDoubleBuffer)
                                          {"Port", "12380"}};
     auto r = std::thread(DataManReader, shape, start, count, steps,
                          readerEngineParams);
-    std::cout << "Reader thread started" << std::endl;
     adios2::Params writerEngineParams = {{"IPAddress", "127.0.0.1"},
                                          {"Port", "12380"},
                                          {"DoubleBuffer", "true"}};
     auto w = std::thread(DataManWriter, shape, start, count, steps,
                          writerEngineParams);
-    std::cout << "Writer thread started" << std::endl;
     w.join();
-    std::cout << "Writer thread ended" << std::endl;
     r.join();
-    std::cout << "Reader thread ended" << std::endl;
 }
 #endif // ZEROMQ
 
@@ -341,8 +308,6 @@ int main(int argc, char **argv)
 #if ADIOS2_USE_MPI
     int mpi_provided;
     MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &mpi_provided);
-    std::cout << "MPI_Init_thread required Mode " << MPI_THREAD_MULTIPLE
-              << " and provided Mode " << mpi_provided << std::endl;
     if (mpi_provided != MPI_THREAD_MULTIPLE)
     {
         MPI_Finalize();
diff --git a/testing/adios2/engine/dataman/TestDataManWriterSingleBuffer.cpp b/testing/adios2/engine/dataman/TestDataManWriterSingleBuffer.cpp
index 83d3b874b4..502ac486a3 100644
--- a/testing/adios2/engine/dataman/TestDataManWriterSingleBuffer.cpp
+++ b/testing/adios2/engine/dataman/TestDataManWriterSingleBuffer.cpp
@@ -63,11 +63,6 @@ void VerifyData(const std::complex *data, const size_t size, size_t step)
     {
         ASSERT_EQ(data[i], tmpdata[i]);
     }
-    if (print_lines < to_print_lines)
-    {
-        PrintData(data, size, step);
-        ++print_lines;
-    }
 }
 
 template 
@@ -79,11 +74,6 @@ void VerifyData(const T *data, const size_t size, size_t step)
     {
         ASSERT_EQ(data[i], tmpdata[i]);
     }
-    if (print_lines < to_print_lines)
-    {
-        PrintData(data, size, step);
-        ++print_lines;
-    }
 }
 
 template 
@@ -97,11 +87,7 @@ void DataManWriter(const Dims &shape, const Dims &start, const Dims &count,
 {
     size_t datasize = std::accumulate(count.begin(), count.end(), 1,
                                       std::multiplies());
-#if ADIOS2_USE_MPI
-    adios2::ADIOS adios(MPI_COMM_SELF);
-#else
     adios2::ADIOS adios;
-#endif
     adios2::IO dataManIO = adios.DeclareIO("WAN");
     dataManIO.SetEngine("DataMan");
     dataManIO.SetParameters(engineParams);
@@ -173,11 +159,7 @@ void DataManReader(const Dims &shape, const Dims &start, const Dims &count,
 {
     size_t datasize = std::accumulate(count.begin(), count.end(), 1,
                                       std::multiplies());
-#if ADIOS2_USE_MPI
-    adios2::ADIOS adios(MPI_COMM_SELF);
-#else
     adios2::ADIOS adios;
-#endif
     adios2::IO dataManIO = adios.DeclareIO("WAN");
     dataManIO.SetEngine("DataMan");
     dataManIO.SetParameters(engineParams);
@@ -203,15 +185,6 @@ void DataManReader(const Dims &shape, const Dims &start, const Dims &count,
             received_steps = true;
             const auto &vars = dataManIO.AvailableVariables();
             ASSERT_EQ(vars.size(), 11);
-            if (print_lines < 10)
-            {
-                std::cout << "All available variables : ";
-                for (const auto &var : vars)
-                {
-                    std::cout << var.first << ", ";
-                }
-                std::cout << std::endl;
-            }
             currentStep = dataManReader.CurrentStep();
             adios2::Variable bpChars =
                 dataManIO.InquireVariable("bpChars");
@@ -275,8 +248,6 @@ void DataManReader(const Dims &shape, const Dims &start, const Dims &count,
         }
         else if (status == adios2::StepStatus::EndOfStream)
         {
-            std::cout << "DataManReader end of stream at Step " << currentStep
-                      << std::endl;
             break;
         }
         else if (status == adios2::StepStatus::NotReady)
@@ -290,10 +261,6 @@ void DataManReader(const Dims &shape, const Dims &start, const Dims &count,
         ASSERT_EQ(110, attInt.Data()[0]);
         ASSERT_NE(111, attInt.Data()[0]);
     }
-    else
-    {
-        std::cout << "no steps received " << std::endl;
-    }
     dataManReader.Close();
 }
 
@@ -317,43 +284,21 @@ TEST_F(DataManEngineTest, WriterSingleBuffer)
                                          {"Port", "12380"}};
     auto r = std::thread(DataManReader, shape, start, count, steps,
                          readerEngineParams);
-    std::cout << "Reader thread started" << std::endl;
     adios2::Params writerEngineParams = {{"IPAddress", "127.0.0.1"},
                                          {"Port", "12380"},
                                          {"DoubleBuffer", "false"}};
     auto w = std::thread(DataManWriter, shape, start, count, steps,
                          writerEngineParams);
-    std::cout << "Writer thread started" << std::endl;
     w.join();
-    std::cout << "Writer thread ended" << std::endl;
     r.join();
-    std::cout << "Reader thread ended" << std::endl;
 }
 #endif // ZEROMQ
 
 int main(int argc, char **argv)
 {
-#if ADIOS2_USE_MPI
-    int mpi_provided;
-    MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &mpi_provided);
-    std::cout << "MPI_Init_thread required Mode " << MPI_THREAD_MULTIPLE
-              << " and provided Mode " << mpi_provided << std::endl;
-    if (mpi_provided != MPI_THREAD_MULTIPLE)
-    {
-        MPI_Finalize();
-        return 0;
-    }
-    MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank);
-    MPI_Comm_size(MPI_COMM_WORLD, &mpiSize);
-#endif
-
     int result;
     ::testing::InitGoogleTest(&argc, argv);
     result = RUN_ALL_TESTS();
 
-#if ADIOS2_USE_MPI
-    MPI_Finalize();
-#endif
-
     return result;
 }
diff --git a/testing/adios2/engine/hdf5/CMakeLists.txt b/testing/adios2/engine/hdf5/CMakeLists.txt
index ca96432f04..1746648163 100644
--- a/testing/adios2/engine/hdf5/CMakeLists.txt
+++ b/testing/adios2/engine/hdf5/CMakeLists.txt
@@ -23,6 +23,10 @@ gtest_add_tests_helper(WriteMemorySelectionRead  ${hdf5_mpi}
   HDF5 Engine.HDF5. ""
 )
 
+gtest_add_tests_helper(Append  ${hdf5_mpi}
+  HDF5 Engine.HDF5. ""
+)
+
 gtest_add_tests_helper(NativeHDF5WriteRead ${hdf5_mpi} "" Engine.HDF5. "")
 if(HDF5_C_INCLUDE_DIRS)
   target_include_directories(Test.Engine.HDF5.NativeHDF5WriteRead${hdf5_sfx}
diff --git a/testing/adios2/engine/hdf5/TestHDF5Append.cpp b/testing/adios2/engine/hdf5/TestHDF5Append.cpp
new file mode 100644
index 0000000000..caec6045ae
--- /dev/null
+++ b/testing/adios2/engine/hdf5/TestHDF5Append.cpp
@@ -0,0 +1,391 @@
+/*
+ * Distributed under the OSI-approved Apache License, Version 2.0.  See
+ * accompanying file Copyright.txt for details.
+ */
+#include 
+#include 
+
+#include 
+#include 
+
+#include 
+
+#include 
+
+#include "../SmallTestData.h"
+
+std::string engineName; // comes from command line
+
+class AppendTimeStepTest : public ::testing::Test
+{
+public:
+    AppendTimeStepTest() = default;
+
+    SmallTestData m_TestData;
+};
+
+//******************************************************************************
+// 1D 1x8 test data
+//******************************************************************************
+
+// ADIOS2 HDF5 write, then append, then read back
+TEST_F(AppendTimeStepTest, ADIOS2HDF5WriteAppendRead)
+{
+    // Each process would write a 1x8 array and all processes would
+    // form a mpiSize * Nx 1D array
+    std::string fname = "appendTest.h5";
+
+    int mpiRank = 0, mpiSize = 1;
+    // Number of rows
+    const std::size_t Nx = 8;
+
+    // Number of steps
+    const std::size_t NSteps = 3;
+
+#if ADIOS2_USE_MPI
+    MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank);
+    MPI_Comm_size(MPI_COMM_WORLD, &mpiSize);
+#endif
+
+#if ADIOS2_USE_MPI
+    adios2::ADIOS adios(MPI_COMM_WORLD);
+#else
+    adios2::ADIOS adios;
+#endif
+
+    // Write test data using HDF5 engine
+    {
+        adios2::IO io = adios.DeclareIO("TestIO");
+
+        // Declare 1D variables (NumOfProcesses * Nx)
+        // The local process' part (start, count) can be defined now or later
+        // before Write().
+        {
+            adios2::Dims shape{static_cast(Nx * mpiSize)};
+            adios2::Dims start{static_cast(Nx * mpiRank)};
+            adios2::Dims count{static_cast(Nx)};
+
+            auto var_iString = io.DefineVariable("iString");
+            auto var_i8 = io.DefineVariable("i8", shape, start, count);
+            auto var_i16 =
+                io.DefineVariable("i16", shape, start, count);
+            auto var_i32 =
+                io.DefineVariable("i32", shape, start, count);
+            auto var_i64 =
+                io.DefineVariable("i64", shape, start, count);
+            auto var_u8 = io.DefineVariable("u8", shape, start, count);
+            auto var_u16 =
+                io.DefineVariable("u16", shape, start, count);
+            auto var_u32 =
+                io.DefineVariable("u32", shape, start, count);
+            auto var_u64 =
+                io.DefineVariable("u64", shape, start, count);
+            auto var_r32 = io.DefineVariable("r32", shape, start, count);
+            auto var_r64 =
+                io.DefineVariable("r64", shape, start, count);
+        }
+
+        if (!engineName.empty())
+            io.SetEngine(engineName);
+        else
+            io.SetEngine("HDF5");
+
+        io.AddTransport("file");
+
+        adios2::Engine engine = io.Open(fname, adios2::Mode::Write);
+
+        for (size_t step = 0; step < NSteps; ++step)
+        {
+            // Generate test data for each process uniquely
+            SmallTestData currentTestData =
+                generateNewSmallTestData(m_TestData, step, mpiRank, mpiSize);
+
+            // Retrieve the variables that previously went out of scope
+            auto var_iString = io.InquireVariable("iString");
+            auto var_i8 = io.InquireVariable("i8");
+            auto var_i16 = io.InquireVariable("i16");
+            auto var_i32 = io.InquireVariable("i32");
+            auto var_i64 = io.InquireVariable("i64");
+            auto var_u8 = io.InquireVariable("u8");
+            auto var_u16 = io.InquireVariable("u16");
+            auto var_u32 = io.InquireVariable("u32");
+            auto var_u64 = io.InquireVariable("u64");
+            auto var_r32 = io.InquireVariable("r32");
+            auto var_r64 = io.InquireVariable("r64");
+
+            // Make a 1D selection to describe the local dimensions of the
+            // variable we write and its offsets in the global spaces
+            adios2::Box sel({mpiRank * Nx}, {Nx});
+
+            var_i8.SetSelection(sel);
+            var_i16.SetSelection(sel);
+            var_i32.SetSelection(sel);
+            var_i64.SetSelection(sel);
+            var_u8.SetSelection(sel);
+            var_u16.SetSelection(sel);
+            var_u32.SetSelection(sel);
+            var_u64.SetSelection(sel);
+            var_r32.SetSelection(sel);
+            var_r64.SetSelection(sel);
+
+            // Write each one
+            // fill in the variable with values from starting index to
+            // starting index + count
+            engine.BeginStep();
+            engine.Put(var_iString, currentTestData.S1);
+            engine.Put(var_i8, currentTestData.I8.data());
+            engine.Put(var_i16, currentTestData.I16.data());
+            engine.Put(var_i32, currentTestData.I32.data());
+            engine.Put(var_i64, currentTestData.I64.data());
+            engine.Put(var_u8, currentTestData.U8.data());
+            engine.Put(var_u16, currentTestData.U16.data());
+            engine.Put(var_u32, currentTestData.U32.data());
+            engine.Put(var_u64, currentTestData.U64.data());
+            engine.Put(var_r32, currentTestData.R32.data());
+            engine.Put(var_r64, currentTestData.R64.data());
+            engine.EndStep();
+        }
+
+        // Close the file
+        engine.Close();
+    }
+
+    size_t ExtraSteps = 2;
+
+    {
+        // Append
+        adios2::IO io = adios.DeclareIO("ioAppend");
+
+        if (!engineName.empty())
+            io.SetEngine(engineName);
+        else
+            io.SetEngine("HDF5");
+
+        adios2::Engine appender = io.Open(fname, adios2::Mode::Append);
+
+        for (size_t step = NSteps; step < NSteps + ExtraSteps; ++step)
+        {
+            // Generate test data for each process uniquely
+            SmallTestData currentTestData =
+                generateNewSmallTestData(m_TestData, step, mpiRank, mpiSize);
+
+            // Retrieve the variables that previously went out of scope
+            auto var_iString = io.InquireVariable("iString");
+            EXPECT_TRUE(var_iString);
+            auto var_i8 = io.InquireVariable("i8");
+            EXPECT_TRUE(var_i8);
+            auto var_i16 = io.InquireVariable("i16");
+            auto var_i32 = io.InquireVariable("i32");
+            auto var_i64 = io.InquireVariable("i64");
+            auto var_u8 = io.InquireVariable("u8");
+            auto var_u16 = io.InquireVariable("u16");
+            auto var_u32 = io.InquireVariable("u32");
+            auto var_u64 = io.InquireVariable("u64");
+            auto var_r32 = io.InquireVariable("r32");
+            auto var_r64 = io.InquireVariable("r64");
+
+            // Make a 1D selection to describe the local dimensions of the
+            // variable we write and its offsets in the global spaces
+            adios2::Box sel({mpiRank * Nx}, {Nx});
+
+            EXPECT_THROW(var_iString.SetSelection(sel), std::invalid_argument);
+            var_i8.SetSelection(sel);
+            var_i16.SetSelection(sel);
+            var_i32.SetSelection(sel);
+            var_i64.SetSelection(sel);
+            var_u8.SetSelection(sel);
+            var_u16.SetSelection(sel);
+            var_u32.SetSelection(sel);
+            var_u64.SetSelection(sel);
+            var_r32.SetSelection(sel);
+            var_r64.SetSelection(sel);
+
+            // Write each one
+            // fill in the variable with values from starting index to
+            // starting index + count
+            appender.BeginStep();
+            appender.Put(var_iString, currentTestData.S1);
+            appender.Put(var_i8, currentTestData.I8.data());
+            appender.Put(var_i16, currentTestData.I16.data());
+            appender.Put(var_i32, currentTestData.I32.data());
+            appender.Put(var_i64, currentTestData.I64.data());
+            appender.Put(var_u8, currentTestData.U8.data());
+            appender.Put(var_u16, currentTestData.U16.data());
+            appender.Put(var_u32, currentTestData.U32.data());
+            appender.Put(var_u64, currentTestData.U64.data());
+            appender.Put(var_r32, currentTestData.R32.data());
+            appender.Put(var_r64, currentTestData.R64.data());
+            appender.EndStep();
+        }
+        appender.Close();
+    }
+
+    {
+        // Read back
+        adios2::IO io = adios.DeclareIO("ioRead");
+        if (!engineName.empty())
+            io.SetEngine(engineName);
+        else
+            io.SetEngine("HDF5");
+
+        adios2::Engine reader = io.Open(fname, adios2::Mode::Read);
+        EXPECT_EQ(reader.Steps(), NSteps + ExtraSteps);
+
+        std::string IString;
+        std::array I8;
+        std::array I16;
+        std::array I32;
+        std::array I64;
+        std::array U8;
+        std::array U16;
+        std::array U32;
+        std::array U64;
+        std::array R32;
+        std::array R64;
+
+        auto var_iString = io.InquireVariable("iString");
+        EXPECT_TRUE(var_iString);
+        EXPECT_EQ(var_iString.Steps(), NSteps + ExtraSteps);
+
+        auto var_i8 = io.InquireVariable("i8");
+        EXPECT_TRUE(var_i8);
+        EXPECT_EQ(var_i8.Steps(), NSteps + ExtraSteps);
+
+        auto var_i16 = io.InquireVariable("i16");
+        EXPECT_TRUE(var_i16);
+        EXPECT_EQ(var_i16.Steps(), NSteps + ExtraSteps);
+
+        auto var_i32 = io.InquireVariable("i32");
+        EXPECT_TRUE(var_i32);
+        EXPECT_EQ(var_i32.Steps(), NSteps + ExtraSteps);
+
+        auto var_i64 = io.InquireVariable("i64");
+        EXPECT_TRUE(var_i64);
+        EXPECT_EQ(var_i64.Steps(), NSteps + ExtraSteps);
+
+        auto var_u8 = io.InquireVariable("u8");
+        EXPECT_TRUE(var_u8);
+        EXPECT_EQ(var_u8.Steps(), NSteps + ExtraSteps);
+
+        auto var_u16 = io.InquireVariable("u16");
+        EXPECT_TRUE(var_u16);
+        EXPECT_EQ(var_u16.Steps(), NSteps + ExtraSteps);
+
+        auto var_u32 = io.InquireVariable("u32");
+        EXPECT_TRUE(var_u32);
+        EXPECT_EQ(var_u32.Steps(), NSteps + ExtraSteps);
+
+        auto var_u64 = io.InquireVariable("u64");
+        EXPECT_TRUE(var_u64);
+        EXPECT_EQ(var_u64.Steps(), NSteps + ExtraSteps);
+
+        auto var_r32 = io.InquireVariable("r32");
+        EXPECT_TRUE(var_r32);
+        EXPECT_EQ(var_r32.Steps(), NSteps + ExtraSteps);
+
+        auto var_r64 = io.InquireVariable("r64");
+        EXPECT_TRUE(var_r64);
+        EXPECT_EQ(var_r64.Steps(), NSteps + ExtraSteps);
+
+        adios2::Box sel({mpiRank * Nx}, {Nx});
+
+        for (size_t step = 0; step < NSteps + ExtraSteps; ++step)
+        {
+            SmallTestData currentTestData =
+                generateNewSmallTestData(m_TestData, step, mpiRank, mpiSize);
+
+            var_i8.SetStepSelection({step, 1});
+            var_i8.SetSelection(sel);
+            reader.Get(var_i8, I8.data());
+
+            var_i16.SetStepSelection({step, 1});
+            var_i16.SetSelection(sel);
+            reader.Get(var_i16, I16.data());
+
+            var_i32.SetStepSelection({step, 1});
+            var_i32.SetSelection(sel);
+            reader.Get(var_i32, I32.data());
+
+            var_i64.SetStepSelection({step, 1});
+            var_i64.SetSelection(sel);
+            reader.Get(var_i64, I64.data());
+
+            var_u8.SetStepSelection({step, 1});
+            var_u8.SetSelection(sel);
+            reader.Get(var_u8, U8.data());
+
+            var_u16.SetStepSelection({step, 1});
+            var_u16.SetSelection(sel);
+            reader.Get(var_u16, U16.data());
+
+            var_u32.SetStepSelection({step, 1});
+            var_u32.SetSelection(sel);
+            reader.Get(var_u32, U32.data());
+
+            var_u64.SetStepSelection({step, 1});
+            var_u64.SetSelection(sel);
+            reader.Get(var_u64, U64.data());
+
+            var_r32.SetStepSelection({step, 1});
+            var_r32.SetSelection(sel);
+            reader.Get(var_r32, R32.data());
+
+            var_r64.SetStepSelection({step, 1});
+            var_r64.SetSelection(sel);
+            reader.Get(var_r64, R64.data());
+
+            reader.Get(var_iString, IString);
+            reader.PerformGets();
+
+            EXPECT_EQ(IString, currentTestData.S1);
+
+            for (size_t i = 0; i < Nx; ++i)
+            {
+                std::stringstream ss;
+                ss << "step=" << step << " i=" << i << " rank=" << mpiRank;
+                std::string msg = ss.str();
+
+                EXPECT_EQ(I8[i], currentTestData.I8[i]) << msg;
+                EXPECT_EQ(I16[i], currentTestData.I16[i]) << msg;
+                EXPECT_EQ(I32[i], currentTestData.I32[i]) << msg;
+                EXPECT_EQ(I64[i], currentTestData.I64[i]) << msg;
+
+                EXPECT_EQ(U8[i], currentTestData.U8[i]) << msg;
+                EXPECT_EQ(U16[i], currentTestData.U16[i]) << msg;
+                EXPECT_EQ(U32[i], currentTestData.U32[i]) << msg;
+                EXPECT_EQ(U64[i], currentTestData.U64[i]) << msg;
+
+                EXPECT_EQ(R32[i], currentTestData.R32[i]) << msg;
+                EXPECT_EQ(R64[i], currentTestData.R64[i]) << msg;
+            }
+        }
+
+        reader.Close();
+    }
+}
+
+//******************************************************************************
+// main
+//******************************************************************************
+
+int main(int argc, char **argv)
+{
+#if ADIOS2_USE_MPI
+    MPI_Init(nullptr, nullptr);
+#endif
+
+    int result;
+    ::testing::InitGoogleTest(&argc, argv);
+
+    if (argc > 1)
+    {
+        engineName = std::string(argv[1]);
+    }
+    result = RUN_ALL_TESTS();
+
+#if ADIOS2_USE_MPI
+    MPI_Finalize();
+#endif
+
+    return result;
+}
diff --git a/testing/adios2/engine/hdf5/TestHDF5WriteMemorySelectionRead.cpp b/testing/adios2/engine/hdf5/TestHDF5WriteMemorySelectionRead.cpp
index 9e90b2d2b9..7ff41fe6e5 100644
--- a/testing/adios2/engine/hdf5/TestHDF5WriteMemorySelectionRead.cpp
+++ b/testing/adios2/engine/hdf5/TestHDF5WriteMemorySelectionRead.cpp
@@ -924,8 +924,8 @@ TEST_P(HDF5WriteMemSelReadVector, HDF5MemorySelectionSteps3D4x2x8)
     HDF5Steps3D8x2x4(GetParam());
 }
 
-INSTANTIATE_TEST_CASE_P(ghostCells, HDF5WriteMemSelReadVector,
-                        ::testing::Values(1));
+INSTANTIATE_TEST_SUITE_P(ghostCells, HDF5WriteMemSelReadVector,
+                         ::testing::Values(1));
 
 int main(int argc, char **argv)
 {
diff --git a/testing/adios2/engine/inline/TestInlineWriteRead.cpp b/testing/adios2/engine/inline/TestInlineWriteRead.cpp
index 3cfbfe0749..e35cf95f21 100644
--- a/testing/adios2/engine/inline/TestInlineWriteRead.cpp
+++ b/testing/adios2/engine/inline/TestInlineWriteRead.cpp
@@ -67,9 +67,6 @@ TEST_F(InlineWriteRead, InlineWriteRead1D8)
 #if ADIOS2_USE_MPI
     MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank);
     MPI_Comm_size(MPI_COMM_WORLD, &mpiSize);
-#endif
-
-#if ADIOS2_USE_MPI
     adios2::ADIOS adios(MPI_COMM_WORLD);
 #else
     adios2::ADIOS adios;
@@ -113,9 +110,7 @@ TEST_F(InlineWriteRead, InlineWriteRead1D8)
         io.SetEngine("Inline");
 
         // writerID parameter makes sure the reader can find the writer.
-        io.SetParameters({{"verbose", "4"},
-                          {"writerID", fname + "_write"},
-                          {"readerID", fname + "_read"}});
+        io.SetParameter("verbose", "4");
 
         adios2::Engine inlineWriter =
             io.Open(fname + "_write", adios2::Mode::Write);
@@ -357,9 +352,6 @@ TEST_F(InlineWriteRead, InlineWriteRead2D2x4)
 #if ADIOS2_USE_MPI
     MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank);
     MPI_Comm_size(MPI_COMM_WORLD, &mpiSize);
-#endif
-
-#if ADIOS2_USE_MPI
     adios2::ADIOS adios(MPI_COMM_WORLD);
 #else
     adios2::ADIOS adios;
@@ -403,9 +395,7 @@ TEST_F(InlineWriteRead, InlineWriteRead2D2x4)
         io.SetEngine("Inline");
 
         // writerID parameter makes sure the reader can find the writer.
-        io.SetParameters({{"verbose", "4"},
-                          {"writerID", fname + "_write"},
-                          {"readerID", fname + "_read"}});
+        io.SetParameter("verbose", "4");
 
         adios2::Engine inlineWriter =
             io.Open(fname + "_write", adios2::Mode::Write);
@@ -630,12 +620,9 @@ TEST_F(InlineWriteRead, InlineWriteReadContracts)
 #if ADIOS2_USE_MPI
     MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank);
     MPI_Comm_size(MPI_COMM_WORLD, &mpiSize);
-#endif
-
-#if ADIOS2_USE_MPI
-    adios2::ADIOS adios(MPI_COMM_WORLD, adios2::DebugON);
+    adios2::ADIOS adios(MPI_COMM_WORLD);
 #else
-    adios2::ADIOS adios(adios2::DebugON);
+    adios2::ADIOS adios;
 #endif
     {
         adios2::IO io = adios.DeclareIO("TestIO");
@@ -657,9 +644,7 @@ TEST_F(InlineWriteRead, InlineWriteReadContracts)
         io.SetEngine("Inline");
 
         // writerID parameter makes sure the reader can find the writer.
-        io.SetParameters({{"verbose", "4"},
-                          {"writerID", fname + "_write"},
-                          {"readerID", fname + "_read"}});
+        io.SetParameter("verbose", "4");
 
         adios2::Engine inlineWriter =
             io.Open(fname + "_write", adios2::Mode::Write);
@@ -757,12 +742,9 @@ TEST_F(InlineWriteRead, InlineWriteReadContracts2)
 #if ADIOS2_USE_MPI
     MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank);
     MPI_Comm_size(MPI_COMM_WORLD, &mpiSize);
-#endif
-
-#if ADIOS2_USE_MPI
-    adios2::ADIOS adios(MPI_COMM_WORLD, adios2::DebugON);
+    adios2::ADIOS adios(MPI_COMM_WORLD);
 #else
-    adios2::ADIOS adios(adios2::DebugON);
+    adios2::ADIOS adios;
 #endif
     {
         adios2::IO io = adios.DeclareIO("TestIO");
@@ -784,9 +766,7 @@ TEST_F(InlineWriteRead, InlineWriteReadContracts2)
         io.SetEngine("Inline");
 
         // writerID parameter makes sure the reader can find the writer.
-        io.SetParameters({{"verbose", "4"},
-                          {"writerID", fname + "_write"},
-                          {"readerID", fname + "_read"}});
+        io.SetParameter("verbose", "4");
 
         adios2::Engine inlineWriter =
             io.Open(fname + "_write", adios2::Mode::Write);
@@ -858,6 +838,30 @@ TEST_F(InlineWriteRead, InlineWriteReadContracts2)
     }
 }
 
+TEST_F(InlineWriteRead, IOInvariants)
+{
+#if ADIOS2_USE_MPI
+    int mpiRank = 0, mpiSize = 1;
+    MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank);
+    MPI_Comm_size(MPI_COMM_WORLD, &mpiSize);
+    adios2::ADIOS adios(MPI_COMM_WORLD);
+#else
+    adios2::ADIOS adios;
+#endif
+    adios2::IO io = adios.DeclareIO("TestIO");
+    io.SetEngine("Inline");
+
+    adios2::Engine inlineWriter = io.Open("writer", adios2::Mode::Write);
+    // The inline engine does not support multiple writers:
+    EXPECT_THROW(io.Open("another_writer", adios2::Mode::Write),
+                 std::exception);
+    // The inline engine does not support append mode:
+    EXPECT_THROW(io.Open("append_mode", adios2::Mode::Append), std::exception);
+    adios2::Engine inlineReader = io.Open("reader", adios2::Mode::Read);
+    // The inline engine does not support more than 2 writers or readers:
+    EXPECT_THROW(io.Open("reader2", adios2::Mode::Read), std::exception);
+}
+
 //******************************************************************************
 // main
 //******************************************************************************
diff --git a/testing/adios2/engine/ssc/CMakeLists.txt b/testing/adios2/engine/ssc/CMakeLists.txt
index 1ff74c3330..3453f837a1 100644
--- a/testing/adios2/engine/ssc/CMakeLists.txt
+++ b/testing/adios2/engine/ssc/CMakeLists.txt
@@ -9,6 +9,18 @@ if(ADIOS2_HAVE_MPI)
   gtest_add_tests_helper(Base MPI_ONLY Ssc Engine.SSC. "")
   SetupTestPipeline(Engine.SSC.SscEngineTest.TestSscBase.MPI "" TRUE)
 
+  gtest_add_tests_helper(BaseUnlocked MPI_ONLY Ssc Engine.SSC. "")
+  SetupTestPipeline(Engine.SSC.SscEngineTest.TestSscBaseUnlocked.MPI "" TRUE)
+
+  gtest_add_tests_helper(LockBeforeEndStep MPI_ONLY Ssc Engine.SSC. "")
+  SetupTestPipeline(Engine.SSC.SscEngineTest.TestSscLockBeforeEndStep.MPI "" TRUE)
+
+  gtest_add_tests_helper(OnlyOneStep MPI_ONLY Ssc Engine.SSC. "")
+  SetupTestPipeline(Engine.SSC.SscEngineTest.TestSscOnlyOneStep.MPI "" TRUE)
+
+  gtest_add_tests_helper(OnlyTwoSteps MPI_ONLY Ssc Engine.SSC. "")
+  SetupTestPipeline(Engine.SSC.SscEngineTest.TestSscOnlyTwoSteps.MPI "" TRUE)
+
   gtest_add_tests_helper(OneSidedFencePush MPI_ONLY Ssc Engine.SSC. "")
   SetupTestPipeline(Engine.SSC.SscEngineTest.TestSscOneSidedFencePush.MPI "" TRUE)
 
@@ -45,13 +57,18 @@ if(ADIOS2_HAVE_MPI)
   gtest_add_tests_helper(MoreWritersThanReaders MPI_ONLY Ssc Engine.SSC. "")
   SetupTestPipeline(Engine.SSC.SscEngineTest.TestSscMoreWritersThanReaders.MPI "" TRUE)
 
-  gtest_add_tests_helper(MultiApp MPI_ONLY Ssc Engine.SSC. "")
-  SetupTestPipeline(Engine.SSC.SscEngineTest.TestSscMultiApp.MPI "" TRUE)
-
   gtest_add_tests_helper(Xgc2Way MPI_ONLY Ssc Engine.SSC. "")
   SetupTestPipeline(Engine.SSC.SscEngineTest.TestSscXgc2Way.MPI "" TRUE)
 
+if(NOT MSVC)
   gtest_add_tests_helper(Xgc3Way MPI_ONLY Ssc Engine.SSC. "")
   SetupTestPipeline(Engine.SSC.SscEngineTest.TestSscXgc3Way.MPI "" TRUE)
 
+  gtest_add_tests_helper(Xgc3WayMatchedSteps MPI_ONLY Ssc Engine.SSC. "")
+  SetupTestPipeline(Engine.SSC.SscEngineTest.TestSscXgc3WayMatchedSteps.MPI "" TRUE)
+endif()
+
+  gtest_add_tests_helper(VaryingSteps MPI_ONLY Ssc Engine.SSC. "")
+  SetupTestPipeline(Engine.SSC.SscEngineTest.TestSscVaryingSteps.MPI "" TRUE)
+
 endif()
diff --git a/testing/adios2/engine/ssc/TestSsc7d.cpp b/testing/adios2/engine/ssc/TestSsc7d.cpp
index 5eda56e2ad..6858cdbdbf 100644
--- a/testing/adios2/engine/ssc/TestSsc7d.cpp
+++ b/testing/adios2/engine/ssc/TestSsc7d.cpp
@@ -25,8 +25,9 @@ void Writer(const Dims &shape, const Dims &start, const Dims &count,
             const size_t steps, const adios2::Params &engineParams,
             const std::string &name)
 {
-    size_t datasize = std::accumulate(count.begin(), count.end(), 1,
-                                      std::multiplies());
+    size_t datasize =
+        std::accumulate(count.begin(), count.end(), static_cast(1),
+                        std::multiplies());
     adios2::ADIOS adios(mpiComm);
     adios2::IO dataManIO = adios.DeclareIO("WAN");
     dataManIO.SetEngine("ssc");
@@ -61,10 +62,11 @@ void Writer(const Dims &shape, const Dims &start, const Dims &count,
     auto bpDComplexes = dataManIO.DefineVariable>(
         "bpDComplexes", shape, start, count);
     dataManIO.DefineAttribute("AttInt", 110);
-    adios2::Engine dataManWriter = dataManIO.Open(name, adios2::Mode::Write);
+    adios2::Engine engine = dataManIO.Open(name, adios2::Mode::Write);
+    engine.LockWriterDefinitions();
     for (int i = 0; i < steps; ++i)
     {
-        dataManWriter.BeginStep();
+        engine.BeginStep();
         GenData(myChars, i, start, count, shape);
         GenData(myUChars, i, start, count, shape);
         GenData(myShorts, i, start, count, shape);
@@ -75,20 +77,19 @@ void Writer(const Dims &shape, const Dims &start, const Dims &count,
         GenData(myDoubles, i, start, count, shape);
         GenData(myComplexes, i, start, count, shape);
         GenData(myDComplexes, i, start, count, shape);
-        dataManWriter.Put(bpChars, myChars.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpUChars, myUChars.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpShorts, myShorts.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpUShorts, myUShorts.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpInts, myInts.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpUInts, myUInts.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpFloats, myFloats.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpDoubles, myDoubles.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpComplexes, myComplexes.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpDComplexes, myDComplexes.data(),
-                          adios2::Mode::Sync);
-        dataManWriter.EndStep();
+        engine.Put(bpChars, myChars.data(), adios2::Mode::Sync);
+        engine.Put(bpUChars, myUChars.data(), adios2::Mode::Sync);
+        engine.Put(bpShorts, myShorts.data(), adios2::Mode::Sync);
+        engine.Put(bpUShorts, myUShorts.data(), adios2::Mode::Sync);
+        engine.Put(bpInts, myInts.data(), adios2::Mode::Sync);
+        engine.Put(bpUInts, myUInts.data(), adios2::Mode::Sync);
+        engine.Put(bpFloats, myFloats.data(), adios2::Mode::Sync);
+        engine.Put(bpDoubles, myDoubles.data(), adios2::Mode::Sync);
+        engine.Put(bpComplexes, myComplexes.data(), adios2::Mode::Sync);
+        engine.Put(bpDComplexes, myDComplexes.data(), adios2::Mode::Sync);
+        engine.EndStep();
     }
-    dataManWriter.Close();
+    engine.Close();
 }
 
 void Reader(const Dims &shape, const Dims &start, const Dims &count,
@@ -99,10 +100,11 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count,
     adios2::IO dataManIO = adios.DeclareIO("Test");
     dataManIO.SetEngine("ssc");
     dataManIO.SetParameters(engineParams);
-    adios2::Engine dataManReader = dataManIO.Open(name, adios2::Mode::Read);
+    adios2::Engine engine = dataManIO.Open(name, adios2::Mode::Read);
 
-    size_t datasize = std::accumulate(shape.begin(), shape.end(), 1,
-                                      std::multiplies());
+    size_t datasize =
+        std::accumulate(shape.begin(), shape.end(), static_cast(1),
+                        std::multiplies());
     std::vector myChars(datasize);
     std::vector myUChars(datasize);
     std::vector myShorts(datasize);
@@ -114,14 +116,16 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count,
     std::vector> myComplexes(datasize);
     std::vector> myDComplexes(datasize);
 
+    engine.LockReaderSelections();
+
     while (true)
     {
-        adios2::StepStatus status = dataManReader.BeginStep(StepMode::Read, 5);
+        adios2::StepStatus status = engine.BeginStep(StepMode::Read, 5);
         if (status == adios2::StepStatus::OK)
         {
             const auto &vars = dataManIO.AvailableVariables();
             ASSERT_EQ(vars.size(), 10);
-            size_t currentStep = dataManReader.CurrentStep();
+            size_t currentStep = engine.CurrentStep();
             adios2::Variable bpChars =
                 dataManIO.InquireVariable("bpChars");
             adios2::Variable bpUChars =
@@ -143,18 +147,16 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count,
             adios2::Variable> bpDComplexes =
                 dataManIO.InquireVariable>("bpDComplexes");
 
-            dataManReader.Get(bpChars, myChars.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpUChars, myUChars.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpShorts, myShorts.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpUShorts, myUShorts.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpInts, myInts.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpUInts, myUInts.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpFloats, myFloats.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpDoubles, myDoubles.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpComplexes, myComplexes.data(),
-                              adios2::Mode::Sync);
-            dataManReader.Get(bpDComplexes, myDComplexes.data(),
-                              adios2::Mode::Sync);
+            engine.Get(bpChars, myChars.data(), adios2::Mode::Sync);
+            engine.Get(bpUChars, myUChars.data(), adios2::Mode::Sync);
+            engine.Get(bpShorts, myShorts.data(), adios2::Mode::Sync);
+            engine.Get(bpUShorts, myUShorts.data(), adios2::Mode::Sync);
+            engine.Get(bpInts, myInts.data(), adios2::Mode::Sync);
+            engine.Get(bpUInts, myUInts.data(), adios2::Mode::Sync);
+            engine.Get(bpFloats, myFloats.data(), adios2::Mode::Sync);
+            engine.Get(bpDoubles, myDoubles.data(), adios2::Mode::Sync);
+            engine.Get(bpComplexes, myComplexes.data(), adios2::Mode::Sync);
+            engine.Get(bpDComplexes, myDComplexes.data(), adios2::Mode::Sync);
             VerifyData(myChars.data(), currentStep, Dims(shape.size(), 0),
                        shape, shape, mpiRank);
             VerifyData(myUChars.data(), currentStep, Dims(shape.size(), 0),
@@ -175,7 +177,7 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count,
                        shape, shape, mpiRank);
             VerifyData(myDComplexes.data(), currentStep, Dims(shape.size(), 0),
                        shape, shape, mpiRank);
-            dataManReader.EndStep();
+            engine.EndStep();
         }
         else if (status == adios2::StepStatus::EndOfStream)
         {
@@ -185,7 +187,7 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count,
             break;
         }
     }
-    dataManReader.Close();
+    engine.Close();
 }
 
 TEST_F(SscEngineTest, TestSsc7d)
diff --git a/testing/adios2/engine/ssc/TestSscBase.cpp b/testing/adios2/engine/ssc/TestSscBase.cpp
index 958010a571..60e047d240 100644
--- a/testing/adios2/engine/ssc/TestSscBase.cpp
+++ b/testing/adios2/engine/ssc/TestSscBase.cpp
@@ -25,8 +25,9 @@ void Writer(const Dims &shape, const Dims &start, const Dims &count,
             const size_t steps, const adios2::Params &engineParams,
             const std::string &name)
 {
-    size_t datasize = std::accumulate(count.begin(), count.end(), 1,
-                                      std::multiplies());
+    size_t datasize =
+        std::accumulate(count.begin(), count.end(), static_cast(1),
+                        std::multiplies());
     adios2::ADIOS adios(mpiComm);
     adios2::IO dataManIO = adios.DeclareIO("WAN");
     dataManIO.SetEngine("ssc");
@@ -63,10 +64,11 @@ void Writer(const Dims &shape, const Dims &start, const Dims &count,
     auto scalarInt = dataManIO.DefineVariable("scalarInt");
     auto stringVar = dataManIO.DefineVariable("stringVar");
     dataManIO.DefineAttribute("AttInt", 110);
-    adios2::Engine dataManWriter = dataManIO.Open(name, adios2::Mode::Write);
+    adios2::Engine engine = dataManIO.Open(name, adios2::Mode::Write);
+    engine.LockWriterDefinitions();
     for (int i = 0; i < steps; ++i)
     {
-        dataManWriter.BeginStep();
+        engine.BeginStep();
         GenData(myChars, i, start, count, shape);
         GenData(myUChars, i, start, count, shape);
         GenData(myShorts, i, start, count, shape);
@@ -77,23 +79,22 @@ void Writer(const Dims &shape, const Dims &start, const Dims &count,
         GenData(myDoubles, i, start, count, shape);
         GenData(myComplexes, i, start, count, shape);
         GenData(myDComplexes, i, start, count, shape);
-        dataManWriter.Put(bpChars, myChars.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpUChars, myUChars.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpShorts, myShorts.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpUShorts, myUShorts.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpInts, myInts.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpUInts, myUInts.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpFloats, myFloats.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpDoubles, myDoubles.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpComplexes, myComplexes.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpDComplexes, myDComplexes.data(),
-                          adios2::Mode::Sync);
-        dataManWriter.Put(scalarInt, i);
+        engine.Put(bpChars, myChars.data(), adios2::Mode::Sync);
+        engine.Put(bpUChars, myUChars.data(), adios2::Mode::Sync);
+        engine.Put(bpShorts, myShorts.data(), adios2::Mode::Sync);
+        engine.Put(bpUShorts, myUShorts.data(), adios2::Mode::Sync);
+        engine.Put(bpInts, myInts.data(), adios2::Mode::Sync);
+        engine.Put(bpUInts, myUInts.data(), adios2::Mode::Sync);
+        engine.Put(bpFloats, myFloats.data(), adios2::Mode::Sync);
+        engine.Put(bpDoubles, myDoubles.data(), adios2::Mode::Sync);
+        engine.Put(bpComplexes, myComplexes.data(), adios2::Mode::Sync);
+        engine.Put(bpDComplexes, myDComplexes.data(), adios2::Mode::Sync);
+        engine.Put(scalarInt, i);
         std::string s = "sample string sample string sample string";
-        dataManWriter.Put(stringVar, s);
-        dataManWriter.EndStep();
+        engine.Put(stringVar, s);
+        engine.EndStep();
     }
-    dataManWriter.Close();
+    engine.Close();
 }
 
 void Reader(const Dims &shape, const Dims &start, const Dims &count,
@@ -104,10 +105,11 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count,
     adios2::IO dataManIO = adios.DeclareIO("Test");
     dataManIO.SetEngine("ssc");
     dataManIO.SetParameters(engineParams);
-    adios2::Engine dataManReader = dataManIO.Open(name, adios2::Mode::Read);
+    adios2::Engine engine = dataManIO.Open(name, adios2::Mode::Read);
 
-    size_t datasize = std::accumulate(count.begin(), count.end(), 1,
-                                      std::multiplies());
+    size_t datasize =
+        std::accumulate(count.begin(), count.end(), static_cast(1),
+                        std::multiplies());
     std::vector myChars(datasize);
     std::vector myUChars(datasize);
     std::vector myShorts(datasize);
@@ -119,26 +121,28 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count,
     std::vector> myComplexes(datasize);
     std::vector> myDComplexes(datasize);
 
+    engine.LockReaderSelections();
+
     while (true)
     {
-        adios2::StepStatus status = dataManReader.BeginStep(StepMode::Read, 5);
+        adios2::StepStatus status = engine.BeginStep(StepMode::Read, 5);
         if (status == adios2::StepStatus::OK)
         {
             auto scalarInt = dataManIO.InquireVariable("scalarInt");
-            auto blocksInfo = dataManReader.BlocksInfo(
-                scalarInt, dataManReader.CurrentStep());
+            auto blocksInfo =
+                engine.BlocksInfo(scalarInt, engine.CurrentStep());
 
             for (const auto &bi : blocksInfo)
             {
                 ASSERT_EQ(bi.IsValue, true);
-                ASSERT_EQ(bi.Value, dataManReader.CurrentStep());
-                ASSERT_EQ(scalarInt.Min(), dataManReader.CurrentStep());
-                ASSERT_EQ(scalarInt.Max(), dataManReader.CurrentStep());
+                ASSERT_EQ(bi.Value, engine.CurrentStep());
+                ASSERT_EQ(scalarInt.Min(), engine.CurrentStep());
+                ASSERT_EQ(scalarInt.Max(), engine.CurrentStep());
             }
 
             const auto &vars = dataManIO.AvailableVariables();
             ASSERT_EQ(vars.size(), 12);
-            size_t currentStep = dataManReader.CurrentStep();
+            size_t currentStep = engine.CurrentStep();
             adios2::Variable bpChars =
                 dataManIO.InquireVariable("bpChars");
             adios2::Variable bpUChars =
@@ -173,20 +177,18 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count,
             bpComplexes.SetSelection({start, count});
             bpDComplexes.SetSelection({start, count});
 
-            dataManReader.Get(bpChars, myChars.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpUChars, myUChars.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpShorts, myShorts.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpUShorts, myUShorts.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpInts, myInts.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpUInts, myUInts.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpFloats, myFloats.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpDoubles, myDoubles.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpComplexes, myComplexes.data(),
-                              adios2::Mode::Sync);
-            dataManReader.Get(bpDComplexes, myDComplexes.data(),
-                              adios2::Mode::Sync);
+            engine.Get(bpChars, myChars.data(), adios2::Mode::Sync);
+            engine.Get(bpUChars, myUChars.data(), adios2::Mode::Sync);
+            engine.Get(bpShorts, myShorts.data(), adios2::Mode::Sync);
+            engine.Get(bpUShorts, myUShorts.data(), adios2::Mode::Sync);
+            engine.Get(bpInts, myInts.data(), adios2::Mode::Sync);
+            engine.Get(bpUInts, myUInts.data(), adios2::Mode::Sync);
+            engine.Get(bpFloats, myFloats.data(), adios2::Mode::Sync);
+            engine.Get(bpDoubles, myDoubles.data(), adios2::Mode::Sync);
+            engine.Get(bpComplexes, myComplexes.data(), adios2::Mode::Sync);
+            engine.Get(bpDComplexes, myDComplexes.data(), adios2::Mode::Sync);
             std::string s;
-            dataManReader.Get(stringVar, s);
+            engine.Get(stringVar, s);
             ASSERT_EQ(s, "sample string sample string sample string");
             ASSERT_EQ(stringVar.Min(),
                       "sample string sample string sample string");
@@ -194,7 +196,7 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count,
                       "sample string sample string sample string");
 
             int i;
-            dataManReader.Get(scalarInt, &i);
+            engine.Get(scalarInt, &i);
             ASSERT_EQ(i, currentStep);
 
             VerifyData(myChars.data(), currentStep, start, count, shape,
@@ -217,7 +219,7 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count,
                        mpiRank);
             VerifyData(myDComplexes.data(), currentStep, start, count, shape,
                        mpiRank);
-            dataManReader.EndStep();
+            engine.EndStep();
         }
         else if (status == adios2::StepStatus::EndOfStream)
         {
@@ -232,7 +234,7 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count,
               << attInt.Data()[0] << ", expected 110" << std::endl;
     ASSERT_EQ(110, attInt.Data()[0]);
     ASSERT_NE(111, attInt.Data()[0]);
-    dataManReader.Close();
+    engine.Close();
 }
 
 TEST_F(SscEngineTest, TestSscBase)
diff --git a/testing/adios2/engine/ssc/TestSscBaseUnlocked.cpp b/testing/adios2/engine/ssc/TestSscBaseUnlocked.cpp
new file mode 100644
index 0000000000..bc89f2a108
--- /dev/null
+++ b/testing/adios2/engine/ssc/TestSscBaseUnlocked.cpp
@@ -0,0 +1,278 @@
+/*
+ * Distributed under the OSI-approved Apache License, Version 2.0.  See
+ * accompanying file Copyright.txt for details.
+ */
+
+#include "TestSscCommon.h"
+#include 
+#include 
+#include 
+#include 
+#include 
+
+using namespace adios2;
+int mpiRank = 0;
+int mpiSize = 1;
+MPI_Comm mpiComm;
+
+class SscEngineTest : public ::testing::Test
+{
+public:
+    SscEngineTest() = default;
+};
+
+void Writer(const Dims &shape, const Dims &start, const Dims &count,
+            const size_t steps, const adios2::Params &engineParams,
+            const std::string &name)
+{
+    size_t datasize =
+        std::accumulate(count.begin(), count.end(), static_cast(1),
+                        std::multiplies());
+    adios2::ADIOS adios(mpiComm);
+    adios2::IO dataManIO = adios.DeclareIO("WAN");
+    dataManIO.SetEngine("ssc");
+    dataManIO.SetParameters(engineParams);
+    std::vector myChars(datasize);
+    std::vector myUChars(datasize);
+    std::vector myShorts(datasize);
+    std::vector myUShorts(datasize);
+    std::vector myInts(datasize);
+    std::vector myUInts(datasize);
+    std::vector myFloats(datasize);
+    std::vector myDoubles(datasize);
+    std::vector> myComplexes(datasize);
+    std::vector> myDComplexes(datasize);
+    auto bpChars =
+        dataManIO.DefineVariable("bpChars", shape, start, count);
+    auto bpUChars = dataManIO.DefineVariable("bpUChars", shape,
+                                                            start, count);
+    auto bpShorts =
+        dataManIO.DefineVariable("bpShorts", shape, start, count);
+    auto bpUShorts = dataManIO.DefineVariable(
+        "bpUShorts", shape, start, count);
+    auto bpInts = dataManIO.DefineVariable("bpInts", shape, start, count);
+    auto bpUInts =
+        dataManIO.DefineVariable("bpUInts", shape, start, count);
+    auto bpFloats =
+        dataManIO.DefineVariable("bpFloats", shape, start, count);
+    auto bpDoubles =
+        dataManIO.DefineVariable("bpDoubles", shape, start, count);
+    auto bpComplexes = dataManIO.DefineVariable>(
+        "bpComplexes", shape, start, count);
+    auto bpDComplexes = dataManIO.DefineVariable>(
+        "bpDComplexes", shape, start, count);
+    auto scalarInt = dataManIO.DefineVariable("scalarInt");
+    auto stringVar = dataManIO.DefineVariable("stringVar");
+    dataManIO.DefineAttribute("AttInt", 110);
+    adios2::Engine engine = dataManIO.Open(name, adios2::Mode::Write);
+    for (int i = 0; i < steps; ++i)
+    {
+        engine.BeginStep();
+        GenData(myChars, i, start, count, shape);
+        GenData(myUChars, i, start, count, shape);
+        GenData(myShorts, i, start, count, shape);
+        GenData(myUShorts, i, start, count, shape);
+        GenData(myInts, i, start, count, shape);
+        GenData(myUInts, i, start, count, shape);
+        GenData(myFloats, i, start, count, shape);
+        GenData(myDoubles, i, start, count, shape);
+        GenData(myComplexes, i, start, count, shape);
+        GenData(myDComplexes, i, start, count, shape);
+        engine.Put(bpChars, myChars.data(), adios2::Mode::Sync);
+        engine.Put(bpUChars, myUChars.data(), adios2::Mode::Sync);
+        engine.Put(bpShorts, myShorts.data(), adios2::Mode::Sync);
+        engine.Put(bpUShorts, myUShorts.data(), adios2::Mode::Sync);
+        engine.Put(bpInts, myInts.data(), adios2::Mode::Sync);
+        engine.Put(bpUInts, myUInts.data(), adios2::Mode::Sync);
+        engine.Put(bpFloats, myFloats.data(), adios2::Mode::Sync);
+        engine.Put(bpDoubles, myDoubles.data(), adios2::Mode::Sync);
+        engine.Put(bpComplexes, myComplexes.data(), adios2::Mode::Sync);
+        engine.Put(bpDComplexes, myDComplexes.data(), adios2::Mode::Sync);
+        engine.Put(scalarInt, i);
+        std::string s = "sample string sample string sample string";
+        engine.Put(stringVar, s);
+        engine.EndStep();
+    }
+    engine.Close();
+}
+
+void Reader(const Dims &shape, const Dims &start, const Dims &count,
+            const size_t steps, const adios2::Params &engineParams,
+            const std::string &name)
+{
+    adios2::ADIOS adios(mpiComm);
+    adios2::IO dataManIO = adios.DeclareIO("Test");
+    dataManIO.SetEngine("ssc");
+    dataManIO.SetParameters(engineParams);
+    adios2::Engine engine = dataManIO.Open(name, adios2::Mode::Read);
+
+    size_t datasize =
+        std::accumulate(count.begin(), count.end(), static_cast(1),
+                        std::multiplies());
+    std::vector myChars(datasize);
+    std::vector myUChars(datasize);
+    std::vector myShorts(datasize);
+    std::vector myUShorts(datasize);
+    std::vector myInts(datasize);
+    std::vector myUInts(datasize);
+    std::vector myFloats(datasize);
+    std::vector myDoubles(datasize);
+    std::vector> myComplexes(datasize);
+    std::vector> myDComplexes(datasize);
+
+    while (true)
+    {
+        adios2::StepStatus status = engine.BeginStep(StepMode::Read, 5);
+        if (status == adios2::StepStatus::OK)
+        {
+            auto scalarInt = dataManIO.InquireVariable("scalarInt");
+            auto blocksInfo =
+                engine.BlocksInfo(scalarInt, engine.CurrentStep());
+
+            for (const auto &bi : blocksInfo)
+            {
+                ASSERT_EQ(bi.IsValue, true);
+                ASSERT_EQ(bi.Value, engine.CurrentStep());
+                ASSERT_EQ(scalarInt.Min(), engine.CurrentStep());
+                ASSERT_EQ(scalarInt.Max(), engine.CurrentStep());
+            }
+
+            const auto &vars = dataManIO.AvailableVariables();
+            ASSERT_EQ(vars.size(), 12);
+            size_t currentStep = engine.CurrentStep();
+            adios2::Variable bpChars =
+                dataManIO.InquireVariable("bpChars");
+            adios2::Variable bpUChars =
+                dataManIO.InquireVariable("bpUChars");
+            adios2::Variable bpShorts =
+                dataManIO.InquireVariable("bpShorts");
+            adios2::Variable bpUShorts =
+                dataManIO.InquireVariable("bpUShorts");
+            adios2::Variable bpInts =
+                dataManIO.InquireVariable("bpInts");
+            adios2::Variable bpUInts =
+                dataManIO.InquireVariable("bpUInts");
+            adios2::Variable bpFloats =
+                dataManIO.InquireVariable("bpFloats");
+            adios2::Variable bpDoubles =
+                dataManIO.InquireVariable("bpDoubles");
+            adios2::Variable> bpComplexes =
+                dataManIO.InquireVariable>("bpComplexes");
+            adios2::Variable> bpDComplexes =
+                dataManIO.InquireVariable>("bpDComplexes");
+            adios2::Variable stringVar =
+                dataManIO.InquireVariable("stringVar");
+
+            bpChars.SetSelection({start, count});
+            bpUChars.SetSelection({start, count});
+            bpShorts.SetSelection({start, count});
+            bpUShorts.SetSelection({start, count});
+            bpInts.SetSelection({start, count});
+            bpUInts.SetSelection({start, count});
+            bpFloats.SetSelection({start, count});
+            bpDoubles.SetSelection({start, count});
+            bpComplexes.SetSelection({start, count});
+            bpDComplexes.SetSelection({start, count});
+
+            engine.Get(bpChars, myChars.data(), adios2::Mode::Sync);
+            engine.Get(bpUChars, myUChars.data(), adios2::Mode::Sync);
+            engine.Get(bpShorts, myShorts.data(), adios2::Mode::Sync);
+            engine.Get(bpUShorts, myUShorts.data(), adios2::Mode::Sync);
+            engine.Get(bpInts, myInts.data(), adios2::Mode::Sync);
+            engine.Get(bpUInts, myUInts.data(), adios2::Mode::Sync);
+            engine.Get(bpFloats, myFloats.data(), adios2::Mode::Sync);
+            engine.Get(bpDoubles, myDoubles.data(), adios2::Mode::Sync);
+            engine.Get(bpComplexes, myComplexes.data(), adios2::Mode::Sync);
+            engine.Get(bpDComplexes, myDComplexes.data(), adios2::Mode::Sync);
+            std::string s;
+            engine.Get(stringVar, s);
+            ASSERT_EQ(s, "sample string sample string sample string");
+            ASSERT_EQ(stringVar.Min(),
+                      "sample string sample string sample string");
+            ASSERT_EQ(stringVar.Max(),
+                      "sample string sample string sample string");
+
+            VerifyData(myChars.data(), currentStep, start, count, shape,
+                       mpiRank);
+            VerifyData(myUChars.data(), currentStep, start, count, shape,
+                       mpiRank);
+            VerifyData(myShorts.data(), currentStep, start, count, shape,
+                       mpiRank);
+            VerifyData(myUShorts.data(), currentStep, start, count, shape,
+                       mpiRank);
+            VerifyData(myInts.data(), currentStep, start, count, shape,
+                       mpiRank);
+            VerifyData(myUInts.data(), currentStep, start, count, shape,
+                       mpiRank);
+            VerifyData(myFloats.data(), currentStep, start, count, shape,
+                       mpiRank);
+            VerifyData(myDoubles.data(), currentStep, start, count, shape,
+                       mpiRank);
+            VerifyData(myComplexes.data(), currentStep, start, count, shape,
+                       mpiRank);
+            VerifyData(myDComplexes.data(), currentStep, start, count, shape,
+                       mpiRank);
+            engine.EndStep();
+        }
+        else if (status == adios2::StepStatus::EndOfStream)
+        {
+            std::cout << "[Rank " + std::to_string(mpiRank) +
+                             "] SscTest reader end of stream!"
+                      << std::endl;
+            break;
+        }
+    }
+    auto attInt = dataManIO.InquireAttribute("AttInt");
+    std::cout << "[Rank " + std::to_string(mpiRank) + "] Attribute received "
+              << attInt.Data()[0] << ", expected 110" << std::endl;
+    ASSERT_EQ(110, attInt.Data()[0]);
+    ASSERT_NE(111, attInt.Data()[0]);
+    engine.Close();
+}
+
+TEST_F(SscEngineTest, TestSscBaseUnlocked)
+{
+    std::string filename = "TestSscBaseUnlocked";
+    adios2::Params engineParams = {};
+
+    int worldRank, worldSize;
+    MPI_Comm_rank(MPI_COMM_WORLD, &worldRank);
+    MPI_Comm_size(MPI_COMM_WORLD, &worldSize);
+    int mpiGroup = worldRank / (worldSize / 2);
+    MPI_Comm_split(MPI_COMM_WORLD, mpiGroup, worldRank, &mpiComm);
+
+    MPI_Comm_rank(mpiComm, &mpiRank);
+    MPI_Comm_size(mpiComm, &mpiSize);
+
+    Dims shape = {10, (size_t)mpiSize * 2};
+    Dims start = {2, (size_t)mpiRank * 2};
+    Dims count = {5, 2};
+    size_t steps = 10;
+
+    if (mpiGroup == 0)
+    {
+        Writer(shape, start, count, steps, engineParams, filename);
+    }
+
+    std::this_thread::sleep_for(std::chrono::milliseconds(1));
+
+    if (mpiGroup == 1)
+    {
+        Reader(shape, start, count, steps, engineParams, filename);
+    }
+
+    MPI_Barrier(MPI_COMM_WORLD);
+}
+
+int main(int argc, char **argv)
+{
+    MPI_Init(&argc, &argv);
+    int worldRank, worldSize;
+    MPI_Comm_rank(MPI_COMM_WORLD, &worldRank);
+    MPI_Comm_size(MPI_COMM_WORLD, &worldSize);
+    ::testing::InitGoogleTest(&argc, argv);
+    int result = RUN_ALL_TESTS();
+
+    MPI_Finalize();
+    return result;
+}
diff --git a/testing/adios2/engine/ssc/TestSscCommon.h b/testing/adios2/engine/ssc/TestSscCommon.h
index 1c451bfc81..93f8560cd9 100644
--- a/testing/adios2/engine/ssc/TestSscCommon.h
+++ b/testing/adios2/engine/ssc/TestSscCommon.h
@@ -12,8 +12,9 @@ template 
 void PrintData(const T *data, const size_t step, const Dims &start,
                const Dims &count, const int rank)
 {
-    size_t size = std::accumulate(count.begin(), count.end(), 1,
-                                  std::multiplies());
+    size_t size =
+        std::accumulate(count.begin(), count.end(), static_cast(1),
+                        std::multiplies());
     std::cout << "Rank: " << rank << " Step: " << step << " Size:" << size
               << "\n";
     size_t printsize = 128;
@@ -40,7 +41,7 @@ void PrintData(const T *data, const size_t step, const Dims &start,
 template 
 void GenDataRecursive(std::vector start, std::vector count,
                       std::vector shape, size_t n0, size_t y,
-                      std::vector &vec)
+                      std::vector> &vec, const size_t step)
 {
     for (size_t i = 0; i < count[0]; i++)
     {
@@ -58,13 +59,49 @@ void GenDataRecursive(std::vector start, std::vector count,
         {
             for (size_t j = 0; j < count_next[0]; j++)
             {
-                vec[i0 * count_next[0] + j] =
-                    z * shape_next[0] + (j + start_next[0]);
+                vec[i0 * count_next[0] + j] = {
+                    static_cast(z * shape_next[0] + (j + start_next[0]) +
+                                   step),
+                    1};
             }
         }
         else
         {
-            GenDataRecursive(start_next, count_next, shape_next, i0, z, vec);
+            GenDataRecursive(start_next, count_next, shape_next, i0, z, vec,
+                             step);
+        }
+    }
+}
+
+template 
+void GenDataRecursive(std::vector start, std::vector count,
+                      std::vector shape, size_t n0, size_t y,
+                      std::vector &vec, const size_t step)
+{
+    for (size_t i = 0; i < count[0]; i++)
+    {
+        size_t i0 = n0 * count[0] + i;
+        size_t z = y * shape[0] + (i + start[0]);
+
+        auto start_next = start;
+        auto count_next = count;
+        auto shape_next = shape;
+        start_next.erase(start_next.begin());
+        count_next.erase(count_next.begin());
+        shape_next.erase(shape_next.begin());
+
+        if (start_next.size() == 1)
+        {
+            for (size_t j = 0; j < count_next[0]; j++)
+            {
+                vec[i0 * count_next[0] + j] = static_cast(
+                    z * shape_next[0] + (j + start_next[0]) + step);
+            }
+        }
+        else
+        {
+            GenDataRecursive(start_next, count_next, shape_next, i0, z, vec,
+                             step);
         }
     }
 }
@@ -74,18 +111,20 @@ void GenData(std::vector &vec, const size_t step,
              const std::vector &start, const std::vector &count,
              const std::vector &shape)
 {
-    size_t total_size = std::accumulate(count.begin(), count.end(), 1,
-                                        std::multiplies());
+    size_t total_size =
+        std::accumulate(count.begin(), count.end(), static_cast(1),
+                        std::multiplies());
     vec.resize(total_size);
-    GenDataRecursive(start, count, shape, 0, 0, vec);
+    GenDataRecursive(start, count, shape, 0, 0, vec, step);
 }
 
 template 
 void VerifyData(const std::complex *data, size_t step, const Dims &start,
                 const Dims &count, const Dims &shape)
 {
-    size_t size = std::accumulate(count.begin(), count.end(), 1,
-                                  std::multiplies());
+    size_t size =
+        std::accumulate(count.begin(), count.end(), static_cast(1),
+                        std::multiplies());
     std::vector> tmpdata(size);
     GenData(tmpdata, step, start, count, shape);
     for (size_t i = 0; i < size; ++i)
@@ -103,8 +142,9 @@ template 
 void VerifyData(const T *data, size_t step, const Dims &start,
                 const Dims &count, const Dims &shape, const int rank)
 {
-    size_t size = std::accumulate(count.begin(), count.end(), 1,
-                                  std::multiplies());
+    size_t size =
+        std::accumulate(count.begin(), count.end(), static_cast(1),
+                        std::multiplies());
     bool compressed = false;
     std::vector tmpdata(size);
     if (printed_lines < to_print_lines)
diff --git a/testing/adios2/engine/ssc/TestSscLockBeforeEndStep.cpp b/testing/adios2/engine/ssc/TestSscLockBeforeEndStep.cpp
new file mode 100644
index 0000000000..f0107e9b1e
--- /dev/null
+++ b/testing/adios2/engine/ssc/TestSscLockBeforeEndStep.cpp
@@ -0,0 +1,284 @@
+/*
+ * Distributed under the OSI-approved Apache License, Version 2.0.  See
+ * accompanying file Copyright.txt for details.
+ */
+
+#include "TestSscCommon.h"
+#include 
+#include 
+#include 
+#include 
+#include 
+
+using namespace adios2;
+int mpiRank = 0;
+int mpiSize = 1;
+MPI_Comm mpiComm;
+
+class SscEngineTest : public ::testing::Test
+{
+public:
+    SscEngineTest() = default;
+};
+
+void Writer(const Dims &shape, const Dims &start, const Dims &count,
+            const size_t steps, const adios2::Params &engineParams,
+            const std::string &name)
+{
+    size_t datasize =
+        std::accumulate(count.begin(), count.end(), static_cast(1),
+                        std::multiplies());
+    adios2::ADIOS adios(mpiComm);
+    adios2::IO dataManIO = adios.DeclareIO("WAN");
+    dataManIO.SetEngine("ssc");
+    dataManIO.SetParameters(engineParams);
+    std::vector myChars(datasize);
+    std::vector myUChars(datasize);
+    std::vector myShorts(datasize);
+    std::vector myUShorts(datasize);
+    std::vector myInts(datasize);
+    std::vector myUInts(datasize);
+    std::vector myFloats(datasize);
+    std::vector myDoubles(datasize);
+    std::vector> myComplexes(datasize);
+    std::vector> myDComplexes(datasize);
+    auto bpChars =
+        dataManIO.DefineVariable("bpChars", shape, start, count);
+    auto bpUChars = dataManIO.DefineVariable("bpUChars", shape,
+                                                            start, count);
+    auto bpShorts =
+        dataManIO.DefineVariable("bpShorts", shape, start, count);
+    auto bpUShorts = dataManIO.DefineVariable(
+        "bpUShorts", shape, start, count);
+    auto bpInts = dataManIO.DefineVariable("bpInts", shape, start, count);
+    auto bpUInts =
+        dataManIO.DefineVariable("bpUInts", shape, start, count);
+    auto bpFloats =
+        dataManIO.DefineVariable("bpFloats", shape, start, count);
+    auto bpDoubles =
+        dataManIO.DefineVariable("bpDoubles", shape, start, count);
+    auto bpComplexes = dataManIO.DefineVariable>(
+        "bpComplexes", shape, start, count);
+    auto bpDComplexes = dataManIO.DefineVariable>(
+        "bpDComplexes", shape, start, count);
+    auto scalarInt = dataManIO.DefineVariable("scalarInt");
+    auto stringVar = dataManIO.DefineVariable("stringVar");
+    dataManIO.DefineAttribute("AttInt", 110);
+    adios2::Engine engine = dataManIO.Open(name, adios2::Mode::Write);
+    for (int i = 0; i < steps; ++i)
+    {
+        engine.BeginStep();
+        GenData(myChars, i, start, count, shape);
+        GenData(myUChars, i, start, count, shape);
+        GenData(myShorts, i, start, count, shape);
+        GenData(myUShorts, i, start, count, shape);
+        GenData(myInts, i, start, count, shape);
+        GenData(myUInts, i, start, count, shape);
+        GenData(myFloats, i, start, count, shape);
+        GenData(myDoubles, i, start, count, shape);
+        GenData(myComplexes, i, start, count, shape);
+        GenData(myDComplexes, i, start, count, shape);
+        engine.Put(bpChars, myChars.data(), adios2::Mode::Sync);
+        engine.Put(bpUChars, myUChars.data(), adios2::Mode::Sync);
+        engine.Put(bpShorts, myShorts.data(), adios2::Mode::Sync);
+        engine.Put(bpUShorts, myUShorts.data(), adios2::Mode::Sync);
+        engine.Put(bpInts, myInts.data(), adios2::Mode::Sync);
+        engine.Put(bpUInts, myUInts.data(), adios2::Mode::Sync);
+        engine.Put(bpFloats, myFloats.data(), adios2::Mode::Sync);
+        engine.Put(bpDoubles, myDoubles.data(), adios2::Mode::Sync);
+        engine.Put(bpComplexes, myComplexes.data(), adios2::Mode::Sync);
+        engine.Put(bpDComplexes, myDComplexes.data(), adios2::Mode::Sync);
+        engine.Put(scalarInt, i);
+        std::string s = "sample string sample string sample string";
+        engine.Put(stringVar, s);
+        engine.LockWriterDefinitions();
+        engine.EndStep();
+    }
+    engine.Close();
+}
+
+void Reader(const Dims &shape, const Dims &start, const Dims &count,
+            const size_t steps, const adios2::Params &engineParams,
+            const std::string &name)
+{
+    adios2::ADIOS adios(mpiComm);
+    adios2::IO dataManIO = adios.DeclareIO("Test");
+    dataManIO.SetEngine("ssc");
+    dataManIO.SetParameters(engineParams);
+    adios2::Engine engine = dataManIO.Open(name, adios2::Mode::Read);
+
+    size_t datasize =
+        std::accumulate(count.begin(), count.end(), static_cast(1),
+                        std::multiplies());
+    std::vector myChars(datasize);
+    std::vector myUChars(datasize);
+    std::vector myShorts(datasize);
+    std::vector myUShorts(datasize);
+    std::vector myInts(datasize);
+    std::vector myUInts(datasize);
+    std::vector myFloats(datasize);
+    std::vector myDoubles(datasize);
+    std::vector> myComplexes(datasize);
+    std::vector> myDComplexes(datasize);
+
+    while (true)
+    {
+        adios2::StepStatus status = engine.BeginStep(StepMode::Read, 5);
+        if (status == adios2::StepStatus::OK)
+        {
+            auto scalarInt = dataManIO.InquireVariable("scalarInt");
+            auto blocksInfo =
+                engine.BlocksInfo(scalarInt, engine.CurrentStep());
+
+            for (const auto &bi : blocksInfo)
+            {
+                ASSERT_EQ(bi.IsValue, true);
+                ASSERT_EQ(bi.Value, engine.CurrentStep());
+                ASSERT_EQ(scalarInt.Min(), engine.CurrentStep());
+                ASSERT_EQ(scalarInt.Max(), engine.CurrentStep());
+            }
+
+            const auto &vars = dataManIO.AvailableVariables();
+            ASSERT_EQ(vars.size(), 12);
+            size_t currentStep = engine.CurrentStep();
+            adios2::Variable bpChars =
+                dataManIO.InquireVariable("bpChars");
+            adios2::Variable bpUChars =
+                dataManIO.InquireVariable("bpUChars");
+            adios2::Variable bpShorts =
+                dataManIO.InquireVariable("bpShorts");
+            adios2::Variable bpUShorts =
+                dataManIO.InquireVariable("bpUShorts");
+            adios2::Variable bpInts =
+                dataManIO.InquireVariable("bpInts");
+            adios2::Variable bpUInts =
+                dataManIO.InquireVariable("bpUInts");
+            adios2::Variable bpFloats =
+                dataManIO.InquireVariable("bpFloats");
+            adios2::Variable bpDoubles =
+                dataManIO.InquireVariable("bpDoubles");
+            adios2::Variable> bpComplexes =
+                dataManIO.InquireVariable>("bpComplexes");
+            adios2::Variable> bpDComplexes =
+                dataManIO.InquireVariable>("bpDComplexes");
+            adios2::Variable stringVar =
+                dataManIO.InquireVariable("stringVar");
+
+            bpChars.SetSelection({start, count});
+            bpUChars.SetSelection({start, count});
+            bpShorts.SetSelection({start, count});
+            bpUShorts.SetSelection({start, count});
+            bpInts.SetSelection({start, count});
+            bpUInts.SetSelection({start, count});
+            bpFloats.SetSelection({start, count});
+            bpDoubles.SetSelection({start, count});
+            bpComplexes.SetSelection({start, count});
+            bpDComplexes.SetSelection({start, count});
+
+            engine.Get(bpChars, myChars.data(), adios2::Mode::Sync);
+            engine.Get(bpUChars, myUChars.data(), adios2::Mode::Sync);
+            engine.Get(bpShorts, myShorts.data(), adios2::Mode::Sync);
+            engine.Get(bpUShorts, myUShorts.data(), adios2::Mode::Sync);
+            engine.Get(bpInts, myInts.data(), adios2::Mode::Sync);
+            engine.Get(bpUInts, myUInts.data(), adios2::Mode::Sync);
+            engine.Get(bpFloats, myFloats.data(), adios2::Mode::Sync);
+            engine.Get(bpDoubles, myDoubles.data(), adios2::Mode::Sync);
+            engine.Get(bpComplexes, myComplexes.data(), adios2::Mode::Sync);
+            engine.Get(bpDComplexes, myDComplexes.data(), adios2::Mode::Sync);
+            std::string s;
+            engine.Get(stringVar, s);
+            ASSERT_EQ(s, "sample string sample string sample string");
+            ASSERT_EQ(stringVar.Min(),
+                      "sample string sample string sample string");
+            ASSERT_EQ(stringVar.Max(),
+                      "sample string sample string sample string");
+
+            int i;
+            engine.Get(scalarInt, &i);
+            ASSERT_EQ(i, currentStep);
+
+            VerifyData(myChars.data(), currentStep, start, count, shape,
+                       mpiRank);
+            VerifyData(myUChars.data(), currentStep, start, count, shape,
+                       mpiRank);
+            VerifyData(myShorts.data(), currentStep, start, count, shape,
+                       mpiRank);
+            VerifyData(myUShorts.data(), currentStep, start, count, shape,
+                       mpiRank);
+            VerifyData(myInts.data(), currentStep, start, count, shape,
+                       mpiRank);
+            VerifyData(myUInts.data(), currentStep, start, count, shape,
+                       mpiRank);
+            VerifyData(myFloats.data(), currentStep, start, count, shape,
+                       mpiRank);
+            VerifyData(myDoubles.data(), currentStep, start, count, shape,
+                       mpiRank);
+            VerifyData(myComplexes.data(), currentStep, start, count, shape,
+                       mpiRank);
+            VerifyData(myDComplexes.data(), currentStep, start, count, shape,
+                       mpiRank);
+            engine.LockReaderSelections();
+            engine.EndStep();
+        }
+        else if (status == adios2::StepStatus::EndOfStream)
+        {
+            std::cout << "[Rank " + std::to_string(mpiRank) +
+                             "] SscTest reader end of stream!"
+                      << std::endl;
+            break;
+        }
+    }
+    auto attInt = dataManIO.InquireAttribute("AttInt");
+    std::cout << "[Rank " + std::to_string(mpiRank) + "] Attribute received "
+              << attInt.Data()[0] << ", expected 110" << std::endl;
+    ASSERT_EQ(110, attInt.Data()[0]);
+    ASSERT_NE(111, attInt.Data()[0]);
+    engine.Close();
+}
+
+TEST_F(SscEngineTest, TestSscLockBeforeEndStep)
+{
+    std::string filename = "TestSscLockBeforeEndStep";
+    adios2::Params engineParams = {};
+
+    int worldRank, worldSize;
+    MPI_Comm_rank(MPI_COMM_WORLD, &worldRank);
+    MPI_Comm_size(MPI_COMM_WORLD, &worldSize);
+    int mpiGroup = worldRank / (worldSize / 2);
+    MPI_Comm_split(MPI_COMM_WORLD, mpiGroup, worldRank, &mpiComm);
+
+    MPI_Comm_rank(mpiComm, &mpiRank);
+    MPI_Comm_size(mpiComm, &mpiSize);
+
+    Dims shape = {10, (size_t)mpiSize * 2};
+    Dims start = {2, (size_t)mpiRank * 2};
+    Dims count = {5, 2};
+    size_t steps = 100;
+
+    if (mpiGroup == 0)
+    {
+        Writer(shape, start, count, steps, engineParams, filename);
+    }
+
+    std::this_thread::sleep_for(std::chrono::milliseconds(1));
+
+    if (mpiGroup == 1)
+    {
+        Reader(shape, start, count, steps, engineParams, filename);
+    }
+
+    MPI_Barrier(MPI_COMM_WORLD);
+}
+
+int main(int argc, char **argv)
+{
+    MPI_Init(&argc, &argv);
+    int worldRank, worldSize;
+    MPI_Comm_rank(MPI_COMM_WORLD, &worldRank);
+    MPI_Comm_size(MPI_COMM_WORLD, &worldSize);
+    ::testing::InitGoogleTest(&argc, argv);
+    int result = RUN_ALL_TESTS();
+
+    MPI_Finalize();
+    return result;
+}
diff --git a/testing/adios2/engine/ssc/TestSscMoreReadersThanWriters.cpp b/testing/adios2/engine/ssc/TestSscMoreReadersThanWriters.cpp
index f5bbe80f6f..fec7c7b8f5 100644
--- a/testing/adios2/engine/ssc/TestSscMoreReadersThanWriters.cpp
+++ b/testing/adios2/engine/ssc/TestSscMoreReadersThanWriters.cpp
@@ -27,8 +27,9 @@ void Writer(const Dims &shape, const Dims &start, const Dims &count,
             const size_t steps, const adios2::Params &engineParams,
             const std::string &name)
 {
-    size_t datasize = std::accumulate(count.begin(), count.end(), 1,
-                                      std::multiplies());
+    size_t datasize =
+        std::accumulate(count.begin(), count.end(), static_cast(1),
+                        std::multiplies());
     adios2::ADIOS adios(mpiComm);
     adios2::IO dataManIO = adios.DeclareIO("WAN");
     dataManIO.SetEngine("ssc");
@@ -63,10 +64,11 @@ void Writer(const Dims &shape, const Dims &start, const Dims &count,
     auto bpDComplexes = dataManIO.DefineVariable>(
         "bpDComplexes", shape, start, count);
     dataManIO.DefineAttribute("AttInt", 110);
-    adios2::Engine dataManWriter = dataManIO.Open(name, adios2::Mode::Write);
+    adios2::Engine engine = dataManIO.Open(name, adios2::Mode::Write);
+    engine.LockWriterDefinitions();
     for (int i = 0; i < steps; ++i)
     {
-        dataManWriter.BeginStep();
+        engine.BeginStep();
         GenData(myChars, i, start, count, shape);
         GenData(myUChars, i, start, count, shape);
         GenData(myShorts, i, start, count, shape);
@@ -77,20 +79,19 @@ void Writer(const Dims &shape, const Dims &start, const Dims &count,
         GenData(myDoubles, i, start, count, shape);
         GenData(myComplexes, i, start, count, shape);
         GenData(myDComplexes, i, start, count, shape);
-        dataManWriter.Put(bpChars, myChars.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpUChars, myUChars.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpShorts, myShorts.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpUShorts, myUShorts.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpInts, myInts.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpUInts, myUInts.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpFloats, myFloats.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpDoubles, myDoubles.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpComplexes, myComplexes.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpDComplexes, myDComplexes.data(),
-                          adios2::Mode::Sync);
-        dataManWriter.EndStep();
+        engine.Put(bpChars, myChars.data(), adios2::Mode::Sync);
+        engine.Put(bpUChars, myUChars.data(), adios2::Mode::Sync);
+        engine.Put(bpShorts, myShorts.data(), adios2::Mode::Sync);
+        engine.Put(bpUShorts, myUShorts.data(), adios2::Mode::Sync);
+        engine.Put(bpInts, myInts.data(), adios2::Mode::Sync);
+        engine.Put(bpUInts, myUInts.data(), adios2::Mode::Sync);
+        engine.Put(bpFloats, myFloats.data(), adios2::Mode::Sync);
+        engine.Put(bpDoubles, myDoubles.data(), adios2::Mode::Sync);
+        engine.Put(bpComplexes, myComplexes.data(), adios2::Mode::Sync);
+        engine.Put(bpDComplexes, myDComplexes.data(), adios2::Mode::Sync);
+        engine.EndStep();
     }
-    dataManWriter.Close();
+    engine.Close();
 }
 
 void Reader(const Dims &shape, const Dims &start, const Dims &count,
@@ -101,10 +102,12 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count,
     adios2::IO dataManIO = adios.DeclareIO("Test");
     dataManIO.SetEngine("ssc");
     dataManIO.SetParameters(engineParams);
-    adios2::Engine dataManReader = dataManIO.Open(name, adios2::Mode::Read);
+    adios2::Engine engine = dataManIO.Open(name, adios2::Mode::Read);
+    engine.LockReaderSelections();
 
-    size_t datasize = std::accumulate(shape.begin(), shape.end(), 1,
-                                      std::multiplies());
+    size_t datasize =
+        std::accumulate(shape.begin(), shape.end(), static_cast(1),
+                        std::multiplies());
     std::vector myChars(datasize);
     std::vector myUChars(datasize);
     std::vector myShorts(datasize);
@@ -118,12 +121,12 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count,
 
     while (true)
     {
-        adios2::StepStatus status = dataManReader.BeginStep(StepMode::Read, 5);
+        adios2::StepStatus status = engine.BeginStep(StepMode::Read, 5);
         if (status == adios2::StepStatus::OK)
         {
             const auto &vars = dataManIO.AvailableVariables();
             ASSERT_EQ(vars.size(), 10);
-            size_t currentStep = dataManReader.CurrentStep();
+            size_t currentStep = engine.CurrentStep();
             adios2::Variable bpChars =
                 dataManIO.InquireVariable("bpChars");
             adios2::Variable bpUChars =
@@ -145,18 +148,16 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count,
             adios2::Variable> bpDComplexes =
                 dataManIO.InquireVariable>("bpDComplexes");
 
-            dataManReader.Get(bpChars, myChars.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpUChars, myUChars.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpShorts, myShorts.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpUShorts, myUShorts.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpInts, myInts.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpUInts, myUInts.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpFloats, myFloats.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpDoubles, myDoubles.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpComplexes, myComplexes.data(),
-                              adios2::Mode::Sync);
-            dataManReader.Get(bpDComplexes, myDComplexes.data(),
-                              adios2::Mode::Sync);
+            engine.Get(bpChars, myChars.data(), adios2::Mode::Sync);
+            engine.Get(bpUChars, myUChars.data(), adios2::Mode::Sync);
+            engine.Get(bpShorts, myShorts.data(), adios2::Mode::Sync);
+            engine.Get(bpUShorts, myUShorts.data(), adios2::Mode::Sync);
+            engine.Get(bpInts, myInts.data(), adios2::Mode::Sync);
+            engine.Get(bpUInts, myUInts.data(), adios2::Mode::Sync);
+            engine.Get(bpFloats, myFloats.data(), adios2::Mode::Sync);
+            engine.Get(bpDoubles, myDoubles.data(), adios2::Mode::Sync);
+            engine.Get(bpComplexes, myComplexes.data(), adios2::Mode::Sync);
+            engine.Get(bpDComplexes, myDComplexes.data(), adios2::Mode::Sync);
             VerifyData(myChars.data(), currentStep, Dims(shape.size(), 0),
                        shape, shape, mpiRank);
             VerifyData(myUChars.data(), currentStep, Dims(shape.size(), 0),
@@ -177,7 +178,7 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count,
                        shape, shape, mpiRank);
             VerifyData(myDComplexes.data(), currentStep, Dims(shape.size(), 0),
                        shape, shape, mpiRank);
-            dataManReader.EndStep();
+            engine.EndStep();
         }
         else if (status == adios2::StepStatus::EndOfStream)
         {
@@ -187,7 +188,7 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count,
             break;
         }
     }
-    dataManReader.Close();
+    engine.Close();
 }
 
 TEST_F(SscEngineTest, TestSscMoreReadersThanWriters)
diff --git a/testing/adios2/engine/ssc/TestSscMoreWritersThanReaders.cpp b/testing/adios2/engine/ssc/TestSscMoreWritersThanReaders.cpp
index d3d2a9e923..8ffa71cd14 100644
--- a/testing/adios2/engine/ssc/TestSscMoreWritersThanReaders.cpp
+++ b/testing/adios2/engine/ssc/TestSscMoreWritersThanReaders.cpp
@@ -25,8 +25,9 @@ void Writer(const Dims &shape, const Dims &start, const Dims &count,
             const size_t steps, const adios2::Params &engineParams,
             const std::string &name)
 {
-    size_t datasize = std::accumulate(count.begin(), count.end(), 1,
-                                      std::multiplies());
+    size_t datasize =
+        std::accumulate(count.begin(), count.end(), static_cast(1),
+                        std::multiplies());
     adios2::ADIOS adios(mpiComm);
     adios2::IO dataManIO = adios.DeclareIO("WAN");
     dataManIO.SetEngine("ssc");
@@ -61,10 +62,11 @@ void Writer(const Dims &shape, const Dims &start, const Dims &count,
     auto bpDComplexes = dataManIO.DefineVariable>(
         "bpDComplexes", shape, start, count);
     dataManIO.DefineAttribute("AttInt", 110);
-    adios2::Engine dataManWriter = dataManIO.Open(name, adios2::Mode::Write);
+    adios2::Engine engine = dataManIO.Open(name, adios2::Mode::Write);
+    engine.LockWriterDefinitions();
     for (int i = 0; i < steps; ++i)
     {
-        dataManWriter.BeginStep();
+        engine.BeginStep();
         GenData(myChars, i, start, count, shape);
         GenData(myUChars, i, start, count, shape);
         GenData(myShorts, i, start, count, shape);
@@ -75,20 +77,19 @@ void Writer(const Dims &shape, const Dims &start, const Dims &count,
         GenData(myDoubles, i, start, count, shape);
         GenData(myComplexes, i, start, count, shape);
         GenData(myDComplexes, i, start, count, shape);
-        dataManWriter.Put(bpChars, myChars.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpUChars, myUChars.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpShorts, myShorts.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpUShorts, myUShorts.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpInts, myInts.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpUInts, myUInts.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpFloats, myFloats.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpDoubles, myDoubles.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpComplexes, myComplexes.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpDComplexes, myDComplexes.data(),
-                          adios2::Mode::Sync);
-        dataManWriter.EndStep();
+        engine.Put(bpChars, myChars.data(), adios2::Mode::Sync);
+        engine.Put(bpUChars, myUChars.data(), adios2::Mode::Sync);
+        engine.Put(bpShorts, myShorts.data(), adios2::Mode::Sync);
+        engine.Put(bpUShorts, myUShorts.data(), adios2::Mode::Sync);
+        engine.Put(bpInts, myInts.data(), adios2::Mode::Sync);
+        engine.Put(bpUInts, myUInts.data(), adios2::Mode::Sync);
+        engine.Put(bpFloats, myFloats.data(), adios2::Mode::Sync);
+        engine.Put(bpDoubles, myDoubles.data(), adios2::Mode::Sync);
+        engine.Put(bpComplexes, myComplexes.data(), adios2::Mode::Sync);
+        engine.Put(bpDComplexes, myDComplexes.data(), adios2::Mode::Sync);
+        engine.EndStep();
     }
-    dataManWriter.Close();
+    engine.Close();
 }
 
 void Reader(const Dims &shape, const Dims &start, const Dims &count,
@@ -99,10 +100,12 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count,
     adios2::IO dataManIO = adios.DeclareIO("Test");
     dataManIO.SetEngine("ssc");
     dataManIO.SetParameters(engineParams);
-    adios2::Engine dataManReader = dataManIO.Open(name, adios2::Mode::Read);
+    adios2::Engine engine = dataManIO.Open(name, adios2::Mode::Read);
+    engine.LockReaderSelections();
 
-    size_t datasize = std::accumulate(shape.begin(), shape.end(), 1,
-                                      std::multiplies());
+    size_t datasize =
+        std::accumulate(shape.begin(), shape.end(), static_cast(1),
+                        std::multiplies());
     std::vector myChars(datasize);
     std::vector myUChars(datasize);
     std::vector myShorts(datasize);
@@ -116,12 +119,12 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count,
 
     while (true)
     {
-        adios2::StepStatus status = dataManReader.BeginStep(StepMode::Read, 5);
+        adios2::StepStatus status = engine.BeginStep(StepMode::Read, 5);
         if (status == adios2::StepStatus::OK)
         {
             const auto &vars = dataManIO.AvailableVariables();
             ASSERT_EQ(vars.size(), 10);
-            size_t currentStep = dataManReader.CurrentStep();
+            size_t currentStep = engine.CurrentStep();
             adios2::Variable bpChars =
                 dataManIO.InquireVariable("bpChars");
             adios2::Variable bpUChars =
@@ -143,18 +146,16 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count,
             adios2::Variable> bpDComplexes =
                 dataManIO.InquireVariable>("bpDComplexes");
 
-            dataManReader.Get(bpChars, myChars.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpUChars, myUChars.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpShorts, myShorts.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpUShorts, myUShorts.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpInts, myInts.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpUInts, myUInts.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpFloats, myFloats.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpDoubles, myDoubles.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpComplexes, myComplexes.data(),
-                              adios2::Mode::Sync);
-            dataManReader.Get(bpDComplexes, myDComplexes.data(),
-                              adios2::Mode::Sync);
+            engine.Get(bpChars, myChars.data(), adios2::Mode::Sync);
+            engine.Get(bpUChars, myUChars.data(), adios2::Mode::Sync);
+            engine.Get(bpShorts, myShorts.data(), adios2::Mode::Sync);
+            engine.Get(bpUShorts, myUShorts.data(), adios2::Mode::Sync);
+            engine.Get(bpInts, myInts.data(), adios2::Mode::Sync);
+            engine.Get(bpUInts, myUInts.data(), adios2::Mode::Sync);
+            engine.Get(bpFloats, myFloats.data(), adios2::Mode::Sync);
+            engine.Get(bpDoubles, myDoubles.data(), adios2::Mode::Sync);
+            engine.Get(bpComplexes, myComplexes.data(), adios2::Mode::Sync);
+            engine.Get(bpDComplexes, myDComplexes.data(), adios2::Mode::Sync);
             VerifyData(myChars.data(), currentStep, Dims(shape.size(), 0),
                        shape, shape, mpiRank);
             VerifyData(myUChars.data(), currentStep, Dims(shape.size(), 0),
@@ -175,7 +176,7 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count,
                        shape, shape, mpiRank);
             VerifyData(myDComplexes.data(), currentStep, Dims(shape.size(), 0),
                        shape, shape, mpiRank);
-            dataManReader.EndStep();
+            engine.EndStep();
         }
         else if (status == adios2::StepStatus::EndOfStream)
         {
@@ -185,7 +186,7 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count,
             break;
         }
     }
-    dataManReader.Close();
+    engine.Close();
 }
 
 TEST_F(SscEngineTest, TestSscMoreWritersThanReaders)
diff --git a/testing/adios2/engine/ssc/TestSscMultiApp.cpp b/testing/adios2/engine/ssc/TestSscMultiApp.cpp
deleted file mode 100644
index 60886ba6d1..0000000000
--- a/testing/adios2/engine/ssc/TestSscMultiApp.cpp
+++ /dev/null
@@ -1,443 +0,0 @@
-/*
- * Distributed under the OSI-approved Apache License, Version 2.0.  See
- * accompanying file Copyright.txt for details.
- */
-
-#include "TestSscCommon.h"
-#include 
-#include 
-#include 
-#include 
-#include 
-
-using namespace adios2;
-int mpiRank = 0;
-int mpiSize = 1;
-int mpiGroup;
-MPI_Comm mpiComm;
-
-class SscEngineTest : public ::testing::Test
-{
-public:
-    SscEngineTest() = default;
-};
-
-void Writer1(const Dims &shape, const Dims &start, const Dims &count,
-             const size_t steps, const adios2::Params &engineParams,
-             const std::string &name)
-{
-    size_t datasize = std::accumulate(count.begin(), count.end(), 1,
-                                      std::multiplies());
-    adios2::ADIOS adios(mpiComm);
-    adios2::IO dataManIO = adios.DeclareIO("WAN");
-    dataManIO.SetEngine("ssc");
-    dataManIO.SetParameters(engineParams);
-    std::vector myChars(datasize);
-    std::vector myUChars(datasize);
-    std::vector myShorts(datasize);
-    std::vector myUShorts(datasize);
-    std::vector myInts(datasize);
-    std::vector myUInts(datasize);
-    std::vector myFloats(datasize);
-    std::vector myDoubles(datasize);
-    std::vector> myComplexes(datasize);
-    std::vector> myDComplexes(datasize);
-    auto bpChars =
-        dataManIO.DefineVariable("bpChars", shape, start, count);
-    auto bpUChars = dataManIO.DefineVariable("bpUChars", shape,
-                                                            start, count);
-    auto bpShorts =
-        dataManIO.DefineVariable("bpShorts", shape, start, count);
-    auto bpUShorts = dataManIO.DefineVariable(
-        "bpUShorts", shape, start, count);
-    auto bpInts = dataManIO.DefineVariable("bpInts", shape, start, count);
-    auto bpUInts =
-        dataManIO.DefineVariable("bpUInts", shape, start, count);
-    auto bpFloats =
-        dataManIO.DefineVariable("bpFloats", shape, start, count);
-    auto bpDoubles =
-        dataManIO.DefineVariable("bpDoubles", shape, start, count);
-    auto bpComplexes = dataManIO.DefineVariable>(
-        "bpComplexes", shape, start, count);
-    auto bpDComplexes = dataManIO.DefineVariable>(
-        "bpDComplexes", shape, start, count);
-    dataManIO.DefineAttribute("AttInt", 110);
-    adios2::Engine dataManWriter = dataManIO.Open(name, adios2::Mode::Write);
-    for (int i = 0; i < steps; ++i)
-    {
-        dataManWriter.BeginStep();
-        GenData(myChars, i, start, count, shape);
-        GenData(myUChars, i, start, count, shape);
-        GenData(myShorts, i, start, count, shape);
-        GenData(myUShorts, i, start, count, shape);
-        GenData(myInts, i, start, count, shape);
-        GenData(myUInts, i, start, count, shape);
-        GenData(myFloats, i, start, count, shape);
-        GenData(myDoubles, i, start, count, shape);
-        GenData(myComplexes, i, start, count, shape);
-        GenData(myDComplexes, i, start, count, shape);
-        dataManWriter.Put(bpChars, myChars.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpUChars, myUChars.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpShorts, myShorts.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpUShorts, myUShorts.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpInts, myInts.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpUInts, myUInts.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpFloats, myFloats.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpDoubles, myDoubles.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpComplexes, myComplexes.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpDComplexes, myDComplexes.data(),
-                          adios2::Mode::Sync);
-        dataManWriter.EndStep();
-    }
-    dataManWriter.Close();
-}
-
-void Writer2(const Dims &shape, const Dims &start, const Dims &count,
-             const size_t steps, const adios2::Params &engineParams,
-             const std::string &name)
-{
-    size_t datasize = std::accumulate(count.begin(), count.end(), 1,
-                                      std::multiplies());
-    adios2::ADIOS adios(mpiComm);
-    adios2::IO dataManIO = adios.DeclareIO("WAN");
-    dataManIO.SetEngine("ssc");
-    dataManIO.SetParameters(engineParams);
-    std::vector myChars(datasize);
-    std::vector myUChars(datasize);
-    std::vector myShorts(datasize);
-    std::vector myUShorts(datasize);
-    std::vector myInts(datasize);
-    std::vector myUInts(datasize);
-    std::vector myFloats(datasize);
-    std::vector myDoubles(datasize);
-    std::vector> myComplexes(datasize);
-    std::vector> myDComplexes(datasize);
-    auto bpChars =
-        dataManIO.DefineVariable("bpChars2", shape, start, count);
-    auto bpUChars = dataManIO.DefineVariable("bpUChars2", shape,
-                                                            start, count);
-    auto bpShorts =
-        dataManIO.DefineVariable("bpShorts2", shape, start, count);
-    auto bpUShorts = dataManIO.DefineVariable(
-        "bpUShorts2", shape, start, count);
-    auto bpInts = dataManIO.DefineVariable("bpInts2", shape, start, count);
-    auto bpUInts =
-        dataManIO.DefineVariable("bpUInts2", shape, start, count);
-    auto bpFloats =
-        dataManIO.DefineVariable("bpFloats2", shape, start, count);
-    auto bpDoubles =
-        dataManIO.DefineVariable("bpDoubles2", shape, start, count);
-    auto bpComplexes = dataManIO.DefineVariable>(
-        "bpComplexes2", shape, start, count);
-    auto bpDComplexes = dataManIO.DefineVariable>(
-        "bpDComplexes2", shape, start, count);
-    dataManIO.DefineAttribute("AttInt2", 111);
-    adios2::Engine dataManWriter = dataManIO.Open(name, adios2::Mode::Write);
-    for (int i = 0; i < steps; ++i)
-    {
-        dataManWriter.BeginStep();
-        GenData(myChars, i, start, count, shape);
-        GenData(myUChars, i, start, count, shape);
-        GenData(myShorts, i, start, count, shape);
-        GenData(myUShorts, i, start, count, shape);
-        GenData(myInts, i, start, count, shape);
-        GenData(myUInts, i, start, count, shape);
-        GenData(myFloats, i, start, count, shape);
-        GenData(myDoubles, i, start, count, shape);
-        GenData(myComplexes, i, start, count, shape);
-        GenData(myDComplexes, i, start, count, shape);
-        dataManWriter.Put(bpChars, myChars.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpUChars, myUChars.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpShorts, myShorts.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpUShorts, myUShorts.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpInts, myInts.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpUInts, myUInts.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpFloats, myFloats.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpDoubles, myDoubles.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpComplexes, myComplexes.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpDComplexes, myDComplexes.data(),
-                          adios2::Mode::Sync);
-        dataManWriter.EndStep();
-    }
-    dataManWriter.Close();
-}
-
-void Reader1(const Dims &shape, const Dims &start, const Dims &count,
-             const size_t steps, const adios2::Params &engineParams,
-             const std::string &name)
-{
-    adios2::ADIOS adios(mpiComm);
-    adios2::IO dataManIO = adios.DeclareIO("Test");
-    dataManIO.SetEngine("ssc");
-    dataManIO.SetParameters(engineParams);
-    adios2::Engine dataManReader = dataManIO.Open(name, adios2::Mode::Read);
-
-    size_t datasize = std::accumulate(shape.begin(), shape.end(), 1,
-                                      std::multiplies());
-    std::vector myChars(datasize);
-    std::vector myUChars(datasize);
-    std::vector myShorts(datasize);
-    std::vector myUShorts(datasize);
-    std::vector myInts(datasize);
-    std::vector myUInts(datasize);
-    std::vector myFloats(datasize);
-    std::vector myDoubles(datasize);
-    std::vector> myComplexes(datasize);
-    std::vector> myDComplexes(datasize);
-
-    while (true)
-    {
-        adios2::StepStatus status = dataManReader.BeginStep(StepMode::Read, 5);
-        if (status == adios2::StepStatus::OK)
-        {
-            const auto &vars = dataManIO.AvailableVariables();
-            ASSERT_EQ(vars.size(), 20);
-            size_t currentStep = dataManReader.CurrentStep();
-            adios2::Variable bpChars =
-                dataManIO.InquireVariable("bpChars");
-            adios2::Variable bpUChars =
-                dataManIO.InquireVariable("bpUChars");
-            adios2::Variable bpShorts =
-                dataManIO.InquireVariable("bpShorts");
-            adios2::Variable bpUShorts =
-                dataManIO.InquireVariable("bpUShorts");
-            adios2::Variable bpInts =
-                dataManIO.InquireVariable("bpInts");
-            adios2::Variable bpUInts =
-                dataManIO.InquireVariable("bpUInts2");
-            adios2::Variable bpFloats =
-                dataManIO.InquireVariable("bpFloats2");
-            adios2::Variable bpDoubles =
-                dataManIO.InquireVariable("bpDoubles2");
-            adios2::Variable> bpComplexes =
-                dataManIO.InquireVariable>("bpComplexes2");
-            adios2::Variable> bpDComplexes =
-                dataManIO.InquireVariable>(
-                    "bpDComplexes2");
-
-            dataManReader.Get(bpChars, myChars.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpUChars, myUChars.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpShorts, myShorts.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpUShorts, myUShorts.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpInts, myInts.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpUInts, myUInts.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpFloats, myFloats.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpDoubles, myDoubles.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpComplexes, myComplexes.data(),
-                              adios2::Mode::Sync);
-            dataManReader.Get(bpDComplexes, myDComplexes.data(),
-                              adios2::Mode::Sync);
-            VerifyData(myChars.data(), currentStep, Dims(shape.size(), 0),
-                       shape, shape, mpiRank);
-            VerifyData(myUChars.data(), currentStep, Dims(shape.size(), 0),
-                       shape, shape, mpiRank);
-            VerifyData(myShorts.data(), currentStep, Dims(shape.size(), 0),
-                       shape, shape, mpiRank);
-            VerifyData(myUShorts.data(), currentStep, Dims(shape.size(), 0),
-                       shape, shape, mpiRank);
-            VerifyData(myInts.data(), currentStep, Dims(shape.size(), 0), shape,
-                       shape, mpiRank);
-            VerifyData(myUInts.data(), currentStep, Dims(shape.size(), 0),
-                       shape, shape, mpiRank);
-            VerifyData(myFloats.data(), currentStep, Dims(shape.size(), 0),
-                       shape, shape, mpiRank);
-            VerifyData(myDoubles.data(), currentStep, Dims(shape.size(), 0),
-                       shape, shape, mpiRank);
-            VerifyData(myComplexes.data(), currentStep, Dims(shape.size(), 0),
-                       shape, shape, mpiRank);
-            VerifyData(myDComplexes.data(), currentStep, Dims(shape.size(), 0),
-                       shape, shape, mpiRank);
-            dataManReader.EndStep();
-        }
-        else if (status == adios2::StepStatus::EndOfStream)
-        {
-            std::cout << "[Rank " + std::to_string(mpiRank) +
-                             "] SscTest reader end of stream!"
-                      << std::endl;
-            break;
-        }
-    }
-    dataManReader.Close();
-}
-
-void Reader2(const Dims &shape, const Dims &start, const Dims &count,
-             const size_t steps, const adios2::Params &engineParams,
-             const std::string &name)
-{
-    adios2::ADIOS adios(mpiComm);
-    adios2::IO dataManIO = adios.DeclareIO("Test");
-    dataManIO.SetEngine("ssc");
-    dataManIO.SetParameters(engineParams);
-    adios2::Engine dataManReader = dataManIO.Open(name, adios2::Mode::Read);
-
-    size_t datasize = std::accumulate(shape.begin(), shape.end(), 1,
-                                      std::multiplies());
-    std::vector myChars(datasize);
-    std::vector myUChars(datasize);
-    std::vector myShorts(datasize);
-    std::vector myUShorts(datasize);
-    std::vector myInts(datasize);
-    std::vector myUInts(datasize);
-    std::vector myFloats(datasize);
-    std::vector myDoubles(datasize);
-    std::vector> myComplexes(datasize);
-    std::vector> myDComplexes(datasize);
-
-    while (true)
-    {
-        adios2::StepStatus status = dataManReader.BeginStep(StepMode::Read, 5);
-        if (status == adios2::StepStatus::OK)
-        {
-            const auto &vars = dataManIO.AvailableVariables();
-            ASSERT_EQ(vars.size(), 20);
-            size_t currentStep = dataManReader.CurrentStep();
-            adios2::Variable bpChars =
-                dataManIO.InquireVariable("bpChars2");
-            adios2::Variable bpUChars =
-                dataManIO.InquireVariable("bpUChars2");
-            adios2::Variable bpShorts =
-                dataManIO.InquireVariable("bpShorts2");
-            adios2::Variable bpUShorts =
-                dataManIO.InquireVariable("bpUShorts2");
-            adios2::Variable bpInts =
-                dataManIO.InquireVariable("bpInts2");
-            adios2::Variable bpUInts =
-                dataManIO.InquireVariable("bpUInts");
-            adios2::Variable bpFloats =
-                dataManIO.InquireVariable("bpFloats");
-            adios2::Variable bpDoubles =
-                dataManIO.InquireVariable("bpDoubles");
-            adios2::Variable> bpComplexes =
-                dataManIO.InquireVariable>("bpComplexes");
-            adios2::Variable> bpDComplexes =
-                dataManIO.InquireVariable>("bpDComplexes");
-
-            dataManReader.Get(bpChars, myChars.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpUChars, myUChars.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpShorts, myShorts.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpUShorts, myUShorts.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpInts, myInts.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpUInts, myUInts.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpFloats, myFloats.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpDoubles, myDoubles.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpComplexes, myComplexes.data(),
-                              adios2::Mode::Sync);
-            dataManReader.Get(bpDComplexes, myDComplexes.data(),
-                              adios2::Mode::Sync);
-            VerifyData(myChars.data(), currentStep, Dims(shape.size(), 0),
-                       shape, shape, mpiRank);
-            VerifyData(myUChars.data(), currentStep, Dims(shape.size(), 0),
-                       shape, shape, mpiRank);
-            VerifyData(myShorts.data(), currentStep, Dims(shape.size(), 0),
-                       shape, shape, mpiRank);
-            VerifyData(myUShorts.data(), currentStep, Dims(shape.size(), 0),
-                       shape, shape, mpiRank);
-            VerifyData(myInts.data(), currentStep, Dims(shape.size(), 0), shape,
-                       shape, mpiRank);
-            VerifyData(myUInts.data(), currentStep, Dims(shape.size(), 0),
-                       shape, shape, mpiRank);
-            VerifyData(myFloats.data(), currentStep, Dims(shape.size(), 0),
-                       shape, shape, mpiRank);
-            VerifyData(myDoubles.data(), currentStep, Dims(shape.size(), 0),
-                       shape, shape, mpiRank);
-            VerifyData(myComplexes.data(), currentStep, Dims(shape.size(), 0),
-                       shape, shape, mpiRank);
-            VerifyData(myDComplexes.data(), currentStep, Dims(shape.size(), 0),
-                       shape, shape, mpiRank);
-            dataManReader.EndStep();
-        }
-        else if (status == adios2::StepStatus::EndOfStream)
-        {
-            std::cout << "[Rank " + std::to_string(mpiRank) +
-                             "] SscTest reader end of stream!"
-                      << std::endl;
-            break;
-        }
-    }
-    dataManReader.Close();
-}
-
-TEST_F(SscEngineTest, TestSscMultiApp)
-{
-    std::string filename = "TestSscMultiApp";
-    adios2::Params engineParams = {{"RendezvousAppCount", "4"}};
-
-    int worldRank, worldSize;
-    Dims start, count, shape;
-    MPI_Comm_rank(MPI_COMM_WORLD, &worldRank);
-    MPI_Comm_size(MPI_COMM_WORLD, &worldSize);
-    if (worldSize < 8)
-    {
-        return;
-    }
-    if (worldRank == 0 or worldRank == 1)
-    {
-        mpiGroup = 0;
-    }
-    else if (worldRank == 2 or worldRank == 3)
-    {
-        mpiGroup = 1;
-    }
-    else if (worldRank == 4 or worldRank == 5)
-    {
-        mpiGroup = 2;
-    }
-    else if (worldRank == 6 or worldRank == 7)
-    {
-        mpiGroup = 3;
-    }
-
-    MPI_Comm_split(MPI_COMM_WORLD, mpiGroup, worldRank, &mpiComm);
-
-    MPI_Comm_rank(mpiComm, &mpiRank);
-    MPI_Comm_size(mpiComm, &mpiSize);
-
-    size_t steps = 20;
-
-    if (mpiGroup == 0)
-    {
-        shape = {2, 10};
-        start = {(size_t)mpiRank, 0};
-        count = {1, 10};
-        Writer1(shape, start, count, steps, engineParams, filename);
-    }
-
-    if (mpiGroup == 1)
-    {
-        shape = {2, 10};
-        start = {0, 0};
-        count = shape;
-        Reader1(shape, start, shape, steps, engineParams, filename);
-    }
-
-    if (mpiGroup == 2)
-    {
-        shape = {2, 10};
-        start = {(size_t)mpiRank, 0};
-        count = {1, 10};
-        Writer2(shape, start, count, steps, engineParams, filename);
-    }
-
-    if (mpiGroup == 3)
-    {
-        shape = {2, 10};
-        start = {0, 0};
-        count = shape;
-        Reader2(shape, start, shape, steps, engineParams, filename);
-    }
-
-    MPI_Barrier(MPI_COMM_WORLD);
-}
-
-int main(int argc, char **argv)
-{
-    MPI_Init(&argc, &argv);
-    int worldRank, worldSize;
-    MPI_Comm_rank(MPI_COMM_WORLD, &worldRank);
-    MPI_Comm_size(MPI_COMM_WORLD, &worldSize);
-    ::testing::InitGoogleTest(&argc, argv);
-    int result = RUN_ALL_TESTS();
-
-    MPI_Finalize();
-    return result;
-}
diff --git a/testing/adios2/engine/ssc/TestSscNoAttributes.cpp b/testing/adios2/engine/ssc/TestSscNoAttributes.cpp
index 1e99ce116b..fe10e59c5c 100644
--- a/testing/adios2/engine/ssc/TestSscNoAttributes.cpp
+++ b/testing/adios2/engine/ssc/TestSscNoAttributes.cpp
@@ -25,8 +25,9 @@ void Writer(const Dims &shape, const Dims &start, const Dims &count,
             const size_t steps, const adios2::Params &engineParams,
             const std::string &name)
 {
-    size_t datasize = std::accumulate(count.begin(), count.end(), 1,
-                                      std::multiplies());
+    size_t datasize =
+        std::accumulate(count.begin(), count.end(), static_cast(1),
+                        std::multiplies());
     adios2::ADIOS adios(mpiComm);
     adios2::IO dataManIO = adios.DeclareIO("staging");
     dataManIO.SetEngine("ssc");
@@ -60,10 +61,11 @@ void Writer(const Dims &shape, const Dims &start, const Dims &count,
         "bpComplexes", shape, start, count);
     auto bpDComplexes = dataManIO.DefineVariable>(
         "bpDComplexes", shape, start, count);
-    adios2::Engine dataManWriter = dataManIO.Open(name, adios2::Mode::Write);
+    adios2::Engine engine = dataManIO.Open(name, adios2::Mode::Write);
+    engine.LockWriterDefinitions();
     for (int i = 0; i < steps; ++i)
     {
-        dataManWriter.BeginStep();
+        engine.BeginStep();
         GenData(myChars, i, start, count, shape);
         GenData(myUChars, i, start, count, shape);
         GenData(myShorts, i, start, count, shape);
@@ -74,20 +76,19 @@ void Writer(const Dims &shape, const Dims &start, const Dims &count,
         GenData(myDoubles, i, start, count, shape);
         GenData(myComplexes, i, start, count, shape);
         GenData(myDComplexes, i, start, count, shape);
-        dataManWriter.Put(bpChars, myChars.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpUChars, myUChars.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpShorts, myShorts.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpUShorts, myUShorts.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpInts, myInts.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpUInts, myUInts.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpFloats, myFloats.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpDoubles, myDoubles.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpComplexes, myComplexes.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpDComplexes, myDComplexes.data(),
-                          adios2::Mode::Sync);
-        dataManWriter.EndStep();
+        engine.Put(bpChars, myChars.data(), adios2::Mode::Sync);
+        engine.Put(bpUChars, myUChars.data(), adios2::Mode::Sync);
+        engine.Put(bpShorts, myShorts.data(), adios2::Mode::Sync);
+        engine.Put(bpUShorts, myUShorts.data(), adios2::Mode::Sync);
+        engine.Put(bpInts, myInts.data(), adios2::Mode::Sync);
+        engine.Put(bpUInts, myUInts.data(), adios2::Mode::Sync);
+        engine.Put(bpFloats, myFloats.data(), adios2::Mode::Sync);
+        engine.Put(bpDoubles, myDoubles.data(), adios2::Mode::Sync);
+        engine.Put(bpComplexes, myComplexes.data(), adios2::Mode::Sync);
+        engine.Put(bpDComplexes, myDComplexes.data(), adios2::Mode::Sync);
+        engine.EndStep();
     }
-    dataManWriter.Close();
+    engine.Close();
 }
 
 void Reader(const Dims &shape, const Dims &start, const Dims &count,
@@ -98,10 +99,12 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count,
     adios2::IO dataManIO = adios.DeclareIO("staging");
     dataManIO.SetEngine("ssc");
     dataManIO.SetParameters(engineParams);
-    adios2::Engine dataManReader = dataManIO.Open(name, adios2::Mode::Read);
+    adios2::Engine engine = dataManIO.Open(name, adios2::Mode::Read);
+    engine.LockReaderSelections();
 
-    size_t datasize = std::accumulate(count.begin(), count.end(), 1,
-                                      std::multiplies());
+    size_t datasize =
+        std::accumulate(count.begin(), count.end(), static_cast(1),
+                        std::multiplies());
     std::vector myChars(datasize);
     std::vector myUChars(datasize);
     std::vector myShorts(datasize);
@@ -115,12 +118,12 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count,
 
     while (true)
     {
-        adios2::StepStatus status = dataManReader.BeginStep(StepMode::Read, 5);
+        adios2::StepStatus status = engine.BeginStep(StepMode::Read, 5);
         if (status == adios2::StepStatus::OK)
         {
             const auto &vars = dataManIO.AvailableVariables();
             ASSERT_EQ(vars.size(), 10);
-            size_t currentStep = dataManReader.CurrentStep();
+            size_t currentStep = engine.CurrentStep();
             //            ASSERT_EQ(i, currentStep);
             adios2::Variable bpChars =
                 dataManIO.InquireVariable("bpChars");
@@ -154,18 +157,16 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count,
             bpComplexes.SetSelection({start, count});
             bpDComplexes.SetSelection({start, count});
 
-            dataManReader.Get(bpChars, myChars.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpUChars, myUChars.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpShorts, myShorts.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpUShorts, myUShorts.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpInts, myInts.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpUInts, myUInts.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpFloats, myFloats.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpDoubles, myDoubles.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpComplexes, myComplexes.data(),
-                              adios2::Mode::Sync);
-            dataManReader.Get(bpDComplexes, myDComplexes.data(),
-                              adios2::Mode::Sync);
+            engine.Get(bpChars, myChars.data(), adios2::Mode::Sync);
+            engine.Get(bpUChars, myUChars.data(), adios2::Mode::Sync);
+            engine.Get(bpShorts, myShorts.data(), adios2::Mode::Sync);
+            engine.Get(bpUShorts, myUShorts.data(), adios2::Mode::Sync);
+            engine.Get(bpInts, myInts.data(), adios2::Mode::Sync);
+            engine.Get(bpUInts, myUInts.data(), adios2::Mode::Sync);
+            engine.Get(bpFloats, myFloats.data(), adios2::Mode::Sync);
+            engine.Get(bpDoubles, myDoubles.data(), adios2::Mode::Sync);
+            engine.Get(bpComplexes, myComplexes.data(), adios2::Mode::Sync);
+            engine.Get(bpDComplexes, myDComplexes.data(), adios2::Mode::Sync);
             VerifyData(myChars.data(), currentStep, start, count, shape,
                        mpiRank);
             VerifyData(myUChars.data(), currentStep, start, count, shape,
@@ -186,7 +187,7 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count,
                        mpiRank);
             VerifyData(myDComplexes.data(), currentStep, start, count, shape,
                        mpiRank);
-            dataManReader.EndStep();
+            engine.EndStep();
         }
         else if (status == adios2::StepStatus::EndOfStream)
         {
@@ -196,7 +197,7 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count,
             break;
         }
     }
-    dataManReader.Close();
+    engine.Close();
 }
 
 TEST_F(SscEngineTest, TestSscNoAttributes)
diff --git a/testing/adios2/engine/ssc/TestSscNoSelection.cpp b/testing/adios2/engine/ssc/TestSscNoSelection.cpp
index 31532ba0cb..d1dc6eba6e 100644
--- a/testing/adios2/engine/ssc/TestSscNoSelection.cpp
+++ b/testing/adios2/engine/ssc/TestSscNoSelection.cpp
@@ -25,8 +25,9 @@ void Writer(const Dims &shape, const Dims &start, const Dims &count,
             const size_t steps, const adios2::Params &engineParams,
             const std::string &name)
 {
-    size_t datasize = std::accumulate(count.begin(), count.end(), 1,
-                                      std::multiplies());
+    size_t datasize =
+        std::accumulate(count.begin(), count.end(), static_cast(1),
+                        std::multiplies());
     adios2::ADIOS adios(mpiComm);
     adios2::IO dataManIO = adios.DeclareIO("WAN");
     dataManIO.SetEngine("ssc");
@@ -61,10 +62,11 @@ void Writer(const Dims &shape, const Dims &start, const Dims &count,
     auto bpDComplexes = dataManIO.DefineVariable>(
         "bpDComplexes", shape, start, count);
     dataManIO.DefineAttribute("AttInt", 110);
-    adios2::Engine dataManWriter = dataManIO.Open(name, adios2::Mode::Write);
+    adios2::Engine engine = dataManIO.Open(name, adios2::Mode::Write);
+    engine.LockWriterDefinitions();
     for (int i = 0; i < steps; ++i)
     {
-        dataManWriter.BeginStep();
+        engine.BeginStep();
         GenData(myChars, i, start, count, shape);
         GenData(myUChars, i, start, count, shape);
         GenData(myShorts, i, start, count, shape);
@@ -75,20 +77,19 @@ void Writer(const Dims &shape, const Dims &start, const Dims &count,
         GenData(myDoubles, i, start, count, shape);
         GenData(myComplexes, i, start, count, shape);
         GenData(myDComplexes, i, start, count, shape);
-        dataManWriter.Put(bpChars, myChars.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpUChars, myUChars.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpShorts, myShorts.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpUShorts, myUShorts.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpInts, myInts.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpUInts, myUInts.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpFloats, myFloats.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpDoubles, myDoubles.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpComplexes, myComplexes.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpDComplexes, myDComplexes.data(),
-                          adios2::Mode::Sync);
-        dataManWriter.EndStep();
+        engine.Put(bpChars, myChars.data(), adios2::Mode::Sync);
+        engine.Put(bpUChars, myUChars.data(), adios2::Mode::Sync);
+        engine.Put(bpShorts, myShorts.data(), adios2::Mode::Sync);
+        engine.Put(bpUShorts, myUShorts.data(), adios2::Mode::Sync);
+        engine.Put(bpInts, myInts.data(), adios2::Mode::Sync);
+        engine.Put(bpUInts, myUInts.data(), adios2::Mode::Sync);
+        engine.Put(bpFloats, myFloats.data(), adios2::Mode::Sync);
+        engine.Put(bpDoubles, myDoubles.data(), adios2::Mode::Sync);
+        engine.Put(bpComplexes, myComplexes.data(), adios2::Mode::Sync);
+        engine.Put(bpDComplexes, myDComplexes.data(), adios2::Mode::Sync);
+        engine.EndStep();
     }
-    dataManWriter.Close();
+    engine.Close();
 }
 
 void Reader(const Dims &shape, const Dims &start, const Dims &count,
@@ -99,10 +100,12 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count,
     adios2::IO dataManIO = adios.DeclareIO("Test");
     dataManIO.SetEngine("ssc");
     dataManIO.SetParameters(engineParams);
-    adios2::Engine dataManReader = dataManIO.Open(name, adios2::Mode::Read);
+    adios2::Engine engine = dataManIO.Open(name, adios2::Mode::Read);
+    engine.LockReaderSelections();
 
-    size_t datasize = std::accumulate(shape.begin(), shape.end(), 1,
-                                      std::multiplies());
+    size_t datasize =
+        std::accumulate(shape.begin(), shape.end(), static_cast(1),
+                        std::multiplies());
     std::vector myChars(datasize);
     std::vector myUChars(datasize);
     std::vector myShorts(datasize);
@@ -116,12 +119,12 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count,
 
     while (true)
     {
-        adios2::StepStatus status = dataManReader.BeginStep(StepMode::Read, 5);
+        adios2::StepStatus status = engine.BeginStep(StepMode::Read, 5);
         if (status == adios2::StepStatus::OK)
         {
             const auto &vars = dataManIO.AvailableVariables();
             ASSERT_EQ(vars.size(), 10);
-            size_t currentStep = dataManReader.CurrentStep();
+            size_t currentStep = engine.CurrentStep();
             adios2::Variable bpChars =
                 dataManIO.InquireVariable("bpChars");
             adios2::Variable bpUChars =
@@ -143,18 +146,16 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count,
             adios2::Variable> bpDComplexes =
                 dataManIO.InquireVariable>("bpDComplexes");
 
-            dataManReader.Get(bpChars, myChars.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpUChars, myUChars.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpShorts, myShorts.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpUShorts, myUShorts.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpInts, myInts.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpUInts, myUInts.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpFloats, myFloats.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpDoubles, myDoubles.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpComplexes, myComplexes.data(),
-                              adios2::Mode::Sync);
-            dataManReader.Get(bpDComplexes, myDComplexes.data(),
-                              adios2::Mode::Sync);
+            engine.Get(bpChars, myChars.data(), adios2::Mode::Sync);
+            engine.Get(bpUChars, myUChars.data(), adios2::Mode::Sync);
+            engine.Get(bpShorts, myShorts.data(), adios2::Mode::Sync);
+            engine.Get(bpUShorts, myUShorts.data(), adios2::Mode::Sync);
+            engine.Get(bpInts, myInts.data(), adios2::Mode::Sync);
+            engine.Get(bpUInts, myUInts.data(), adios2::Mode::Sync);
+            engine.Get(bpFloats, myFloats.data(), adios2::Mode::Sync);
+            engine.Get(bpDoubles, myDoubles.data(), adios2::Mode::Sync);
+            engine.Get(bpComplexes, myComplexes.data(), adios2::Mode::Sync);
+            engine.Get(bpDComplexes, myDComplexes.data(), adios2::Mode::Sync);
             VerifyData(myChars.data(), currentStep, Dims(shape.size(), 0),
                        shape, shape, mpiRank);
             VerifyData(myUChars.data(), currentStep, Dims(shape.size(), 0),
@@ -175,7 +176,7 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count,
                        shape, shape, mpiRank);
             VerifyData(myDComplexes.data(), currentStep, Dims(shape.size(), 0),
                        shape, shape, mpiRank);
-            dataManReader.EndStep();
+            engine.EndStep();
         }
         else if (status == adios2::StepStatus::EndOfStream)
         {
@@ -191,7 +192,7 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count,
               << attInt.Data()[0] << ", expected 110" << std::endl;
     ASSERT_EQ(110, attInt.Data()[0]);
     ASSERT_NE(111, attInt.Data()[0]);
-    dataManReader.Close();
+    engine.Close();
 }
 
 TEST_F(SscEngineTest, TestSscNoSelection)
diff --git a/testing/adios2/engine/ssc/TestSscOneSidedFencePull.cpp b/testing/adios2/engine/ssc/TestSscOneSidedFencePull.cpp
index 8583fba09f..418e86651e 100644
--- a/testing/adios2/engine/ssc/TestSscOneSidedFencePull.cpp
+++ b/testing/adios2/engine/ssc/TestSscOneSidedFencePull.cpp
@@ -25,8 +25,9 @@ void Writer(const Dims &shape, const Dims &start, const Dims &count,
             const size_t steps, const adios2::Params &engineParams,
             const std::string &name)
 {
-    size_t datasize = std::accumulate(count.begin(), count.end(), 1,
-                                      std::multiplies());
+    size_t datasize =
+        std::accumulate(count.begin(), count.end(), static_cast(1),
+                        std::multiplies());
     adios2::ADIOS adios(mpiComm);
     adios2::IO dataManIO = adios.DeclareIO("WAN");
     dataManIO.SetEngine("ssc");
@@ -62,10 +63,11 @@ void Writer(const Dims &shape, const Dims &start, const Dims &count,
         "bpDComplexes", shape, start, count);
     auto scalarInt = dataManIO.DefineVariable("scalarInt");
     dataManIO.DefineAttribute("AttInt", 110);
-    adios2::Engine dataManWriter = dataManIO.Open(name, adios2::Mode::Write);
+    adios2::Engine engine = dataManIO.Open(name, adios2::Mode::Write);
+    engine.LockWriterDefinitions();
     for (int i = 0; i < steps; ++i)
     {
-        dataManWriter.BeginStep();
+        engine.BeginStep();
         GenData(myChars, i, start, count, shape);
         GenData(myUChars, i, start, count, shape);
         GenData(myShorts, i, start, count, shape);
@@ -76,21 +78,20 @@ void Writer(const Dims &shape, const Dims &start, const Dims &count,
         GenData(myDoubles, i, start, count, shape);
         GenData(myComplexes, i, start, count, shape);
         GenData(myDComplexes, i, start, count, shape);
-        dataManWriter.Put(bpChars, myChars.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpUChars, myUChars.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpShorts, myShorts.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpUShorts, myUShorts.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpInts, myInts.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpUInts, myUInts.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpFloats, myFloats.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpDoubles, myDoubles.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpComplexes, myComplexes.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpDComplexes, myDComplexes.data(),
-                          adios2::Mode::Sync);
-        dataManWriter.Put(scalarInt, i);
-        dataManWriter.EndStep();
+        engine.Put(bpChars, myChars.data(), adios2::Mode::Sync);
+        engine.Put(bpUChars, myUChars.data(), adios2::Mode::Sync);
+        engine.Put(bpShorts, myShorts.data(), adios2::Mode::Sync);
+        engine.Put(bpUShorts, myUShorts.data(), adios2::Mode::Sync);
+        engine.Put(bpInts, myInts.data(), adios2::Mode::Sync);
+        engine.Put(bpUInts, myUInts.data(), adios2::Mode::Sync);
+        engine.Put(bpFloats, myFloats.data(), adios2::Mode::Sync);
+        engine.Put(bpDoubles, myDoubles.data(), adios2::Mode::Sync);
+        engine.Put(bpComplexes, myComplexes.data(), adios2::Mode::Sync);
+        engine.Put(bpDComplexes, myDComplexes.data(), adios2::Mode::Sync);
+        engine.Put(scalarInt, i);
+        engine.EndStep();
     }
-    dataManWriter.Close();
+    engine.Close();
 }
 
 void Reader(const Dims &shape, const Dims &start, const Dims &count,
@@ -101,10 +102,12 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count,
     adios2::IO dataManIO = adios.DeclareIO("Test");
     dataManIO.SetEngine("ssc");
     dataManIO.SetParameters(engineParams);
-    adios2::Engine dataManReader = dataManIO.Open(name, adios2::Mode::Read);
+    adios2::Engine engine = dataManIO.Open(name, adios2::Mode::Read);
+    engine.LockReaderSelections();
 
-    size_t datasize = std::accumulate(count.begin(), count.end(), 1,
-                                      std::multiplies());
+    size_t datasize =
+        std::accumulate(count.begin(), count.end(), static_cast(1),
+                        std::multiplies());
     std::vector myChars(datasize);
     std::vector myUChars(datasize);
     std::vector myShorts(datasize);
@@ -118,12 +121,12 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count,
 
     while (true)
     {
-        adios2::StepStatus status = dataManReader.BeginStep(StepMode::Read, 5);
+        adios2::StepStatus status = engine.BeginStep(StepMode::Read, 5);
         if (status == adios2::StepStatus::OK)
         {
             const auto &vars = dataManIO.AvailableVariables();
             ASSERT_EQ(vars.size(), 11);
-            size_t currentStep = dataManReader.CurrentStep();
+            size_t currentStep = engine.CurrentStep();
             adios2::Variable bpChars =
                 dataManIO.InquireVariable("bpChars");
             adios2::Variable bpUChars =
@@ -157,20 +160,18 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count,
             bpComplexes.SetSelection({start, count});
             bpDComplexes.SetSelection({start, count});
 
-            dataManReader.Get(bpChars, myChars.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpUChars, myUChars.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpShorts, myShorts.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpUShorts, myUShorts.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpInts, myInts.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpUInts, myUInts.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpFloats, myFloats.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpDoubles, myDoubles.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpComplexes, myComplexes.data(),
-                              adios2::Mode::Sync);
-            dataManReader.Get(bpDComplexes, myDComplexes.data(),
-                              adios2::Mode::Sync);
+            engine.Get(bpChars, myChars.data(), adios2::Mode::Sync);
+            engine.Get(bpUChars, myUChars.data(), adios2::Mode::Sync);
+            engine.Get(bpShorts, myShorts.data(), adios2::Mode::Sync);
+            engine.Get(bpUShorts, myUShorts.data(), adios2::Mode::Sync);
+            engine.Get(bpInts, myInts.data(), adios2::Mode::Sync);
+            engine.Get(bpUInts, myUInts.data(), adios2::Mode::Sync);
+            engine.Get(bpFloats, myFloats.data(), adios2::Mode::Sync);
+            engine.Get(bpDoubles, myDoubles.data(), adios2::Mode::Sync);
+            engine.Get(bpComplexes, myComplexes.data(), adios2::Mode::Sync);
+            engine.Get(bpDComplexes, myDComplexes.data(), adios2::Mode::Sync);
             int i;
-            dataManReader.Get(scalarInt, &i);
+            engine.Get(scalarInt, &i);
 
             ASSERT_EQ(i, currentStep);
 
@@ -194,7 +195,7 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count,
                        mpiRank);
             VerifyData(myDComplexes.data(), currentStep, start, count, shape,
                        mpiRank);
-            dataManReader.EndStep();
+            engine.EndStep();
         }
         else if (status == adios2::StepStatus::EndOfStream)
         {
@@ -209,7 +210,7 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count,
               << attInt.Data()[0] << ", expected 110" << std::endl;
     ASSERT_EQ(110, attInt.Data()[0]);
     ASSERT_NE(111, attInt.Data()[0]);
-    dataManReader.Close();
+    engine.Close();
 }
 
 TEST_F(SscEngineTest, TestSscOneSidedFencePull)
diff --git a/testing/adios2/engine/ssc/TestSscOneSidedFencePush.cpp b/testing/adios2/engine/ssc/TestSscOneSidedFencePush.cpp
index 581e842b5c..23120cab5b 100644
--- a/testing/adios2/engine/ssc/TestSscOneSidedFencePush.cpp
+++ b/testing/adios2/engine/ssc/TestSscOneSidedFencePush.cpp
@@ -25,8 +25,9 @@ void Writer(const Dims &shape, const Dims &start, const Dims &count,
             const size_t steps, const adios2::Params &engineParams,
             const std::string &name)
 {
-    size_t datasize = std::accumulate(count.begin(), count.end(), 1,
-                                      std::multiplies());
+    size_t datasize =
+        std::accumulate(count.begin(), count.end(), static_cast(1),
+                        std::multiplies());
     adios2::ADIOS adios(mpiComm);
     adios2::IO dataManIO = adios.DeclareIO("WAN");
     dataManIO.SetEngine("ssc");
@@ -62,10 +63,11 @@ void Writer(const Dims &shape, const Dims &start, const Dims &count,
         "bpDComplexes", shape, start, count);
     auto scalarInt = dataManIO.DefineVariable("scalarInt");
     dataManIO.DefineAttribute("AttInt", 110);
-    adios2::Engine dataManWriter = dataManIO.Open(name, adios2::Mode::Write);
+    adios2::Engine engine = dataManIO.Open(name, adios2::Mode::Write);
+    engine.LockWriterDefinitions();
     for (int i = 0; i < steps; ++i)
     {
-        dataManWriter.BeginStep();
+        engine.BeginStep();
         GenData(myChars, i, start, count, shape);
         GenData(myUChars, i, start, count, shape);
         GenData(myShorts, i, start, count, shape);
@@ -76,21 +78,20 @@ void Writer(const Dims &shape, const Dims &start, const Dims &count,
         GenData(myDoubles, i, start, count, shape);
         GenData(myComplexes, i, start, count, shape);
         GenData(myDComplexes, i, start, count, shape);
-        dataManWriter.Put(bpChars, myChars.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpUChars, myUChars.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpShorts, myShorts.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpUShorts, myUShorts.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpInts, myInts.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpUInts, myUInts.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpFloats, myFloats.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpDoubles, myDoubles.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpComplexes, myComplexes.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpDComplexes, myDComplexes.data(),
-                          adios2::Mode::Sync);
-        dataManWriter.Put(scalarInt, i);
-        dataManWriter.EndStep();
+        engine.Put(bpChars, myChars.data(), adios2::Mode::Sync);
+        engine.Put(bpUChars, myUChars.data(), adios2::Mode::Sync);
+        engine.Put(bpShorts, myShorts.data(), adios2::Mode::Sync);
+        engine.Put(bpUShorts, myUShorts.data(), adios2::Mode::Sync);
+        engine.Put(bpInts, myInts.data(), adios2::Mode::Sync);
+        engine.Put(bpUInts, myUInts.data(), adios2::Mode::Sync);
+        engine.Put(bpFloats, myFloats.data(), adios2::Mode::Sync);
+        engine.Put(bpDoubles, myDoubles.data(), adios2::Mode::Sync);
+        engine.Put(bpComplexes, myComplexes.data(), adios2::Mode::Sync);
+        engine.Put(bpDComplexes, myDComplexes.data(), adios2::Mode::Sync);
+        engine.Put(scalarInt, i);
+        engine.EndStep();
     }
-    dataManWriter.Close();
+    engine.Close();
 }
 
 void Reader(const Dims &shape, const Dims &start, const Dims &count,
@@ -101,10 +102,12 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count,
     adios2::IO dataManIO = adios.DeclareIO("Test");
     dataManIO.SetEngine("ssc");
     dataManIO.SetParameters(engineParams);
-    adios2::Engine dataManReader = dataManIO.Open(name, adios2::Mode::Read);
+    adios2::Engine engine = dataManIO.Open(name, adios2::Mode::Read);
+    engine.LockReaderSelections();
 
-    size_t datasize = std::accumulate(count.begin(), count.end(), 1,
-                                      std::multiplies());
+    size_t datasize =
+        std::accumulate(count.begin(), count.end(), static_cast(1),
+                        std::multiplies());
     std::vector myChars(datasize);
     std::vector myUChars(datasize);
     std::vector myShorts(datasize);
@@ -118,12 +121,12 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count,
 
     while (true)
     {
-        adios2::StepStatus status = dataManReader.BeginStep(StepMode::Read, 5);
+        adios2::StepStatus status = engine.BeginStep(StepMode::Read, 5);
         if (status == adios2::StepStatus::OK)
         {
             const auto &vars = dataManIO.AvailableVariables();
             ASSERT_EQ(vars.size(), 11);
-            size_t currentStep = dataManReader.CurrentStep();
+            size_t currentStep = engine.CurrentStep();
             adios2::Variable bpChars =
                 dataManIO.InquireVariable("bpChars");
             adios2::Variable bpUChars =
@@ -157,20 +160,18 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count,
             bpComplexes.SetSelection({start, count});
             bpDComplexes.SetSelection({start, count});
 
-            dataManReader.Get(bpChars, myChars.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpUChars, myUChars.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpShorts, myShorts.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpUShorts, myUShorts.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpInts, myInts.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpUInts, myUInts.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpFloats, myFloats.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpDoubles, myDoubles.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpComplexes, myComplexes.data(),
-                              adios2::Mode::Sync);
-            dataManReader.Get(bpDComplexes, myDComplexes.data(),
-                              adios2::Mode::Sync);
+            engine.Get(bpChars, myChars.data(), adios2::Mode::Sync);
+            engine.Get(bpUChars, myUChars.data(), adios2::Mode::Sync);
+            engine.Get(bpShorts, myShorts.data(), adios2::Mode::Sync);
+            engine.Get(bpUShorts, myUShorts.data(), adios2::Mode::Sync);
+            engine.Get(bpInts, myInts.data(), adios2::Mode::Sync);
+            engine.Get(bpUInts, myUInts.data(), adios2::Mode::Sync);
+            engine.Get(bpFloats, myFloats.data(), adios2::Mode::Sync);
+            engine.Get(bpDoubles, myDoubles.data(), adios2::Mode::Sync);
+            engine.Get(bpComplexes, myComplexes.data(), adios2::Mode::Sync);
+            engine.Get(bpDComplexes, myDComplexes.data(), adios2::Mode::Sync);
             int i;
-            dataManReader.Get(scalarInt, &i);
+            engine.Get(scalarInt, &i);
 
             ASSERT_EQ(i, currentStep);
 
@@ -194,7 +195,7 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count,
                        mpiRank);
             VerifyData(myDComplexes.data(), currentStep, start, count, shape,
                        mpiRank);
-            dataManReader.EndStep();
+            engine.EndStep();
         }
         else if (status == adios2::StepStatus::EndOfStream)
         {
@@ -209,7 +210,7 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count,
               << attInt.Data()[0] << ", expected 110" << std::endl;
     ASSERT_EQ(110, attInt.Data()[0]);
     ASSERT_NE(111, attInt.Data()[0]);
-    dataManReader.Close();
+    engine.Close();
 }
 
 TEST_F(SscEngineTest, TestSscOneSidedFencePush)
diff --git a/testing/adios2/engine/ssc/TestSscOneSidedPostPull.cpp b/testing/adios2/engine/ssc/TestSscOneSidedPostPull.cpp
index 39596a0e69..d73c1b609e 100644
--- a/testing/adios2/engine/ssc/TestSscOneSidedPostPull.cpp
+++ b/testing/adios2/engine/ssc/TestSscOneSidedPostPull.cpp
@@ -25,8 +25,9 @@ void Writer(const Dims &shape, const Dims &start, const Dims &count,
             const size_t steps, const adios2::Params &engineParams,
             const std::string &name)
 {
-    size_t datasize = std::accumulate(count.begin(), count.end(), 1,
-                                      std::multiplies());
+    size_t datasize =
+        std::accumulate(count.begin(), count.end(), static_cast(1),
+                        std::multiplies());
     adios2::ADIOS adios(mpiComm);
     adios2::IO dataManIO = adios.DeclareIO("WAN");
     dataManIO.SetEngine("ssc");
@@ -62,10 +63,11 @@ void Writer(const Dims &shape, const Dims &start, const Dims &count,
         "bpDComplexes", shape, start, count);
     auto scalarInt = dataManIO.DefineVariable("scalarInt");
     dataManIO.DefineAttribute("AttInt", 110);
-    adios2::Engine dataManWriter = dataManIO.Open(name, adios2::Mode::Write);
+    adios2::Engine engine = dataManIO.Open(name, adios2::Mode::Write);
+    engine.LockWriterDefinitions();
     for (int i = 0; i < steps; ++i)
     {
-        dataManWriter.BeginStep();
+        engine.BeginStep();
         GenData(myChars, i, start, count, shape);
         GenData(myUChars, i, start, count, shape);
         GenData(myShorts, i, start, count, shape);
@@ -76,21 +78,20 @@ void Writer(const Dims &shape, const Dims &start, const Dims &count,
         GenData(myDoubles, i, start, count, shape);
         GenData(myComplexes, i, start, count, shape);
         GenData(myDComplexes, i, start, count, shape);
-        dataManWriter.Put(bpChars, myChars.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpUChars, myUChars.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpShorts, myShorts.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpUShorts, myUShorts.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpInts, myInts.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpUInts, myUInts.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpFloats, myFloats.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpDoubles, myDoubles.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpComplexes, myComplexes.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpDComplexes, myDComplexes.data(),
-                          adios2::Mode::Sync);
-        dataManWriter.Put(scalarInt, i);
-        dataManWriter.EndStep();
+        engine.Put(bpChars, myChars.data(), adios2::Mode::Sync);
+        engine.Put(bpUChars, myUChars.data(), adios2::Mode::Sync);
+        engine.Put(bpShorts, myShorts.data(), adios2::Mode::Sync);
+        engine.Put(bpUShorts, myUShorts.data(), adios2::Mode::Sync);
+        engine.Put(bpInts, myInts.data(), adios2::Mode::Sync);
+        engine.Put(bpUInts, myUInts.data(), adios2::Mode::Sync);
+        engine.Put(bpFloats, myFloats.data(), adios2::Mode::Sync);
+        engine.Put(bpDoubles, myDoubles.data(), adios2::Mode::Sync);
+        engine.Put(bpComplexes, myComplexes.data(), adios2::Mode::Sync);
+        engine.Put(bpDComplexes, myDComplexes.data(), adios2::Mode::Sync);
+        engine.Put(scalarInt, i);
+        engine.EndStep();
     }
-    dataManWriter.Close();
+    engine.Close();
 }
 
 void Reader(const Dims &shape, const Dims &start, const Dims &count,
@@ -101,10 +102,12 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count,
     adios2::IO dataManIO = adios.DeclareIO("Test");
     dataManIO.SetEngine("ssc");
     dataManIO.SetParameters(engineParams);
-    adios2::Engine dataManReader = dataManIO.Open(name, adios2::Mode::Read);
+    adios2::Engine engine = dataManIO.Open(name, adios2::Mode::Read);
+    engine.LockReaderSelections();
 
-    size_t datasize = std::accumulate(count.begin(), count.end(), 1,
-                                      std::multiplies());
+    size_t datasize =
+        std::accumulate(count.begin(), count.end(), static_cast(1),
+                        std::multiplies());
     std::vector myChars(datasize);
     std::vector myUChars(datasize);
     std::vector myShorts(datasize);
@@ -118,12 +121,12 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count,
 
     while (true)
     {
-        adios2::StepStatus status = dataManReader.BeginStep(StepMode::Read, 5);
+        adios2::StepStatus status = engine.BeginStep(StepMode::Read, 5);
         if (status == adios2::StepStatus::OK)
         {
             const auto &vars = dataManIO.AvailableVariables();
             ASSERT_EQ(vars.size(), 11);
-            size_t currentStep = dataManReader.CurrentStep();
+            size_t currentStep = engine.CurrentStep();
             adios2::Variable bpChars =
                 dataManIO.InquireVariable("bpChars");
             adios2::Variable bpUChars =
@@ -157,20 +160,18 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count,
             bpComplexes.SetSelection({start, count});
             bpDComplexes.SetSelection({start, count});
 
-            dataManReader.Get(bpChars, myChars.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpUChars, myUChars.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpShorts, myShorts.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpUShorts, myUShorts.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpInts, myInts.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpUInts, myUInts.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpFloats, myFloats.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpDoubles, myDoubles.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpComplexes, myComplexes.data(),
-                              adios2::Mode::Sync);
-            dataManReader.Get(bpDComplexes, myDComplexes.data(),
-                              adios2::Mode::Sync);
+            engine.Get(bpChars, myChars.data(), adios2::Mode::Sync);
+            engine.Get(bpUChars, myUChars.data(), adios2::Mode::Sync);
+            engine.Get(bpShorts, myShorts.data(), adios2::Mode::Sync);
+            engine.Get(bpUShorts, myUShorts.data(), adios2::Mode::Sync);
+            engine.Get(bpInts, myInts.data(), adios2::Mode::Sync);
+            engine.Get(bpUInts, myUInts.data(), adios2::Mode::Sync);
+            engine.Get(bpFloats, myFloats.data(), adios2::Mode::Sync);
+            engine.Get(bpDoubles, myDoubles.data(), adios2::Mode::Sync);
+            engine.Get(bpComplexes, myComplexes.data(), adios2::Mode::Sync);
+            engine.Get(bpDComplexes, myDComplexes.data(), adios2::Mode::Sync);
             int i;
-            dataManReader.Get(scalarInt, &i);
+            engine.Get(scalarInt, &i);
 
             ASSERT_EQ(i, currentStep);
 
@@ -194,7 +195,7 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count,
                        mpiRank);
             VerifyData(myDComplexes.data(), currentStep, start, count, shape,
                        mpiRank);
-            dataManReader.EndStep();
+            engine.EndStep();
         }
         else if (status == adios2::StepStatus::EndOfStream)
         {
@@ -209,7 +210,7 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count,
               << attInt.Data()[0] << ", expected 110" << std::endl;
     ASSERT_EQ(110, attInt.Data()[0]);
     ASSERT_NE(111, attInt.Data()[0]);
-    dataManReader.Close();
+    engine.Close();
 }
 
 TEST_F(SscEngineTest, TestSscOneSidedPostPull)
diff --git a/testing/adios2/engine/ssc/TestSscOneSidedPostPush.cpp b/testing/adios2/engine/ssc/TestSscOneSidedPostPush.cpp
index f049be2b98..056dbcc544 100644
--- a/testing/adios2/engine/ssc/TestSscOneSidedPostPush.cpp
+++ b/testing/adios2/engine/ssc/TestSscOneSidedPostPush.cpp
@@ -25,8 +25,9 @@ void Writer(const Dims &shape, const Dims &start, const Dims &count,
             const size_t steps, const adios2::Params &engineParams,
             const std::string &name)
 {
-    size_t datasize = std::accumulate(count.begin(), count.end(), 1,
-                                      std::multiplies());
+    size_t datasize =
+        std::accumulate(count.begin(), count.end(), static_cast(1),
+                        std::multiplies());
     adios2::ADIOS adios(mpiComm);
     adios2::IO dataManIO = adios.DeclareIO("WAN");
     dataManIO.SetEngine("ssc");
@@ -62,10 +63,11 @@ void Writer(const Dims &shape, const Dims &start, const Dims &count,
         "bpDComplexes", shape, start, count);
     auto scalarInt = dataManIO.DefineVariable("scalarInt");
     dataManIO.DefineAttribute("AttInt", 110);
-    adios2::Engine dataManWriter = dataManIO.Open(name, adios2::Mode::Write);
+    adios2::Engine engine = dataManIO.Open(name, adios2::Mode::Write);
+    engine.LockWriterDefinitions();
     for (int i = 0; i < steps; ++i)
     {
-        dataManWriter.BeginStep();
+        engine.BeginStep();
         GenData(myChars, i, start, count, shape);
         GenData(myUChars, i, start, count, shape);
         GenData(myShorts, i, start, count, shape);
@@ -76,21 +78,20 @@ void Writer(const Dims &shape, const Dims &start, const Dims &count,
         GenData(myDoubles, i, start, count, shape);
         GenData(myComplexes, i, start, count, shape);
         GenData(myDComplexes, i, start, count, shape);
-        dataManWriter.Put(bpChars, myChars.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpUChars, myUChars.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpShorts, myShorts.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpUShorts, myUShorts.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpInts, myInts.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpUInts, myUInts.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpFloats, myFloats.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpDoubles, myDoubles.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpComplexes, myComplexes.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpDComplexes, myDComplexes.data(),
-                          adios2::Mode::Sync);
-        dataManWriter.Put(scalarInt, i);
-        dataManWriter.EndStep();
+        engine.Put(bpChars, myChars.data(), adios2::Mode::Sync);
+        engine.Put(bpUChars, myUChars.data(), adios2::Mode::Sync);
+        engine.Put(bpShorts, myShorts.data(), adios2::Mode::Sync);
+        engine.Put(bpUShorts, myUShorts.data(), adios2::Mode::Sync);
+        engine.Put(bpInts, myInts.data(), adios2::Mode::Sync);
+        engine.Put(bpUInts, myUInts.data(), adios2::Mode::Sync);
+        engine.Put(bpFloats, myFloats.data(), adios2::Mode::Sync);
+        engine.Put(bpDoubles, myDoubles.data(), adios2::Mode::Sync);
+        engine.Put(bpComplexes, myComplexes.data(), adios2::Mode::Sync);
+        engine.Put(bpDComplexes, myDComplexes.data(), adios2::Mode::Sync);
+        engine.Put(scalarInt, i);
+        engine.EndStep();
     }
-    dataManWriter.Close();
+    engine.Close();
 }
 
 void Reader(const Dims &shape, const Dims &start, const Dims &count,
@@ -101,10 +102,12 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count,
     adios2::IO dataManIO = adios.DeclareIO("Test");
     dataManIO.SetEngine("ssc");
     dataManIO.SetParameters(engineParams);
-    adios2::Engine dataManReader = dataManIO.Open(name, adios2::Mode::Read);
+    adios2::Engine engine = dataManIO.Open(name, adios2::Mode::Read);
+    engine.LockReaderSelections();
 
-    size_t datasize = std::accumulate(count.begin(), count.end(), 1,
-                                      std::multiplies());
+    size_t datasize =
+        std::accumulate(count.begin(), count.end(), static_cast(1),
+                        std::multiplies());
     std::vector myChars(datasize);
     std::vector myUChars(datasize);
     std::vector myShorts(datasize);
@@ -118,12 +121,12 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count,
 
     while (true)
     {
-        adios2::StepStatus status = dataManReader.BeginStep(StepMode::Read, 5);
+        adios2::StepStatus status = engine.BeginStep(StepMode::Read, 5);
         if (status == adios2::StepStatus::OK)
         {
             const auto &vars = dataManIO.AvailableVariables();
             ASSERT_EQ(vars.size(), 11);
-            size_t currentStep = dataManReader.CurrentStep();
+            size_t currentStep = engine.CurrentStep();
             adios2::Variable bpChars =
                 dataManIO.InquireVariable("bpChars");
             adios2::Variable bpUChars =
@@ -157,20 +160,18 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count,
             bpComplexes.SetSelection({start, count});
             bpDComplexes.SetSelection({start, count});
 
-            dataManReader.Get(bpChars, myChars.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpUChars, myUChars.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpShorts, myShorts.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpUShorts, myUShorts.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpInts, myInts.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpUInts, myUInts.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpFloats, myFloats.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpDoubles, myDoubles.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpComplexes, myComplexes.data(),
-                              adios2::Mode::Sync);
-            dataManReader.Get(bpDComplexes, myDComplexes.data(),
-                              adios2::Mode::Sync);
+            engine.Get(bpChars, myChars.data(), adios2::Mode::Sync);
+            engine.Get(bpUChars, myUChars.data(), adios2::Mode::Sync);
+            engine.Get(bpShorts, myShorts.data(), adios2::Mode::Sync);
+            engine.Get(bpUShorts, myUShorts.data(), adios2::Mode::Sync);
+            engine.Get(bpInts, myInts.data(), adios2::Mode::Sync);
+            engine.Get(bpUInts, myUInts.data(), adios2::Mode::Sync);
+            engine.Get(bpFloats, myFloats.data(), adios2::Mode::Sync);
+            engine.Get(bpDoubles, myDoubles.data(), adios2::Mode::Sync);
+            engine.Get(bpComplexes, myComplexes.data(), adios2::Mode::Sync);
+            engine.Get(bpDComplexes, myDComplexes.data(), adios2::Mode::Sync);
             int i;
-            dataManReader.Get(scalarInt, &i);
+            engine.Get(scalarInt, &i);
 
             ASSERT_EQ(i, currentStep);
 
@@ -194,7 +195,7 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count,
                        mpiRank);
             VerifyData(myDComplexes.data(), currentStep, start, count, shape,
                        mpiRank);
-            dataManReader.EndStep();
+            engine.EndStep();
         }
         else if (status == adios2::StepStatus::EndOfStream)
         {
@@ -209,7 +210,7 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count,
               << attInt.Data()[0] << ", expected 110" << std::endl;
     ASSERT_EQ(110, attInt.Data()[0]);
     ASSERT_NE(111, attInt.Data()[0]);
-    dataManReader.Close();
+    engine.Close();
 }
 
 TEST_F(SscEngineTest, TestSscOneSidedPostPush)
diff --git a/testing/adios2/engine/ssc/TestSscOnlyOneStep.cpp b/testing/adios2/engine/ssc/TestSscOnlyOneStep.cpp
new file mode 100644
index 0000000000..3b1088621c
--- /dev/null
+++ b/testing/adios2/engine/ssc/TestSscOnlyOneStep.cpp
@@ -0,0 +1,284 @@
+/*
+ * Distributed under the OSI-approved Apache License, Version 2.0.  See
+ * accompanying file Copyright.txt for details.
+ */
+
+#include "TestSscCommon.h"
+#include 
+#include 
+#include 
+#include 
+#include 
+
+using namespace adios2;
+int mpiRank = 0;
+int mpiSize = 1;
+MPI_Comm mpiComm;
+
+class SscEngineTest : public ::testing::Test
+{
+public:
+    SscEngineTest() = default;
+};
+
+void Writer(const Dims &shape, const Dims &start, const Dims &count,
+            const size_t steps, const adios2::Params &engineParams,
+            const std::string &name)
+{
+    size_t datasize =
+        std::accumulate(count.begin(), count.end(), static_cast(1),
+                        std::multiplies());
+    adios2::ADIOS adios(mpiComm);
+    adios2::IO dataManIO = adios.DeclareIO("WAN");
+    dataManIO.SetEngine("ssc");
+    dataManIO.SetParameters(engineParams);
+    std::vector myChars(datasize);
+    std::vector myUChars(datasize);
+    std::vector myShorts(datasize);
+    std::vector myUShorts(datasize);
+    std::vector myInts(datasize);
+    std::vector myUInts(datasize);
+    std::vector myFloats(datasize);
+    std::vector myDoubles(datasize);
+    std::vector> myComplexes(datasize);
+    std::vector> myDComplexes(datasize);
+    auto bpChars =
+        dataManIO.DefineVariable("bpChars", shape, start, count);
+    auto bpUChars = dataManIO.DefineVariable("bpUChars", shape,
+                                                            start, count);
+    auto bpShorts =
+        dataManIO.DefineVariable("bpShorts", shape, start, count);
+    auto bpUShorts = dataManIO.DefineVariable(
+        "bpUShorts", shape, start, count);
+    auto bpInts = dataManIO.DefineVariable("bpInts", shape, start, count);
+    auto bpUInts =
+        dataManIO.DefineVariable("bpUInts", shape, start, count);
+    auto bpFloats =
+        dataManIO.DefineVariable("bpFloats", shape, start, count);
+    auto bpDoubles =
+        dataManIO.DefineVariable("bpDoubles", shape, start, count);
+    auto bpComplexes = dataManIO.DefineVariable>(
+        "bpComplexes", shape, start, count);
+    auto bpDComplexes = dataManIO.DefineVariable>(
+        "bpDComplexes", shape, start, count);
+    auto scalarInt = dataManIO.DefineVariable("scalarInt");
+    auto stringVar = dataManIO.DefineVariable("stringVar");
+    dataManIO.DefineAttribute("AttInt", 110);
+    adios2::Engine engine = dataManIO.Open(name, adios2::Mode::Write);
+    engine.LockWriterDefinitions();
+    for (int i = 0; i < steps; ++i)
+    {
+        engine.BeginStep();
+        GenData(myChars, i, start, count, shape);
+        GenData(myUChars, i, start, count, shape);
+        GenData(myShorts, i, start, count, shape);
+        GenData(myUShorts, i, start, count, shape);
+        GenData(myInts, i, start, count, shape);
+        GenData(myUInts, i, start, count, shape);
+        GenData(myFloats, i, start, count, shape);
+        GenData(myDoubles, i, start, count, shape);
+        GenData(myComplexes, i, start, count, shape);
+        GenData(myDComplexes, i, start, count, shape);
+        engine.Put(bpChars, myChars.data(), adios2::Mode::Sync);
+        engine.Put(bpUChars, myUChars.data(), adios2::Mode::Sync);
+        engine.Put(bpShorts, myShorts.data(), adios2::Mode::Sync);
+        engine.Put(bpUShorts, myUShorts.data(), adios2::Mode::Sync);
+        engine.Put(bpInts, myInts.data(), adios2::Mode::Sync);
+        engine.Put(bpUInts, myUInts.data(), adios2::Mode::Sync);
+        engine.Put(bpFloats, myFloats.data(), adios2::Mode::Sync);
+        engine.Put(bpDoubles, myDoubles.data(), adios2::Mode::Sync);
+        engine.Put(bpComplexes, myComplexes.data(), adios2::Mode::Sync);
+        engine.Put(bpDComplexes, myDComplexes.data(), adios2::Mode::Sync);
+        engine.Put(scalarInt, i);
+        std::string s = "sample string sample string sample string";
+        engine.Put(stringVar, s);
+        engine.EndStep();
+    }
+    engine.Close();
+}
+
+void Reader(const Dims &shape, const Dims &start, const Dims &count,
+            const size_t steps, const adios2::Params &engineParams,
+            const std::string &name)
+{
+    adios2::ADIOS adios(mpiComm);
+    adios2::IO dataManIO = adios.DeclareIO("Test");
+    dataManIO.SetEngine("ssc");
+    dataManIO.SetParameters(engineParams);
+    adios2::Engine engine = dataManIO.Open(name, adios2::Mode::Read);
+    engine.LockReaderSelections();
+
+    size_t datasize =
+        std::accumulate(count.begin(), count.end(), static_cast(1),
+                        std::multiplies());
+    std::vector myChars(datasize);
+    std::vector myUChars(datasize);
+    std::vector myShorts(datasize);
+    std::vector myUShorts(datasize);
+    std::vector myInts(datasize);
+    std::vector myUInts(datasize);
+    std::vector myFloats(datasize);
+    std::vector myDoubles(datasize);
+    std::vector> myComplexes(datasize);
+    std::vector> myDComplexes(datasize);
+
+    while (true)
+    {
+        adios2::StepStatus status = engine.BeginStep(StepMode::Read, 5);
+        if (status == adios2::StepStatus::OK)
+        {
+            auto scalarInt = dataManIO.InquireVariable("scalarInt");
+            auto blocksInfo =
+                engine.BlocksInfo(scalarInt, engine.CurrentStep());
+
+            for (const auto &bi : blocksInfo)
+            {
+                ASSERT_EQ(bi.IsValue, true);
+                ASSERT_EQ(bi.Value, engine.CurrentStep());
+                ASSERT_EQ(scalarInt.Min(), engine.CurrentStep());
+                ASSERT_EQ(scalarInt.Max(), engine.CurrentStep());
+            }
+
+            const auto &vars = dataManIO.AvailableVariables();
+            ASSERT_EQ(vars.size(), 12);
+            size_t currentStep = engine.CurrentStep();
+            adios2::Variable bpChars =
+                dataManIO.InquireVariable("bpChars");
+            adios2::Variable bpUChars =
+                dataManIO.InquireVariable("bpUChars");
+            adios2::Variable bpShorts =
+                dataManIO.InquireVariable("bpShorts");
+            adios2::Variable bpUShorts =
+                dataManIO.InquireVariable("bpUShorts");
+            adios2::Variable bpInts =
+                dataManIO.InquireVariable("bpInts");
+            adios2::Variable bpUInts =
+                dataManIO.InquireVariable("bpUInts");
+            adios2::Variable bpFloats =
+                dataManIO.InquireVariable("bpFloats");
+            adios2::Variable bpDoubles =
+                dataManIO.InquireVariable("bpDoubles");
+            adios2::Variable> bpComplexes =
+                dataManIO.InquireVariable>("bpComplexes");
+            adios2::Variable> bpDComplexes =
+                dataManIO.InquireVariable>("bpDComplexes");
+            adios2::Variable stringVar =
+                dataManIO.InquireVariable("stringVar");
+
+            bpChars.SetSelection({start, count});
+            bpUChars.SetSelection({start, count});
+            bpShorts.SetSelection({start, count});
+            bpUShorts.SetSelection({start, count});
+            bpInts.SetSelection({start, count});
+            bpUInts.SetSelection({start, count});
+            bpFloats.SetSelection({start, count});
+            bpDoubles.SetSelection({start, count});
+            bpComplexes.SetSelection({start, count});
+            bpDComplexes.SetSelection({start, count});
+
+            engine.Get(bpChars, myChars.data(), adios2::Mode::Sync);
+            engine.Get(bpUChars, myUChars.data(), adios2::Mode::Sync);
+            engine.Get(bpShorts, myShorts.data(), adios2::Mode::Sync);
+            engine.Get(bpUShorts, myUShorts.data(), adios2::Mode::Sync);
+            engine.Get(bpInts, myInts.data(), adios2::Mode::Sync);
+            engine.Get(bpUInts, myUInts.data(), adios2::Mode::Sync);
+            engine.Get(bpFloats, myFloats.data(), adios2::Mode::Sync);
+            engine.Get(bpDoubles, myDoubles.data(), adios2::Mode::Sync);
+            engine.Get(bpComplexes, myComplexes.data(), adios2::Mode::Sync);
+            engine.Get(bpDComplexes, myDComplexes.data(), adios2::Mode::Sync);
+            std::string s;
+            engine.Get(stringVar, s);
+            ASSERT_EQ(s, "sample string sample string sample string");
+            ASSERT_EQ(stringVar.Min(),
+                      "sample string sample string sample string");
+            ASSERT_EQ(stringVar.Max(),
+                      "sample string sample string sample string");
+
+            int i;
+            engine.Get(scalarInt, &i);
+            ASSERT_EQ(i, currentStep);
+
+            VerifyData(myChars.data(), currentStep, start, count, shape,
+                       mpiRank);
+            VerifyData(myUChars.data(), currentStep, start, count, shape,
+                       mpiRank);
+            VerifyData(myShorts.data(), currentStep, start, count, shape,
+                       mpiRank);
+            VerifyData(myUShorts.data(), currentStep, start, count, shape,
+                       mpiRank);
+            VerifyData(myInts.data(), currentStep, start, count, shape,
+                       mpiRank);
+            VerifyData(myUInts.data(), currentStep, start, count, shape,
+                       mpiRank);
+            VerifyData(myFloats.data(), currentStep, start, count, shape,
+                       mpiRank);
+            VerifyData(myDoubles.data(), currentStep, start, count, shape,
+                       mpiRank);
+            VerifyData(myComplexes.data(), currentStep, start, count, shape,
+                       mpiRank);
+            VerifyData(myDComplexes.data(), currentStep, start, count, shape,
+                       mpiRank);
+            engine.EndStep();
+        }
+        else if (status == adios2::StepStatus::EndOfStream)
+        {
+            std::cout << "[Rank " + std::to_string(mpiRank) +
+                             "] SscTest reader end of stream!"
+                      << std::endl;
+            break;
+        }
+    }
+    auto attInt = dataManIO.InquireAttribute("AttInt");
+    std::cout << "[Rank " + std::to_string(mpiRank) + "] Attribute received "
+              << attInt.Data()[0] << ", expected 110" << std::endl;
+    ASSERT_EQ(110, attInt.Data()[0]);
+    ASSERT_NE(111, attInt.Data()[0]);
+    engine.Close();
+}
+
+TEST_F(SscEngineTest, TestSscOnlyOneStep)
+{
+    std::string filename = "TestSscOnlyOneStep";
+    adios2::Params engineParams = {};
+
+    int worldRank, worldSize;
+    MPI_Comm_rank(MPI_COMM_WORLD, &worldRank);
+    MPI_Comm_size(MPI_COMM_WORLD, &worldSize);
+    int mpiGroup = worldRank / (worldSize / 2);
+    MPI_Comm_split(MPI_COMM_WORLD, mpiGroup, worldRank, &mpiComm);
+
+    MPI_Comm_rank(mpiComm, &mpiRank);
+    MPI_Comm_size(mpiComm, &mpiSize);
+
+    Dims shape = {10, (size_t)mpiSize * 2};
+    Dims start = {2, (size_t)mpiRank * 2};
+    Dims count = {5, 2};
+    size_t steps = 1;
+
+    if (mpiGroup == 0)
+    {
+        Writer(shape, start, count, steps, engineParams, filename);
+    }
+
+    std::this_thread::sleep_for(std::chrono::milliseconds(1));
+
+    if (mpiGroup == 1)
+    {
+        Reader(shape, start, count, steps, engineParams, filename);
+    }
+
+    MPI_Barrier(MPI_COMM_WORLD);
+}
+
+int main(int argc, char **argv)
+{
+    MPI_Init(&argc, &argv);
+    int worldRank, worldSize;
+    MPI_Comm_rank(MPI_COMM_WORLD, &worldRank);
+    MPI_Comm_size(MPI_COMM_WORLD, &worldSize);
+    ::testing::InitGoogleTest(&argc, argv);
+    int result = RUN_ALL_TESTS();
+
+    MPI_Finalize();
+    return result;
+}
diff --git a/testing/adios2/engine/ssc/TestSscOnlyTwoSteps.cpp b/testing/adios2/engine/ssc/TestSscOnlyTwoSteps.cpp
new file mode 100644
index 0000000000..44968d7542
--- /dev/null
+++ b/testing/adios2/engine/ssc/TestSscOnlyTwoSteps.cpp
@@ -0,0 +1,284 @@
+/*
+ * Distributed under the OSI-approved Apache License, Version 2.0.  See
+ * accompanying file Copyright.txt for details.
+ */
+
+#include "TestSscCommon.h"
+#include 
+#include 
+#include 
+#include 
+#include 
+
+using namespace adios2;
+int mpiRank = 0;
+int mpiSize = 1;
+MPI_Comm mpiComm;
+
+class SscEngineTest : public ::testing::Test
+{
+public:
+    SscEngineTest() = default;
+};
+
+void Writer(const Dims &shape, const Dims &start, const Dims &count,
+            const size_t steps, const adios2::Params &engineParams,
+            const std::string &name)
+{
+    size_t datasize =
+        std::accumulate(count.begin(), count.end(), static_cast(1),
+                        std::multiplies());
+    adios2::ADIOS adios(mpiComm);
+    adios2::IO dataManIO = adios.DeclareIO("WAN");
+    dataManIO.SetEngine("ssc");
+    dataManIO.SetParameters(engineParams);
+    std::vector myChars(datasize);
+    std::vector myUChars(datasize);
+    std::vector myShorts(datasize);
+    std::vector myUShorts(datasize);
+    std::vector myInts(datasize);
+    std::vector myUInts(datasize);
+    std::vector myFloats(datasize);
+    std::vector myDoubles(datasize);
+    std::vector> myComplexes(datasize);
+    std::vector> myDComplexes(datasize);
+    auto bpChars =
+        dataManIO.DefineVariable("bpChars", shape, start, count);
+    auto bpUChars = dataManIO.DefineVariable("bpUChars", shape,
+                                                            start, count);
+    auto bpShorts =
+        dataManIO.DefineVariable("bpShorts", shape, start, count);
+    auto bpUShorts = dataManIO.DefineVariable(
+        "bpUShorts", shape, start, count);
+    auto bpInts = dataManIO.DefineVariable("bpInts", shape, start, count);
+    auto bpUInts =
+        dataManIO.DefineVariable("bpUInts", shape, start, count);
+    auto bpFloats =
+        dataManIO.DefineVariable("bpFloats", shape, start, count);
+    auto bpDoubles =
+        dataManIO.DefineVariable("bpDoubles", shape, start, count);
+    auto bpComplexes = dataManIO.DefineVariable>(
+        "bpComplexes", shape, start, count);
+    auto bpDComplexes = dataManIO.DefineVariable>(
+        "bpDComplexes", shape, start, count);
+    auto scalarInt = dataManIO.DefineVariable("scalarInt");
+    auto stringVar = dataManIO.DefineVariable("stringVar");
+    dataManIO.DefineAttribute("AttInt", 110);
+    adios2::Engine engine = dataManIO.Open(name, adios2::Mode::Write);
+    engine.LockWriterDefinitions();
+    for (int i = 0; i < steps; ++i)
+    {
+        engine.BeginStep();
+        GenData(myChars, i, start, count, shape);
+        GenData(myUChars, i, start, count, shape);
+        GenData(myShorts, i, start, count, shape);
+        GenData(myUShorts, i, start, count, shape);
+        GenData(myInts, i, start, count, shape);
+        GenData(myUInts, i, start, count, shape);
+        GenData(myFloats, i, start, count, shape);
+        GenData(myDoubles, i, start, count, shape);
+        GenData(myComplexes, i, start, count, shape);
+        GenData(myDComplexes, i, start, count, shape);
+        engine.Put(bpChars, myChars.data(), adios2::Mode::Sync);
+        engine.Put(bpUChars, myUChars.data(), adios2::Mode::Sync);
+        engine.Put(bpShorts, myShorts.data(), adios2::Mode::Sync);
+        engine.Put(bpUShorts, myUShorts.data(), adios2::Mode::Sync);
+        engine.Put(bpInts, myInts.data(), adios2::Mode::Sync);
+        engine.Put(bpUInts, myUInts.data(), adios2::Mode::Sync);
+        engine.Put(bpFloats, myFloats.data(), adios2::Mode::Sync);
+        engine.Put(bpDoubles, myDoubles.data(), adios2::Mode::Sync);
+        engine.Put(bpComplexes, myComplexes.data(), adios2::Mode::Sync);
+        engine.Put(bpDComplexes, myDComplexes.data(), adios2::Mode::Sync);
+        engine.Put(scalarInt, i);
+        std::string s = "sample string sample string sample string";
+        engine.Put(stringVar, s);
+        engine.EndStep();
+    }
+    engine.Close();
+}
+
+void Reader(const Dims &shape, const Dims &start, const Dims &count,
+            const size_t steps, const adios2::Params &engineParams,
+            const std::string &name)
+{
+    adios2::ADIOS adios(mpiComm);
+    adios2::IO dataManIO = adios.DeclareIO("Test");
+    dataManIO.SetEngine("ssc");
+    dataManIO.SetParameters(engineParams);
+    adios2::Engine engine = dataManIO.Open(name, adios2::Mode::Read);
+    engine.LockReaderSelections();
+
+    size_t datasize =
+        std::accumulate(count.begin(), count.end(), static_cast(1),
+                        std::multiplies());
+    std::vector myChars(datasize);
+    std::vector myUChars(datasize);
+    std::vector myShorts(datasize);
+    std::vector myUShorts(datasize);
+    std::vector myInts(datasize);
+    std::vector myUInts(datasize);
+    std::vector myFloats(datasize);
+    std::vector myDoubles(datasize);
+    std::vector> myComplexes(datasize);
+    std::vector> myDComplexes(datasize);
+
+    while (true)
+    {
+        adios2::StepStatus status = engine.BeginStep(StepMode::Read, 5);
+        if (status == adios2::StepStatus::OK)
+        {
+            auto scalarInt = dataManIO.InquireVariable("scalarInt");
+            auto blocksInfo =
+                engine.BlocksInfo(scalarInt, engine.CurrentStep());
+
+            for (const auto &bi : blocksInfo)
+            {
+                ASSERT_EQ(bi.IsValue, true);
+                ASSERT_EQ(bi.Value, engine.CurrentStep());
+                ASSERT_EQ(scalarInt.Min(), engine.CurrentStep());
+                ASSERT_EQ(scalarInt.Max(), engine.CurrentStep());
+            }
+
+            const auto &vars = dataManIO.AvailableVariables();
+            ASSERT_EQ(vars.size(), 12);
+            size_t currentStep = engine.CurrentStep();
+            adios2::Variable bpChars =
+                dataManIO.InquireVariable("bpChars");
+            adios2::Variable bpUChars =
+                dataManIO.InquireVariable("bpUChars");
+            adios2::Variable bpShorts =
+                dataManIO.InquireVariable("bpShorts");
+            adios2::Variable bpUShorts =
+                dataManIO.InquireVariable("bpUShorts");
+            adios2::Variable bpInts =
+                dataManIO.InquireVariable("bpInts");
+            adios2::Variable bpUInts =
+                dataManIO.InquireVariable("bpUInts");
+            adios2::Variable bpFloats =
+                dataManIO.InquireVariable("bpFloats");
+            adios2::Variable bpDoubles =
+                dataManIO.InquireVariable("bpDoubles");
+            adios2::Variable> bpComplexes =
+                dataManIO.InquireVariable>("bpComplexes");
+            adios2::Variable> bpDComplexes =
+                dataManIO.InquireVariable>("bpDComplexes");
+            adios2::Variable stringVar =
+                dataManIO.InquireVariable("stringVar");
+
+            bpChars.SetSelection({start, count});
+            bpUChars.SetSelection({start, count});
+            bpShorts.SetSelection({start, count});
+            bpUShorts.SetSelection({start, count});
+            bpInts.SetSelection({start, count});
+            bpUInts.SetSelection({start, count});
+            bpFloats.SetSelection({start, count});
+            bpDoubles.SetSelection({start, count});
+            bpComplexes.SetSelection({start, count});
+            bpDComplexes.SetSelection({start, count});
+
+            engine.Get(bpChars, myChars.data(), adios2::Mode::Sync);
+            engine.Get(bpUChars, myUChars.data(), adios2::Mode::Sync);
+            engine.Get(bpShorts, myShorts.data(), adios2::Mode::Sync);
+            engine.Get(bpUShorts, myUShorts.data(), adios2::Mode::Sync);
+            engine.Get(bpInts, myInts.data(), adios2::Mode::Sync);
+            engine.Get(bpUInts, myUInts.data(), adios2::Mode::Sync);
+            engine.Get(bpFloats, myFloats.data(), adios2::Mode::Sync);
+            engine.Get(bpDoubles, myDoubles.data(), adios2::Mode::Sync);
+            engine.Get(bpComplexes, myComplexes.data(), adios2::Mode::Sync);
+            engine.Get(bpDComplexes, myDComplexes.data(), adios2::Mode::Sync);
+            std::string s;
+            engine.Get(stringVar, s);
+            ASSERT_EQ(s, "sample string sample string sample string");
+            ASSERT_EQ(stringVar.Min(),
+                      "sample string sample string sample string");
+            ASSERT_EQ(stringVar.Max(),
+                      "sample string sample string sample string");
+
+            int i;
+            engine.Get(scalarInt, &i);
+            ASSERT_EQ(i, currentStep);
+
+            VerifyData(myChars.data(), currentStep, start, count, shape,
+                       mpiRank);
+            VerifyData(myUChars.data(), currentStep, start, count, shape,
+                       mpiRank);
+            VerifyData(myShorts.data(), currentStep, start, count, shape,
+                       mpiRank);
+            VerifyData(myUShorts.data(), currentStep, start, count, shape,
+                       mpiRank);
+            VerifyData(myInts.data(), currentStep, start, count, shape,
+                       mpiRank);
+            VerifyData(myUInts.data(), currentStep, start, count, shape,
+                       mpiRank);
+            VerifyData(myFloats.data(), currentStep, start, count, shape,
+                       mpiRank);
+            VerifyData(myDoubles.data(), currentStep, start, count, shape,
+                       mpiRank);
+            VerifyData(myComplexes.data(), currentStep, start, count, shape,
+                       mpiRank);
+            VerifyData(myDComplexes.data(), currentStep, start, count, shape,
+                       mpiRank);
+            engine.EndStep();
+        }
+        else if (status == adios2::StepStatus::EndOfStream)
+        {
+            std::cout << "[Rank " + std::to_string(mpiRank) +
+                             "] SscTest reader end of stream!"
+                      << std::endl;
+            break;
+        }
+    }
+    auto attInt = dataManIO.InquireAttribute("AttInt");
+    std::cout << "[Rank " + std::to_string(mpiRank) + "] Attribute received "
+              << attInt.Data()[0] << ", expected 110" << std::endl;
+    ASSERT_EQ(110, attInt.Data()[0]);
+    ASSERT_NE(111, attInt.Data()[0]);
+    engine.Close();
+}
+
+TEST_F(SscEngineTest, TestSscOnlyTwoSteps)
+{
+    std::string filename = "TestSscOnlyTwoSteps";
+    adios2::Params engineParams = {};
+
+    int worldRank, worldSize;
+    MPI_Comm_rank(MPI_COMM_WORLD, &worldRank);
+    MPI_Comm_size(MPI_COMM_WORLD, &worldSize);
+    int mpiGroup = worldRank / (worldSize / 2);
+    MPI_Comm_split(MPI_COMM_WORLD, mpiGroup, worldRank, &mpiComm);
+
+    MPI_Comm_rank(mpiComm, &mpiRank);
+    MPI_Comm_size(mpiComm, &mpiSize);
+
+    Dims shape = {10, (size_t)mpiSize * 2};
+    Dims start = {2, (size_t)mpiRank * 2};
+    Dims count = {5, 2};
+    size_t steps = 2;
+
+    if (mpiGroup == 0)
+    {
+        Writer(shape, start, count, steps, engineParams, filename);
+    }
+
+    std::this_thread::sleep_for(std::chrono::milliseconds(1));
+
+    if (mpiGroup == 1)
+    {
+        Reader(shape, start, count, steps, engineParams, filename);
+    }
+
+    MPI_Barrier(MPI_COMM_WORLD);
+}
+
+int main(int argc, char **argv)
+{
+    MPI_Init(&argc, &argv);
+    int worldRank, worldSize;
+    MPI_Comm_rank(MPI_COMM_WORLD, &worldRank);
+    MPI_Comm_size(MPI_COMM_WORLD, &worldSize);
+    ::testing::InitGoogleTest(&argc, argv);
+    int result = RUN_ALL_TESTS();
+
+    MPI_Finalize();
+    return result;
+}
diff --git a/testing/adios2/engine/ssc/TestSscReaderMultiblock.cpp b/testing/adios2/engine/ssc/TestSscReaderMultiblock.cpp
index b1d7c00dc5..58cd6b1df1 100644
--- a/testing/adios2/engine/ssc/TestSscReaderMultiblock.cpp
+++ b/testing/adios2/engine/ssc/TestSscReaderMultiblock.cpp
@@ -25,8 +25,9 @@ void Writer(const Dims &shape, const Dims &start, const Dims &count,
             const size_t steps, const adios2::Params &engineParams,
             const std::string &name)
 {
-    size_t datasize = std::accumulate(count.begin(), count.end(), 1,
-                                      std::multiplies());
+    size_t datasize =
+        std::accumulate(count.begin(), count.end(), static_cast(1),
+                        std::multiplies());
     adios2::ADIOS adios(mpiComm);
     adios2::IO dataManIO = adios.DeclareIO("WAN");
     dataManIO.SetEngine("ssc");
@@ -62,10 +63,11 @@ void Writer(const Dims &shape, const Dims &start, const Dims &count,
         "bpDComplexes", shape, start, count);
     auto scalarInt = dataManIO.DefineVariable("scalarInt");
     dataManIO.DefineAttribute("AttInt", 110);
-    adios2::Engine dataManWriter = dataManIO.Open(name, adios2::Mode::Write);
+    adios2::Engine engine = dataManIO.Open(name, adios2::Mode::Write);
+    engine.LockWriterDefinitions();
     for (int i = 0; i < steps; ++i)
     {
-        dataManWriter.BeginStep();
+        engine.BeginStep();
 
         Dims startTmp = {(size_t)mpiRank * 2, 0};
 
@@ -91,18 +93,17 @@ void Writer(const Dims &shape, const Dims &start, const Dims &count,
         bpComplexes.SetSelection({startTmp, count});
         bpDComplexes.SetSelection({startTmp, count});
 
-        dataManWriter.Put(bpChars, myChars.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpUChars, myUChars.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpShorts, myShorts.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpUShorts, myUShorts.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpInts, myInts.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpUInts, myUInts.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpFloats, myFloats.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpDoubles, myDoubles.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpComplexes, myComplexes.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpDComplexes, myDComplexes.data(),
-                          adios2::Mode::Sync);
-        dataManWriter.Put(scalarInt, i);
+        engine.Put(bpChars, myChars.data(), adios2::Mode::Sync);
+        engine.Put(bpUChars, myUChars.data(), adios2::Mode::Sync);
+        engine.Put(bpShorts, myShorts.data(), adios2::Mode::Sync);
+        engine.Put(bpUShorts, myUShorts.data(), adios2::Mode::Sync);
+        engine.Put(bpInts, myInts.data(), adios2::Mode::Sync);
+        engine.Put(bpUInts, myUInts.data(), adios2::Mode::Sync);
+        engine.Put(bpFloats, myFloats.data(), adios2::Mode::Sync);
+        engine.Put(bpDoubles, myDoubles.data(), adios2::Mode::Sync);
+        engine.Put(bpComplexes, myComplexes.data(), adios2::Mode::Sync);
+        engine.Put(bpDComplexes, myDComplexes.data(), adios2::Mode::Sync);
+        engine.Put(scalarInt, i);
 
         startTmp = {(size_t)mpiRank * 2 + 1, 0};
 
@@ -128,22 +129,21 @@ void Writer(const Dims &shape, const Dims &start, const Dims &count,
         bpComplexes.SetSelection({startTmp, count});
         bpDComplexes.SetSelection({startTmp, count});
 
-        dataManWriter.Put(bpChars, myChars.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpUChars, myUChars.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpShorts, myShorts.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpUShorts, myUShorts.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpInts, myInts.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpUInts, myUInts.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpFloats, myFloats.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpDoubles, myDoubles.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpComplexes, myComplexes.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpDComplexes, myDComplexes.data(),
-                          adios2::Mode::Sync);
-        dataManWriter.Put(scalarInt, i);
+        engine.Put(bpChars, myChars.data(), adios2::Mode::Sync);
+        engine.Put(bpUChars, myUChars.data(), adios2::Mode::Sync);
+        engine.Put(bpShorts, myShorts.data(), adios2::Mode::Sync);
+        engine.Put(bpUShorts, myUShorts.data(), adios2::Mode::Sync);
+        engine.Put(bpInts, myInts.data(), adios2::Mode::Sync);
+        engine.Put(bpUInts, myUInts.data(), adios2::Mode::Sync);
+        engine.Put(bpFloats, myFloats.data(), adios2::Mode::Sync);
+        engine.Put(bpDoubles, myDoubles.data(), adios2::Mode::Sync);
+        engine.Put(bpComplexes, myComplexes.data(), adios2::Mode::Sync);
+        engine.Put(bpDComplexes, myDComplexes.data(), adios2::Mode::Sync);
+        engine.Put(scalarInt, i);
 
-        dataManWriter.EndStep();
+        engine.EndStep();
     }
-    dataManWriter.Close();
+    engine.Close();
 }
 
 void Reader(const Dims &shape, const Dims &start, const Dims &count,
@@ -154,10 +154,12 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count,
     adios2::IO dataManIO = adios.DeclareIO("Test");
     dataManIO.SetEngine("ssc");
     dataManIO.SetParameters(engineParams);
-    adios2::Engine dataManReader = dataManIO.Open(name, adios2::Mode::Read);
+    adios2::Engine engine = dataManIO.Open(name, adios2::Mode::Read);
+    engine.LockReaderSelections();
 
-    size_t datasize = std::accumulate(shape.begin(), shape.end(), 1,
-                                      std::multiplies());
+    size_t datasize =
+        std::accumulate(shape.begin(), shape.end(), static_cast(1),
+                        std::multiplies());
     std::vector myChars(datasize);
     std::vector myUChars(datasize);
     std::vector myShorts(datasize);
@@ -171,12 +173,12 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count,
 
     while (true)
     {
-        adios2::StepStatus status = dataManReader.BeginStep(StepMode::Read, 5);
+        adios2::StepStatus status = engine.BeginStep(StepMode::Read, 5);
         if (status == adios2::StepStatus::OK)
         {
             const auto &vars = dataManIO.AvailableVariables();
             ASSERT_EQ(vars.size(), 11);
-            size_t currentStep = dataManReader.CurrentStep();
+            size_t currentStep = engine.CurrentStep();
             adios2::Variable bpChars =
                 dataManIO.InquireVariable("bpChars");
             adios2::Variable bpUChars =
@@ -200,7 +202,7 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count,
             auto scalarInt = dataManIO.InquireVariable("scalarInt");
 
             int i;
-            dataManReader.Get(scalarInt, &i);
+            engine.Get(scalarInt, &i);
             ASSERT_EQ(i, currentStep);
 
             adios2::Dims startTmp = start;
@@ -217,18 +219,16 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count,
             bpDoubles.SetSelection({startTmp, countTmp});
             bpComplexes.SetSelection({startTmp, countTmp});
             bpDComplexes.SetSelection({startTmp, countTmp});
-            dataManReader.Get(bpChars, myChars.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpUChars, myUChars.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpShorts, myShorts.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpUShorts, myUShorts.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpInts, myInts.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpUInts, myUInts.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpFloats, myFloats.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpDoubles, myDoubles.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpComplexes, myComplexes.data(),
-                              adios2::Mode::Sync);
-            dataManReader.Get(bpDComplexes, myDComplexes.data(),
-                              adios2::Mode::Sync);
+            engine.Get(bpChars, myChars.data(), adios2::Mode::Sync);
+            engine.Get(bpUChars, myUChars.data(), adios2::Mode::Sync);
+            engine.Get(bpShorts, myShorts.data(), adios2::Mode::Sync);
+            engine.Get(bpUShorts, myUShorts.data(), adios2::Mode::Sync);
+            engine.Get(bpInts, myInts.data(), adios2::Mode::Sync);
+            engine.Get(bpUInts, myUInts.data(), adios2::Mode::Sync);
+            engine.Get(bpFloats, myFloats.data(), adios2::Mode::Sync);
+            engine.Get(bpDoubles, myDoubles.data(), adios2::Mode::Sync);
+            engine.Get(bpComplexes, myComplexes.data(), adios2::Mode::Sync);
+            engine.Get(bpDComplexes, myDComplexes.data(), adios2::Mode::Sync);
             VerifyData(myChars.data(), currentStep, startTmp, countTmp, shape,
                        mpiRank);
             VerifyData(myUChars.data(), currentStep, startTmp, countTmp, shape,
@@ -262,18 +262,16 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count,
             bpDoubles.SetSelection({startTmp, countTmp});
             bpComplexes.SetSelection({startTmp, countTmp});
             bpDComplexes.SetSelection({startTmp, countTmp});
-            dataManReader.Get(bpChars, myChars.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpUChars, myUChars.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpShorts, myShorts.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpUShorts, myUShorts.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpInts, myInts.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpUInts, myUInts.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpFloats, myFloats.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpDoubles, myDoubles.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpComplexes, myComplexes.data(),
-                              adios2::Mode::Sync);
-            dataManReader.Get(bpDComplexes, myDComplexes.data(),
-                              adios2::Mode::Sync);
+            engine.Get(bpChars, myChars.data(), adios2::Mode::Sync);
+            engine.Get(bpUChars, myUChars.data(), adios2::Mode::Sync);
+            engine.Get(bpShorts, myShorts.data(), adios2::Mode::Sync);
+            engine.Get(bpUShorts, myUShorts.data(), adios2::Mode::Sync);
+            engine.Get(bpInts, myInts.data(), adios2::Mode::Sync);
+            engine.Get(bpUInts, myUInts.data(), adios2::Mode::Sync);
+            engine.Get(bpFloats, myFloats.data(), adios2::Mode::Sync);
+            engine.Get(bpDoubles, myDoubles.data(), adios2::Mode::Sync);
+            engine.Get(bpComplexes, myComplexes.data(), adios2::Mode::Sync);
+            engine.Get(bpDComplexes, myDComplexes.data(), adios2::Mode::Sync);
             VerifyData(myChars.data(), currentStep, startTmp, countTmp, shape,
                        mpiRank);
             VerifyData(myUChars.data(), currentStep, startTmp, countTmp, shape,
@@ -295,7 +293,7 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count,
             VerifyData(myDComplexes.data(), currentStep, startTmp, countTmp,
                        shape, mpiRank);
 
-            dataManReader.EndStep();
+            engine.EndStep();
         }
         else if (status == adios2::StepStatus::EndOfStream)
         {
@@ -310,7 +308,7 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count,
               << attInt.Data()[0] << ", expected 110" << std::endl;
     ASSERT_EQ(110, attInt.Data()[0]);
     ASSERT_NE(111, attInt.Data()[0]);
-    dataManReader.Close();
+    engine.Close();
 }
 
 TEST_F(SscEngineTest, TestSscReaderMultiblock)
diff --git a/testing/adios2/engine/ssc/TestSscUnbalanced.cpp b/testing/adios2/engine/ssc/TestSscUnbalanced.cpp
index ba22d88b63..a78a1104d8 100644
--- a/testing/adios2/engine/ssc/TestSscUnbalanced.cpp
+++ b/testing/adios2/engine/ssc/TestSscUnbalanced.cpp
@@ -25,8 +25,9 @@ void Writer(const Dims &shape, const Dims &start, const Dims &count,
             const size_t steps, const adios2::Params &engineParams,
             const std::string &name)
 {
-    size_t datasize = std::accumulate(count.begin(), count.end(), 1,
-                                      std::multiplies());
+    size_t datasize =
+        std::accumulate(count.begin(), count.end(), static_cast(1),
+                        std::multiplies());
     adios2::ADIOS adios(mpiComm);
     adios2::IO dataManIO = adios.DeclareIO("WAN");
     dataManIO.SetEngine("ssc");
@@ -62,10 +63,11 @@ void Writer(const Dims &shape, const Dims &start, const Dims &count,
         "bpDComplexes", shape, start, count);
     auto scalarInt = dataManIO.DefineVariable("scalarInt");
     dataManIO.DefineAttribute("AttInt", 110);
-    adios2::Engine dataManWriter = dataManIO.Open(name, adios2::Mode::Write);
+    adios2::Engine engine = dataManIO.Open(name, adios2::Mode::Write);
+    engine.LockWriterDefinitions();
     for (int i = 0; i < steps; ++i)
     {
-        dataManWriter.BeginStep();
+        engine.BeginStep();
         GenData(myChars, i, start, count, shape);
         GenData(myUChars, i, start, count, shape);
         GenData(myShorts, i, start, count, shape);
@@ -76,21 +78,20 @@ void Writer(const Dims &shape, const Dims &start, const Dims &count,
         GenData(myDoubles, i, start, count, shape);
         GenData(myComplexes, i, start, count, shape);
         GenData(myDComplexes, i, start, count, shape);
-        dataManWriter.Put(bpChars, myChars.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpUChars, myUChars.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpShorts, myShorts.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpUShorts, myUShorts.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpInts, myInts.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpUInts, myUInts.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpFloats, myFloats.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpDoubles, myDoubles.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpComplexes, myComplexes.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpDComplexes, myDComplexes.data(),
-                          adios2::Mode::Sync);
-        dataManWriter.Put(scalarInt, i);
-        dataManWriter.EndStep();
+        engine.Put(bpChars, myChars.data(), adios2::Mode::Sync);
+        engine.Put(bpUChars, myUChars.data(), adios2::Mode::Sync);
+        engine.Put(bpShorts, myShorts.data(), adios2::Mode::Sync);
+        engine.Put(bpUShorts, myUShorts.data(), adios2::Mode::Sync);
+        engine.Put(bpInts, myInts.data(), adios2::Mode::Sync);
+        engine.Put(bpUInts, myUInts.data(), adios2::Mode::Sync);
+        engine.Put(bpFloats, myFloats.data(), adios2::Mode::Sync);
+        engine.Put(bpDoubles, myDoubles.data(), adios2::Mode::Sync);
+        engine.Put(bpComplexes, myComplexes.data(), adios2::Mode::Sync);
+        engine.Put(bpDComplexes, myDComplexes.data(), adios2::Mode::Sync);
+        engine.Put(scalarInt, i);
+        engine.EndStep();
     }
-    dataManWriter.Close();
+    engine.Close();
 }
 
 void Reader(const Dims &shape, const Dims &start, const Dims &count,
@@ -101,10 +102,12 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count,
     adios2::IO dataManIO = adios.DeclareIO("Test");
     dataManIO.SetEngine("ssc");
     dataManIO.SetParameters(engineParams);
-    adios2::Engine dataManReader = dataManIO.Open(name, adios2::Mode::Read);
+    adios2::Engine engine = dataManIO.Open(name, adios2::Mode::Read);
+    engine.LockReaderSelections();
 
-    size_t datasize = std::accumulate(count.begin(), count.end(), 1,
-                                      std::multiplies());
+    size_t datasize =
+        std::accumulate(count.begin(), count.end(), static_cast(1),
+                        std::multiplies());
     std::vector myChars(datasize);
     std::vector myUChars(datasize);
     std::vector myShorts(datasize);
@@ -118,12 +121,12 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count,
 
     while (true)
     {
-        adios2::StepStatus status = dataManReader.BeginStep(StepMode::Read, 5);
+        adios2::StepStatus status = engine.BeginStep(StepMode::Read, 5);
         if (status == adios2::StepStatus::OK)
         {
             const auto &vars = dataManIO.AvailableVariables();
             ASSERT_EQ(vars.size(), 11);
-            size_t currentStep = dataManReader.CurrentStep();
+            size_t currentStep = engine.CurrentStep();
             adios2::Variable bpChars =
                 dataManIO.InquireVariable("bpChars");
             adios2::Variable bpUChars =
@@ -157,20 +160,18 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count,
             bpComplexes.SetSelection({start, count});
             bpDComplexes.SetSelection({start, count});
 
-            dataManReader.Get(bpChars, myChars.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpUChars, myUChars.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpShorts, myShorts.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpUShorts, myUShorts.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpInts, myInts.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpUInts, myUInts.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpFloats, myFloats.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpDoubles, myDoubles.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpComplexes, myComplexes.data(),
-                              adios2::Mode::Sync);
-            dataManReader.Get(bpDComplexes, myDComplexes.data(),
-                              adios2::Mode::Sync);
+            engine.Get(bpChars, myChars.data(), adios2::Mode::Sync);
+            engine.Get(bpUChars, myUChars.data(), adios2::Mode::Sync);
+            engine.Get(bpShorts, myShorts.data(), adios2::Mode::Sync);
+            engine.Get(bpUShorts, myUShorts.data(), adios2::Mode::Sync);
+            engine.Get(bpInts, myInts.data(), adios2::Mode::Sync);
+            engine.Get(bpUInts, myUInts.data(), adios2::Mode::Sync);
+            engine.Get(bpFloats, myFloats.data(), adios2::Mode::Sync);
+            engine.Get(bpDoubles, myDoubles.data(), adios2::Mode::Sync);
+            engine.Get(bpComplexes, myComplexes.data(), adios2::Mode::Sync);
+            engine.Get(bpDComplexes, myDComplexes.data(), adios2::Mode::Sync);
             int i;
-            dataManReader.Get(scalarInt, &i);
+            engine.Get(scalarInt, &i);
 
             ASSERT_EQ(i, currentStep);
 
@@ -194,7 +195,7 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count,
                        mpiRank);
             VerifyData(myDComplexes.data(), currentStep, start, count, shape,
                        mpiRank);
-            dataManReader.EndStep();
+            engine.EndStep();
         }
         else if (status == adios2::StepStatus::EndOfStream)
         {
@@ -209,7 +210,7 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count,
               << attInt.Data()[0] << ", expected 110" << std::endl;
     ASSERT_EQ(110, attInt.Data()[0]);
     ASSERT_NE(111, attInt.Data()[0]);
-    dataManReader.Close();
+    engine.Close();
 }
 
 TEST_F(SscEngineTest, TestSscUnbalanced)
diff --git a/testing/adios2/engine/ssc/TestSscVaryingSteps.cpp b/testing/adios2/engine/ssc/TestSscVaryingSteps.cpp
new file mode 100644
index 0000000000..07b2d87063
--- /dev/null
+++ b/testing/adios2/engine/ssc/TestSscVaryingSteps.cpp
@@ -0,0 +1,334 @@
+/*
+ * Distributed under the OSI-approved Apache License, Version 2.0.  See
+ * accompanying file Copyright.txt for details.
+ */
+
+#include "TestSscCommon.h"
+#include 
+#include 
+#include 
+#include 
+#include 
+
+using namespace adios2;
+int mpiRank = 0;
+int mpiSize = 1;
+MPI_Comm mpiComm;
+
+class SscEngineTest : public ::testing::Test
+{
+public:
+    SscEngineTest() = default;
+};
+
+void Writer(const Dims &shape, const Dims &start, const Dims &count,
+            const size_t steps, const adios2::Params &engineParams,
+            const std::string &name)
+{
+    size_t datasize =
+        std::accumulate(count.begin(), count.end(), static_cast(1),
+                        std::multiplies());
+    adios2::ADIOS adios(mpiComm);
+    adios2::IO dataManIO = adios.DeclareIO("WAN");
+    dataManIO.SetEngine("ssc");
+    dataManIO.SetParameters(engineParams);
+    std::vector myChars(datasize);
+    std::vector myUChars(datasize);
+    std::vector myShorts(datasize);
+    std::vector myUShorts(datasize);
+    std::vector myInts(datasize);
+    std::vector myUInts(datasize);
+    std::vector myFloats(datasize);
+    std::vector myDoubles(datasize);
+    std::vector> myComplexes(datasize);
+    std::vector> myDComplexes(datasize);
+    auto bpChars =
+        dataManIO.DefineVariable("bpChars", shape, start, count);
+    auto bpUChars = dataManIO.DefineVariable("bpUChars", shape,
+                                                            start, count);
+    auto bpShorts =
+        dataManIO.DefineVariable("bpShorts", shape, start, count);
+    auto bpUShorts = dataManIO.DefineVariable(
+        "bpUShorts", shape, start, count);
+    auto bpInts = dataManIO.DefineVariable("bpInts", shape, start, count);
+    auto bpUInts =
+        dataManIO.DefineVariable("bpUInts", shape, start, count);
+    auto bpFloats =
+        dataManIO.DefineVariable("bpFloats", shape, start, count);
+    auto bpDoubles =
+        dataManIO.DefineVariable("bpDoubles", shape, start, count);
+    auto bpComplexes = dataManIO.DefineVariable>(
+        "bpComplexes", shape, start, count);
+    auto bpDComplexes = dataManIO.DefineVariable>(
+        "bpDComplexes", shape, start, count);
+    auto scalarInt = dataManIO.DefineVariable("scalarInt");
+    auto stringVar = dataManIO.DefineVariable("stringVar");
+    dataManIO.DefineAttribute("AttInt", 110);
+    adios2::Engine engine = dataManIO.Open(name, adios2::Mode::Write);
+    //    engine.LockWriterDefinitions();
+    for (int i = 0; i < steps; ++i)
+    {
+        engine.BeginStep();
+
+        Dims vstart = {0, (size_t)mpiRank * 2};
+        //        Dims vcount = {(size_t)i+1, 2};
+        //        Dims vshape = {(size_t)i+1, (size_t)mpiSize * 2};
+        Dims vcount = {10, 2};
+        Dims vshape = {10, (size_t)mpiSize * 2};
+
+        bpChars.SetShape(vshape);
+        bpChars.SetSelection({vstart, vcount});
+        bpUChars.SetShape(vshape);
+        bpUChars.SetSelection({vstart, vcount});
+        bpShorts.SetShape(vshape);
+        bpShorts.SetSelection({vstart, vcount});
+        bpUShorts.SetShape(vshape);
+        bpUShorts.SetSelection({vstart, vcount});
+        bpInts.SetShape(vshape);
+        bpInts.SetSelection({vstart, vcount});
+        bpUInts.SetShape(vshape);
+        bpUInts.SetSelection({vstart, vcount});
+        bpFloats.SetShape(vshape);
+        bpFloats.SetSelection({vstart, vcount});
+        bpDoubles.SetShape(vshape);
+        bpDoubles.SetSelection({vstart, vcount});
+        bpComplexes.SetShape(vshape);
+        bpComplexes.SetSelection({vstart, vcount});
+        bpDComplexes.SetShape(vshape);
+        bpDComplexes.SetSelection({vstart, vcount});
+
+        GenData(myChars, i, vstart, vcount, vshape);
+        GenData(myUChars, i, vstart, vcount, vshape);
+        GenData(myShorts, i, vstart, vcount, vshape);
+        GenData(myUShorts, i, vstart, vcount, vshape);
+        GenData(myInts, i, vstart, vcount, vshape);
+        GenData(myUInts, i, vstart, vcount, vshape);
+        GenData(myFloats, i, vstart, vcount, vshape);
+        GenData(myDoubles, i, vstart, vcount, vshape);
+        GenData(myComplexes, i, vstart, vcount, vshape);
+        GenData(myDComplexes, i, vstart, vcount, vshape);
+
+        engine.Put(bpChars, myChars.data(), adios2::Mode::Sync);
+        engine.Put(bpUChars, myUChars.data(), adios2::Mode::Sync);
+        engine.Put(bpShorts, myShorts.data(), adios2::Mode::Sync);
+        engine.Put(bpUShorts, myUShorts.data(), adios2::Mode::Sync);
+        engine.Put(bpInts, myInts.data(), adios2::Mode::Sync);
+        engine.Put(bpUInts, myUInts.data(), adios2::Mode::Sync);
+        engine.Put(bpFloats, myFloats.data(), adios2::Mode::Sync);
+        engine.Put(bpDoubles, myDoubles.data(), adios2::Mode::Sync);
+        engine.Put(bpComplexes, myComplexes.data(), adios2::Mode::Sync);
+        engine.Put(bpDComplexes, myDComplexes.data(), adios2::Mode::Sync);
+        engine.Put(scalarInt, i);
+        std::string s = "sample string sample string sample string";
+        engine.Put(stringVar, s);
+        engine.EndStep();
+    }
+    engine.Close();
+}
+
+void Reader(const Dims &shape, const Dims &start, const Dims &count,
+            const size_t steps, const adios2::Params &engineParams,
+            const std::string &name)
+{
+    adios2::ADIOS adios(mpiComm);
+    adios2::IO dataManIO = adios.DeclareIO("Test");
+    dataManIO.SetEngine("ssc");
+    dataManIO.SetParameters(engineParams);
+    adios2::Engine engine = dataManIO.Open(name, adios2::Mode::Read);
+    //    engine.LockReaderSelections();
+
+    size_t datasize =
+        std::accumulate(count.begin(), count.end(), static_cast(1),
+                        std::multiplies());
+    std::vector myChars(datasize);
+    std::vector myUChars(datasize);
+    std::vector myShorts(datasize);
+    std::vector myUShorts(datasize);
+    std::vector myInts(datasize);
+    std::vector myUInts(datasize);
+    std::vector myFloats(datasize);
+    std::vector myDoubles(datasize);
+    std::vector> myComplexes(datasize);
+    std::vector> myDComplexes(datasize);
+
+    while (true)
+    {
+        adios2::StepStatus status = engine.BeginStep(StepMode::Read, 5);
+        if (status == adios2::StepStatus::OK)
+        {
+            auto scalarInt = dataManIO.InquireVariable("scalarInt");
+            auto blocksInfo =
+                engine.BlocksInfo(scalarInt, engine.CurrentStep());
+
+            for (const auto &bi : blocksInfo)
+            {
+                ASSERT_EQ(bi.IsValue, true);
+                ASSERT_EQ(bi.Value, engine.CurrentStep());
+                ASSERT_EQ(scalarInt.Min(), engine.CurrentStep());
+                ASSERT_EQ(scalarInt.Max(), engine.CurrentStep());
+            }
+
+            const auto &vars = dataManIO.AvailableVariables();
+            ASSERT_EQ(vars.size(), 12);
+            size_t currentStep = engine.CurrentStep();
+            adios2::Variable bpChars =
+                dataManIO.InquireVariable("bpChars");
+            adios2::Variable bpUChars =
+                dataManIO.InquireVariable("bpUChars");
+            adios2::Variable bpShorts =
+                dataManIO.InquireVariable("bpShorts");
+            adios2::Variable bpUShorts =
+                dataManIO.InquireVariable("bpUShorts");
+            adios2::Variable bpInts =
+                dataManIO.InquireVariable("bpInts");
+            adios2::Variable bpUInts =
+                dataManIO.InquireVariable("bpUInts");
+            adios2::Variable bpFloats =
+                dataManIO.InquireVariable("bpFloats");
+            adios2::Variable bpDoubles =
+                dataManIO.InquireVariable("bpDoubles");
+            adios2::Variable> bpComplexes =
+                dataManIO.InquireVariable>("bpComplexes");
+            adios2::Variable> bpDComplexes =
+                dataManIO.InquireVariable>("bpDComplexes");
+            adios2::Variable stringVar =
+                dataManIO.InquireVariable("stringVar");
+
+            auto vshape = bpChars.Shape();
+            myChars.resize(std::accumulate(vshape.begin(), vshape.end(),
+                                           static_cast(1),
+                                           std::multiplies()));
+            myUChars.resize(std::accumulate(vshape.begin(), vshape.end(),
+                                            static_cast(1),
+                                            std::multiplies()));
+            myShorts.resize(std::accumulate(vshape.begin(), vshape.end(),
+                                            static_cast(1),
+                                            std::multiplies()));
+            myUShorts.resize(std::accumulate(vshape.begin(), vshape.end(),
+                                             static_cast(1),
+                                             std::multiplies()));
+            myInts.resize(std::accumulate(vshape.begin(), vshape.end(),
+                                          static_cast(1),
+                                          std::multiplies()));
+            myUInts.resize(std::accumulate(vshape.begin(), vshape.end(),
+                                           static_cast(1),
+                                           std::multiplies()));
+            myFloats.resize(std::accumulate(vshape.begin(), vshape.end(),
+                                            static_cast(1),
+                                            std::multiplies()));
+            myDoubles.resize(std::accumulate(vshape.begin(), vshape.end(),
+                                             static_cast(1),
+                                             std::multiplies()));
+            myComplexes.resize(std::accumulate(vshape.begin(), vshape.end(),
+                                               static_cast(1),
+                                               std::multiplies()));
+            myDComplexes.resize(std::accumulate(vshape.begin(), vshape.end(),
+                                                static_cast(1),
+                                                std::multiplies()));
+
+            engine.Get(bpChars, myChars.data(), adios2::Mode::Sync);
+            engine.Get(bpUChars, myUChars.data(), adios2::Mode::Sync);
+            engine.Get(bpShorts, myShorts.data(), adios2::Mode::Sync);
+            engine.Get(bpUShorts, myUShorts.data(), adios2::Mode::Sync);
+            engine.Get(bpInts, myInts.data(), adios2::Mode::Sync);
+            engine.Get(bpUInts, myUInts.data(), adios2::Mode::Sync);
+            engine.Get(bpFloats, myFloats.data(), adios2::Mode::Sync);
+            engine.Get(bpDoubles, myDoubles.data(), adios2::Mode::Sync);
+            engine.Get(bpComplexes, myComplexes.data(), adios2::Mode::Sync);
+            engine.Get(bpDComplexes, myDComplexes.data(), adios2::Mode::Sync);
+            std::string s;
+            engine.Get(stringVar, s);
+            ASSERT_EQ(s, "sample string sample string sample string");
+            ASSERT_EQ(stringVar.Min(),
+                      "sample string sample string sample string");
+            ASSERT_EQ(stringVar.Max(),
+                      "sample string sample string sample string");
+
+            int i;
+            engine.Get(scalarInt, &i);
+            ASSERT_EQ(i, currentStep);
+
+            VerifyData(myChars.data(), currentStep, {0, 0}, vshape, vshape,
+                       mpiRank);
+            VerifyData(myUChars.data(), currentStep, {0, 0}, vshape, vshape,
+                       mpiRank);
+            VerifyData(myShorts.data(), currentStep, {0, 0}, vshape, vshape,
+                       mpiRank);
+            VerifyData(myUShorts.data(), currentStep, {0, 0}, vshape, vshape,
+                       mpiRank);
+            VerifyData(myInts.data(), currentStep, {0, 0}, vshape, vshape,
+                       mpiRank);
+            VerifyData(myUInts.data(), currentStep, {0, 0}, vshape, vshape,
+                       mpiRank);
+            VerifyData(myFloats.data(), currentStep, {0, 0}, vshape, vshape,
+                       mpiRank);
+            VerifyData(myDoubles.data(), currentStep, {0, 0}, vshape, vshape,
+                       mpiRank);
+            VerifyData(myComplexes.data(), currentStep, {0, 0}, vshape, vshape,
+                       mpiRank);
+            VerifyData(myDComplexes.data(), currentStep, {0, 0}, vshape, vshape,
+                       mpiRank);
+            engine.EndStep();
+        }
+        else if (status == adios2::StepStatus::EndOfStream)
+        {
+            std::cout << "[Rank " + std::to_string(mpiRank) +
+                             "] SscTest reader end of stream!"
+                      << std::endl;
+            break;
+        }
+    }
+    auto attInt = dataManIO.InquireAttribute("AttInt");
+    std::cout << "[Rank " + std::to_string(mpiRank) + "] Attribute received "
+              << attInt.Data()[0] << ", expected 110" << std::endl;
+    ASSERT_EQ(110, attInt.Data()[0]);
+    ASSERT_NE(111, attInt.Data()[0]);
+    engine.Close();
+}
+
+TEST_F(SscEngineTest, TestSscVaryingSteps)
+{
+    std::string filename = "TestSscVaryingSteps";
+    adios2::Params engineParams = {};
+
+    int worldRank, worldSize;
+    MPI_Comm_rank(MPI_COMM_WORLD, &worldRank);
+    MPI_Comm_size(MPI_COMM_WORLD, &worldSize);
+    int mpiGroup = worldRank / (worldSize / 2);
+    MPI_Comm_split(MPI_COMM_WORLD, mpiGroup, worldRank, &mpiComm);
+
+    MPI_Comm_rank(mpiComm, &mpiRank);
+    MPI_Comm_size(mpiComm, &mpiSize);
+
+    Dims shape = {1, (size_t)mpiSize * 2};
+    Dims start = {0, (size_t)mpiRank * 2};
+    Dims count = {1, 2};
+    size_t steps = 100;
+
+    if (mpiGroup == 0)
+    {
+        Writer(shape, start, count, steps, engineParams, filename);
+    }
+
+    std::this_thread::sleep_for(std::chrono::milliseconds(1));
+
+    if (mpiGroup == 1)
+    {
+        Reader(shape, start, count, steps, engineParams, filename);
+    }
+
+    MPI_Barrier(MPI_COMM_WORLD);
+}
+
+int main(int argc, char **argv)
+{
+    MPI_Init(&argc, &argv);
+    int worldRank, worldSize;
+    MPI_Comm_rank(MPI_COMM_WORLD, &worldRank);
+    MPI_Comm_size(MPI_COMM_WORLD, &worldSize);
+    ::testing::InitGoogleTest(&argc, argv);
+    int result = RUN_ALL_TESTS();
+
+    MPI_Finalize();
+    return result;
+}
diff --git a/testing/adios2/engine/ssc/TestSscWriterMultiblock.cpp b/testing/adios2/engine/ssc/TestSscWriterMultiblock.cpp
index 64212afc14..806a53e1d4 100644
--- a/testing/adios2/engine/ssc/TestSscWriterMultiblock.cpp
+++ b/testing/adios2/engine/ssc/TestSscWriterMultiblock.cpp
@@ -25,8 +25,9 @@ void Writer(const Dims &shape, const Dims &start, const Dims &count,
             const size_t steps, const adios2::Params &engineParams,
             const std::string &name)
 {
-    size_t datasize = std::accumulate(count.begin(), count.end(), 1,
-                                      std::multiplies());
+    size_t datasize =
+        std::accumulate(count.begin(), count.end(), static_cast(1),
+                        std::multiplies());
     adios2::ADIOS adios(mpiComm);
     adios2::IO dataManIO = adios.DeclareIO("WAN");
     dataManIO.SetEngine("ssc");
@@ -62,10 +63,11 @@ void Writer(const Dims &shape, const Dims &start, const Dims &count,
         "bpDComplexes", shape, start, count);
     auto scalarInt = dataManIO.DefineVariable("scalarInt");
     dataManIO.DefineAttribute("AttInt", 110);
-    adios2::Engine dataManWriter = dataManIO.Open(name, adios2::Mode::Write);
+    adios2::Engine engine = dataManIO.Open(name, adios2::Mode::Write);
+    engine.LockWriterDefinitions();
     for (int i = 0; i < steps; ++i)
     {
-        dataManWriter.BeginStep();
+        engine.BeginStep();
 
         Dims start = {(size_t)mpiRank * 2, 0};
 
@@ -91,18 +93,17 @@ void Writer(const Dims &shape, const Dims &start, const Dims &count,
         bpComplexes.SetSelection({start, count});
         bpDComplexes.SetSelection({start, count});
 
-        dataManWriter.Put(bpChars, myChars.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpUChars, myUChars.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpShorts, myShorts.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpUShorts, myUShorts.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpInts, myInts.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpUInts, myUInts.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpFloats, myFloats.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpDoubles, myDoubles.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpComplexes, myComplexes.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpDComplexes, myDComplexes.data(),
-                          adios2::Mode::Sync);
-        dataManWriter.Put(scalarInt, i);
+        engine.Put(bpChars, myChars.data(), adios2::Mode::Sync);
+        engine.Put(bpUChars, myUChars.data(), adios2::Mode::Sync);
+        engine.Put(bpShorts, myShorts.data(), adios2::Mode::Sync);
+        engine.Put(bpUShorts, myUShorts.data(), adios2::Mode::Sync);
+        engine.Put(bpInts, myInts.data(), adios2::Mode::Sync);
+        engine.Put(bpUInts, myUInts.data(), adios2::Mode::Sync);
+        engine.Put(bpFloats, myFloats.data(), adios2::Mode::Sync);
+        engine.Put(bpDoubles, myDoubles.data(), adios2::Mode::Sync);
+        engine.Put(bpComplexes, myComplexes.data(), adios2::Mode::Sync);
+        engine.Put(bpDComplexes, myDComplexes.data(), adios2::Mode::Sync);
+        engine.Put(scalarInt, i);
 
         start = {(size_t)mpiRank * 2 + 1, 0};
 
@@ -128,22 +129,21 @@ void Writer(const Dims &shape, const Dims &start, const Dims &count,
         bpComplexes.SetSelection({start, count});
         bpDComplexes.SetSelection({start, count});
 
-        dataManWriter.Put(bpChars, myChars.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpUChars, myUChars.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpShorts, myShorts.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpUShorts, myUShorts.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpInts, myInts.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpUInts, myUInts.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpFloats, myFloats.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpDoubles, myDoubles.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpComplexes, myComplexes.data(), adios2::Mode::Sync);
-        dataManWriter.Put(bpDComplexes, myDComplexes.data(),
-                          adios2::Mode::Sync);
-        dataManWriter.Put(scalarInt, i);
+        engine.Put(bpChars, myChars.data(), adios2::Mode::Sync);
+        engine.Put(bpUChars, myUChars.data(), adios2::Mode::Sync);
+        engine.Put(bpShorts, myShorts.data(), adios2::Mode::Sync);
+        engine.Put(bpUShorts, myUShorts.data(), adios2::Mode::Sync);
+        engine.Put(bpInts, myInts.data(), adios2::Mode::Sync);
+        engine.Put(bpUInts, myUInts.data(), adios2::Mode::Sync);
+        engine.Put(bpFloats, myFloats.data(), adios2::Mode::Sync);
+        engine.Put(bpDoubles, myDoubles.data(), adios2::Mode::Sync);
+        engine.Put(bpComplexes, myComplexes.data(), adios2::Mode::Sync);
+        engine.Put(bpDComplexes, myDComplexes.data(), adios2::Mode::Sync);
+        engine.Put(scalarInt, i);
 
-        dataManWriter.EndStep();
+        engine.EndStep();
     }
-    dataManWriter.Close();
+    engine.Close();
 }
 
 void Reader(const Dims &shape, const Dims &start, const Dims &count,
@@ -154,10 +154,12 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count,
     adios2::IO dataManIO = adios.DeclareIO("Test");
     dataManIO.SetEngine("ssc");
     dataManIO.SetParameters(engineParams);
-    adios2::Engine dataManReader = dataManIO.Open(name, adios2::Mode::Read);
+    adios2::Engine engine = dataManIO.Open(name, adios2::Mode::Read);
+    engine.LockReaderSelections();
 
-    size_t datasize = std::accumulate(shape.begin(), shape.end(), 1,
-                                      std::multiplies());
+    size_t datasize =
+        std::accumulate(shape.begin(), shape.end(), static_cast(1),
+                        std::multiplies());
     std::vector myChars(datasize);
     std::vector myUChars(datasize);
     std::vector myShorts(datasize);
@@ -171,12 +173,12 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count,
 
     while (true)
     {
-        adios2::StepStatus status = dataManReader.BeginStep(StepMode::Read, 5);
+        adios2::StepStatus status = engine.BeginStep(StepMode::Read, 5);
         if (status == adios2::StepStatus::OK)
         {
             const auto &vars = dataManIO.AvailableVariables();
             ASSERT_EQ(vars.size(), 11);
-            size_t currentStep = dataManReader.CurrentStep();
+            size_t currentStep = engine.CurrentStep();
             adios2::Variable bpChars =
                 dataManIO.InquireVariable("bpChars");
             adios2::Variable bpUChars =
@@ -199,20 +201,18 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count,
                 dataManIO.InquireVariable>("bpDComplexes");
             auto scalarInt = dataManIO.InquireVariable("scalarInt");
 
-            dataManReader.Get(bpChars, myChars.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpUChars, myUChars.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpShorts, myShorts.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpUShorts, myUShorts.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpInts, myInts.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpUInts, myUInts.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpFloats, myFloats.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpDoubles, myDoubles.data(), adios2::Mode::Sync);
-            dataManReader.Get(bpComplexes, myComplexes.data(),
-                              adios2::Mode::Sync);
-            dataManReader.Get(bpDComplexes, myDComplexes.data(),
-                              adios2::Mode::Sync);
+            engine.Get(bpChars, myChars.data(), adios2::Mode::Sync);
+            engine.Get(bpUChars, myUChars.data(), adios2::Mode::Sync);
+            engine.Get(bpShorts, myShorts.data(), adios2::Mode::Sync);
+            engine.Get(bpUShorts, myUShorts.data(), adios2::Mode::Sync);
+            engine.Get(bpInts, myInts.data(), adios2::Mode::Sync);
+            engine.Get(bpUInts, myUInts.data(), adios2::Mode::Sync);
+            engine.Get(bpFloats, myFloats.data(), adios2::Mode::Sync);
+            engine.Get(bpDoubles, myDoubles.data(), adios2::Mode::Sync);
+            engine.Get(bpComplexes, myComplexes.data(), adios2::Mode::Sync);
+            engine.Get(bpDComplexes, myDComplexes.data(), adios2::Mode::Sync);
             int i;
-            dataManReader.Get(scalarInt, &i);
+            engine.Get(scalarInt, &i);
 
             ASSERT_EQ(i, currentStep);
 
@@ -236,7 +236,7 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count,
                        mpiRank);
             VerifyData(myDComplexes.data(), currentStep, start, shape, shape,
                        mpiRank);
-            dataManReader.EndStep();
+            engine.EndStep();
         }
         else if (status == adios2::StepStatus::EndOfStream)
         {
@@ -251,7 +251,7 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count,
               << attInt.Data()[0] << ", expected 110" << std::endl;
     ASSERT_EQ(110, attInt.Data()[0]);
     ASSERT_NE(111, attInt.Data()[0]);
-    dataManReader.Close();
+    engine.Close();
 }
 
 TEST_F(SscEngineTest, TestSscWriterMultiblock)
diff --git a/testing/adios2/engine/ssc/TestSscXgc2Way.cpp b/testing/adios2/engine/ssc/TestSscXgc2Way.cpp
index 2bc221d379..e2c9cfa9c1 100644
--- a/testing/adios2/engine/ssc/TestSscXgc2Way.cpp
+++ b/testing/adios2/engine/ssc/TestSscXgc2Way.cpp
@@ -25,8 +25,9 @@ class SscEngineTest : public ::testing::Test
 void xgc(const Dims &shape, const Dims &start, const Dims &count,
          const size_t steps, const adios2::Params &engineParams)
 {
-    size_t datasize = std::accumulate(count.begin(), count.end(), 1,
-                                      std::multiplies());
+    size_t datasize =
+        std::accumulate(count.begin(), count.end(), static_cast(1),
+                        std::multiplies());
 
     std::vector x_to_g_data(datasize);
     std::vector g_to_x_data;
@@ -47,6 +48,8 @@ void xgc(const Dims &shape, const Dims &start, const Dims &count,
     adios2::Engine x_to_g_engine =
         x_to_g_io.Open("x_to_g", adios2::Mode::Write);
     adios2::Engine g_to_x_engine = g_to_x_io.Open("g_to_x", adios2::Mode::Read);
+    x_to_g_engine.LockWriterDefinitions();
+    g_to_x_engine.LockReaderSelections();
 
     for (int i = 0; i < steps; ++i)
     {
@@ -59,7 +62,8 @@ void xgc(const Dims &shape, const Dims &start, const Dims &count,
         auto g_to_x_var = g_to_x_io.InquireVariable("g_to_x");
         auto readShape = g_to_x_var.Shape();
         g_to_x_data.resize(std::accumulate(readShape.begin(), readShape.end(),
-                                           1, std::multiplies()));
+                                           static_cast(1),
+                                           std::multiplies()));
         g_to_x_engine.Get(g_to_x_var, g_to_x_data.data(), adios2::Mode::Sync);
         VerifyData(g_to_x_data.data(), i, Dims(readShape.size(), 0), readShape,
                    readShape, mpiRank);
@@ -90,9 +94,12 @@ void gene(const Dims &shape, const Dims &start, const Dims &count,
     adios2::Engine x_to_g_engine = x_to_g_io.Open("x_to_g", adios2::Mode::Read);
     adios2::Engine g_to_x_engine =
         g_to_x_io.Open("g_to_x", adios2::Mode::Write);
+    g_to_x_engine.LockWriterDefinitions();
+    x_to_g_engine.LockReaderSelections();
 
-    size_t datasize = std::accumulate(shape.begin(), shape.end(), 1,
-                                      std::multiplies());
+    size_t datasize =
+        std::accumulate(shape.begin(), shape.end(), static_cast(1),
+                        std::multiplies());
     std::vector x_to_g_data;
     std::vector g_to_x_data(datasize);
 
@@ -102,7 +109,8 @@ void gene(const Dims &shape, const Dims &start, const Dims &count,
         auto x_to_g_var = x_to_g_io.InquireVariable("x_to_g");
         auto readShape = x_to_g_var.Shape();
         x_to_g_data.resize(std::accumulate(readShape.begin(), readShape.end(),
-                                           1, std::multiplies()));
+                                           static_cast(1),
+                                           std::multiplies()));
         x_to_g_engine.Get(x_to_g_var, x_to_g_data.data(), adios2::Mode::Sync);
         VerifyData(x_to_g_data.data(), i, Dims(readShape.size(), 0), readShape,
                    readShape, mpiRank);
diff --git a/testing/adios2/engine/ssc/TestSscXgc3Way.cpp b/testing/adios2/engine/ssc/TestSscXgc3Way.cpp
index 0164d274f3..c67f0d8898 100644
--- a/testing/adios2/engine/ssc/TestSscXgc3Way.cpp
+++ b/testing/adios2/engine/ssc/TestSscXgc3Way.cpp
@@ -25,8 +25,9 @@ class SscEngineTest : public ::testing::Test
 void coupler(const Dims &shape, const Dims &start, const Dims &count,
              const size_t steps, const adios2::Params &engineParams)
 {
-    size_t datasize = std::accumulate(count.begin(), count.end(), 1,
-                                      std::multiplies());
+    size_t datasize =
+        std::accumulate(count.begin(), count.end(), static_cast(1),
+                        std::multiplies());
 
     adios2::ADIOS adios(mpiComm);
 
@@ -57,14 +58,18 @@ void coupler(const Dims &shape, const Dims &start, const Dims &count,
         c_to_g_io.DefineVariable("c_to_g", shape, start, count);
 
     adios2::Engine x_to_c_engine = x_to_c_io.Open("x_to_c", adios2::Mode::Read);
+    x_to_c_engine.LockReaderSelections();
 
     adios2::Engine c_to_x_engine =
         c_to_x_io.Open("c_to_x", adios2::Mode::Write);
+    c_to_x_engine.LockWriterDefinitions();
 
     adios2::Engine c_to_g_engine =
         c_to_g_io.Open("c_to_g", adios2::Mode::Write);
+    c_to_g_engine.LockWriterDefinitions();
 
     adios2::Engine g_to_c_engine = g_to_c_io.Open("g_to_c", adios2::Mode::Read);
+    g_to_c_engine.LockReaderSelections();
 
     for (int i = 0; i < steps; ++i)
     {
@@ -72,7 +77,8 @@ void coupler(const Dims &shape, const Dims &start, const Dims &count,
         auto x_to_c_var = x_to_c_io.InquireVariable("x_to_c");
         auto readShape = x_to_c_var.Shape();
         x_to_c_data.resize(std::accumulate(readShape.begin(), readShape.end(),
-                                           1, std::multiplies()));
+                                           static_cast(1),
+                                           std::multiplies()));
         x_to_c_engine.Get(x_to_c_var, x_to_c_data.data(), adios2::Mode::Sync);
         VerifyData(x_to_c_data.data(), i, Dims(readShape.size(), 0), readShape,
                    readShape, mpiRank);
@@ -87,7 +93,8 @@ void coupler(const Dims &shape, const Dims &start, const Dims &count,
         auto g_to_c_var = g_to_c_io.InquireVariable("g_to_c");
         readShape = g_to_c_var.Shape();
         g_to_c_data.resize(std::accumulate(readShape.begin(), readShape.end(),
-                                           1, std::multiplies()));
+                                           static_cast(1),
+                                           std::multiplies()));
         g_to_c_engine.Get(g_to_c_var, g_to_c_data.data(), adios2::Mode::Sync);
         VerifyData(g_to_c_data.data(), i, Dims(readShape.size(), 0), readShape,
                    readShape, mpiRank);
@@ -110,8 +117,9 @@ void coupler(const Dims &shape, const Dims &start, const Dims &count,
 void xgc(const Dims &shape, const Dims &start, const Dims &count,
          const size_t steps, const adios2::Params &engineParams)
 {
-    size_t datasize = std::accumulate(count.begin(), count.end(), 1,
-                                      std::multiplies());
+    size_t datasize =
+        std::accumulate(count.begin(), count.end(), static_cast(1),
+                        std::multiplies());
 
     std::vector x_to_c_data(datasize);
     std::vector c_to_x_data;
@@ -131,7 +139,10 @@ void xgc(const Dims &shape, const Dims &start, const Dims &count,
 
     adios2::Engine x_to_c_engine =
         x_to_c_io.Open("x_to_c", adios2::Mode::Write);
+    x_to_c_engine.LockWriterDefinitions();
+
     adios2::Engine c_to_x_engine = c_to_x_io.Open("c_to_x", adios2::Mode::Read);
+    c_to_x_engine.LockReaderSelections();
 
     for (int i = 0; i < steps; ++i)
     {
@@ -144,7 +155,8 @@ void xgc(const Dims &shape, const Dims &start, const Dims &count,
         auto c_to_x_var = c_to_x_io.InquireVariable("c_to_x");
         auto readShape = c_to_x_var.Shape();
         c_to_x_data.resize(std::accumulate(readShape.begin(), readShape.end(),
-                                           1, std::multiplies()));
+                                           static_cast(1),
+                                           std::multiplies()));
         c_to_x_engine.Get(c_to_x_var, c_to_x_data.data(), adios2::Mode::Sync);
         VerifyData(c_to_x_data.data(), i, Dims(readShape.size(), 0), readShape,
                    readShape, mpiRank);
@@ -173,11 +185,14 @@ void gene(const Dims &shape, const Dims &start, const Dims &count,
         g_to_c_io.DefineVariable("g_to_c", shape, start, count);
 
     adios2::Engine c_to_g_engine = c_to_g_io.Open("c_to_g", adios2::Mode::Read);
+    c_to_g_engine.LockReaderSelections();
     adios2::Engine g_to_c_engine =
         g_to_c_io.Open("g_to_c", adios2::Mode::Write);
+    g_to_c_engine.LockWriterDefinitions();
 
-    size_t datasize = std::accumulate(shape.begin(), shape.end(), 1,
-                                      std::multiplies());
+    size_t datasize =
+        std::accumulate(shape.begin(), shape.end(), static_cast(1),
+                        std::multiplies());
     std::vector c_to_g_data;
     std::vector g_to_c_data(datasize);
 
@@ -187,7 +202,8 @@ void gene(const Dims &shape, const Dims &start, const Dims &count,
         auto c_to_g_var = c_to_g_io.InquireVariable("c_to_g");
         auto readShape = c_to_g_var.Shape();
         c_to_g_data.resize(std::accumulate(readShape.begin(), readShape.end(),
-                                           1, std::multiplies()));
+                                           static_cast(1),
+                                           std::multiplies()));
         c_to_g_engine.Get(c_to_g_var, c_to_g_data.data(), adios2::Mode::Sync);
         VerifyData(c_to_g_data.data(), i, Dims(readShape.size(), 0), readShape,
                    readShape, mpiRank);
diff --git a/testing/adios2/engine/ssc/TestSscXgc3WayMatchedSteps.cpp b/testing/adios2/engine/ssc/TestSscXgc3WayMatchedSteps.cpp
new file mode 100644
index 0000000000..9442249023
--- /dev/null
+++ b/testing/adios2/engine/ssc/TestSscXgc3WayMatchedSteps.cpp
@@ -0,0 +1,300 @@
+/*
+ * Distributed under the OSI-approved Apache License, Version 2.0.  See
+ * accompanying file Copyright.txt for details.
+ */
+
+#include "TestSscCommon.h"
+#include 
+#include 
+#include 
+#include 
+#include 
+
+using namespace adios2;
+int mpiRank = 0;
+int mpiSize = 1;
+int mpiGroup;
+MPI_Comm mpiComm;
+
+class SscEngineTest : public ::testing::Test
+{
+public:
+    SscEngineTest() = default;
+};
+
+void coupler(const Dims &shape, const Dims &start, const Dims &count,
+             const size_t steps, const adios2::Params &engineParams)
+{
+    size_t datasize =
+        std::accumulate(count.begin(), count.end(), static_cast(1),
+                        std::multiplies());
+
+    adios2::ADIOS adios(mpiComm);
+
+    adios2::IO x_to_c_io = adios.DeclareIO("x_to_c");
+    x_to_c_io.SetEngine("ssc");
+    x_to_c_io.SetParameters(engineParams);
+
+    adios2::IO c_to_x_io = adios.DeclareIO("c_to_x");
+    c_to_x_io.SetEngine("ssc");
+    c_to_x_io.SetParameters(engineParams);
+
+    adios2::IO g_to_c_io = adios.DeclareIO("g_to_c");
+    g_to_c_io.SetEngine("ssc");
+    g_to_c_io.SetParameters(engineParams);
+
+    adios2::IO c_to_g_io = adios.DeclareIO("c_to_g");
+    c_to_g_io.SetEngine("ssc");
+    c_to_g_io.SetParameters(engineParams);
+
+    std::vector x_to_c_data;
+    std::vector c_to_x_data(datasize);
+    std::vector g_to_c_data;
+    std::vector c_to_g_data(datasize);
+
+    auto c_to_x_var =
+        c_to_x_io.DefineVariable("c_to_x", shape, start, count);
+    auto c_to_g_var =
+        c_to_g_io.DefineVariable("c_to_g", shape, start, count);
+
+    adios2::Engine x_to_c_engine = x_to_c_io.Open("x_to_c", adios2::Mode::Read);
+    x_to_c_engine.LockReaderSelections();
+
+    adios2::Engine c_to_x_engine =
+        c_to_x_io.Open("c_to_x", adios2::Mode::Write);
+    c_to_x_engine.LockWriterDefinitions();
+
+    adios2::Engine c_to_g_engine =
+        c_to_g_io.Open("c_to_g", adios2::Mode::Write);
+    c_to_g_engine.LockWriterDefinitions();
+
+    adios2::Engine g_to_c_engine = g_to_c_io.Open("g_to_c", adios2::Mode::Read);
+    g_to_c_engine.LockReaderSelections();
+
+    for (int i = 0; i < steps; ++i)
+    {
+        x_to_c_engine.BeginStep();
+        auto x_to_c_var = x_to_c_io.InquireVariable("x_to_c");
+        auto readShape = x_to_c_var.Shape();
+        x_to_c_data.resize(std::accumulate(readShape.begin(), readShape.end(),
+                                           static_cast(1),
+                                           std::multiplies()));
+        x_to_c_engine.Get(x_to_c_var, x_to_c_data.data(), adios2::Mode::Sync);
+        VerifyData(x_to_c_data.data(), i, Dims(readShape.size(), 0), readShape,
+                   readShape, mpiRank);
+        x_to_c_engine.EndStep();
+
+        c_to_g_engine.BeginStep();
+        GenData(c_to_g_data, i, start, count, shape);
+        c_to_g_engine.Put(c_to_g_var, c_to_g_data.data(), adios2::Mode::Sync);
+        c_to_g_engine.EndStep();
+
+        g_to_c_engine.BeginStep();
+        auto g_to_c_var = g_to_c_io.InquireVariable("g_to_c");
+        readShape = g_to_c_var.Shape();
+        g_to_c_data.resize(std::accumulate(readShape.begin(), readShape.end(),
+                                           static_cast(1),
+                                           std::multiplies()));
+        g_to_c_engine.Get(g_to_c_var, g_to_c_data.data(), adios2::Mode::Sync);
+        VerifyData(g_to_c_data.data(), i, Dims(readShape.size(), 0), readShape,
+                   readShape, mpiRank);
+        g_to_c_engine.EndStep();
+
+        c_to_x_engine.BeginStep();
+        GenData(c_to_x_data, i, start, count, shape);
+        c_to_x_engine.Put(c_to_x_var, c_to_x_data.data(), adios2::Mode::Sync);
+        c_to_x_engine.EndStep();
+    }
+
+    x_to_c_engine.Close();
+    c_to_g_engine.Close();
+    g_to_c_engine.Close();
+    c_to_x_engine.Close();
+}
+
+void xgc(const Dims &shape, const Dims &start, const Dims &count,
+         const size_t steps, const adios2::Params &engineParams)
+{
+    size_t datasize =
+        std::accumulate(count.begin(), count.end(), static_cast(1),
+                        std::multiplies());
+
+    std::vector x_to_c_data(datasize);
+    std::vector c_to_x_data;
+
+    adios2::ADIOS adios(mpiComm);
+
+    adios2::IO x_to_c_io = adios.DeclareIO("x_to_c");
+    x_to_c_io.SetEngine("ssc");
+    x_to_c_io.SetParameters(engineParams);
+
+    adios2::IO c_to_x_io = adios.DeclareIO("c_to_x");
+    c_to_x_io.SetEngine("ssc");
+    c_to_x_io.SetParameters(engineParams);
+
+    auto x_to_c_var =
+        x_to_c_io.DefineVariable("x_to_c", shape, start, count);
+
+    adios2::Engine x_to_c_engine =
+        x_to_c_io.Open("x_to_c", adios2::Mode::Write);
+    x_to_c_engine.LockWriterDefinitions();
+
+    adios2::Engine c_to_x_engine = c_to_x_io.Open("c_to_x", adios2::Mode::Read);
+    c_to_x_engine.LockReaderSelections();
+
+    for (int i = 0; i < steps; ++i)
+    {
+        x_to_c_engine.BeginStep();
+        GenData(x_to_c_data, i, start, count, shape);
+        x_to_c_engine.Put(x_to_c_var, x_to_c_data.data(), adios2::Mode::Sync);
+        x_to_c_engine.EndStep();
+
+        c_to_x_engine.BeginStep();
+        auto c_to_x_var = c_to_x_io.InquireVariable("c_to_x");
+        auto readShape = c_to_x_var.Shape();
+        c_to_x_data.resize(std::accumulate(readShape.begin(), readShape.end(),
+                                           static_cast(1),
+                                           std::multiplies()));
+        c_to_x_engine.Get(c_to_x_var, c_to_x_data.data(), adios2::Mode::Sync);
+        VerifyData(c_to_x_data.data(), i, Dims(readShape.size(), 0), readShape,
+                   readShape, mpiRank);
+        c_to_x_engine.EndStep();
+    }
+
+    x_to_c_engine.Close();
+    c_to_x_engine.Close();
+}
+
+void gene(const Dims &shape, const Dims &start, const Dims &count,
+          const size_t steps, const adios2::Params &engineParams)
+{
+    adios2::ADIOS adios(mpiComm);
+
+    adios2::IO c_to_g_io = adios.DeclareIO("c_to_g");
+    c_to_g_io.SetEngine("ssc");
+    c_to_g_io.SetParameters(engineParams);
+
+    adios2::IO g_to_c_io = adios.DeclareIO("g_to_c");
+    g_to_c_io.SetEngine("ssc");
+    g_to_c_io.SetParameters(engineParams);
+
+    auto g_to_c_var =
+        g_to_c_io.DefineVariable("g_to_c", shape, start, count);
+
+    adios2::Engine c_to_g_engine = c_to_g_io.Open("c_to_g", adios2::Mode::Read);
+    c_to_g_engine.LockReaderSelections();
+    adios2::Engine g_to_c_engine =
+        g_to_c_io.Open("g_to_c", adios2::Mode::Write);
+    g_to_c_engine.LockWriterDefinitions();
+
+    size_t datasize =
+        std::accumulate(shape.begin(), shape.end(), static_cast(1),
+                        std::multiplies());
+    std::vector c_to_g_data;
+    std::vector g_to_c_data(datasize);
+
+    for (int i = 0; i < steps; ++i)
+    {
+        c_to_g_engine.BeginStep(StepMode::Read, 5);
+        auto c_to_g_var = c_to_g_io.InquireVariable("c_to_g");
+        auto readShape = c_to_g_var.Shape();
+        c_to_g_data.resize(std::accumulate(readShape.begin(), readShape.end(),
+                                           static_cast(1),
+                                           std::multiplies()));
+        c_to_g_engine.Get(c_to_g_var, c_to_g_data.data(), adios2::Mode::Sync);
+        VerifyData(c_to_g_data.data(), i, Dims(readShape.size(), 0), readShape,
+                   readShape, mpiRank);
+        c_to_g_engine.EndStep();
+
+        g_to_c_engine.BeginStep();
+        GenData(g_to_c_data, i, start, count, shape);
+        g_to_c_engine.Put(g_to_c_var, g_to_c_data.data(), adios2::Mode::Sync);
+        g_to_c_engine.EndStep();
+    }
+
+    c_to_g_engine.Close();
+    g_to_c_engine.Close();
+}
+
+TEST_F(SscEngineTest, TestSscXgc3WayMatchedSteps)
+{
+
+    Dims start, count, shape;
+    int worldRank, worldSize;
+    MPI_Comm_rank(MPI_COMM_WORLD, &worldRank);
+    MPI_Comm_size(MPI_COMM_WORLD, &worldSize);
+
+    if (worldSize < 4)
+    {
+        return;
+    }
+
+    if (worldRank == 0)
+    {
+        mpiGroup = 0;
+    }
+    else if (worldRank == 1)
+    {
+        mpiGroup = 1;
+    }
+    else
+    {
+        mpiGroup = 2;
+    }
+
+    MPI_Comm_split(MPI_COMM_WORLD, mpiGroup, worldRank, &mpiComm);
+
+    MPI_Comm_rank(mpiComm, &mpiRank);
+    MPI_Comm_size(mpiComm, &mpiSize);
+
+    size_t steps = 20;
+
+    if (mpiGroup == 0)
+    {
+        shape = {(size_t)mpiSize, 10};
+        start = {(size_t)mpiRank, 0};
+        count = {1, 10};
+        adios2::Params engineParams = {{"RendezvousAppCount", "2"},
+                                       {"MaxStreamsPerApp", "4"},
+                                       {"OpenTimeoutSecs", "3"},
+                                       {"Verbose", "0"}};
+        coupler(shape, start, count, steps, engineParams);
+    }
+
+    if (mpiGroup == 1)
+    {
+        shape = {(size_t)mpiSize, 10};
+        start = {(size_t)mpiRank, 0};
+        count = {1, 10};
+        adios2::Params engineParams = {{"RendezvousAppCount", "2"},
+                                       {"MaxStreamsPerApp", "4"},
+                                       {"OpenTimeoutSecs", "3"},
+                                       {"Verbose", "0"}};
+        gene(shape, start, shape, steps, engineParams);
+    }
+
+    if (mpiGroup == 2)
+    {
+        shape = {(size_t)mpiSize, 10};
+        start = {(size_t)mpiRank, 0};
+        count = {1, 10};
+        adios2::Params engineParams = {{"RendezvousAppCount", "2"},
+                                       {"MaxStreamsPerApp", "4"},
+                                       {"OpenTimeoutSecs", "3"},
+                                       {"Verbose", "0"}};
+        xgc(shape, start, count, steps, engineParams);
+    }
+
+    MPI_Barrier(MPI_COMM_WORLD);
+}
+
+int main(int argc, char **argv)
+{
+    MPI_Init(&argc, &argv);
+
+    ::testing::InitGoogleTest(&argc, argv);
+    int result = RUN_ALL_TESTS();
+
+    MPI_Finalize();
+    return result;
+}
diff --git a/testing/adios2/engine/staging-common/CMakeLists.txt b/testing/adios2/engine/staging-common/CMakeLists.txt
index 296a998f62..f2902fd894 100644
--- a/testing/adios2/engine/staging-common/CMakeLists.txt
+++ b/testing/adios2/engine/staging-common/CMakeLists.txt
@@ -9,6 +9,10 @@ gtest_add_tests_helper(StagingMPMD MPI_ONLY "" Engine.Staging. ".InSituMPI" EXTR
 if(ADIOS2_HAVE_SST)
   gtest_add_tests_helper(StagingMPMD MPI_ONLY "" Engine.Staging. ".SST.FFS" EXTRA_ARGS "SST" "MarshalMethod=FFS")
   gtest_add_tests_helper(StagingMPMD MPI_ONLY "" Engine.Staging. ".SST.BP" EXTRA_ARGS "SST" "MarshalMethod=BP")
+  gtest_add_tests_helper(Threads MPI_NONE "" Engine.Staging. ".SST.FFS" EXTRA_ARGS "SST"  "--engine_params" "MarshalMethod=FFS")
+  gtest_add_tests_helper(Threads MPI_NONE "" Engine.Staging. ".SST.BP" EXTRA_ARGS "SST"  "--engine_params" "MarshalMethod=BP")
+  gtest_add_tests_helper(Threads MPI_NONE "" Engine.Staging. ".BP4_stream" EXTRA_ARGS "BP4"  "--engine_params" "OpenTimeoutSecs=5")
+  gtest_add_tests_helper(Threads MPI_NONE "" Engine.Staging. ".FileStream" EXTRA_ARGS "FileStream")
 endif()
 
 foreach(helper
@@ -50,9 +54,17 @@ if(ADIOS2_HAVE_Fortran)
     TestCommonReadF.F90
     $
   )
+  target_compile_definitions(TestCommonRead_f PRIVATE
+    $<$:ADIOS2_HAVE_FORTRAN_F03_ARGS>
+    $<$:ADIOS2_HAVE_FORTRAN_GNU_ARGS>
+  )
   set_target_properties(TestCommonWrite_f TestCommonRead_f PROPERTIES
     LINKER_LANGUAGE Fortran
   )
+  target_compile_definitions(TestCommonWrite_f PRIVATE
+    $<$:ADIOS2_HAVE_FORTRAN_F03_ARGS>
+    $<$:ADIOS2_HAVE_FORTRAN_GNU_ARGS>
+  )
   if(ADIOS2_HAVE_MPI)
     target_link_libraries(TestCommonWrite_f adios2_fortran_mpi MPI::MPI_Fortran)
     target_link_libraries(TestCommonRead_f adios2_fortran_mpi MPI::MPI_Fortran)
@@ -91,7 +103,7 @@ if(ADIOS2_HAVE_MPI AND MPIEXEC_EXECUTABLE)
    endforeach()
 endif()
 
-set (SIMPLE_TESTS "1x1;NoReaderNoWait;TimeoutOnOpen;1x1.NoData;1x1.Modes;1x1.Attrs;1x1.Local;1x1.SharedNothing;1x1.SharedIO;1x1.SharedVar;1x1.SharedNothingSync;1x1.SharedIOSync;1x1.SharedVarSync;1x1EarlyExit")
+set (SIMPLE_TESTS "1x1;NoReaderNoWait;TimeoutOnOpen;1x1.NoData;1x1.Modes;1x1.Attrs;1x1.Local;1x1.SharedNothing;1x1.SharedIO;1x1.SharedVar;1x1.SharedNothingSync;1x1.SharedIOSync;1x1.SharedVarSync;1x1EarlyExit;CumulativeAttr.1x1")
 
 set (SIMPLE_FORTRAN_TESTS "")
 if(ADIOS2_HAVE_Fortran)
@@ -122,12 +134,18 @@ endif()
 set (ALL_SIMPLE_TESTS "")
 list (APPEND ALL_SIMPLE_TESTS ${SIMPLE_TESTS} ${SIMPLE_FORTRAN_TESTS} ${SIMPLE_MPI_TESTS} ${SIMPLE_ZFP_TESTS})
 
+set (SST_SPECIFIC_TESTS  "")
+list (APPEND SST_SPECIFIC_TESTS  "1x1.SstRUDP;1x1.LocalMultiblock")
+if (ADIOS2_HAVE_MPI)
+  list (APPEND SST_SPECIFIC_TESTS  "2x3.SstRUDP;2x1.LocalMultiblock;5x3.LocalMultiblock;")
+endif()
+
 #
 #   Setup tests for SST engine
 #
 SET (BASIC_SST_TESTS "")
 if(ADIOS2_HAVE_SST)
-    list (APPEND BASIC_SST_TESTS ${ALL_SIMPLE_TESTS} ${SPECIAL_TESTS})
+  list (APPEND BASIC_SST_TESTS ${ALL_SIMPLE_TESTS} ${SPECIAL_TESTS} ${SST_SPECIFIC_TESTS})
 endif()
 
 
@@ -154,7 +172,7 @@ endforeach()
 #
 #   Setup tests for InSituMPI engine
 #
-if(ADIOS2_HAVE_MPI AND ADIOS2_RUN_MPMD_TESTS)
+if(ADIOS2_HAVE_MPI AND ADIOS2_RUN_MPI_MPMD_TESTS)
     set (SIMPLE_IMPI_TESTS "1x1;TimeoutOnOpen;1x1.Modes;1x1.Attrs;1x1.Local;1x1.SharedNothing;1x1.SharedIO;1x1.SharedVar;1x1.SharedNothingSync;1x1.SharedIOSync;1x1.SharedVarSync;2x1.SharedNothing;2x1.SharedIO;2x1.SharedVar;2x1.SharedNothingSync;2x1.SharedIOSync;2x1.SharedVarSync")
     set (INSITU_TESTS ${SIMPLE_IMPI_TESTS} ${SIMPLE_FORTRAN_TESTS} ${SIMPLE_MPI_TESTS} ${SIMPLE_ZFP_TESTS})
     # Tests that don't work for InSitu
@@ -184,8 +202,8 @@ if(ADIOS2_HAVE_MPI AND ADIOS2_RUN_MPMD_TESTS)
     endforeach()
 endif()
 
-if(ADIOS2_HAVE_SSC AND ADIOS2_RUN_MPMD_TESTS)
-    set (SSC_BASE_TESTS "1x1;1x1.Attrs;2x1;1x2;2x1ZeroDataVar;2x1ZeroDataR64;3x5;5x3;3x5LockGeometry;TimeoutOnOpen")
+if(ADIOS2_HAVE_SSC AND ADIOS2_RUN_MPI_MPMD_TESTS)
+    set (SSC_BASE_TESTS "1x1;1x1.Attrs;2x1;1x2;2x1ZeroDataVar;2x1ZeroDataR64;3x5;5x3;3x5LockGeometry")
     set (SSC_TESTS  ${SIMPLE_FORTRAN_TESTS} ${SIMPLE_MPI_FORTRAN_TESTS} ${SSC_BASE_TESTS})
     foreach(test ${SSC_TESTS})
         add_common_test(${test} SSC)
@@ -221,7 +239,7 @@ endif()
 if(NOT MSVC)    # not on windows
     # BP4 streaming tests start with all the simple tests, but with a timeout added on open
     LIST (APPEND BP4_STREAM_TESTS ${ALL_SIMPLE_TESTS} ${SPECIAL_TESTS})
-    MutateTestSet( BP4_STREAM_TESTS "BPS" reader "OpenTimeoutSecs=10" "${BP4_STREAM_TESTS}")
+    MutateTestSet( BP4_STREAM_TESTS "BPS" reader "OpenTimeoutSecs=10,BeginStepPollingFrequencySecs=0.1" "${BP4_STREAM_TESTS}")
     # SharedVars fail with BP4_streaming*
     list (FILTER BP4_STREAM_TESTS EXCLUDE REGEX ".*SharedVar.BPS$")
     # Discard not a feature of BP4
@@ -236,6 +254,8 @@ if(NOT MSVC)    # not on windows
     list (FILTER BP4_STREAM_TESTS EXCLUDE REGEX ".*Preload.*BPS$")
     # KillWriter fails with BP4
     list (FILTER BP4_STREAM_TESTS EXCLUDE REGEX ".*KillWriter.*BPS$")
+    # KillReaders  We swear this isn't necessary for BP4 streaming
+    list (FILTER BP4_STREAM_TESTS EXCLUDE REGEX ".*KillReaders.*BPS$")
     # SharedVars fail with BP4_streaming*
     list (FILTER BP4_STREAM_TESTS EXCLUDE REGEX ".*SharedVar.BPS$")
     # Local fail with BP4_streaming*
@@ -243,7 +263,7 @@ if(NOT MSVC)    # not on windows
     # Fortran scalar reads don't work for BP4 streaming
     list (FILTER BP4_STREAM_TESTS EXCLUDE REGEX "(FtoF|CtoF).*")
     # Local Vars have a heisen failure for BP4 streaming
-    list (FILTER BP4_STREAM_TESTS EXCLUDE REGEX ".*LocalVarying.BPS$")
+#list (FILTER BP4_STREAM_TESTS EXCLUDE REGEX ".*LocalVarying.BPS$")
     # The nobody-writes-data-in-a-timestep tests don't work for any BP file engine
     list (FILTER BP4_STREAM_TESTS EXCLUDE REGEX ".*NoData.BPS$")
     
@@ -259,6 +279,37 @@ if(NOT MSVC)    # not on windows
     
 endif()
 
+#
+#   Setup streaming tests for FileStream virtual engine (BP4+StreamReader=true)
+#
+if(NOT MSVC)    # not on windows
+    # FileStream streaming tests start with all the simple tests, but with a timeout added on open
+    LIST (APPEND FILESTREAM_TESTS ${SIMPLE_TESTS} ${SIMPLE_MPI_TESTS})
+    MutateTestSet( FILESTREAM_TESTS "FS" reader "OpenTimeoutSecs=10,BeginStepPollingFrequencySecs=0.1" "${FILESTREAM_TESTS}")
+    # SharedVars fail with file_streaming*
+    list (FILTER FILESTREAM_TESTS EXCLUDE REGEX ".*SharedVar.FS$")
+    # SharedVars fail with file_streaming*
+    list (FILTER FILESTREAM_TESTS EXCLUDE REGEX ".*SharedVar.FS$")
+    # Local fail with file_streaming*
+    list (FILTER FILESTREAM_TESTS EXCLUDE REGEX ".*Local.FS$")
+    # The nobody-writes-data-in-a-timestep tests don't work for any BP-file based engine
+    list (FILTER FILESTREAM_TESTS EXCLUDE REGEX ".*NoData.FS$")
+    # Don't need to repeat tests that are identical for BP4 and FileStream
+    list (FILTER FILESTREAM_TESTS EXCLUDE REGEX ".*NoReaderNoWait.FS$")
+    list (FILTER FILESTREAM_TESTS EXCLUDE REGEX ".*TimeoutOnOpen.FS$")
+    list (FILTER FILESTREAM_TESTS EXCLUDE REGEX ".*NoReaderNoWait.FS$")
+
+    foreach(test ${FILESTREAM_TESTS})
+        add_common_test(${test} FileStream)
+    endforeach()
+
+    MutateTestSet( FileStream_BBSTREAM_TESTS "BB" writer "BurstBufferPath=bb,BurstBufferVerbose=2" "${FILESTREAM_TESTS}")
+    foreach(test ${FileStream_BBSTREAM_TESTS})
+        add_common_test(${test} FileStream)
+    endforeach()
+
+endif()
+
 #
 #   Setup tests for HDF5 engine
 #
diff --git a/testing/adios2/engine/staging-common/ParseArgs.h b/testing/adios2/engine/staging-common/ParseArgs.h
index 9a1f0e37f3..8f606b2613 100644
--- a/testing/adios2/engine/staging-common/ParseArgs.h
+++ b/testing/adios2/engine/staging-common/ParseArgs.h
@@ -33,9 +33,11 @@ int LongFirstDelay = 0;
 int FirstTimestepMustBeZero = 0;
 int LockGeometry = 0;
 bool VaryingDataSize = false;
+bool AdvancingAttrs = false;
 int NoData = 0;
 int NoDataNode = -1;
 int EarlyExit = 0;
+int LocalCount = 1;
 
 std::string shutdown_name = "DieTest";
 adios2::Mode GlobalWriteMode = adios2::Mode::Deferred;
@@ -218,6 +220,10 @@ static void ParseArgs(int argc, char **argv)
         {
             VaryingDataSize = true;
         }
+        else if (std::string(argv[1]) == "--advancing_attributes")
+        {
+            AdvancingAttrs = true;
+        }
         else if (std::string(argv[1]) == "--long_first_delay")
         {
             LongFirstDelay = 1;
@@ -251,6 +257,15 @@ static void ParseArgs(int argc, char **argv)
             argv++;
             argc--;
         }
+        else if (std::string(argv[1]) == "--local_count")
+        {
+            std::istringstream ss(argv[2]);
+            if (!(ss >> LocalCount))
+                std::cerr << "Invalid number for --local_count argument"
+                          << argv[1] << '\n';
+            argv++;
+            argc--;
+        }
         else if (std::string(argv[1]) == "--early_exit")
         {
             EarlyExit++;
diff --git a/testing/adios2/engine/staging-common/TestCommonClient.cpp b/testing/adios2/engine/staging-common/TestCommonClient.cpp
index 2fb4513e1c..14e7d379b8 100644
--- a/testing/adios2/engine/staging-common/TestCommonClient.cpp
+++ b/testing/adios2/engine/staging-common/TestCommonClient.cpp
@@ -287,6 +287,31 @@ TEST_F(SstReadTest, ADIOS2SstRead)
             EXPECT_EQ(validateCommonTestData(myStart, myLength, currentStep, 0),
                       0);
             write_times.push_back(write_time);
+            if (AdvancingAttrs)
+            {
+                /* we only succeed if every attribute from every prior step is
+                 * there, but not the next few */
+                for (int step = 0; step <= currentStep + 2; step++)
+                {
+                    const std::string r64_Single =
+                        std::string("r64_PerStep_") + std::to_string(step);
+                    auto attr_r64 = io.InquireAttribute(r64_Single);
+                    std::cout << "Testing for attribute " << r64_Single
+                              << std::endl;
+                    if (step <= currentStep)
+                    {
+                        EXPECT_TRUE(attr_r64);
+                        ASSERT_EQ(attr_r64.Data().size() == 1, true);
+                        ASSERT_EQ(attr_r64.Type(), adios2::GetType());
+                        ASSERT_EQ(attr_r64.Data().front(),
+                                  (double)(step * 10.0));
+                    }
+                    else
+                    {
+                        EXPECT_FALSE(attr_r64);
+                    }
+                }
+            }
         }
         catch (...)
         {
@@ -305,6 +330,8 @@ TEST_F(SstReadTest, ADIOS2SstRead)
                 break;
             }
         }
+        std::cout << "Reader finished with step " << ExpectedStep - 1
+                  << std::endl;
         if (IncreasingDelay && !DelayWhileHoldingStep)
         {
             std::this_thread::sleep_for(std::chrono::milliseconds(
diff --git a/testing/adios2/engine/staging-common/TestCommonRead.cpp b/testing/adios2/engine/staging-common/TestCommonRead.cpp
index 95fad19497..ee8656ab19 100644
--- a/testing/adios2/engine/staging-common/TestCommonRead.cpp
+++ b/testing/adios2/engine/staging-common/TestCommonRead.cpp
@@ -332,7 +332,8 @@ TEST_F(CommonReadTest, ADIOS2CommonRead1D8)
 
                 engine.Get(var_r32, in_R32.data());
                 engine.Get(var_r64, in_R64.data());
-                engine.Get(var_time, (int64_t *)&write_time);
+                if (!mpiRank)
+                    engine.Get(var_time, (int64_t *)&write_time);
             }
             else
             {
@@ -360,9 +361,42 @@ TEST_F(CommonReadTest, ADIOS2CommonRead1D8)
 
         if (!NoData)
         {
-            EXPECT_EQ(validateCommonTestData(myStart, myLength, t, !var_c32),
-                      0);
-            write_times.push_back(write_time);
+            int result = validateCommonTestData(myStart, myLength, t, !var_c32);
+            if (result != 0)
+            {
+                std::cout << "Read Data Validation failed on node " << mpiRank
+                          << " timestep " << t << std::endl;
+            }
+            EXPECT_EQ(result, 0);
+            if (AdvancingAttrs)
+            {
+                /* we only succeed if every attribute from every prior step is
+                 * there, but not the next few */
+                for (int step = 0; step <= currentStep + 2; step++)
+                {
+                    const std::string r64_Single =
+                        std::string("r64_PerStep_") + std::to_string(step);
+                    auto attr_r64 = io.InquireAttribute(r64_Single);
+                    std::cout << "Testing for attribute " << r64_Single
+                              << std::endl;
+                    if (step <= currentStep)
+                    {
+                        EXPECT_TRUE(attr_r64);
+                        ASSERT_EQ(attr_r64.Data().size() == 1, true);
+                        ASSERT_EQ(attr_r64.Type(), adios2::GetType());
+                        ASSERT_EQ(attr_r64.Data().front(),
+                                  (double)(step * 10.0));
+                    }
+                    else
+                    {
+                        // The file engines let attributes appear early, so only
+                        // enforce non-appearance if that changes.
+                        // EXPECT_FALSE(attr_r64);
+                    }
+                }
+            }
+            if (!mpiRank)
+                write_times.push_back(write_time);
         }
         else
         {
@@ -388,7 +422,7 @@ TEST_F(CommonReadTest, ADIOS2CommonRead1D8)
     }
 
     EXPECT_EQ(t, NSteps);
-    if (!NoData)
+    if (!NoData && !mpiRank)
     {
         if ((write_times.back() - write_times.front()) > 1)
         {
diff --git a/testing/adios2/engine/staging-common/TestCommonReadAttrs.cpp b/testing/adios2/engine/staging-common/TestCommonReadAttrs.cpp
index babf056719..db823f6598 100644
--- a/testing/adios2/engine/staging-common/TestCommonReadAttrs.cpp
+++ b/testing/adios2/engine/staging-common/TestCommonReadAttrs.cpp
@@ -302,7 +302,13 @@ TEST_F(CommonReadTest, ADIOS2CommonRead1D8)
         engine.Get(var_time, (int64_t *)&write_time);
         engine.EndStep();
 
-        EXPECT_EQ(validateCommonTestData(myStart, myLength, t, !var_c32), 0);
+        int result = validateCommonTestData(myStart, myLength, t, !var_c32);
+        if (result != 0)
+        {
+            std::cout << "Read Data Validation failed on node " << mpiRank
+                      << " timestep " << t << std::endl;
+        }
+        EXPECT_EQ(result, 0);
         write_times.push_back(write_time);
         ++t;
     }
diff --git a/testing/adios2/engine/staging-common/TestCommonReadF.F90 b/testing/adios2/engine/staging-common/TestCommonReadF.F90
index 0df6ba8593..1956d4d98c 100644
--- a/testing/adios2/engine/staging-common/TestCommonReadF.F90
+++ b/testing/adios2/engine/staging-common/TestCommonReadF.F90
@@ -12,13 +12,15 @@ program TestSstRead
   use adios2
   implicit none
 
-#ifndef __GFORTRAN__
-#ifndef __GNUC__
-  interface
-     integer function iargc()
-     end function iargc
-  end interface
-#endif
+#if defined(ADIOS2_HAVE_FORTRAN_F03_ARGS)
+# define ADIOS2_ARGC() command_argument_count()
+# define ADIOS2_ARGV(i, v) call get_command_argument(i, v)
+#elif defined(ADIOS2_HAVE_FORTRAN_GNU_ARGS)
+# define ADIOS2_ARGC() iargc()
+# define ADIOS2_ARGV(i, v) call getarg(i, v)
+#else
+# define ADIOS2_ARGC() 1
+# define ADIOS2_ARGV(i, v)
 #endif
 
   integer :: numargs
@@ -26,7 +28,7 @@ end function iargc
   integer(kind = 8), dimension(1)::shape_dims, start_dims, count_dims
   integer(kind = 8), dimension(2)::shape_dims2, start_dims2, count_dims2
   integer(kind = 8), dimension(2)::shape_dims3, start_dims3, count_dims3
-  integer:: irank, isize, ierr, i, insteps, status
+  integer:: irank, isize, ierr, i, insteps, step_status
 
   character(len=256) :: filename, engine, params
 
@@ -43,7 +45,7 @@ end function iargc
   integer(kind = 8), dimension(:), allocatable::shape_in
   integer::key, color
   
-  numargs = iargc()
+  numargs = ADIOS2_ARGC()
 
   if ( numargs < 2 ) then
      call usage()
@@ -51,13 +53,13 @@ end function iargc
   endif
 
 
-  call getarg(1, engine)
-  call getarg(2, filename)
+  ADIOS2_ARGV(1, engine)
+  ADIOS2_ARGV(2, filename)
   if ( numargs > 2 ) then
-     call getarg(3, params)
+     ADIOS2_ARGV(3, params)
   endif
 
-  insteps = 1;
+  insteps = 0;
 
 #if ADIOS2_USE_MPI
   !Launch MPI
@@ -86,157 +88,175 @@ end function iargc
 
   if (numargs > 2) then
      call adios2_set_parameters(ioRead, params, ierr)
-  endif 
+  endif
   call adios2_set_engine(ioRead, engine, ierr)
 
   ! Open sstReader engine
   call adios2_open(sstReader, ioRead, filename, adios2_mode_read, ierr)
 
-  call adios2_begin_step(sstReader, ierr)
-
-  call adios2_inquire_variable(variables(1), ioRead, "i8", ierr)
-  if (variables(1)%name /= 'i8') stop 'i8 not recognized'
-  if (variables(1)%type /= adios2_type_integer1) stop 'i8 type not recognized'
-  call adios2_variable_shape(shape_in, ndims, variables(1),  ierr)
-  if (ndims /= 1) stop 'i8 ndims is not 1'
-  if (modulo(shape_in(1), int(nx, 8)) /= 0) stop 'i8 shape_in read failed'
-  writerSize = shape_in(1) / nx
-  deallocate(shape_in)
-
-  call adios2_inquire_variable(variables(2), ioRead, "i16", ierr)
-  if (variables(2)%name /= 'i16') stop 'i16 not recognized'
-  if (variables(2)%type /= adios2_type_integer2) stop 'i16 type not recognized'
-  call adios2_variable_shape( shape_in, ndims, variables(2), ierr)
-  if (ndims /= 1) stop 'i16 ndims is not 1'
-  if (shape_in(1) /= nx*writerSize) stop 'i16 shape_in read failed'
-  deallocate(shape_in)
-  
-  call adios2_inquire_variable(variables(3), ioRead, "i32", ierr)
-  if (variables(3)%name /= 'i32') stop 'i32 not recognized'
-  if (variables(3)%type /= adios2_type_integer4) stop 'i32 type not recognized'
-  call adios2_variable_shape(shape_in, ndims, variables(3), ierr)
-  if (ndims /= 1) stop 'i32 ndims is not 1'
-  if (shape_in(1) /= nx*writerSize) stop 'i32 shape_in read failed'
-  deallocate(shape_in)
-
-  call adios2_inquire_variable(variables(4), ioRead, "i64", ierr)
-  if (variables(4)%name /= 'i64') stop 'i64 not recognized'
-  if (variables(4)%type /= adios2_type_integer8) stop 'i64 type not recognized'
-  call adios2_variable_shape(shape_in, ndims, variables(4), ierr)
-  if (ndims /= 1) stop 'i64 ndims is not 1'
-  if (shape_in(1) /= nx*writerSize) stop 'i64 shape_in read failed'
-  deallocate(shape_in)
-  
-  call adios2_inquire_variable(variables(5), ioRead, "r32", ierr)
-  if (variables(5)%name /= 'r32') stop 'r32 not recognized'
-  if (variables(5)%type /= adios2_type_real) stop 'r32 type not recognized'
-  call adios2_variable_shape(shape_in, ndims, variables(5), ierr)
-  if (ndims /= 1) stop 'r32 ndims is not 1'
-  if (shape_in(1) /= nx*writerSize) stop 'r32 shape_in read failed'
-  deallocate(shape_in)
-
-  call adios2_inquire_variable(variables(6), ioRead, "r64", ierr)
-  if (variables(6)%name /= 'r64') stop 'r64 not recognized'
-  if (variables(6)%type /= adios2_type_dp) stop 'r64 type not recognized'
-  call adios2_variable_shape(shape_in, ndims, variables(6), ierr)
-  if (ndims /= 1) stop 'r64 ndims is not 1'
-  if (shape_in(1) /= nx*writerSize) stop 'r64 shape_in read failed'
-  deallocate(shape_in)
-
-  call adios2_inquire_variable(variables(7), ioRead, "r64_2d", ierr)
-  if (variables(7)%name /= 'r64_2d') stop 'r64_2d not recognized'
-  if (variables(7)%type /= adios2_type_dp) stop 'r64_2d type not recognized'
-  call adios2_variable_shape( shape_in, ndims, variables(7), ierr)
-  if (ndims /= 2) stop 'r64_2d ndims is not 2'
-  if (shape_in(1) /= 2) stop 'r64_2d shape_in(1) read failed'
-  if (shape_in(2) /= nx*writerSize) stop 'r64_2d shape_in(2) read failed'
-  deallocate(shape_in)
-
-  call adios2_inquire_variable(variables(8), ioRead, "r64_2d_rev", ierr)
-  if (variables(8)%name /= 'r64_2d_rev') stop 'r64_2d_rev not recognized'
-  if (variables(8)%type /= adios2_type_dp) stop 'r64_2d_rev type not recognized'
-  call adios2_variable_shape(shape_in, ndims, variables(8), ierr)
-  if (ndims /= 2) stop 'r64_2d_rev ndims is not 2'
-  if (shape_in(1) /= nx*writerSize) stop 'r64_2d_rev shape_in(2) read failed'
-  if (shape_in(2) /= 2) stop 'r64_2d_rev shape_in(1) read failed'
-  deallocate(shape_in)
-
-  call adios2_inquire_variable(variables(10), ioRead, "c32", ierr)
-  if (variables(10)%name /= 'c32') stop 'c32 not recognized'
-  if (variables(10)%type /= adios2_type_complex) stop 'c32 type not recognized'
-  call adios2_variable_shape(shape_in, ndims, variables(10), ierr)
-  if (ndims /= 1) stop 'c32 ndims is not 1'
-  if (shape_in(1) /= nx*writerSize) stop 'c32 shape_in read failed'
-  deallocate(shape_in)
-
-  call adios2_inquire_variable(variables(11), ioRead, "c64", ierr)
-  if (variables(11)%name /= 'c64') stop 'c64 not recognized'
-  if (variables(11)%type /= adios2_type_complex_dp) stop 'c64 type not recognized'
-  call adios2_variable_shape(shape_in, ndims, variables(11), ierr)
-  if (ndims /= 1) stop 'c64 ndims is not 1'
-  if (shape_in(1) /= nx*writerSize) stop 'c64 shape_in read failed'
-  deallocate(shape_in)
-
-  call adios2_inquire_variable(variables(12), ioRead, "scalar_r64", ierr)
-  if (variables(12)%name /= 'scalar_r64') stop 'scalar_r64 not recognized'
-  if (variables(12)%type /= adios2_type_dp) stop 'scalar_r64 type not recognized'
-  
-  myStart = (writerSize * Nx / isize) * irank
-  myLength = ((writerSize * Nx + isize - 1) / isize)
-
-  if (myStart + myLength > writerSize * Nx) then
-      myLength = writerSize * Nx - myStart
-  end if
-  allocate (in_I8(myLength));
-  allocate (in_I16(myLength));
-  allocate (in_I32(myLength));
-  allocate (in_I64(myLength));
-  allocate (in_R32(myLength));
-  allocate (in_R64(myLength));
-  allocate (in_C32(myLength));
-  allocate (in_C64(myLength));
-  allocate (in_R64_2d(2, myLength));
-  allocate (in_R64_2d_rev(myLength, 2));
-
-  start_dims(1) = myStart
-  count_dims(1) = myLength
-
-  start_dims2(1) = 0
-  count_dims2(1) = 2
-  start_dims2(2) = myStart
-  count_dims2(2) = myLength
-
-  start_dims3(1) = myStart
-  count_dims3(1) = myLength
-  start_dims3(2) = 0
-  count_dims3(2) = 2
-
-  call adios2_set_selection(variables(1), 1, start_dims, count_dims, ierr)
-  call adios2_set_selection(variables(2), 1, start_dims, count_dims, ierr)
-  call adios2_set_selection(variables(3), 1, start_dims, count_dims, ierr)
-  call adios2_set_selection(variables(4), 1, start_dims, count_dims, ierr)
-  call adios2_set_selection(variables(5), 1, start_dims, count_dims, ierr)
-  call adios2_set_selection(variables(6), 1, start_dims, count_dims, ierr)
-  call adios2_set_selection(variables(7), 2, start_dims2, count_dims2, ierr)
-  call adios2_set_selection(variables(8), 2, start_dims3, count_dims3, ierr)
-  call adios2_set_selection(variables(10), 1, start_dims, count_dims, ierr)
-  call adios2_set_selection(variables(11), 1, start_dims, count_dims, ierr)
-
-  call adios2_get(sstReader, variables(1), in_I8, ierr)
-  call adios2_get(sstReader, variables(2), in_I16, ierr)
-  call adios2_get(sstReader, variables(3), in_I32, ierr)
-  call adios2_get(sstReader, variables(4), in_I64, ierr)
-  call adios2_get(sstReader, variables(5), in_R32, ierr)
-  call adios2_get(sstReader, variables(6), in_R64, ierr)
-  call adios2_get(sstReader, variables(7), in_R64_2d, ierr)
-  call adios2_get(sstReader, variables(8), in_R64_2d_rev, ierr)
-  call adios2_get(sstReader, variables(10), in_C32, ierr)
-  call adios2_get(sstReader, variables(11), in_C64, ierr)
-  call adios2_get(sstReader, variables(12), in_scalar_R64, ierr)
-
-  call adios2_end_step(sstReader, ierr)
-  
-  call validateTestData(myStart, myLength, 0)
+  do
+    call adios2_begin_step(sstReader, adios2_step_mode_read, -1., step_status, ierr)
+
+    if(step_status == adios2_step_status_end_of_stream) exit
+
+
+    call adios2_inquire_variable(variables(1), ioRead, "i8", ierr)
+    if (variables(1)%name /= 'i8') stop 'i8 not recognized'
+    if (variables(1)%type /= adios2_type_integer1) stop 'i8 type not recognized'
+    call adios2_variable_shape(shape_in, ndims, variables(1),  ierr)
+    if (ndims /= 1) stop 'i8 ndims is not 1'
+    if (modulo(shape_in(1), int(nx, 8)) /= 0) stop 'i8 shape_in read failed'
+    writerSize = shape_in(1) / nx
+    deallocate(shape_in)
+
+    call adios2_inquire_variable(variables(2), ioRead, "i16", ierr)
+    if (variables(2)%name /= 'i16') stop 'i16 not recognized'
+    if (variables(2)%type /= adios2_type_integer2) stop 'i16 type not recognized'
+    call adios2_variable_shape( shape_in, ndims, variables(2), ierr)
+    if (ndims /= 1) stop 'i16 ndims is not 1'
+    if (shape_in(1) /= nx*writerSize) stop 'i16 shape_in read failed'
+    deallocate(shape_in)
+
+    call adios2_inquire_variable(variables(3), ioRead, "i32", ierr)
+    if (variables(3)%name /= 'i32') stop 'i32 not recognized'
+    if (variables(3)%type /= adios2_type_integer4) stop 'i32 type not recognized'
+    call adios2_variable_shape(shape_in, ndims, variables(3), ierr)
+    if (ndims /= 1) stop 'i32 ndims is not 1'
+    if (shape_in(1) /= nx*writerSize) stop 'i32 shape_in read failed'
+    deallocate(shape_in)
+
+    call adios2_inquire_variable(variables(4), ioRead, "i64", ierr)
+    if (variables(4)%name /= 'i64') stop 'i64 not recognized'
+    if (variables(4)%type /= adios2_type_integer8) stop 'i64 type not recognized'
+    call adios2_variable_shape(shape_in, ndims, variables(4), ierr)
+    if (ndims /= 1) stop 'i64 ndims is not 1'
+    if (shape_in(1) /= nx*writerSize) stop 'i64 shape_in read failed'
+    deallocate(shape_in)
+
+    call adios2_inquire_variable(variables(5), ioRead, "r32", ierr)
+    if (variables(5)%name /= 'r32') stop 'r32 not recognized'
+    if (variables(5)%type /= adios2_type_real) stop 'r32 type not recognized'
+    call adios2_variable_shape(shape_in, ndims, variables(5), ierr)
+    if (ndims /= 1) stop 'r32 ndims is not 1'
+    if (shape_in(1) /= nx*writerSize) stop 'r32 shape_in read failed'
+    deallocate(shape_in)
+
+    call adios2_inquire_variable(variables(6), ioRead, "r64", ierr)
+    if (variables(6)%name /= 'r64') stop 'r64 not recognized'
+    if (variables(6)%type /= adios2_type_dp) stop 'r64 type not recognized'
+    call adios2_variable_shape(shape_in, ndims, variables(6), ierr)
+    if (ndims /= 1) stop 'r64 ndims is not 1'
+    if (shape_in(1) /= nx*writerSize) stop 'r64 shape_in read failed'
+    deallocate(shape_in)
+
+    call adios2_inquire_variable(variables(7), ioRead, "r64_2d", ierr)
+    if (variables(7)%name /= 'r64_2d') stop 'r64_2d not recognized'
+    if (variables(7)%type /= adios2_type_dp) stop 'r64_2d type not recognized'
+    call adios2_variable_shape( shape_in, ndims, variables(7), ierr)
+    if (ndims /= 2) stop 'r64_2d ndims is not 2'
+    if (shape_in(1) /= 2) stop 'r64_2d shape_in(1) read failed'
+    if (shape_in(2) /= nx*writerSize) stop 'r64_2d shape_in(2) read failed'
+    deallocate(shape_in)
+
+    call adios2_inquire_variable(variables(8), ioRead, "r64_2d_rev", ierr)
+    if (variables(8)%name /= 'r64_2d_rev') stop 'r64_2d_rev not recognized'
+    if (variables(8)%type /= adios2_type_dp) stop 'r64_2d_rev type not recognized'
+    call adios2_variable_shape(shape_in, ndims, variables(8), ierr)
+    if (ndims /= 2) stop 'r64_2d_rev ndims is not 2'
+    if (shape_in(1) /= nx*writerSize) stop 'r64_2d_rev shape_in(2) read failed'
+    if (shape_in(2) /= 2) stop 'r64_2d_rev shape_in(1) read failed'
+    deallocate(shape_in)
+
+    call adios2_inquire_variable(variables(10), ioRead, "c32", ierr)
+    if (variables(10)%name /= 'c32') stop 'c32 not recognized'
+    if (variables(10)%type /= adios2_type_complex) stop 'c32 type not recognized'
+    call adios2_variable_shape(shape_in, ndims, variables(10), ierr)
+    if (ndims /= 1) stop 'c32 ndims is not 1'
+    if (shape_in(1) /= nx*writerSize) stop 'c32 shape_in read failed'
+    deallocate(shape_in)
+
+    call adios2_inquire_variable(variables(11), ioRead, "c64", ierr)
+    if (variables(11)%name /= 'c64') stop 'c64 not recognized'
+    if (variables(11)%type /= adios2_type_complex_dp) stop 'c64 type not recognized'
+    call adios2_variable_shape(shape_in, ndims, variables(11), ierr)
+    if (ndims /= 1) stop 'c64 ndims is not 1'
+    if (shape_in(1) /= nx*writerSize) stop 'c64 shape_in read failed'
+    deallocate(shape_in)
+
+    call adios2_inquire_variable(variables(12), ioRead, "scalar_r64", ierr)
+    if (variables(12)%name /= 'scalar_r64') stop 'scalar_r64 not recognized'
+    if (variables(12)%type /= adios2_type_dp) stop 'scalar_r64 type not recognized'
+
+    myStart = (writerSize * Nx / isize) * irank
+    myLength = ((writerSize * Nx + isize - 1) / isize)
+
+    if (myStart + myLength > writerSize * Nx) then
+        myLength = writerSize * Nx - myStart
+    end if
+    allocate (in_I8(myLength));
+    allocate (in_I16(myLength));
+    allocate (in_I32(myLength));
+    allocate (in_I64(myLength));
+    allocate (in_R32(myLength));
+    allocate (in_R64(myLength));
+    allocate (in_C32(myLength));
+    allocate (in_C64(myLength));
+    allocate (in_R64_2d(2, myLength));
+    allocate (in_R64_2d_rev(myLength, 2));
+
+    start_dims(1) = myStart
+    count_dims(1) = myLength
+
+    start_dims2(1) = 0
+    count_dims2(1) = 2
+    start_dims2(2) = myStart
+    count_dims2(2) = myLength
+
+    start_dims3(1) = myStart
+    count_dims3(1) = myLength
+    start_dims3(2) = 0
+    count_dims3(2) = 2
+
+    call adios2_set_selection(variables(1), 1, start_dims, count_dims, ierr)
+    call adios2_set_selection(variables(2), 1, start_dims, count_dims, ierr)
+    call adios2_set_selection(variables(3), 1, start_dims, count_dims, ierr)
+    call adios2_set_selection(variables(4), 1, start_dims, count_dims, ierr)
+    call adios2_set_selection(variables(5), 1, start_dims, count_dims, ierr)
+    call adios2_set_selection(variables(6), 1, start_dims, count_dims, ierr)
+    call adios2_set_selection(variables(7), 2, start_dims2, count_dims2, ierr)
+    call adios2_set_selection(variables(8), 2, start_dims3, count_dims3, ierr)
+    call adios2_set_selection(variables(10), 1, start_dims, count_dims, ierr)
+    call adios2_set_selection(variables(11), 1, start_dims, count_dims, ierr)
+
+    call adios2_get(sstReader, variables(1), in_I8, ierr)
+    call adios2_get(sstReader, variables(2), in_I16, ierr)
+    call adios2_get(sstReader, variables(3), in_I32, ierr)
+    call adios2_get(sstReader, variables(4), in_I64, ierr)
+    call adios2_get(sstReader, variables(5), in_R32, ierr)
+    call adios2_get(sstReader, variables(6), in_R64, ierr)
+    call adios2_get(sstReader, variables(7), in_R64_2d, ierr)
+    call adios2_get(sstReader, variables(8), in_R64_2d_rev, ierr)
+    call adios2_get(sstReader, variables(10), in_C32, ierr)
+    call adios2_get(sstReader, variables(11), in_C64, ierr)
+    call adios2_get(sstReader, variables(12), in_scalar_R64, ierr)
+
+    call adios2_end_step(sstReader, ierr)
+
+    if(insteps == 0) then
+      call validateTestData(myStart, myLength, 0)
+    endif
+    deallocate (in_I8);
+    deallocate (in_I16);
+    deallocate (in_I32);
+    deallocate (in_I64);
+    deallocate (in_R32);
+    deallocate (in_R64);
+    deallocate (in_C32);
+    deallocate (in_C64);
+    deallocate (in_R64_2d);
+    deallocate (in_R64_2d_rev);
+    insteps = insteps + 1
+  end do
 
   call adios2_close(sstReader, ierr)
 
diff --git a/testing/adios2/engine/staging-common/TestCommonReadLocal.cpp b/testing/adios2/engine/staging-common/TestCommonReadLocal.cpp
index 385f9cfe49..b42ad8d5bd 100644
--- a/testing/adios2/engine/staging-common/TestCommonReadLocal.cpp
+++ b/testing/adios2/engine/staging-common/TestCommonReadLocal.cpp
@@ -166,9 +166,16 @@ TEST_F(CommonReadTest, ADIOS2CommonRead1D8)
             hisLength);
 
         var_r32.SetBlockSelection(rankToRead);
-        ASSERT_EQ(
-            engine.BlocksInfo(var_r32, currentStep).at(rankToRead).Count[0],
-            hisLength);
+        for (int index = 0; index < LocalCount; index++)
+        {
+            int indexToRead = rankToRead * LocalCount;
+            std::cout << "Testing blocks info for var_r32 at index "
+                      << indexToRead << std::endl;
+            ASSERT_EQ(engine.BlocksInfo(var_r32, currentStep)
+                          .at(indexToRead)
+                          .Count[0],
+                      hisLength);
+        }
         var_r64.SetBlockSelection(rankToRead);
         ASSERT_EQ(
             engine.BlocksInfo(var_r64, currentStep).at(rankToRead).Count[0],
@@ -219,10 +226,21 @@ TEST_F(CommonReadTest, ADIOS2CommonRead1D8)
         var_time.SetSelection(sel_time);
 
         in_I8.resize(hisLength);
+        if (LocalCount == 1)
+        {
+            in_R32.resize(hisLength);
+        }
+        else
+        {
+            in_R32_blocks.resize(LocalCount);
+            for (int index = 0; index < LocalCount; index++)
+            {
+                in_R32_blocks[index].resize(hisLength);
+            }
+        }
         in_I16.resize(hisLength);
         in_I32.resize(hisLength);
         in_I64.resize(hisLength);
-        in_R32.resize(hisLength);
         in_R64.resize(hisLength);
         in_C32.resize(hisLength);
         in_C64.resize(hisLength);
@@ -233,9 +251,22 @@ TEST_F(CommonReadTest, ADIOS2CommonRead1D8)
         engine.Get(var_i32, in_I32.data());
         engine.Get(var_i64, in_I64.data());
 
+        if (LocalCount == 1)
+        {
+            engine.Get(var_r32, in_R32.data());
+        }
+        else
+        {
+            for (int index = 0; index < LocalCount; index++)
+            {
+                int indexToRead = rankToRead * LocalCount + index;
+                auto block = in_R32_blocks[index];
+                var_r32.SetBlockSelection(indexToRead);
+                engine.Get(var_r32, in_R32_blocks[index].data());
+            }
+        }
         engine.Get(scalar_r64, in_scalar_R64);
 
-        engine.Get(var_r32, in_R32.data());
         engine.Get(var_r64, in_R64.data());
         if (var_c32)
             engine.Get(var_c32, in_C32.data());
@@ -249,9 +280,14 @@ TEST_F(CommonReadTest, ADIOS2CommonRead1D8)
         engine.Get(var_time, (int64_t *)&write_time);
         engine.EndStep();
 
-        EXPECT_EQ(validateCommonTestData(hisStart, hisLength, t, !var_c32,
-                                         VaryingDataSize, rankToRead),
-                  0);
+        int result = validateCommonTestData(hisStart, hisLength, t, !var_c32,
+                                            VaryingDataSize, rankToRead);
+        if (result != 0)
+        {
+            std::cout << "Read Data Validation failed on node " << mpiRank
+                      << " timestep " << t << std::endl;
+        }
+        EXPECT_EQ(result, 0);
         write_times.push_back(write_time);
         ++t;
     }
diff --git a/testing/adios2/engine/staging-common/TestCommonReadR64.cpp b/testing/adios2/engine/staging-common/TestCommonReadR64.cpp
index 4adecbbe53..5b1a42c48f 100644
--- a/testing/adios2/engine/staging-common/TestCommonReadR64.cpp
+++ b/testing/adios2/engine/staging-common/TestCommonReadR64.cpp
@@ -119,7 +119,13 @@ TEST_F(CommonReadTest, ADIOS2CommonRead)
         in_R64.resize(myLength);
         engine.Get(var_r64, in_R64.data());
         engine.EndStep();
-        EXPECT_EQ(validateCommonTestR64(myStart, myLength, t), 0);
+        int result = validateCommonTestR64(myStart, myLength, t);
+        if (result != 0)
+        {
+            std::cout << "Read Data Validation failed on node " << mpiRank
+                      << " timestep " << t << std::endl;
+        }
+        EXPECT_EQ(result, 0);
 
         ++t;
     }
diff --git a/testing/adios2/engine/staging-common/TestCommonReadShared.cpp b/testing/adios2/engine/staging-common/TestCommonReadShared.cpp
index 988a543710..8c7d099576 100644
--- a/testing/adios2/engine/staging-common/TestCommonReadShared.cpp
+++ b/testing/adios2/engine/staging-common/TestCommonReadShared.cpp
@@ -117,12 +117,16 @@ TEST_F(CommonReadTest, ADIOS2CommonRead1D8)
         engine1.EndStep();
         engine2.EndStep();
 
-        EXPECT_EQ(validateSimpleForwardData(in_R64_1, 0, myStart, myLength,
-                                            (writerSize + 1) * Nx),
-                  0);
-        EXPECT_EQ(validateSimpleReverseData(in_R64_2, 0, myStart, myLength,
-                                            (writerSize + 1) * Nx),
-                  0);
+        int result = validateSimpleForwardData(in_R64_1, 0, myStart, myLength,
+                                               (writerSize + 1) * Nx);
+        result |= validateSimpleReverseData(in_R64_2, 0, myStart, myLength,
+                                            (writerSize + 1) * Nx);
+        if (result != 0)
+        {
+            std::cout << "Read Data Validation failed on node " << mpiRank
+                      << " timestep " << t << std::endl;
+        }
+        EXPECT_EQ(result, 0);
 
         ++t;
     }
diff --git a/testing/adios2/engine/staging-common/TestCommonServer.cpp b/testing/adios2/engine/staging-common/TestCommonServer.cpp
index b8cdf5d9fc..a85bddbc04 100644
--- a/testing/adios2/engine/staging-common/TestCommonServer.cpp
+++ b/testing/adios2/engine/staging-common/TestCommonServer.cpp
@@ -158,7 +158,14 @@ TEST_F(CommonServerTest, ADIOS2CommonServer)
             // we'll never change our data decomposition
             engine.LockWriterDefinitions();
         }
+        if (AdvancingAttrs)
+        {
+            const std::string r64_Single =
+                std::string("r64_PerStep_") + std::to_string(step);
+            io.DefineAttribute(r64_Single, (double)(step * 10.0));
+        }
         engine.EndStep();
+        std::cout << "Writer finished step " << step << std::endl;
         std::this_thread::sleep_for(std::chrono::milliseconds(
             DelayMS)); /* sleep for DelayMS milliseconds */
         step++;
diff --git a/testing/adios2/engine/staging-common/TestCommonWrite.cpp b/testing/adios2/engine/staging-common/TestCommonWrite.cpp
index 4983d374f6..f5ee2985ca 100644
--- a/testing/adios2/engine/staging-common/TestCommonWrite.cpp
+++ b/testing/adios2/engine/staging-common/TestCommonWrite.cpp
@@ -224,6 +224,13 @@ TEST_F(CommonWriteTest, ADIOS2CommonWrite)
             // we'll never change our data decomposition
             engine.LockWriterDefinitions();
         }
+        if (AdvancingAttrs)
+        {
+            const std::string r64_Single =
+                std::string("r64_PerStep_") + std::to_string(step);
+            io.DefineAttribute(r64_Single, (double)(step * 10.0));
+            std::cout << "Dumping attribute " << r64_Single << std::endl;
+        }
         engine.EndStep();
         std::this_thread::sleep_for(std::chrono::milliseconds(
             DelayMS)); /* sleep for DelayMS milliseconds */
diff --git a/testing/adios2/engine/staging-common/TestCommonWriteF.F90 b/testing/adios2/engine/staging-common/TestCommonWriteF.F90
index 622aa018ed..9739302db6 100644
--- a/testing/adios2/engine/staging-common/TestCommonWriteF.F90
+++ b/testing/adios2/engine/staging-common/TestCommonWriteF.F90
@@ -11,13 +11,15 @@ program TestSstWrite
   use adios2
   implicit none
 
-#ifndef __GFORTRAN__
-#ifndef __GNUC__
-  interface
-     integer function iargc()
-     end function iargc
-  end interface
-#endif
+#if defined(ADIOS2_HAVE_FORTRAN_F03_ARGS)
+# define ADIOS2_ARGC() command_argument_count()
+# define ADIOS2_ARGV(i, v) call get_command_argument(i, v)
+#elif defined(ADIOS2_HAVE_FORTRAN_GNU_ARGS)
+# define ADIOS2_ARGC() iargc()
+# define ADIOS2_ARGV(i, v) call getarg(i, v)
+#else
+# define ADIOS2_ARGC() 1
+# define ADIOS2_ARGV(i, v)
 #endif
 
   integer :: numargs
@@ -42,17 +44,17 @@ end function iargc
   integer(kind = 8)::localtime
   integer::color, key, testComm
 
-  numargs = iargc()
+  numargs = ADIOS2_ARGC()
   if ( numargs < 2 ) then
      call usage()
      call exit(1)
   endif
 
 
-  call getarg(1, engine)
-  call getarg(2, filename)
+  ADIOS2_ARGV(1, engine)
+  ADIOS2_ARGV(2, filename)
   if ( numargs > 2 ) then
-     call getarg(3, params)
+     ADIOS2_ARGV(3, params)
   endif
 
 #if ADIOS2_USE_MPI
@@ -184,8 +186,10 @@ end function iargc
      call adios2_put(sstWriter, variables(6), data_R64, ierr)
      call adios2_put(sstWriter, variables(7), data_R64_2d, ierr)
      call adios2_put(sstWriter, variables(8), data_R64_2d_rev, ierr)
+#if !defined(_CRAYFTN)
      localtime = 0    ! should be time(), but non-portable and value is unused
      call adios2_put(sstWriter, variables(9), loc(localtime), ierr)
+#endif
      call adios2_put(sstWriter, variables(10), data_C32, ierr)
      call adios2_put(sstWriter, variables(11), data_C64, ierr)
      call adios2_end_step(sstWriter, ierr)
diff --git a/testing/adios2/engine/staging-common/TestCommonWriteLocal.cpp b/testing/adios2/engine/staging-common/TestCommonWriteLocal.cpp
index b253b2aa67..7cf4654bdb 100644
--- a/testing/adios2/engine/staging-common/TestCommonWriteLocal.cpp
+++ b/testing/adios2/engine/staging-common/TestCommonWriteLocal.cpp
@@ -160,6 +160,14 @@ TEST_F(CommonWriteTest, ADIOS2CommonWrite)
         engine.Put(var_i32, data_I32.data(), sync);
         engine.Put(var_i64, data_I64.data(), sync);
         engine.Put(var_r32, data_R32.data(), sync);
+        for (int index = 1; index < LocalCount; index++)
+        {
+            for (int i = 0; i < data_R32.size(); i++)
+            {
+                data_R32[i] += 1000.0;
+            }
+            engine.Put(var_r32, data_R32.data(), sync);
+        }
         engine.Put(var_r64, data_R64.data(), sync);
         engine.Put(var_c32, data_C32.data(), sync);
         engine.Put(var_c64, data_C64.data(), sync);
@@ -192,7 +200,7 @@ int main(int argc, char **argv)
     int result;
     ::testing::InitGoogleTest(&argc, argv);
 
-    DelayMS = 0; // zero for common writer
+    DelayMS = 0; // zero default for common writer
 
     ParseArgs(argc, argv);
 
diff --git a/testing/adios2/engine/staging-common/TestData.h b/testing/adios2/engine/staging-common/TestData.h
index 21bf03ca59..8a0b75d8b3 100644
--- a/testing/adios2/engine/staging-common/TestData.h
+++ b/testing/adios2/engine/staging-common/TestData.h
@@ -40,6 +40,7 @@ std::vector in_I16;
 std::vector in_I32;
 std::vector in_I64;
 std::vector in_R32;
+std::vector> in_R32_blocks;
 std::vector in_R64;
 std::vector> in_C32;
 std::vector> in_C64;
@@ -174,7 +175,7 @@ void generateCommonTestData(int step, int rank, int size, int Nx, int r64_Nx)
 
 int validateCommonTestData(int start, int length, size_t step,
                            int missing_end_data, bool varying = false,
-                           int writerRank = 0)
+                           int writerRank = 0, int LocalCount = 1)
 {
     int failures = 0;
     if (in_scalar_R64 != 1.5 * (step + 1))
@@ -191,8 +192,8 @@ int validateCommonTestData(int start, int length, size_t step,
             if (in_I8[i] != (int8_t)((i + start) * 10 + step))
             {
                 std::cout << "Expected 0x" << std::hex
-                          << (int8_t)((i + start) * 10 + step) << ", got 0x"
-                          << std::hex << in_I8[i] << " for in_I8[" << i
+                          << (int16_t)((i + start) * 10 + step) << ", got 0x"
+                          << std::hex << (int16_t)in_I8[i] << " for in_I8[" << i
                           << "](global[" << i + start << "])" << std::endl;
                 failures++;
             }
@@ -222,13 +223,33 @@ int validateCommonTestData(int start, int length, size_t step,
             failures++;
         }
 
-        if (in_R32[i] != (float)((i + start) * 10 + step))
+        if (in_R32_blocks.size() == 0)
         {
-            std::cout << "Expected " << (float)((i + start) * 10 + step)
-                      << ", got " << in_R32[i] << " for in_R32[" << i
-                      << "](global[" << i + start << "])" << std::endl;
-            failures++;
+            if (in_R32[i] != (float)((i + start) * 10 + step))
+            {
+                std::cout << "Expected " << (float)((i + start) * 10 + step)
+                          << ", got " << in_R32[i] << " for in_R32[" << i
+                          << "](global[" << i + start << "])" << std::endl;
+                failures++;
+            }
         }
+        else
+        {
+            for (int j = 0; j < in_R32_blocks.size(); j++)
+            {
+                if (in_R32_blocks[j][i] !=
+                    (float)((i + start) * 10 + step + 1000.0 * j))
+                {
+                    std::cout << "Expected "
+                              << (float)((i + start) * 10 + step + 1000.0 * j)
+                              << ", got " << in_R32_blocks[j][i]
+                              << " for in_R32[" << i << "][" << j << "(global["
+                              << i + start << "])" << std::endl;
+                    failures++;
+                }
+            }
+        }
+
         if (in_R64[i] != (double)((i + start) * 10 + step))
         {
             std::cout << "Expected " << (double)((i + start) * 10 + step)
diff --git a/testing/adios2/engine/staging-common/TestStagingMPMD.cpp b/testing/adios2/engine/staging-common/TestStagingMPMD.cpp
index e9e10ce739..762583d79f 100644
--- a/testing/adios2/engine/staging-common/TestStagingMPMD.cpp
+++ b/testing/adios2/engine/staging-common/TestStagingMPMD.cpp
@@ -40,7 +40,7 @@ struct RunParams
     : npx_w{xw}, npy_w{yw}, npx_r{xr}, npy_r{yr} {};
 };
 
-/* This function is executed by INSTANTIATE_TEST_CASE_P
+/* This function is executed by INSTANTIATE_TEST_SUITE_P
    before main() and MPI_Init()!!! */
 std::vector CreateRunParams()
 {
@@ -346,8 +346,8 @@ TEST_P(TestStagingMPMD, SlowReader)
     TestCommon(p, 4, 0, 100, -1.0);
 }
 
-INSTANTIATE_TEST_CASE_P(NxM, TestStagingMPMD,
-                        ::testing::ValuesIn(CreateRunParams()));
+INSTANTIATE_TEST_SUITE_P(NxM, TestStagingMPMD,
+                         ::testing::ValuesIn(CreateRunParams()));
 
 void threadTimeoutRun(size_t t)
 {
diff --git a/testing/adios2/engine/staging-common/TestSupp.cmake b/testing/adios2/engine/staging-common/TestSupp.cmake
index bb24cc2e19..c1865e18c2 100644
--- a/testing/adios2/engine/staging-common/TestSupp.cmake
+++ b/testing/adios2/engine/staging-common/TestSupp.cmake
@@ -64,6 +64,7 @@ set (STAGING_COMMON_TEST_SUPP_VERBOSE OFF)
 
 set (1x1_CMD "run_test.py.$ -nw 1 -nr 1")
 set (1x1.NoPreload_CMD "run_test.py.$ -nw 1 -nr 1 --rarg=PreloadMode=SstPreloadNone,RENGINE_PARAMS")
+set (1x1.SstRUDP_CMD "run_test.py.$ -nw 1 -nr 1 --rarg=DataTransport=WAN,WANDataTransport=enet,RENGINE_PARAMS --warg=DataTransport=WAN,WANDataTransport=enet,WENGINE_PARAMS")
 set (1x1.NoData_CMD "run_test.py.$ -nw 1 -nr 1 --warg=--no_data --rarg=--no_data")
 set (2x2.NoData_CMD "run_test.py.$ -nw 2 -nr 2 --warg=--no_data --rarg=--no_data")
 set (2x2.HalfNoData_CMD "run_test.py.$ -nw 2 -nr 2 --warg=--no_data --warg=--no_data_node --warg=1 --rarg=--no_data --rarg=--no_data_node --rarg=1" )
@@ -75,6 +76,7 @@ set (2x1ZeroDataVar_CMD "run_test.py.$ -nw 2 -nr 1 --warg=--zero_data_va
 set (2x1ZeroDataR64_CMD "run_test.py.$ -nw 2 -nr 1  -r $ --warg=--zero_data_var")
 set (2x1.NoPreload_CMD "run_test.py.$ -nw 2 -nr 1 --rarg=PreloadMode=SstPreloadNone,RENGINE_PARAMS")
 set (2x3.ForcePreload_CMD "run_test.py.$ -nw 2 -nr 3 --rarg=PreloadMode=SstPreloadOn,RENGINE_PARAMS")
+set (2x3.SstRUDP_CMD "run_test.py.$ -nw 2 -nr 3 --rarg=DataTransport=WAN,WANDataTransport=enet,RENGINE_PARAMS --warg=DataTransport=WAN,WANDataTransport=enet,WENGINE_PARAMS")
 set (1x2_CMD "run_test.py.$ -nw 1 -nr 2")
 set (3x5_CMD "run_test.py.$ -nw 3 -nr 5")
 set (3x5LockGeometry_CMD "run_test.py.$ -nw 3 -nr 5 --warg=--num_steps --warg=50 --warg=--ms_delay --warg=10 --rarg=--num_steps --rarg=50 --warg=--lock_geometry --rarg=--lock_geometry")
@@ -89,6 +91,9 @@ set (3x5.Local_CMD "run_test.py.$ -nw 3 -nr 5  -w $ -nw 5 -nr 3  -w $ -r $")
 set (1x1.LocalVarying_CMD "run_test.py.$ -nw 1 -nr 1  -w $ --warg=--nx --warg=20 --warg=--varying_data_size -r $ --rarg=--nx --rarg=20 --rarg=--varying_data_size ")
 set (5x3.LocalVarying_CMD "run_test.py.$ -nw 5 -nr 3  -w $ --warg=--nx --warg=20 --warg=--varying_data_size -r $ --rarg=--nx --rarg=20 --rarg=--varying_data_size ")
+set (1x1.LocalMultiblock_CMD "run_test.py.$ -nw 1 -nr 1  -w $ -r $ --warg=--local_count --warg=2 --rarg=--local_count --rarg=2")
+set (2x1.LocalMultiblock_CMD "run_test.py.$ -nw 2 -nr 1  -w $ -r $ --warg=--local_count --warg=3 --rarg=--local_count --rarg=3")
+set (5x3.LocalMultiblock_CMD "run_test.py.$ -nw 5 -nr 3  -w $ -r $ --warg=--local_count --warg=5 --rarg=--local_count --rarg=5")
 set (DelayedReader_3x5_CMD "run_test.py.$ -rd 5 -nw 3 -nr 5")
 set (FtoC.3x5_CMD "run_test.py.$ -nw 3 -nr 5  -w $")
 set (FtoF.3x5_CMD "run_test.py.$ -nw 3 -nr 5  -w $ -r $")
@@ -161,6 +166,9 @@ set (LatestReaderHold.1x1_CMD "run_test.py.$ --test_protocol one_client
 # A faster writer and a queue policy that will cause timesteps to be discarded
 set (DiscardWriter.1x1_CMD "run_test.py.$ --test_protocol one_client -nw 1 -nr 1 --warg=--engine_params --warg=QueueLimit=1,QueueFullPolicy=discard,WENGINE_PARAMS --warg=--ms_delay --warg=250 --rarg=--discard")
 
+# Readers using Advancing attributes
+set (CumulativeAttr.1x1_CMD "run_test.py.$ -nw 1 -nr 1 --warg=--advancing_attributes --rarg=--advancing_attributes")
+
 function(remove_engine_params_placeholder dst_str src_str )
     string(REGEX REPLACE "([^ 		  ]*),WENGINE_PARAMS" "\\1" src_str "${src_str}")
     string(REGEX REPLACE "([^ 		  ]*),RENGINE_PARAMS" "\\1" src_str "${src_str}")
@@ -230,7 +238,7 @@ function(add_common_test basename engine)
     remove_engine_params_placeholder(command  "${command}")
     separate_arguments(command)
     list(INSERT command 2 "${engine}" "${testname}")
-    if(NOT ADIOS2_RUN_MPMD_TESTS)
+    if(NOT ADIOS2_RUN_MPI_MPMD_TESTS)
       list(APPEND command "--disable_mpmd")
     endif()
     add_test(NAME ${testname} COMMAND ${command})
@@ -246,7 +254,10 @@ function(add_common_test basename engine)
        set (timeout "30")
     endif()
 
-    set_tests_properties(${testname} PROPERTIES TIMEOUT ${timeout} ${${basename}_PROPERTIES} )
+    set_tests_properties(${testname} PROPERTIES
+        TIMEOUT ${timeout} ${${basename}_PROPERTIES}
+        RUN_SERIAL TRUE
+    )
 endfunction()
 
 function(from_hex HEX DEC)
diff --git a/testing/adios2/engine/staging-common/TestThreads.cpp b/testing/adios2/engine/staging-common/TestThreads.cpp
new file mode 100644
index 0000000000..d4d4af26a8
--- /dev/null
+++ b/testing/adios2/engine/staging-common/TestThreads.cpp
@@ -0,0 +1,193 @@
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include 
+
+#include "ParseArgs.h"
+
+using dt = long long;
+
+int value_errors = 0;
+
+std::mutex StdOutMtx;
+
+int Read(int ID)
+{
+    adios2::ADIOS adios;
+    adios2::IO io = adios.DeclareIO("IO");
+
+    {
+        std::lock_guard guard(StdOutMtx);
+        std::cout << "Reader: engine = " << engine << std::endl;
+    }
+    io.SetEngine(engine);
+    io.SetParameters(engineParams);
+
+    {
+        std::lock_guard guard(StdOutMtx);
+        std::cout << "Reader: call Open" << std::endl;
+    }
+
+    try
+    {
+        std::string FName = "File" + std::to_string(ID);
+        adios2::Engine Reader = io.Open(FName, adios2::Mode::Read);
+        {
+            std::lock_guard guard(StdOutMtx);
+            std::cout << "Reader: passed Open" << std::endl;
+        }
+        std::array ar;
+
+        auto status = Reader.BeginStep();
+        {
+            std::lock_guard guard(StdOutMtx);
+            std::cout << "Reader: passed BeginStep";
+        }
+        if (status == adios2::StepStatus::EndOfStream)
+        {
+            {
+                std::lock_guard guard(StdOutMtx);
+                std::cout << " with EndOfStream " << std::endl;
+            }
+            return false;
+        }
+        {
+            std::lock_guard guard(StdOutMtx);
+            std::cout << " with success " << std::endl;
+        }
+
+        adios2::Variable
 var = io.InquireVariable
("data");
+        Reader.Get(var, ar.begin());
+        Reader.EndStep();
+        dt expect = 0;
+        for (auto &val : ar)
+        {
+            if (val != expect)
+            {
+                value_errors++;
+            }
+            expect++;
+        }
+
+        Reader.Close();
+        {
+            std::lock_guard guard(StdOutMtx);
+            std::cout << "Reader got " << expect << " values, " << value_errors
+                      << " were incorrect" << std::endl;
+        }
+    }
+    catch (std::exception &e)
+    {
+        {
+            std::lock_guard guard(StdOutMtx);
+            std::cout << "Reader: Exception: " << e.what() << std::endl;
+        }
+        return false;
+    }
+    return true;
+}
+
+bool Write(int ID)
+{
+    adios2::ADIOS adios;
+    adios2::IO io = adios.DeclareIO("IO");
+    io.SetEngine(engine);
+    io.SetParameters(engineParams);
+    {
+        std::lock_guard guard(StdOutMtx);
+        std::cout << "Writer: engine = " << engine << std::endl;
+    }
+    auto var = io.DefineVariable
("data", adios2::Dims{100, 10},
+                                     adios2::Dims{0, 0}, adios2::Dims{100, 10});
+
+    std::array ar;
+    std::iota(ar.begin(), ar.end(), 0);
+
+    try
+    {
+        std::string FName = "File" + std::to_string(ID);
+        adios2::Engine Writer = io.Open(FName, adios2::Mode::Write);
+
+        {
+            std::lock_guard guard(StdOutMtx);
+            std::cout << "Writer completed Open() " << std::endl;
+        }
+        Writer.BeginStep();
+        Writer.Put
(var, ar.begin());
+        Writer.EndStep();
+        Writer.Close();
+        {
+            std::lock_guard guard(StdOutMtx);
+            std::cout << "Writer completed Close() " << std::endl;
+        }
+    }
+    catch (std::exception &e)
+    {
+        {
+            std::lock_guard guard(StdOutMtx);
+            std::cout << "Writer: Exception: " << e.what() << std::endl;
+        }
+        return false;
+    }
+    return true;
+}
+
+class TestThreads : public ::testing::Test
+{
+public:
+    TestThreads() = default;
+};
+
+TEST_F(TestThreads, Basic)
+{
+    auto read_fut = std::async(std::launch::async, Read, 0);
+    auto write_fut = std::async(std::launch::async, Write, 0);
+    bool reader_success = read_fut.get();
+    bool writer_success = write_fut.get();
+    EXPECT_TRUE(reader_success);
+    EXPECT_TRUE(writer_success);
+    EXPECT_EQ(value_errors, 0)
+        << "We got " << value_errors << " erroneous values at the reader";
+}
+
+//  This test tries to push up to the limits to see if we're leaking FDs, but it
+//  runs slowly, commenting it out until needed.
+//
+// TEST_F(TestThreads, Repeated)
+// {
+//     auto high_write_fut = std::async(std::launch::async, Write, 0);
+//     for (int i = 0; i < 1024; i++)
+//     {
+//         auto read_fut = std::async(std::launch::async, Read, i + 1);
+//         auto write_fut = std::async(std::launch::async, Write, i + 1);
+//         bool reader_success = read_fut.get();
+//         bool writer_success = write_fut.get();
+//         EXPECT_TRUE(reader_success);
+//         EXPECT_TRUE(writer_success);
+//         EXPECT_EQ(value_errors, 0)
+//             << "We got " << value_errors << " erroneous values at the
+//             reader";
+//         std::cout << "finished pair " << i << std::endl;
+//     }
+//     auto high_read_fut = std::async(std::launch::async, Read, 0);
+//     bool reader_success = high_read_fut.get();
+//     bool writer_success = high_write_fut.get();
+//     EXPECT_TRUE(reader_success);
+//     EXPECT_TRUE(writer_success);
+//     EXPECT_EQ(value_errors, 0);
+// }
+
+int main(int argc, char **argv)
+{
+    ::testing::InitGoogleTest(&argc, argv);
+    ParseArgs(argc, argv);
+
+    int result;
+    result = RUN_ALL_TESTS();
+    return result;
+}
diff --git a/testing/adios2/hierarchy/CMakeLists.txt b/testing/adios2/hierarchy/CMakeLists.txt
new file mode 100644
index 0000000000..c699ec9ac5
--- /dev/null
+++ b/testing/adios2/hierarchy/CMakeLists.txt
@@ -0,0 +1,6 @@
+#------------------------------------------------------------------------------#
+#Distributed under the OSI - approved Apache License, Version 2.0. See
+#accompanying file Copyright.txt for details.
+#------------------------------------------------------------------------------#
+
+gtest_add_tests_helper(HierarchicalReading MPI_NONE "" "" . "")
\ No newline at end of file
diff --git a/testing/adios2/hierarchy/TestHierarchicalReading.cpp b/testing/adios2/hierarchy/TestHierarchicalReading.cpp
new file mode 100644
index 0000000000..0637494ac9
--- /dev/null
+++ b/testing/adios2/hierarchy/TestHierarchicalReading.cpp
@@ -0,0 +1,102 @@
+/*
+ * Distributed under the OSI-approved Apache License, Version 2.0.  See
+ * accompanying file Copyright.txt for details.
+ */
+#include 
+#include 
+
+#include 
+#include 
+
+#include 
+#include 
+
+#include 
+
+class ADIOSHierarchicalReadVariableTest : public ::testing::Test
+{
+public:
+    ADIOSHierarchicalReadVariableTest() = default;
+};
+
+TEST_F(ADIOSHierarchicalReadVariableTest, Read)
+{
+    std::string filename = "ADIOSHierarchicalReadVariable.bp";
+
+    // Number of steps
+    const std::size_t NSteps = 1;
+
+#if ADIOS2_USE_MPI
+    MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank);
+    MPI_Comm_size(MPI_COMM_WORLD, &mpiSize);
+#endif
+
+    // Write test data using BP
+    {
+#if ADIOS2_USE_MPI
+        adios2::ADIOS adios(MPI_COMM_WORLD);
+#else
+        adios2::ADIOS adios;
+#endif
+        adios2::IO io = adios.DeclareIO("TestIO");
+        io.SetEngine("BPFile");
+
+        io.AddTransport("file");
+        adios2::Engine engine = io.Open(filename, adios2::Mode::Write);
+        const adios2::Dims shape = {10};
+        const adios2::Dims start = {0};
+        const adios2::Dims count = {10};
+
+        auto var1 = io.DefineVariable(
+            "group1/group2/group3/group4/variable1", shape, start, count);
+        auto var2 = io.DefineVariable(
+            "group1/group2/group3/group4/variable2", shape, start, count);
+        auto var3 = io.DefineVariable(
+            "group1/group2/group3/group4/variable3", shape, start, count);
+        auto var4 = io.DefineVariable(
+            "group1/group2/group3/group4/variable4", shape, start, count);
+        auto var5 = io.DefineVariable(
+            "group1/group2/group3/group4/variable5", shape, start, count);
+        std::vector Ints = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+        for (size_t step = 0; step < NSteps; ++step)
+        {
+            engine.BeginStep();
+
+            engine.Put(var1, Ints.data());
+            engine.Put(var2, Ints.data());
+            engine.Put(var3, Ints.data());
+            engine.Put(var4, Ints.data());
+            engine.Put(var5, Ints.data());
+
+            engine.EndStep();
+        }
+
+        for (int step = 0; step < NSteps; step++)
+        {
+            engine.BeginStep();
+            auto g = io.InquireGroup("group1", '/');
+            auto res = g.AvailableGroups();
+            EXPECT_EQ(res[0], "group2");
+            g.setPath("group1/group2");
+            res = g.AvailableGroups();
+            EXPECT_EQ(res[0], "group3");
+            g.setPath("group1/group2/group3");
+            res = g.AvailableGroups();
+            EXPECT_EQ(res[0], "group4");
+            g.setPath("group1/group2/group3/group4");
+            res = g.AvailableGroups();
+            EXPECT_EQ(res.size(), 0);
+            res = g.AvailableVariables();
+            EXPECT_EQ(res.size(), 5);
+            res = g.AvailableAttributes();
+            EXPECT_EQ(res.size(), 0);
+            engine.EndStep();
+        }
+        engine.Close();
+    }
+}
+int main(int argc, char **argv)
+{
+    ::testing::InitGoogleTest(&argc, argv);
+    return RUN_ALL_TESTS();
+}
diff --git a/testing/adios2/interface/CMakeLists.txt b/testing/adios2/interface/CMakeLists.txt
index 3726dbf7f1..db718690ca 100644
--- a/testing/adios2/interface/CMakeLists.txt
+++ b/testing/adios2/interface/CMakeLists.txt
@@ -7,5 +7,5 @@ gtest_add_tests_helper(Interface MPI_ALLOW ADIOS Interface. "")
 gtest_add_tests_helper(Write MPI_ALLOW ADIOSInterface Interface. "")
 gtest_add_tests_helper(DefineVariable MPI_ALLOW ADIOS Interface. "")
 gtest_add_tests_helper(DefineAttribute MPI_ALLOW ADIOS Interface. "")
-gtest_add_tests_helper(Selection MPI_NOEXEC ADIOS Interface. "")
+gtest_add_tests_helper(Selection MPI_NONE ADIOS Interface. "")
 gtest_add_tests_helper(NoMpi MPI_NONE ADIOS Interface. "")
diff --git a/testing/adios2/interface/TestADIOSInterface.cpp b/testing/adios2/interface/TestADIOSInterface.cpp
index c8152ad39a..2842423766 100644
--- a/testing/adios2/interface/TestADIOSInterface.cpp
+++ b/testing/adios2/interface/TestADIOSInterface.cpp
@@ -303,7 +303,7 @@ class ADIOS2_CXX11_API_MultiBlock : public ADIOS2_CXX11_API_IO
     std::vector m_Selections;
 };
 
-TYPED_TEST_CASE(ADIOS2_CXX11_API_MultiBlock, MultiBlockTypes);
+TYPED_TEST_SUITE(ADIOS2_CXX11_API_MultiBlock, MultiBlockTypes);
 
 TYPED_TEST(ADIOS2_CXX11_API_MultiBlock, Put)
 {
diff --git a/testing/adios2/performance/CMakeLists.txt b/testing/adios2/performance/CMakeLists.txt
index f70378a329..bb672a0968 100644
--- a/testing/adios2/performance/CMakeLists.txt
+++ b/testing/adios2/performance/CMakeLists.txt
@@ -5,3 +5,4 @@
 
 add_subdirectory(manyvars)
 add_subdirectory(query)
+add_subdirectory(metadata)
diff --git a/testing/adios2/performance/manyvars/PerfManyVars.c b/testing/adios2/performance/manyvars/PerfManyVars.c
index 592c0181e3..cc2635624e 100644
--- a/testing/adios2/performance/manyvars/PerfManyVars.c
+++ b/testing/adios2/performance/manyvars/PerfManyVars.c
@@ -251,7 +251,7 @@ int main(int argc, char **argv)
 
 void define_vars()
 {
-    int i, block;
+    int i;
 
     size_t shape[2] = {gdim1, gdim2};
     size_t count[2] = {ldim1, ldim2};
@@ -269,8 +269,6 @@ void define_vars()
 
 int write_file(int step)
 {
-    int64_t fh;
-    uint64_t groupsize = 0, totalsize;
     int block, v, i;
     double tb, te;
     size_t count[2] = {ldim1, ldim2};
@@ -361,15 +359,14 @@ void reset_readvars()
 int read_file()
 {
     adios2_variable *vi;
-    int err = 0, v, n;
+    int err = 0, v;
     int block, step, i; // loop variables
     int iMacro;         // loop variable in macros
-    double tb, te, tsched;
+    double tb, te;
     double tsb, ts; // time for just scheduling for one step/block
 
     size_t start[2] = {offs1, offs2};
     size_t count[2] = {ldim1, ldim2};
-    uint64_t ndim;
 
     reset_readvars();
 
diff --git a/testing/adios2/performance/manyvars/TestManyVars.cpp b/testing/adios2/performance/manyvars/TestManyVars.cpp
index d9ef1d85a5..bd473b4a19 100644
--- a/testing/adios2/performance/manyvars/TestManyVars.cpp
+++ b/testing/adios2/performance/manyvars/TestManyVars.cpp
@@ -45,7 +45,7 @@ struct RunParams
     : nvars{nv}, nblocks{nb}, nsteps{ns} {};
 };
 
-/* This function is executed by INSTANTIATE_TEST_CASE_P
+/* This function is executed by INSTANTIATE_TEST_SUITE_P
    before main() and MPI_Init()!!! */
 std::vector CreateRunParams()
 {
@@ -479,8 +479,8 @@ TEST_P(TestManyVars, DontRedefineVars)
     ASSERT_EQ(err, 0);
 }
 
-INSTANTIATE_TEST_CASE_P(NxM, TestManyVars,
-                        ::testing::ValuesIn(CreateRunParams()));
+INSTANTIATE_TEST_SUITE_P(NxM, TestManyVars,
+                         ::testing::ValuesIn(CreateRunParams()));
 
 //******************************************************************************
 // main
diff --git a/testing/adios2/performance/metadata/CMakeLists.txt b/testing/adios2/performance/metadata/CMakeLists.txt
new file mode 100644
index 0000000000..d0a3661800
--- /dev/null
+++ b/testing/adios2/performance/metadata/CMakeLists.txt
@@ -0,0 +1,10 @@
+#------------------------------------------------------------------------------#
+# Distributed under the OSI-approved Apache License, Version 2.0.  See
+# accompanying file Copyright.txt for details.
+#------------------------------------------------------------------------------#
+
+if(ADIOS2_HAVE_MPI)
+  # just for executing manually for performance studies
+  add_executable(PerfMetaData PerfMetaData.cpp)
+  target_link_libraries(PerfMetaData adios2::cxx11_mpi MPI::MPI_C)
+endif()
diff --git a/testing/adios2/performance/metadata/PerfMetaData.cpp b/testing/adios2/performance/metadata/PerfMetaData.cpp
new file mode 100644
index 0000000000..88f221a58d
--- /dev/null
+++ b/testing/adios2/performance/metadata/PerfMetaData.cpp
@@ -0,0 +1,533 @@
+/*
+ * Distributed under the OSI-approved Apache License, Version 2.0.  See
+ * accompanying file Copyright.txt for details.
+ */
+#include 
+#include 
+#include 
+#include 
+
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include 
+
+#ifndef _WIN32
+#include "strings.h"
+#else
+#define strcasecmp _stricmp
+#endif
+
+std::string engine = "sst";
+adios2::Params engineParams = {}; // parsed from command line
+
+int NSteps = 10;
+int NumVars = 100;
+int NumArrays = 100;
+int NumAttrs = 100;
+int NumBlocks = 1;
+int ReaderDelay = 20;
+int WriterSize;
+
+int LastVarSize;
+int LastArraySize;
+int LastAttrsSize;
+
+typedef enum
+{
+    WriterConsolidation,
+    ReaderInstallation,
+    ReaderTraversal
+} TestModeEnum;
+
+TestModeEnum TestMode = WriterConsolidation;
+
+static std::string Trim(std::string &str)
+{
+    size_t first = str.find_first_not_of(' ');
+    size_t last = str.find_last_not_of(' ');
+    return str.substr(first, (last - first + 1));
+}
+
+/*
+ * Engine parameters spec is a poor-man's JSON.  name:value pairs are separated
+ * by equal.  White space is trimmed off front and back.  No quotes or anything
+ * fancy allowed.
+ */
+static adios2::Params ParseEngineParams(std::string Input)
+{
+    std::istringstream ss(Input);
+    std::string Param;
+    adios2::Params Ret = {};
+
+    while (std::getline(ss, Param, ','))
+    {
+        std::istringstream ss2(Param);
+        std::string ParamName;
+        std::string ParamValue;
+        std::getline(ss2, ParamName, '=');
+        if (!std::getline(ss2, ParamValue, '='))
+        {
+            throw std::invalid_argument("Engine parameter \"" + Param +
+                                        "\" missing value");
+        }
+        Ret[Trim(ParamName)] = Trim(ParamValue);
+    }
+    return Ret;
+}
+
+static void Usage()
+{
+    std::cout << "PerfMetaData  " << std::endl;
+    std::cout << "  --num_steps " << std::endl;
+    std::cout << "  --num_vars " << std::endl;
+    std::cout << "  --num_arrays " << std::endl;
+    std::cout << "  --num_attrs " << std::endl;
+    std::cout << "  --num_attrs " << std::endl;
+    std::cout << "  --e3sm   (Approx E3SM characteristics)" << std::endl;
+    std::cout << "  --warpx   (Approx WarpX characteristics)" << std::endl;
+    std::cout << "  --engine " << std::endl;
+    std::cout << "  --engine_params " << std::endl;
+}
+
+static void ParseArgs(int argc, char **argv, int rank)
+{
+    int bare_arg = 0;
+    while (argc > 1)
+    {
+        if (std::string(argv[1]) == "--num_steps")
+        {
+            std::istringstream ss(argv[2]);
+            if (!(ss >> NSteps))
+                std::cerr << "Invalid number for num_steps " << argv[1] << '\n';
+            argv++;
+            argc--;
+        }
+        else if (std::string(argv[1]) == "--num_vars")
+        {
+            std::istringstream ss(argv[2]);
+            if (!(ss >> NumVars))
+                std::cerr << "Invalid number for number of variables "
+                          << argv[1] << '\n';
+            argv++;
+            argc--;
+        }
+        else if (std::string(argv[1]) == "--num_arrays")
+        {
+            std::istringstream ss(argv[2]);
+            if (!(ss >> NumArrays))
+                std::cerr << "Invalid number for number of arrays" << argv[1]
+                          << '\n';
+            argv++;
+            argc--;
+        }
+        else if (std::string(argv[1]) == "--num_attrs")
+        {
+            std::istringstream ss(argv[2]);
+            if (!(ss >> NumAttrs))
+                std::cerr << "Invalid number for number of attrs" << argv[1]
+                          << '\n';
+            argv++;
+            argc--;
+        }
+        else if (std::string(argv[1]) == "--num_blocks")
+        {
+            std::istringstream ss(argv[2]);
+            if (!(ss >> NumBlocks))
+                std::cerr << "Invalid number for number of blocks" << argv[1]
+                          << '\n';
+            argv++;
+            argc--;
+        }
+        else if (std::string(argv[1]) == "--e3sm")
+        {
+            NumArrays = 535;
+            NumVars = 1000;
+            NumAttrs = 3800;
+            NumBlocks = 1;
+            if (rank == 0)
+                std::cout
+                    << "E3SM mode.  Full run: 960 Timesteps, at 1344 ranks"
+                    << std::endl;
+        }
+        else if (std::string(argv[1]) == "--warpx")
+        {
+            NumArrays = 24;
+            NumVars = 1000;
+            NumAttrs = 0;
+            NumBlocks = 86;
+            if (rank == 0)
+                std::cout << "E3SM mode.  Full run: 50 Timesteps, at 3000 ranks"
+                          << std::endl;
+        }
+        else if (std::string(argv[1]) == "--test_mode")
+        {
+            std::string mode = argv[2];
+            std::transform(mode.begin(), mode.end(), mode.begin(),
+                           [](unsigned char c) { return std::tolower(c); });
+            if (mode == "writerconsolidation")
+            {
+                TestMode = WriterConsolidation;
+            }
+            else if (mode == "readerinstallation")
+            {
+                TestMode = ReaderInstallation;
+            }
+            else if (mode == "readertraversal")
+            {
+                TestMode = ReaderTraversal;
+            }
+            else
+            {
+                std::cerr << "Unknown test mode \"" << argv[2]
+                          << "\", must be one of WriterConsolidation, "
+                             "ReaderInstallation, or ReaderTraversal"
+                          << std::endl;
+            }
+            argv++;
+            argc--;
+        }
+        else if (std::string(argv[1]) == "--engine_params")
+        {
+            engineParams = ParseEngineParams(argv[2]);
+            argv++;
+            argc--;
+        }
+        else if (std::string(argv[1]) == "--engine")
+        {
+            engine = std::string(argv[2]);
+            argv++;
+            argc--;
+        }
+        else if (std::string(argv[1]) == "--reader_delay")
+        {
+            std::istringstream ss(argv[2]);
+            if (!(ss >> ReaderDelay))
+                std::cerr << "Invalid number for ms_delay " << argv[1] << '\n';
+            argv++;
+            argc--;
+        }
+        else
+        {
+            if (std::string(argv[1], 2) == "--")
+            {
+                if (rank == 0)
+                    Usage();
+                exit(0);
+            }
+            if (bare_arg == 0)
+            {
+                /* first arg without -- is engine */
+                engine = std::string(argv[1]);
+                bare_arg++;
+            }
+            else if (bare_arg == 1)
+            {
+                engineParams = ParseEngineParams(argv[1]);
+                bare_arg++;
+            }
+            else
+            {
+
+                throw std::invalid_argument("Unknown argument \"" +
+                                            std::string(argv[1]) + "\"");
+            }
+        }
+        argv++;
+        argc--;
+    }
+}
+
+MPI_Comm testComm;
+std::chrono::duration elapsed;
+std::chrono::duration InstallTime, TraversalTime;
+
+void DoWriter(adios2::Params writerParams)
+{
+    // form a mpiSize * Nx 1D array
+    int mpiRank = 0, mpiSize = 1;
+
+#if ADIOS2_USE_MPI
+    MPI_Comm_rank(testComm, &mpiRank);
+    MPI_Comm_size(testComm, &mpiSize);
+#endif
+
+    // Write test data using ADIOS2
+
+    adios2::ADIOS adios(testComm);
+    adios2::IO io = adios.DeclareIO("TestIO");
+
+    io.SetEngine(engine);
+    io.SetParameters(writerParams);
+    adios2::Engine writer = io.Open("MetaDataTest", adios2::Mode::Write);
+    std::chrono::time_point start, finish;
+    std::vector myFloats = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+    adios2::Variable *Floats = new adios2::Variable[NumVars];
+    adios2::Variable *FloatArrays =
+        new adios2::Variable[NumArrays];
+    if (mpiRank == 0)
+    {
+        // attributes and globals on rank 0
+        for (int i = 0; i < NumVars; i++)
+        {
+            std::string varname = "Variable" + std::to_string(i);
+            Floats[i] = io.DefineVariable(varname);
+        }
+        for (int i = 0; i < NumAttrs; i++)
+        {
+            std::string varname = "Attribute" + std::to_string(i);
+            io.DefineAttribute(varname, 0.0);
+        }
+    }
+    for (int i = 0; i < NumArrays; i++)
+    {
+        std::string varname = "Array" + std::to_string(i);
+        if (NumBlocks == 1)
+        {
+            FloatArrays[i] = io.DefineVariable(
+                varname, {(unsigned long)mpiSize}, {(unsigned long)mpiRank},
+                {1}, adios2::ConstantDims);
+        }
+        else
+        {
+            FloatArrays[i] = io.DefineVariable(varname, {}, {}, {1},
+                                                      adios2::ConstantDims);
+        }
+    }
+    start = std::chrono::high_resolution_clock::now();
+    for (int j = 0; j < NSteps; j++)
+    {
+        writer.BeginStep();
+        if (mpiRank == 0)
+        {
+            // globals on rank 0
+            for (int i = 0; i < NumVars; i++)
+            {
+                writer.Put(Floats[i], myFloats.data()[0]);
+            }
+        }
+        for (int i = 0; i < NumArrays; i++)
+        {
+            for (int j = 0; j < NumBlocks; j++)
+            {
+                writer.Put(FloatArrays[i], myFloats.data());
+            }
+        }
+        writer.EndStep();
+    }
+    finish = std::chrono::high_resolution_clock::now();
+    elapsed = finish - start;
+    writer.Close();
+}
+
+void DoReader()
+{
+    // form a mpiSize * Nx 1D array
+    int mpiRank = 0, mpiSize = 1;
+
+#if ADIOS2_USE_MPI
+    MPI_Comm_rank(testComm, &mpiRank);
+    MPI_Comm_size(testComm, &mpiSize);
+#endif
+
+    adios2::ADIOS adios(testComm);
+    adios2::IO io = adios.DeclareIO("TestIO");
+
+    io.SetEngine(engine);
+    engineParams["SpeculativePreloadMode"] = "Off";
+    engineParams["ReaderShortCircuitReads"] = "On";
+    io.SetParameters(engineParams);
+    adios2::Engine reader = io.Open("MetaDataTest", adios2::Mode::Read);
+    std::chrono::time_point startTS,
+        endBeginStep, finishTS;
+    std::vector myFloats = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+    std::this_thread::sleep_for(std::chrono::seconds(ReaderDelay));
+    std::vector in;
+
+    in.resize(WriterSize);
+    while (1)
+    {
+        startTS = std::chrono::high_resolution_clock::now();
+        adios2::StepStatus status = reader.BeginStep();
+        endBeginStep = std::chrono::high_resolution_clock::now();
+        if (status != adios2::StepStatus::OK)
+        {
+            break;
+        }
+        //	if (DoGets) {
+        std::vector> Floats;
+        std::vector> FloatArrays;
+        std::vector> Attributes;
+        int Cont = 1;
+        int i = 0;
+        while (Cont)
+        {
+            std::string varname = "Variable" + std::to_string(i++);
+            adios2::Variable tmp = io.InquireVariable(varname);
+            if (tmp)
+            {
+                Floats.push_back(tmp);
+            }
+            else
+            {
+                Cont = 0;
+            }
+        }
+        Cont = 1;
+        i = 0;
+        while (Cont)
+        {
+            std::string varname = "Array" + std::to_string(i++);
+            adios2::Variable tmp = io.InquireVariable(varname);
+            if (tmp)
+            {
+                FloatArrays.push_back(tmp);
+            }
+            else
+            {
+                Cont = 0;
+            }
+        }
+        i = 0;
+        Cont = 1;
+        while (Cont)
+        {
+            std::string varname = "Attribute" + std::to_string(i++);
+            adios2::Attribute tmp = io.InquireAttribute(varname);
+            if (tmp)
+            {
+                Attributes.push_back(tmp);
+            }
+            else
+            {
+                Cont = 0;
+            }
+        }
+        LastVarSize = (int)Floats.size();
+        for (auto Var : Floats)
+        {
+            reader.Get(Var, in.data());
+        }
+        LastArraySize = (int)FloatArrays.size();
+        for (auto Var : FloatArrays)
+        {
+            if (Var.ShapeID() == adios2::ShapeID::GlobalArray)
+            {
+                reader.Get(Var, in.data());
+            }
+            else
+            {
+                // local, go through blocks
+                for (int rank = 0; rank < WriterSize; rank++)
+                {
+                    for (auto blk :
+                         reader.BlocksInfo(Var, reader.CurrentStep()))
+                    {
+                        Var.SetBlockSelection(blk.BlockID);
+                        reader.Get(Var, in.data());
+                    }
+                }
+            }
+        }
+        LastAttrsSize = (int)Attributes.size();
+        for (auto Attr : Attributes)
+        {
+            if (Attr.Data().front() != 0.0)
+            {
+                std::cerr << "Bad attr data" << std::endl;
+            }
+        }
+        //	}
+        reader.EndStep();
+        finishTS = std::chrono::high_resolution_clock::now();
+        InstallTime += (endBeginStep - startTS);
+        TraversalTime += (finishTS - endBeginStep);
+    }
+    reader.Close();
+}
+
+int main(int argc, char **argv)
+{
+    int key;
+    std::string MeasurementString;
+
+    MPI_Init(nullptr, nullptr);
+
+    MPI_Comm_rank(MPI_COMM_WORLD, &key);
+    MPI_Comm_size(MPI_COMM_WORLD, &WriterSize);
+
+    ParseArgs(argc, argv, key);
+
+    if (WriterSize < 2)
+    {
+        std::cerr << "PerfMetaData cannot run with MPI size < 2" << std::endl;
+        exit(1);
+    }
+    if ((NumBlocks > 1) && (key == 0))
+    {
+        std::cerr
+            << "Warning, metadata info for FFS not valid for num_blocks > 1"
+            << std::endl;
+    }
+    unsigned int color = 0;
+    if (key > 0)
+    {
+        color = 1;
+        WriterSize--;
+    }
+    MPI_Comm_split(MPI_COMM_WORLD, color, key, &testComm);
+
+    // first all writer ranks do Writer calcs with no reader
+    if (key > 0)
+    {
+        adios2::Params ConsolidationParams =
+            engineParams; // parsed from command line
+        ConsolidationParams["RendezvousReaderCount"] = "0";
+        DoWriter(ConsolidationParams);
+        if (key == 1)
+        {
+            std::cout << "Metadata Consolidation Time " << elapsed.count()
+                      << " seconds." << std::endl;
+        }
+        ReaderDelay = (int)elapsed.count() + 5;
+    }
+
+    MPI_Bcast(&ReaderDelay, 1, MPI_INT, 1, MPI_COMM_WORLD);
+    // next do writers and single reader
+    if (key == 0)
+    {
+        DoReader();
+    }
+    else
+    {
+        DoWriter(engineParams);
+    }
+
+    if (key == 0)
+    {
+        std::cout << "Metadata Installation Time " << InstallTime.count()
+                  << " seconds." << std::endl;
+
+        std::cout << "Metadata Traversal Time " << TraversalTime.count()
+                  << " seconds." << std::endl;
+
+        std::cout << "Parameters Nsteps=" << NSteps
+                  << ", NumArrays=" << NumArrays << ", NumVArs=" << NumVars
+                  << ", NumAttrs=" << NumAttrs << ", NumBlocks=" << NumBlocks
+                  << std::endl;
+        if ((NumArrays != LastArraySize) || (NumVars != LastVarSize) ||
+            (NumAttrs != LastAttrsSize))
+        {
+            std::cout << "Arrays=" << LastArraySize << ", Vars=" << LastVarSize
+                      << ", Attrs=" << LastAttrsSize
+                      << ", NumBlocks=" << NumBlocks << std::endl;
+            std::cout << "Inconsistency" << std::endl;
+        }
+    }
+    MPI_Finalize();
+
+    return 0;
+}
diff --git a/testing/adios2/performance/metadata/README b/testing/adios2/performance/metadata/README
new file mode 100644
index 0000000000..55883e4b81
--- /dev/null
+++ b/testing/adios2/performance/metadata/README
@@ -0,0 +1,29 @@
+
+We'd like to emphasize metadata processing in these tests and hopefully isolate the following costs -
+
+Writer side:
+	metadata composition	- anything done to combine metadata rank contributions
+
+Reader side:
+	metadata installation	- creation of variables, blocks info 
+	metadata traversal	- generating read requests, etc.
+
+
+plan:
+
+(Likely most effective for SST, but hopefully useful for other engines.)
+
+For measuring writer-side metadata composition.
+  - in sst (other staging engines?) measure writer completion time with zero readers (metadata composition happens, but data is never delivered anywhere).
+  - for file engines, just make data small to de-emphasize it and measure write time (to /dev/null ?)
+
+
+for measuring reader-side metadata installation time.
+  -  in sst, open both sides of the stream, but delay reader some amount of time so that all metadata arrives, then measure time for BeginStep/EndStep without any Get()s.
+     Must turn off preload to avoid data interference, and delay should be sufficiently long (whatever that means)
+  -  in file engine.  All parsing/installation *might* happen in Open(), so we should include that?  Obviously without a delay, because the file is already written
+
+
+for measuring reader-side traversal
+  - in sst, same as above, but add Get()s and disable actual reads of data (obviously don't check for data correctness)
+
diff --git a/testing/adios2/transports/TestFile.cpp b/testing/adios2/transports/TestFile.cpp
index fe341348c7..dc1b5429c4 100644
--- a/testing/adios2/transports/TestFile.cpp
+++ b/testing/adios2/transports/TestFile.cpp
@@ -76,7 +76,7 @@ TEST_P(BufferTest, WriteRead)
 }
 
 #ifdef __unix__
-INSTANTIATE_TEST_CASE_P(
+INSTANTIATE_TEST_SUITE_P(
     TransportTests, BufferTest,
     ::testing::Values(std::make_tuple("fstream", "true", "posix", "false"),
                       std::make_tuple("fstream", "false", "posix", "false"),
@@ -106,7 +106,7 @@ INSTANTIATE_TEST_CASE_P(
                       std::make_tuple("fstream", "false", "fstream", "true"),
                       std::make_tuple("fstream", "false", "fstream", "false")));
 #else
-INSTANTIATE_TEST_CASE_P(
+INSTANTIATE_TEST_SUITE_P(
     TransportTests, BufferTest,
     ::testing::Values(std::make_tuple("stdio", "true", "stdio", "true"),
                       std::make_tuple("stdio", "true", "stdio", "false"),
diff --git a/testing/adios2/xml/CMakeLists.txt b/testing/adios2/xml/CMakeLists.txt
index 1890f14916..e7d40fdfc2 100644
--- a/testing/adios2/xml/CMakeLists.txt
+++ b/testing/adios2/xml/CMakeLists.txt
@@ -4,13 +4,9 @@
 #------------------------------------------------------------------------------#
 
 gtest_add_tests_helper(XMLConfig MPI_ALLOW "" "" "")
-gtest_add_tests_helper(XMLConfigSerial MPI_NOEXEC "" "" "")
 
-foreach(tgt
-    ${Test.XMLConfig-TARGETS}
-    ${Test.XMLConfigSerial-TARGETS}
-    )
+foreach(tgt IN LISTS Test.XMLConfig-TARGETS)
   target_compile_definitions(${tgt} PRIVATE
     "XML_CONFIG_DIR=${CMAKE_CURRENT_SOURCE_DIR}"
-    )
+  )
 endforeach()
diff --git a/testing/adios2/yaml/CMakeLists.txt b/testing/adios2/yaml/CMakeLists.txt
index f980de0388..e51a2897bf 100644
--- a/testing/adios2/yaml/CMakeLists.txt
+++ b/testing/adios2/yaml/CMakeLists.txt
@@ -4,13 +4,9 @@
 #------------------------------------------------------------------------------#
 
 gtest_add_tests_helper(YAMLConfig MPI_ALLOW "" "" "")
-gtest_add_tests_helper(YAMLConfigSerial MPI_NOEXEC "" "" "")
 
-foreach(tgt
-    ${Test.YAMLConfig-TARGETS}
-    ${Test.YAMLConfigSerial-TARGETS}
-    )
+foreach(tgt IN LISTS Test.YAMLConfig-TARGETS)
   target_compile_definitions(${tgt} PRIVATE
     "YAML_CONFIG_DIR=${CMAKE_CURRENT_SOURCE_DIR}"
-    )
+  )
 endforeach()
diff --git a/testing/examples/heatTransfer/CMakeLists.txt b/testing/examples/heatTransfer/CMakeLists.txt
index 44b7d864af..01c032688b 100644
--- a/testing/examples/heatTransfer/CMakeLists.txt
+++ b/testing/examples/heatTransfer/CMakeLists.txt
@@ -15,7 +15,7 @@ if(ADIOS2_HAVE_ZFP)
   include(${CMAKE_CURRENT_SOURCE_DIR}/TestBPFileMx1_zfp.cmake)
 endif()
 
-if(ADIOS2_RUN_MPMD_TESTS)
+if(ADIOS2_RUN_MPI_MPMD_TESTS)
   if(ADIOS2_HAVE_SST)
     include(${CMAKE_CURRENT_SOURCE_DIR}/TestSSTBPMxM.cmake)
     include(${CMAKE_CURRENT_SOURCE_DIR}/TestSSTBPMxN.cmake)
diff --git a/testing/install/C/Makefile b/testing/install/C/Makefile
index b8734c2a80..87fc7114c5 100644
--- a/testing/install/C/Makefile
+++ b/testing/install/C/Makefile
@@ -39,7 +39,7 @@ adios_c_serial_test: main_nompi.o
 main_nompi.o: main_nompi.c
 	$(CC) $(CFLAGS) $(ISYSROOT) -o main_nompi.o -c main_nompi.c $(shell adios2-config$(ADIOS2_EXECUTABLE_SUFFIX) -s --c-flags)
 
-adios_c_serial_test_2: main_nompi.o
+adios_c_serial_test_2: main_nompi.c
 	$(CXX) $(LDFLAGS) $(CXXFLAGS) $(ISYSROOT) -o adios_c_serial_test_2 main_nompi.c $(shell adios2-config$(ADIOS2_EXECUTABLE_SUFFIX) -s -c)
 
 
@@ -53,5 +53,5 @@ adios_c_mpi_test: main_mpi.o
 main_mpi.o: main_mpi.c
 	$(CC) $(CFLAGS) $(ISYSROOT) -o main_mpi.o -c main_mpi.c $(shell adios2-config$(ADIOS2_EXECUTABLE_SUFFIX) -m --c-flags)
 
-adios_c_mpi_test_2: main_mpi.o
+adios_c_mpi_test_2: main_mpi.c
 	$(CXX) $(LDFLAGS) $(CXXFLAGS) $(ISYSROOT) -o adios_c_mpi_test_2 main_mpi.c $(shell adios2-config$(ADIOS2_EXECUTABLE_SUFFIX) -m -c)
diff --git a/testing/install/CXX11/Makefile b/testing/install/CXX11/Makefile
index f499770fb7..1ed2765ec6 100644
--- a/testing/install/CXX11/Makefile
+++ b/testing/install/CXX11/Makefile
@@ -39,7 +39,7 @@ adios_cxx11_serial_test: main_nompi.o
 main_nompi.o: main_nompi.cxx
 	$(CXX) $(CXXFLAGS) $(ISYSROOT) -o main_nompi.o -c main_nompi.cxx $(shell adios2-config$(ADIOS2_EXECUTABLE_SUFFIX) -s --cxx-flags)
 
-adios_cxx11_serial_test_2: main_nompi.o
+adios_cxx11_serial_test_2: main_nompi.cxx
 	$(CXX) $(LDFLAGS) $(CXXFLAGS) $(ISYSROOT) -o adios_cxx11_serial_test_2 main_nompi.cxx $(shell adios2-config$(ADIOS2_EXECUTABLE_SUFFIX) -s -x)
 
 
@@ -53,5 +53,5 @@ adios_cxx11_mpi_test: main_mpi.o
 main_mpi.o: main_mpi.cxx
 	$(CXX) $(CXXFLAGS) $(ISYSROOT) -o main_mpi.o -c main_mpi.cxx $(shell adios2-config$(ADIOS2_EXECUTABLE_SUFFIX) -m --cxx-flags)
 
-adios_cxx11_mpi_test_2: main_mpi.o
+adios_cxx11_mpi_test_2: main_mpi.cxx
 	$(CXX) $(LDFLAGS) $(CXXFLAGS) $(ISYSROOT) -o adios_cxx11_mpi_test_2 main_mpi.cxx $(shell adios2-config$(ADIOS2_EXECUTABLE_SUFFIX) -m -x)
diff --git a/testing/install/Fortran/Makefile b/testing/install/Fortran/Makefile
index 5149a077b6..fbb5b529fc 100644
--- a/testing/install/Fortran/Makefile
+++ b/testing/install/Fortran/Makefile
@@ -39,7 +39,7 @@ adios_fortran_serial_test: main_nompi.o
 main_nompi.o: main_nompi.f90
 	$(FC) $(FFLAGS) -o main_nompi.o -c main_nompi.f90 $(shell adios2-config$(ADIOS2_EXECUTABLE_SUFFIX) -s --fortran-flags)
 
-adios_fortran_serial_test_2: main_nompi.o
+adios_fortran_serial_test_2: main_nompi.f90
 	$(FC) $(FFLAGS) -o adios_fortran_serial_test_2 main_nompi.f90 $(shell adios2-config$(ADIOS2_EXECUTABLE_SUFFIX) -s -f)
 
 
@@ -53,5 +53,5 @@ adios_fortran_mpi_test: main_mpi.o
 main_mpi.o: main_mpi.f90
 	$(FC) $(FFLAGS) -o main_mpi.o -c main_mpi.f90 $(shell adios2-config$(ADIOS2_EXECUTABLE_SUFFIX) -m --fortran-flags)
 
-adios_fortran_mpi_test_2: main_mpi.o
+adios_fortran_mpi_test_2: main_mpi.f90
 	$(FC) $(FFLAGS) -o adios_fortran_mpi_test_2 main_mpi.f90 $(shell adios2-config$(ADIOS2_EXECUTABLE_SUFFIX) -m -f)
diff --git a/testing/utils/cwriter/TestUtilsCWriter.bplsh.expected.txt b/testing/utils/cwriter/TestUtilsCWriter.bplsh.expected.txt
index 1c8ef7917d..60c97a0443 100644
--- a/testing/utils/cwriter/TestUtilsCWriter.bplsh.expected.txt
+++ b/testing/utils/cwriter/TestUtilsCWriter.bplsh.expected.txt
@@ -10,7 +10,7 @@ The time dimension is the first dimension then.
   --attrs     | -a           List/match attributes too
   --attrsonly | -A           List attributes only
   --meshes    | -m           List meshes
-  --timestep  | -t           Print values of timestep elements
+  --timestep  | -t           Read content step by step (stream reading)
   --dump      | -d           Dump matched variables/attributes
                                To match attributes too, add option -a
   --regexp    | -e           Treat masks as extended regular expressions
diff --git a/testing/utils/iotest/CMakeLists.txt b/testing/utils/iotest/CMakeLists.txt
index 2a7a5acc0f..0a70b5bc58 100644
--- a/testing/utils/iotest/CMakeLists.txt
+++ b/testing/utils/iotest/CMakeLists.txt
@@ -132,7 +132,7 @@ if(ADIOS2_HAVE_HDF5 AND HDF5_IS_PARALLEL)
   )
 endif()
 
-if(ADIOS2_RUN_MPMD_TESTS)
+if(ADIOS2_RUN_MPI_MPMD_TESTS)
   if(MPIEXEC_MAX_NUMPROCS GREATER 1) 
     #------------------------------------------
     #  Pipe2 InsituMPI version 
@@ -349,7 +349,7 @@ SetupTestPipeline(
 #-------------------------------------------------
 #  BurstBuffer BP4 streaming test nproc = 2 
 #-------------------------------------------------
-if(ADIOS2_RUN_MPMD_TESTS)
+if(ADIOS2_RUN_MPI_MPMD_TESTS AND NOT WIN32)
   add_test(NAME Utils.IOTest.BurstBuffer.1to1.BP4.Stream.Run
     COMMAND ${MPIEXEC_COMMAND} 
         ${MPIEXEC_NUMPROC_FLAG} 1 $
diff --git a/thirdparty/CMakeLists.txt b/thirdparty/CMakeLists.txt
index 509afdbd4e..364655d512 100644
--- a/thirdparty/CMakeLists.txt
+++ b/thirdparty/CMakeLists.txt
@@ -1,10 +1,10 @@
 # Disable warnings in thirdparty code
 if(CMAKE_C_COMPILER_ID MATCHES
-    "^(GNU|Clang|AppleClang|XLClang|XL|VisualAge|SunPro|HP|Intel)$")
+    "^(GNU|Clang|AppleClang|XLClang|XL|VisualAge|SunPro|HP|Intel|PGI|NVIDIA)$")
   set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -w")
 endif()
 if(CMAKE_CXX_COMPILER_ID MATCHES
-    "^(GNU|Clang|AppleClang|XLClang|XL|VisualAge|SunPro|HP|Intel)$")
+    "^(GNU|Clang|AppleClang|XLClang|XL|VisualAge|SunPro|HP|Intel|PGI|NVIDIA)$")
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -w")
 endif()
 
@@ -41,7 +41,7 @@ if(BUILD_TESTING)
     adios2_add_thirdparty_target(gtest GTest::GTest)
   else()
     add_subdirectory(GTest)
-    adios2_add_thirdparty_target(gtest gtest_interface)
+    adios2_add_thirdparty_target(gtest gtest)
   endif()
 endif()
 
diff --git a/thirdparty/EVPath/EVPath/cm.c b/thirdparty/EVPath/EVPath/cm.c
index 53f2db52a3..7295cde845 100644
--- a/thirdparty/EVPath/EVPath/cm.c
+++ b/thirdparty/EVPath/EVPath/cm.c
@@ -334,6 +334,15 @@ INT_CMget_contact_list(CManager cm)
     return (cm->contact_lists[0]);
 }
 
+extern attr_list
+INT_CMderef_and_copy_list(CManager cm, attr_list attrs)
+{
+  // done inside the CM lock, so a safe way to convert a shared list to an owned list
+  attr_list ret = attr_copy_list(attrs);
+  free_attr_list(attrs);
+  return ret;
+}
+
 extern attr_list
 INT_CMget_specific_contact_list(CManager cm, attr_list attrs)
 {
@@ -2543,7 +2552,11 @@ timeout_conn(CManager cm, void *client_data)
      local_format = FFS_target_from_encode(conn->cm->FFScontext, data_buffer);
      original_format = FFSTypeHandle_from_encode(conn->cm->FFScontext, data_buffer);
      if (local_format == NULL) {
-	 fprintf(stderr, "invalid format in incoming buffer\n");
+	 if (conn->cm->unregistered_format_handler) {
+	     conn->cm->unregistered_format_handler(conn, name_of_FMformat(FMFormat_of_original(original_format)));
+	 } else {
+	     fprintf(stderr, "No conversion found for incoming CM message\n");
+	 }
 	 return 0;
      }
      CMtrace_out(cm, CMDataVerbose, "CM - Receiving record of type %s, FFSformat %p\n", 
@@ -2610,11 +2623,12 @@ timeout_conn(CManager cm, void *client_data)
 	 }
 	 fprintf(cm->CMTrace_file, "CM - record type %s, contents are:\n  ", name_of_FMformat(FMFormat_of_original(cm_format->format)));
 	 r = FMfdump_data(cm->CMTrace_file, FMFormat_of_original(cm_format->format), decode_buffer, dump_char_limit);
-	 if (r && !warned) {
+	 if (!r && !warned) {
 	     printf("\n\n  ****  Warning **** CM record dump truncated\n");
-	     printf("  To change size limits, set CMDumpSize environment variable.\n\n\n");
+	     printf("  To change size limits, set CMDumpSize environment variable.\n");
 	     warned++;
 	 }
+	 fprintf(cm->CMTrace_file, "\n=======\n");
      }
      if (attrs == NULL) {
 	 attrs = CMcreate_attr_list(cm);
@@ -2707,6 +2721,12 @@ timeout_conn(CManager cm, void *client_data)
      }
  }
 
+extern void
+INT_CMregister_invalid_message_handler(CManager cm, CMUnregCMHandler handler)
+{
+    cm->unregistered_format_handler = handler;
+}
+
  extern int
  INT_CMwrite(CMConnection conn, CMFormat format, void *data)
  {
@@ -3245,11 +3265,12 @@ timeout_conn(CManager cm, void *client_data)
 	 }
 	 fprintf(cm->CMTrace_file, "CM - record type %s, contents are:\n  ", name_of_FMformat(format->fmformat));
 	 r = FMfdump_data(cm->CMTrace_file, format->fmformat, data, dump_char_limit);
-	 if (r && !warned) {
+	 if (!r && !warned) {
 	     fprintf(cm->CMTrace_file, "\n\n  ****  Warning **** CM record dump truncated\n");
-	     fprintf(cm->CMTrace_file, "  To change size limits, set CMDumpSize environment variable.\n\n\n");
+	     fprintf(cm->CMTrace_file, "  To change size limits, set CMDumpSize environment variable.\n");
 	     warned++;
 	 }
+	 fprintf(cm->CMTrace_file, "\n=======\n");
      }
 
      /* encode data with CM context */
@@ -3380,11 +3401,12 @@ timeout_conn(CManager cm, void *client_data)
 	     r = FMfdump_encoded_data(cm->CMTrace_file, format->fmformat,
 				      event->encoded_event, dump_char_limit);
 	 }	    
-	 if (r && !warned) {
+	 if (!r && !warned) {
 	     fprintf(cm->CMTrace_file, "\n\n  ****  Warning **** CM record dump truncated\n");
-	     fprintf(cm->CMTrace_file, "  To change size limits, set CMDumpSize environment variable.\n\n\n");
+	     fprintf(cm->CMTrace_file, "  To change size limits, set CMDumpSize environment variable.\n");
 	     warned++;
 	 }
+	 fprintf(cm->CMTrace_file, "\n=======\n");
      }
 
      if (!event->encoded_event) {
diff --git a/thirdparty/EVPath/EVPath/cm_internal.h b/thirdparty/EVPath/EVPath/cm_internal.h
index 351f400655..b33d37abbe 100644
--- a/thirdparty/EVPath/EVPath/cm_internal.h
+++ b/thirdparty/EVPath/EVPath/cm_internal.h
@@ -136,6 +136,7 @@ typedef struct _CManager {
 
     func_entry *shutdown_functions;
     CMperf_upcall perf_upcall;
+    CMUnregCMHandler unregistered_format_handler;
 
     struct _event_path_data *evp;
     FILE * CMTrace_file;
@@ -415,6 +416,7 @@ void *INT_CMCondition_get_client_data(CManager cm, int condition);
 int INT_CMCondition_wait(CManager cm, int condition);
 extern void INT_CMCondition_fail(CManager cm, int condition);
 extern attr_list INT_CMget_contact_list(CManager cm);
+extern attr_list INT_CMderef_and_copy_list(CManager cm, attr_list attrs);
 extern void INT_CMregister_non_CM_message_handler(int header, CMNonCMHandler handler);
 extern void *INT_CMtake_buffer(CManager cm, void *data);
 extern void INT_CMreturn_buffer(CManager cm, void *data);
@@ -565,3 +567,6 @@ extern void wait_for_pending_write(CMConnection conn);
 extern int
 INT_CMinstall_pull_schedule(CManager cm, struct timeval *base_time, 
 			    struct timeval *period, CMavail_period_ptr avail);
+extern void
+INT_CMregister_invalid_message_handler(CManager cm, CMUnregCMHandler handler);
+
diff --git a/thirdparty/EVPath/EVPath/cmsockets.c b/thirdparty/EVPath/EVPath/cmsockets.c
index 3dfbe8b0ce..d32a99b2f1 100644
--- a/thirdparty/EVPath/EVPath/cmsockets.c
+++ b/thirdparty/EVPath/EVPath/cmsockets.c
@@ -85,8 +85,9 @@ typedef struct func_list_item {
 typedef struct socket_client_data {
     CManager cm;
     char *hostname;
-    int listen_port;
-    int conn_sock;
+    int listen_count;
+    int *listen_fds;
+    int *listen_ports;
     attr_list characteristics;
     CMtrans_services svc;
 } *socket_client_data_ptr;
@@ -482,7 +483,10 @@ attr_list conn_attr_list;
 #endif
 
     {
-	int local_listen_port = htons(sd->listen_port);
+	int local_listen_port = 0;
+	if (sd->listen_count) {
+	    local_listen_port = htons(sd->listen_ports[0]);
+	}
 	if (write(sock, &local_listen_port, 4) != 4) {
 	    svc->trace_out(cm, "Write failed\n");
 	    return -1;
@@ -601,8 +605,14 @@ attr_list attrs;
 	svc->trace_out(cm, "CMself check - Host IP addrs don't match, %lx, %lx", IP, host_addr);
 	return 0;
     }
-    if (int_port_num != sd->listen_port) {
-	svc->trace_out(cm, "CMself check - Ports don't match, %d, %d", int_port_num, sd->listen_port);
+    int port_match = 0;
+    for (int i = 0; i < sd->listen_count; i++) {
+	if (int_port_num == sd->listen_ports[i]) {
+	    port_match = sd->listen_ports[i];
+	}
+    }
+    if (!port_match) {
+	svc->trace_out(cm, "CMself check - Ports don't match, %d, %d", int_port_num, port_match);
 	return 0;
     }
     svc->trace_out(cm, "CMself check returning TRUE");
@@ -668,7 +678,7 @@ attr_list listen_info;
     unsigned int length;
     struct sockaddr_in sock_addr;
     int sock_opt_val = 1;
-    int conn_sock;
+    int conn_sock = 0;
     int attr_port_num = 0;
     u_short port_num = 0;
     int port_range_low, port_range_high;
@@ -676,11 +686,6 @@ attr_list listen_info;
     int IP;
     char host_name[256];
 
-    conn_sock = socket(AF_INET, SOCK_STREAM, 0);
-    if (conn_sock == SOCKET_ERROR) {
-	fprintf(stderr, "Cannot open INET socket\n");
-	return NULL;
-    }
     if (sd->cm) {
 	/* assert CM is locked */
 	assert(CM_LOCKED(svc, sd->cm));
@@ -707,87 +712,114 @@ attr_list listen_info;
     sock_addr.sin_family = AF_INET;
     sock_addr.sin_addr.s_addr = INADDR_ANY;
     sock_addr.sin_port = htons(port_num);
-    if (sock_addr.sin_port != 0) {
-	/* specific port requested. set REUSEADDR, REUSEPORT because previous server might have died badly */
-	if (setsockopt(conn_sock, SOL_SOCKET, SO_REUSEADDR, (char *) &sock_opt_val,
-		       sizeof(sock_opt_val)) != 0) {
-	    fprintf(stderr, "Failed to set REUSEADDR on INET socket before bind\n");
-	    perror("setsockopt(SO_REUSEADDR) failed");
-	    return NULL;
+
+    for (int i = 0; i < sd->listen_count; i++) {
+	if ((sd->listen_ports[i] == port_num) ||
+	    ((sd->listen_ports[i] >= port_range_low) &&
+	     (sd->listen_ports[i] <= port_range_high))) {
+	    conn_sock = sd->listen_fds[i];
 	}
-#ifdef SO_REUSEPORT
-	sock_opt_val = 1;
-	if (setsockopt(conn_sock, SOL_SOCKET, SO_REUSEPORT, (const char*)&sock_opt_val, sizeof(sock_opt_val)) != 0) {
-	    fprintf(stderr, "Failed to set REUSEADDR on INET socket before bind\n");
-	    perror("setsockopt(SO_REUSEPORT) failed");
+    }
+    if (!conn_sock) {
+	conn_sock = socket(AF_INET, SOCK_STREAM, 0);
+	if (conn_sock == SOCKET_ERROR) {
+	    fprintf(stderr, "Cannot open INET socket\n");
 	    return NULL;
 	}
+	if (sock_addr.sin_port != 0) {
+	    /* specific port requested. set REUSEADDR, REUSEPORT because previous server might have died badly */
+	    if (setsockopt(conn_sock, SOL_SOCKET, SO_REUSEADDR, (char *) &sock_opt_val,
+			   sizeof(sock_opt_val)) != 0) {
+		fprintf(stderr, "Failed to set REUSEADDR on INET socket before bind\n");
+		perror("setsockopt(SO_REUSEADDR) failed");
+		return NULL;
+	    }
+#ifdef SO_REUSEPORT
+	    sock_opt_val = 1;
+	    if (setsockopt(conn_sock, SOL_SOCKET, SO_REUSEPORT, (const char*)&sock_opt_val, sizeof(sock_opt_val)) != 0) {
+		fprintf(stderr, "Failed to set REUSEADDR on INET socket before bind\n");
+		perror("setsockopt(SO_REUSEPORT) failed");
+		return NULL;
+	    }
 #endif
-	svc->trace_out(cm, "CMSocket trying to bind selected port %d", port_num);
-	if (bind(conn_sock, (struct sockaddr *) &sock_addr,
-		 sizeof sock_addr) == SOCKET_ERROR) {
-	    fprintf(stderr, "Cannot bind INET socket\n");
+	    svc->trace_out(cm, "CMSocket trying to bind selected port %d", port_num);
+	    if (bind(conn_sock, (struct sockaddr *) &sock_addr,
+		     sizeof sock_addr) == SOCKET_ERROR) {
+		fprintf(stderr, "Cannot bind INET socket\n");
+		return NULL;
+	    }
+	} else if (port_range_high == -1) {
+	    /* bind to any port, range unconstrained */
+	    sock_addr.sin_port = 0;
+	    svc->trace_out(cm, "CMSocket trying to bind to any available port");
+	    if (bind(conn_sock, (struct sockaddr *) &sock_addr,
+		     sizeof sock_addr) == SOCKET_ERROR) {
+		fprintf(stderr, "Cannot bind INET socket\n");
+		return NULL;
+	    }
+	} else {
+	    long seedval = time(NULL) + getpid();
+	    /* port num is free.  Constrain to range to standards */
+	    int size = port_range_high - port_range_low;
+	    int tries = 30;
+	    int result = SOCKET_ERROR;
+	    srand48(seedval);
+	    while (tries > 0) {
+		int target = port_range_low + size * drand48();
+		sock_addr.sin_port = htons(target);
+		svc->trace_out(cm, "CMSocket trying to bind port %d", target);
+		result = bind(conn_sock, (struct sockaddr *) &sock_addr,
+			      sizeof sock_addr);
+		tries--;
+		if (result != SOCKET_ERROR) tries = 0;
+		if (tries%5 == 4) {
+		    /* try reseeding in case we're in sync with another process */
+		    srand48(time(NULL) + getpid());
+		}
+		if (tries == 20) {
+		    /* damn, tried a lot, increase the range (This might violate specified range) */
+		    size *= 10;
+		}
+		if (tries == 10) {
+		    /* damn, tried a lot more, increase the range (This might violate specified range) */
+		    size *= 10;
+		}
+	    }
+	    if (result == SOCKET_ERROR) {
+		fprintf(stderr, "Cannot bind INET socket\n");
+		return NULL;
+	    }
+	}
+	/* begin listening for conns and set the backlog */
+	if (listen(conn_sock, FD_SETSIZE)) {
+	    fprintf(stderr, "listen failed\n");
 	    return NULL;
 	}
-    } else if (port_range_high == -1) {
-	/* bind to any port, range unconstrained */
-	sock_addr.sin_port = 0;
-	svc->trace_out(cm, "CMSocket trying to bind to any available port");
-	if (bind(conn_sock, (struct sockaddr *) &sock_addr,
-		 sizeof sock_addr) == SOCKET_ERROR) {
-	    fprintf(stderr, "Cannot bind INET socket\n");
+	svc->trace_out(cm, "CMSockets Adding socket_accept_conn as action on fd %d", conn_sock);
+	svc->fd_add_select(cm, conn_sock, socket_accept_conn,
+			   (void *) trans, (void *) (long)conn_sock);
+
+	length = sizeof(sock_addr);
+	if (getsockname(conn_sock, (struct sockaddr *) &sock_addr, &length) < 0) {
+	    fprintf(stderr, "Cannot get socket name\n");
 	    return NULL;
 	}
+	sd->listen_fds = realloc(sd->listen_fds,
+				 sizeof(int)*(sd->listen_count+1));
+	sd->listen_ports = realloc(sd->listen_ports,
+				 sizeof(int)*(sd->listen_count+1));
+	sd->listen_fds[sd->listen_count] = conn_sock;
+	sd->listen_ports[sd->listen_count] = ntohs(sock_addr.sin_port);
+	sd->listen_count++;
     } else {
-	long seedval = time(NULL) + getpid();
-	/* port num is free.  Constrain to range to standards */
-	int size = port_range_high - port_range_low;
-	int tries = 30;
-	int result = SOCKET_ERROR;
-	srand48(seedval);
-	while (tries > 0) {
-	    int target = port_range_low + size * drand48();
-	    sock_addr.sin_port = htons(target);
-	    svc->trace_out(cm, "CMSocket trying to bind port %d", target);
-	    result = bind(conn_sock, (struct sockaddr *) &sock_addr,
-			  sizeof sock_addr);
-	    tries--;
-	    if (result != SOCKET_ERROR) tries = 0;
-	    if (tries%5 == 4) {
-		/* try reseeding in case we're in sync with another process */
-		srand48(time(NULL) + getpid());
-	    }
-	    if (tries == 20) {
-		/* damn, tried a lot, increase the range (This might violate specified range) */
-		size *= 10;
-	    }
-	    if (tries == 10) {
-		/* damn, tried a lot more, increase the range (This might violate specified range) */
-		size *= 10;
-	    }
-	}
-	if (result == SOCKET_ERROR) {
-	    fprintf(stderr, "Cannot bind INET socket\n");
+	length = sizeof(sock_addr);
+	if (getsockname(conn_sock, (struct sockaddr *) &sock_addr, &length) < 0) {
+	    fprintf(stderr, "Cannot get socket name\n");
 	    return NULL;
 	}
-    }
-    length = sizeof sock_addr;
-    if (getsockname(conn_sock, (struct sockaddr *) &sock_addr, &length) < 0) {
-	fprintf(stderr, "Cannot get socket name\n");
-	return NULL;
-    }
-    /* begin listening for conns and set the backlog */
-    if (listen(conn_sock, FD_SETSIZE)) {
-	fprintf(stderr, "listen failed\n");
-	return NULL;
+	svc->trace_out(cm, "CMSockets reusing prior listen, fd %d, port %d\n", conn_sock, ntohs(sock_addr.sin_port));
     }
     /* set the port num as one we can be contacted at */
-
-    svc->trace_out(cm, "CMSockets Adding socket_accept_conn as action on fd %d", conn_sock);
-    svc->fd_add_select(cm, conn_sock, socket_accept_conn,
-		       (void *) trans, (void *) (long)conn_sock);
-
-    sd->conn_sock = conn_sock;
     {
 	int int_port_num = ntohs(sock_addr.sin_port);
 	attr_list ret_list;
@@ -799,7 +831,6 @@ attr_list listen_info;
 	if (sd->hostname != NULL)
 	    svc->free_func(sd->hostname);
 	sd->hostname = strdup(host_name);
-	sd->listen_port = int_port_num;
 	if ((IP != 0) && (!use_hostname)) {
 	    add_attr(ret_list, CM_IP_ADDR, Attr_Int4,
 		     (attr_value) (long)IP);
@@ -1141,7 +1172,11 @@ free_socket_data(CManager cm, void *sdv)
     if (sd->hostname != NULL)
 	svc->free_func(sd->hostname);
     free_attr_list(sd->characteristics);
-    close(sd->conn_sock);
+    for(int i = 0 ; i < sd->listen_count; i++) {
+	close(sd->listen_fds[i]);
+    }
+    svc->free_func(sd->listen_fds);
+    svc->free_func(sd->listen_ports);
     svc->free_func(sd);
 }
 
@@ -1189,9 +1224,11 @@ libcmsockets_LTX_initialize(CManager cm, CMtrans_services svc, transport_entry t
     socket_data = svc->malloc_func(sizeof(struct socket_client_data));
     socket_data->cm = cm;
     socket_data->hostname = NULL;
-    socket_data->listen_port = -1;
     socket_data->svc = svc;
     socket_data->characteristics = create_attr_list();
+    socket_data->listen_count = 0;
+    socket_data->listen_fds = malloc(sizeof(int));
+    socket_data->listen_ports = malloc(sizeof(int));
     add_int_attr(socket_data->characteristics, CM_TRANSPORT_RELIABLE, 1);
     svc->add_shutdown_task(cm, free_socket_data, (void *) socket_data, FREE_TASK);
     return (void *) socket_data;
diff --git a/thirdparty/EVPath/EVPath/evpath.h b/thirdparty/EVPath/EVPath/evpath.h
index dbd121b3ce..e9151e1e14 100644
--- a/thirdparty/EVPath/EVPath/evpath.h
+++ b/thirdparty/EVPath/EVPath/evpath.h
@@ -256,6 +256,15 @@ CM_insert_contact_info (CManager cm, attr_list attrs);
 extern attr_list
 CMget_specific_contact_list (CManager cm, attr_list attrs);
 
+/*!
+ * get a thread-owned version of a shared attribute list in a thread-safe way
+ * \param cm  the CManager which owns the attribute list.
+ * \param attrs the shared attribute list (from CMget_contact list, etc.)
+ * \return a single-owner contact list
+ */
+extern attr_list
+CMderef_and_copy_list(CManager cm, attr_list attrs);
+
 /*!
  * check to see if this is contact information for this CM.
  *
@@ -605,6 +614,16 @@ typedef int (*CMNonCMHandler) (CMConnection conn, CMTransport transport,
 extern void
 CMregister_non_CM_message_handler (int header, CMNonCMHandler handler);
 
+/*!
+ * register a handler for CM messages that don't match registered handlers.
+ *
+ */
+typedef void (*CMUnregCMHandler) (CMConnection conn, char *format_name);
+
+/*NOLOCK*/
+extern void
+CMregister_invalid_message_handler (CManager cm, CMUnregCMHandler handler);
+
 /*!
  * return the pointer to the static transport services structure.
  *
diff --git a/thirdparty/GTest/CMakeLists.txt b/thirdparty/GTest/CMakeLists.txt
index 879e0c32ac..e438cbf416 100644
--- a/thirdparty/GTest/CMakeLists.txt
+++ b/thirdparty/GTest/CMakeLists.txt
@@ -7,16 +7,6 @@ if(BUILD_SHARED_LIBS)
 endif()
 set(BUILD_SHARED_LIBS OFF)
 
-set(CMAKE_POLICY_DEFAULT_CMP0042 NEW)
-
 add_subdirectory(googletest EXCLUDE_FROM_ALL)
 
-add_library(gtest_interface INTERFACE)
-target_link_libraries(gtest_interface INTERFACE gtest)
-if(BUILD_SHARED_LIBS)
-  target_compile_definitions(gtest_interface INTERFACE
-    GTEST_LINKED_AS_SHARED_LIBRARY=1
-  )
-endif()
-
 message_end_thirdparty()
diff --git a/thirdparty/GTest/googletest/.clang-format b/thirdparty/GTest/googletest/.clang-format
new file mode 100644
index 0000000000..5b9bfe6d22
--- /dev/null
+++ b/thirdparty/GTest/googletest/.clang-format
@@ -0,0 +1,4 @@
+# Run manually to reformat a file:
+# clang-format -i --style=file 
+Language:        Cpp
+BasedOnStyle:  Google
diff --git a/thirdparty/GTest/googletest/.gitignore b/thirdparty/GTest/googletest/.gitignore
index b294d3b602..f08cb72a33 100644
--- a/thirdparty/GTest/googletest/.gitignore
+++ b/thirdparty/GTest/googletest/.gitignore
@@ -12,6 +12,7 @@ bazel-testlogs
 *.pyc
 
 # Visual Studio files
+.vs
 *.sdf
 *.opensdf
 *.VC.opendb
@@ -34,7 +35,50 @@ googletest/m4/ltoptions.m4
 googletest/m4/ltsugar.m4
 googletest/m4/ltversion.m4
 googletest/m4/lt~obsolete.m4
+googlemock/m4
 
 # Ignore generated directories.
 googlemock/fused-src/
 googletest/fused-src/
+
+# macOS files
+.DS_Store
+googletest/.DS_Store
+googletest/xcode/.DS_Store
+
+# Ignore cmake generated directories and files.
+CMakeFiles
+CTestTestfile.cmake
+Makefile
+cmake_install.cmake
+googlemock/CMakeFiles
+googlemock/CTestTestfile.cmake
+googlemock/Makefile
+googlemock/cmake_install.cmake
+googlemock/gtest
+/bin
+/googlemock/gmock.dir
+/googlemock/gmock_main.dir
+/googlemock/RUN_TESTS.vcxproj.filters
+/googlemock/RUN_TESTS.vcxproj
+/googlemock/INSTALL.vcxproj.filters
+/googlemock/INSTALL.vcxproj
+/googlemock/gmock_main.vcxproj.filters
+/googlemock/gmock_main.vcxproj
+/googlemock/gmock.vcxproj.filters
+/googlemock/gmock.vcxproj
+/googlemock/gmock.sln
+/googlemock/ALL_BUILD.vcxproj.filters
+/googlemock/ALL_BUILD.vcxproj
+/lib
+/Win32
+/ZERO_CHECK.vcxproj.filters
+/ZERO_CHECK.vcxproj
+/RUN_TESTS.vcxproj.filters
+/RUN_TESTS.vcxproj
+/INSTALL.vcxproj.filters
+/INSTALL.vcxproj
+/googletest-distribution.sln
+/CMakeCache.txt
+/ALL_BUILD.vcxproj.filters
+/ALL_BUILD.vcxproj
diff --git a/thirdparty/GTest/googletest/.travis.yml b/thirdparty/GTest/googletest/.travis.yml
index 2fbb3b169e..d7b23b9499 100644
--- a/thirdparty/GTest/googletest/.travis.yml
+++ b/thirdparty/GTest/googletest/.travis.yml
@@ -1,9 +1,8 @@
 # Build matrix / environment variable are explained on:
-# http://about.travis-ci.org/docs/user/build-configuration/
+# https://docs.travis-ci.com/user/customizing-the-build/
 # This file can be validated on:
 # http://lint.travis-ci.org/
 
-sudo: false
 language: cpp
 
 # Define the matrix explicitly, manually expanding the combinations of (os, compiler, env).
@@ -11,48 +10,33 @@ language: cpp
 matrix:
   include:
     - os: linux
+      before_install: chmod -R +x ./ci/*platformio.sh
+      install: ./ci/install-platformio.sh
+      script: ./ci/build-platformio.sh
+    - os: linux
+      dist: bionic
       compiler: gcc
-      sudo : true
       install: ./ci/install-linux.sh && ./ci/log-config.sh
       script: ./ci/build-linux-bazel.sh
     - os: linux
+      dist: bionic
       compiler: clang
-      sudo : true
       install: ./ci/install-linux.sh && ./ci/log-config.sh
       script: ./ci/build-linux-bazel.sh
     - os: linux
-      group: deprecated-2017Q4
-      compiler: gcc
-      install: ./ci/install-linux.sh && ./ci/log-config.sh
-      script: ./ci/build-linux-autotools.sh
-    - os: linux
-      group: deprecated-2017Q4
+      dist: bionic
       compiler: gcc
-      env: BUILD_TYPE=Debug VERBOSE=1 CXX_FLAGS=-std=c++11
-    - os: linux
-      group: deprecated-2017Q4
-      compiler: clang
-      env: BUILD_TYPE=Debug VERBOSE=1
+      env: BUILD_TYPE=Debug VERBOSE=1 CXX_FLAGS="-std=c++11 -Wdeprecated"
     - os: linux
-      group: deprecated-2017Q4
+      dist: bionic
       compiler: clang
-      env: BUILD_TYPE=Release VERBOSE=1 CXX_FLAGS=-std=c++11
+      env: BUILD_TYPE=Release VERBOSE=1 CXX_FLAGS="-std=c++11 -Wdeprecated" NO_EXCEPTION=ON NO_RTTI=ON COMPILER_IS_GNUCXX=ON
     - os: osx
       compiler: gcc
-      env: BUILD_TYPE=Debug VERBOSE=1
-      if: type != pull_request
-    - os: osx
-      compiler: gcc
-      env: BUILD_TYPE=Release VERBOSE=1 CXX_FLAGS=-std=c++11
-      if: type != pull_request
+      env: BUILD_TYPE=Release VERBOSE=1 CXX_FLAGS="-std=c++11 -Wdeprecated" HOMEBREW_LOGS=~/homebrew-logs HOMEBREW_TEMP=~/homebrew-temp
     - os: osx
       compiler: clang
-      env: BUILD_TYPE=Debug VERBOSE=1
-      if: type != pull_request
-    - os: osx
-      compiler: clang
-      env: BUILD_TYPE=Release VERBOSE=1 CXX_FLAGS=-std=c++11
-      if: type != pull_request
+      env: BUILD_TYPE=Release VERBOSE=1 CXX_FLAGS="-std=c++11 -Wdeprecated" HOMEBREW_LOGS=~/homebrew-logs HOMEBREW_TEMP=~/homebrew-temp
 
 # These are the install and build (script) phases for the most common entries in the matrix.  They could be included
 # in each entry in the matrix, but that is just repetitive.
@@ -63,19 +47,19 @@ install:
 
 script: ./ci/travis.sh
 
-# For sudo=false builds this section installs the necessary dependencies.
+# This section installs the necessary dependencies.
 addons:
   apt:
-    # List of whitelisted in travis packages for ubuntu-precise can be found here:
-    #   https://github.com/travis-ci/apt-package-whitelist/blob/master/ubuntu-precise
-    # List of whitelisted in travis apt-sources:
-    #   https://github.com/travis-ci/apt-source-whitelist/blob/master/ubuntu.json
-    sources:
-    - ubuntu-toolchain-r-test
-    - llvm-toolchain-precise-3.7
     packages:
-    - g++-4.9
-    - clang-3.7
+    - g++
+    - clang
+    update: true
+  homebrew:
+    packages:
+    - ccache
+    - gcc@4.9
+    - llvm@4
+    update: true
 
 notifications:
   email: false
diff --git a/thirdparty/GTest/googletest/BUILD.bazel b/thirdparty/GTest/googletest/BUILD.bazel
index 6d828294c0..8099642a85 100644
--- a/thirdparty/GTest/googletest/BUILD.bazel
+++ b/thirdparty/GTest/googletest/BUILD.bazel
@@ -28,22 +28,19 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #
-# Author: misterg@google.com (Gennadiy Civil)
-#
 #   Bazel Build for Google C++ Testing Framework(Google Test)
 
+load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test")
+
 package(default_visibility = ["//visibility:public"])
 
 licenses(["notice"])
 
-config_setting(
-    name = "windows",
-    values = { "cpu": "x64_windows" },
-)
+exports_files(["LICENSE"])
 
 config_setting(
-    name = "windows_msvc",
-    values = {"cpu": "x64_windows_msvc"},
+    name = "windows",
+    constraint_values = ["@bazel_tools//platforms:windows"],
 )
 
 config_setting(
@@ -51,6 +48,12 @@ config_setting(
     values = {"define": "absl=1"},
 )
 
+# Library that defines the FRIEND_TEST macro.
+cc_library(
+    name = "gtest_prod",
+    hdrs = ["googletest/include/gtest/gtest_prod.h"],
+    includes = ["googletest/include"],
+)
 
 # Google Test including Google Mock
 cc_library(
@@ -70,17 +73,22 @@ cc_library(
             "googlemock/src/gmock_main.cc",
         ],
     ),
-    hdrs =glob([
+    hdrs = glob([
         "googletest/include/gtest/*.h",
         "googlemock/include/gmock/*.h",
     ]),
-    copts = select(
-        {
-            ":windows": [],
-            ":windows_msvc": [],
-            "//conditions:default": ["-pthread"],
-        },
-    ),
+    copts = select({
+        ":windows": [],
+        "//conditions:default": ["-pthread"],
+    }),
+    defines = select({
+        ":has_absl": ["GTEST_HAS_ABSL=1"],
+        "//conditions:default": [],
+    }),
+    features = select({
+        ":windows": ["windows_export_all_symbols"],
+        "//conditions:default": [],
+    }),
     includes = [
         "googlemock",
         "googlemock/include",
@@ -89,33 +97,29 @@ cc_library(
     ],
     linkopts = select({
         ":windows": [],
-        ":windows_msvc": [],
-        "//conditions:default": [
-            "-pthread",
-        ],
+        "//conditions:default": ["-pthread"],
     }),
-    defines = select ({
-        ":has_absl": [
-        "GTEST_HAS_ABSL=1",
-        ],
-        "//conditions:default": [],
-    }
-    ),
-    deps = select ({
+    deps = select({
         ":has_absl": [
-        "@com_google_absl//absl/types:optional",
-        "@com_google_absl//absl/strings"
+            "@com_google_absl//absl/debugging:failure_signal_handler",
+            "@com_google_absl//absl/debugging:stacktrace",
+            "@com_google_absl//absl/debugging:symbolize",
+            "@com_google_absl//absl/strings",
+            "@com_google_absl//absl/types:any",
+            "@com_google_absl//absl/types:optional",
+            "@com_google_absl//absl/types:variant",
         ],
         "//conditions:default": [],
-    }
-    )
+    }),
 )
 
 cc_library(
     name = "gtest_main",
-    srcs = [
-        "googlemock/src/gmock_main.cc",
-    ],
+    srcs = ["googlemock/src/gmock_main.cc"],
+    features = select({
+        ":windows": ["windows_export_all_symbols"],
+        "//conditions:default": [],
+    }),
     deps = [":gtest"],
 )
 
@@ -134,14 +138,18 @@ cc_library(
         "googletest/samples/sample3-inl.h",
         "googletest/samples/sample4.h",
     ],
+    features = select({
+        ":windows": ["windows_export_all_symbols"],
+        "//conditions:default": [],
+    }),
 )
 
 cc_test(
     name = "gtest_samples",
     size = "small",
-    #All Samples except:
-    #sample9 ( main )
-    #sample10 (main and takes a command line option and needs to be separate)
+    # All Samples except:
+    #   sample9 (main)
+    #   sample10 (main and takes a command line option and needs to be separate)
     srcs = [
         "googletest/samples/sample1_unittest.cc",
         "googletest/samples/sample2_unittest.cc",
@@ -152,6 +160,7 @@ cc_test(
         "googletest/samples/sample7_unittest.cc",
         "googletest/samples/sample8_unittest.cc",
     ],
+    linkstatic = 0,
     deps = [
         "gtest_sample_lib",
         ":gtest_main",
@@ -169,7 +178,5 @@ cc_test(
     name = "sample10_unittest",
     size = "small",
     srcs = ["googletest/samples/sample10_unittest.cc"],
-    deps = [
-        ":gtest",
-    ],
+    deps = [":gtest"],
 )
diff --git a/thirdparty/GTest/googletest/CMakeLists.txt b/thirdparty/GTest/googletest/CMakeLists.txt
index f8a97faaeb..e516b4b7a6 100644
--- a/thirdparty/GTest/googletest/CMakeLists.txt
+++ b/thirdparty/GTest/googletest/CMakeLists.txt
@@ -1,33 +1,32 @@
-cmake_minimum_required(VERSION 2.6.4)
+# Note: CMake support is community-based. The maintainers do not use CMake
+# internally.
+
+cmake_minimum_required(VERSION 2.8.8)
 
 if (POLICY CMP0048)
   cmake_policy(SET CMP0048 NEW)
 endif (POLICY CMP0048)
 
-project( googletest-distribution )
+project(googletest-distribution)
+set(GOOGLETEST_VERSION 1.10.0)
+
+if (CMAKE_VERSION VERSION_GREATER "3.0.2")
+  if(NOT CYGWIN AND NOT MSYS AND NOT ${CMAKE_SYSTEM_NAME} STREQUAL QNX)
+    set(CMAKE_CXX_EXTENSIONS OFF)
+  endif()
+endif()
 
 enable_testing()
 
 include(CMakeDependentOption)
-if (CMAKE_VERSION VERSION_LESS 2.8.5)
-  set(CMAKE_INSTALL_BINDIR "bin" CACHE STRING "User executables (bin)")
-  set(CMAKE_INSTALL_LIBDIR "lib${LIB_SUFFIX}" CACHE STRING "Object code libraries (lib)")
-  set(CMAKE_INSTALL_INCLUDEDIR "include" CACHE STRING "C header files (include)")
-  mark_as_advanced(CMAKE_INSTALL_BINDIR CMAKE_INSTALL_LIBDIR CMAKE_INSTALL_INCLUDEDIR)
-else()
-  include(GNUInstallDirs)
-endif()
-
-option(BUILD_GTEST "Builds the googletest subproject" OFF)
+include(GNUInstallDirs)
 
 #Note that googlemock target already builds googletest
 option(BUILD_GMOCK "Builds the googlemock subproject" ON)
-
-cmake_dependent_option(INSTALL_GTEST "Enable installation of googletest. (Projects embedding googletest may want to turn this OFF.)" ON "BUILD_GTEST OR BUILD_GMOCK" OFF)
-cmake_dependent_option(INSTALL_GMOCK "Enable installation of googlemock. (Projects embedding googlemock may want to turn this OFF.)" ON "BUILD_GMOCK" OFF)
+option(INSTALL_GTEST "Enable installation of googletest. (Projects embedding googletest may want to turn this OFF.)" ON)
 
 if(BUILD_GMOCK)
   add_subdirectory( googlemock )
-elseif(BUILD_GTEST)
+else()
   add_subdirectory( googletest )
 endif()
diff --git a/thirdparty/GTest/googletest/CONTRIBUTING.md b/thirdparty/GTest/googletest/CONTRIBUTING.md
index 0ebdfcc6f0..fe4790d7f6 100644
--- a/thirdparty/GTest/googletest/CONTRIBUTING.md
+++ b/thirdparty/GTest/googletest/CONTRIBUTING.md
@@ -2,16 +2,16 @@
 
 ## Contributor License Agreements
 
-We'd love to accept your patches! Before we can take them, we
-have to jump a couple of legal hurdles.
+We'd love to accept your patches! Before we can take them, we have to jump a
+couple of legal hurdles.
 
 Please fill out either the individual or corporate Contributor License Agreement
 (CLA).
 
-  * If you are an individual writing original source code and you're sure you
+*   If you are an individual writing original source code and you're sure you
     own the intellectual property, then you'll need to sign an
     [individual CLA](https://developers.google.com/open-source/cla/individual).
-  * If you work for a company that wants to allow you to contribute your work,
+*   If you work for a company that wants to allow you to contribute your work,
     then you'll need to sign a
     [corporate CLA](https://developers.google.com/open-source/cla/corporate).
 
@@ -19,142 +19,124 @@ Follow either of the two links above to access the appropriate CLA and
 instructions for how to sign and return it. Once we receive it, we'll be able to
 accept your pull requests.
 
+## Are you a Googler?
+
+If you are a Googler, please make an attempt to submit an internal change rather
+than a GitHub Pull Request. If you are not able to submit an internal change a
+PR is acceptable as an alternative.
+
 ## Contributing A Patch
 
-1. Submit an issue describing your proposed change to the
-   [issue tracker](https://github.com/google/googletest).
-1. Please don't mix more than one logical change per submittal,
-   because it makes the history hard to follow. If you want to make a
-   change that doesn't have a corresponding issue in the issue
-   tracker, please create one.
-1. Also, coordinate with team members that are listed on the issue in
-   question. This ensures that work isn't being duplicated and
-   communicating your plan early also generally leads to better
-   patches.
-1. If your proposed change is accepted, and you haven't already done so, sign a
-   Contributor License Agreement (see details above).
-1. Fork the desired repo, develop and test your code changes.
-1. Ensure that your code adheres to the existing style in the sample to which
-   you are contributing.
-1. Ensure that your code has an appropriate set of unit tests which all pass.
-1. Submit a pull request.
-
-If you are a Googler, it is preferable to first create an internal change and
-have it reviewed and submitted, and then create an upstreaming pull
-request here. 
-
-## The Google Test and Google Mock Communities ##
+1.  Submit an issue describing your proposed change to the
+    [issue tracker](https://github.com/google/googletest/issues).
+2.  Please don't mix more than one logical change per submittal, because it
+    makes the history hard to follow. If you want to make a change that doesn't
+    have a corresponding issue in the issue tracker, please create one.
+3.  Also, coordinate with team members that are listed on the issue in question.
+    This ensures that work isn't being duplicated and communicating your plan
+    early also generally leads to better patches.
+4.  If your proposed change is accepted, and you haven't already done so, sign a
+    Contributor License Agreement (see details above).
+5.  Fork the desired repo, develop and test your code changes.
+6.  Ensure that your code adheres to the existing style in the sample to which
+    you are contributing.
+7.  Ensure that your code has an appropriate set of unit tests which all pass.
+8.  Submit a pull request.
+
+## The Google Test and Google Mock Communities
 
 The Google Test community exists primarily through the
-[discussion group](http://groups.google.com/group/googletestframework)
-and the GitHub repository.
-Likewise, the Google Mock community exists primarily through their own
-[discussion group](http://groups.google.com/group/googlemock).
-You are definitely encouraged to contribute to the
-discussion and you can also help us to keep the effectiveness of the
-group high by following and promoting the guidelines listed here.
-
-### Please Be Friendly ###
-
-Showing courtesy and respect to others is a vital part of the Google
-culture, and we strongly encourage everyone participating in Google
-Test development to join us in accepting nothing less. Of course,
-being courteous is not the same as failing to constructively disagree
-with each other, but it does mean that we should be respectful of each
-other when enumerating the 42 technical reasons that a particular
-proposal may not be the best choice. There's never a reason to be
-antagonistic or dismissive toward anyone who is sincerely trying to
+[discussion group](http://groups.google.com/group/googletestframework) and the
+GitHub repository. Likewise, the Google Mock community exists primarily through
+their own [discussion group](http://groups.google.com/group/googlemock). You are
+definitely encouraged to contribute to the discussion and you can also help us
+to keep the effectiveness of the group high by following and promoting the
+guidelines listed here.
+
+### Please Be Friendly
+
+Showing courtesy and respect to others is a vital part of the Google culture,
+and we strongly encourage everyone participating in Google Test development to
+join us in accepting nothing less. Of course, being courteous is not the same as
+failing to constructively disagree with each other, but it does mean that we
+should be respectful of each other when enumerating the 42 technical reasons
+that a particular proposal may not be the best choice. There's never a reason to
+be antagonistic or dismissive toward anyone who is sincerely trying to
 contribute to a discussion.
 
-Sure, C++ testing is serious business and all that, but it's also
-a lot of fun. Let's keep it that way. Let's strive to be one of the
-friendliest communities in all of open source.
+Sure, C++ testing is serious business and all that, but it's also a lot of fun.
+Let's keep it that way. Let's strive to be one of the friendliest communities in
+all of open source.
 
-As always, discuss Google Test in the official GoogleTest discussion group.
-You don't have to actually submit code in order to sign up. Your participation
+As always, discuss Google Test in the official GoogleTest discussion group. You
+don't have to actually submit code in order to sign up. Your participation
 itself is a valuable contribution.
 
 ## Style
 
-To keep the source consistent, readable, diffable and easy to merge,
-we use a fairly rigid coding style, as defined by the [google-styleguide](https://github.com/google/styleguide) project.  All patches will be expected
-to conform to the style outlined [here](https://google.github.io/styleguide/cppguide.html).
+To keep the source consistent, readable, diffable and easy to merge, we use a
+fairly rigid coding style, as defined by the
+[google-styleguide](https://github.com/google/styleguide) project. All patches
+will be expected to conform to the style outlined
+[here](https://google.github.io/styleguide/cppguide.html). Use
+[.clang-format](https://github.com/google/googletest/blob/master/.clang-format)
+to check your formatting.
 
-## Requirements for Contributors ###
+## Requirements for Contributors
 
-If you plan to contribute a patch, you need to build Google Test,
-Google Mock, and their own tests from a git checkout, which has
-further requirements:
+If you plan to contribute a patch, you need to build Google Test, Google Mock,
+and their own tests from a git checkout, which has further requirements:
 
-  * [Python](https://www.python.org/) v2.3 or newer (for running some of
-    the tests and re-generating certain source files from templates)
-  * [CMake](https://cmake.org/) v2.6.4 or newer
-  * [GNU Build System](https://en.wikipedia.org/wiki/GNU_Build_System)
-    including automake (>= 1.9), autoconf (>= 2.59), and
-    libtool / libtoolize.
+*   [Python](https://www.python.org/) v2.3 or newer (for running some of the
+    tests and re-generating certain source files from templates)
+*   [CMake](https://cmake.org/) v2.6.4 or newer
 
-## Developing Google Test ##
+## Developing Google Test and Google Mock
 
-This section discusses how to make your own changes to Google Test.
+This section discusses how to make your own changes to the Google Test project.
 
-### Testing Google Test Itself ###
+### Testing Google Test and Google Mock Themselves
 
 To make sure your changes work as intended and don't break existing
-functionality, you'll want to compile and run Google Test's own tests.
-For that you can use CMake:
+functionality, you'll want to compile and run Google Test and GoogleMock's own
+tests. For that you can use CMake:
 
     mkdir mybuild
     cd mybuild
-    cmake -Dgtest_build_tests=ON ${GTEST_DIR}
-
-Make sure you have Python installed, as some of Google Test's tests
-are written in Python.  If the cmake command complains about not being
-able to find Python (`Could NOT find PythonInterp (missing:
-PYTHON_EXECUTABLE)`), try telling it explicitly where your Python
-executable can be found:
+    cmake -Dgtest_build_tests=ON -Dgmock_build_tests=ON ${GTEST_REPO_DIR}
 
-    cmake -DPYTHON_EXECUTABLE=path/to/python -Dgtest_build_tests=ON ${GTEST_DIR}
+To choose between building only Google Test or Google Mock, you may modify your
+cmake command to be one of each
 
-Next, you can build Google Test and all of its own tests.  On \*nix,
-this is usually done by 'make'.  To run the tests, do
+    cmake -Dgtest_build_tests=ON ${GTEST_DIR} # sets up Google Test tests
+    cmake -Dgmock_build_tests=ON ${GMOCK_DIR} # sets up Google Mock tests
 
-    make test
-
-All tests should pass.
+Make sure you have Python installed, as some of Google Test's tests are written
+in Python. If the cmake command complains about not being able to find Python
+(`Could NOT find PythonInterp (missing: PYTHON_EXECUTABLE)`), try telling it
+explicitly where your Python executable can be found:
 
-### Regenerating Source Files ##
+    cmake -DPYTHON_EXECUTABLE=path/to/python ...
 
-Some of Google Test's source files are generated from templates (not
-in the C++ sense) using a script.
-For example, the
-file include/gtest/internal/gtest-type-util.h.pump is used to generate
-gtest-type-util.h in the same directory.
+Next, you can build Google Test and / or Google Mock and all desired tests. On
+\*nix, this is usually done by
 
-You don't need to worry about regenerating the source files
-unless you need to modify them.  You would then modify the
-corresponding `.pump` files and run the '[pump.py](googletest/scripts/pump.py)'
-generator script.  See the [Pump Manual](googletest/docs/PumpManual.md).
+    make
 
-## Developing Google Mock ###
+To run the tests, do
 
-This section discusses how to make your own changes to Google Mock.
-
-#### Testing Google Mock Itself ####
-
-To make sure your changes work as intended and don't break existing
-functionality, you'll want to compile and run Google Test's own tests.
-For that you'll need Autotools.  First, make sure you have followed
-the instructions above to configure Google Mock.
-Then, create a build output directory and enter it.  Next,
+    make test
 
-    ${GMOCK_DIR}/configure  # try --help for more info
+All tests should pass.
 
-Once you have successfully configured Google Mock, the build steps are
-standard for GNU-style OSS packages.
+### Regenerating Source Files
 
-    make        # Standard makefile following GNU conventions
-    make check  # Builds and runs all tests - all should pass.
+Some of Google Test's source files are generated from templates (not in the C++
+sense) using a script. For example, the file
+*googlemock/include/gmock/gmock-generated-actions.h.pump* is used to generate
+*gmock-generated-actions.h* in the same directory.
 
-Note that when building your project against Google Mock, you are building
-against Google Test as well.  There is no need to configure Google Test
-separately.
+You don't need to worry about regenerating the source files unless you need to
+modify them. You would then modify the corresponding `.pump` files and run the
+'[pump.py](googlemock/scripts/pump.py)' generator script. See the
+[Pump Manual](googlemock/docs/pump_manual.md).
diff --git a/thirdparty/GTest/googletest/Makefile.am b/thirdparty/GTest/googletest/Makefile.am
deleted file mode 100644
index 433eefeb08..0000000000
--- a/thirdparty/GTest/googletest/Makefile.am
+++ /dev/null
@@ -1,14 +0,0 @@
-## Process this file with automake to produce Makefile.in
-ACLOCAL_AMFLAGS = -I m4
-
-AUTOMAKE_OPTIONS = foreign
-
-# Build . before src so that our all-local and clean-local hooks kicks in at
-# the right time.
-SUBDIRS = googletest googlemock
-
-EXTRA_DIST = \
-  BUILD.bazel \
-  CMakeLists.txt \
-  README.md \
-  WORKSPACE
diff --git a/thirdparty/GTest/googletest/README.md b/thirdparty/GTest/googletest/README.md
index 157316c03f..4ecfd4441f 100644
--- a/thirdparty/GTest/googletest/README.md
+++ b/thirdparty/GTest/googletest/README.md
@@ -1,73 +1,87 @@
+# Google Test
 
-# Google Test #
+#### OSS Builds Status:
 
-[![Build Status](https://travis-ci.org/google/googletest.svg?branch=master)](https://travis-ci.org/google/googletest)
+[![Build Status](https://api.travis-ci.org/google/googletest.svg?branch=master)](https://travis-ci.org/google/googletest)
 [![Build status](https://ci.appveyor.com/api/projects/status/4o38plt0xbo1ubc8/branch/master?svg=true)](https://ci.appveyor.com/project/GoogleTestAppVeyor/googletest/branch/master)
 
-Welcome to **Google Test**, Google's C++ test framework!
+### Announcements:
 
-This repository is a merger of the formerly separate GoogleTest and
-GoogleMock projects. These were so closely related that it makes sense to
-maintain and release them together.
+#### Release 1.10.x
 
-Please see the project page above for more information as well as the
-mailing list for questions, discussions, and development.  There is
-also an IRC channel on [OFTC](https://webchat.oftc.net/) (irc.oftc.net) #gtest available.  Please
-join us!
+[Release 1.10.x](https://github.com/google/googletest/releases/tag/release-1.10.0)
+is now available.
 
-Getting started information for **Google Test** is available in the 
-[Google Test Primer](googletest/docs/Primer.md) documentation.
+#### Coming Soon
+
+*   Post 1.10.x googletest will follow
+    [Abseil Live at Head philosophy](https://abseil.io/about/philosophy)
+*   We are also planning to take a dependency on
+    [Abseil](https://github.com/abseil/abseil-cpp).
+
+## Welcome to **Google Test**, Google's C++ test framework!
+
+This repository is a merger of the formerly separate GoogleTest and GoogleMock
+projects. These were so closely related that it makes sense to maintain and
+release them together.
+
+### Getting started:
+
+The information for **Google Test** is available in the
+[Google Test Primer](googletest/docs/primer.md) documentation.
 
 **Google Mock** is an extension to Google Test for writing and using C++ mock
-classes.  See the separate [Google Mock documentation](googlemock/README.md).
+classes. See the separate [Google Mock documentation](googlemock/README.md).
 
-More detailed documentation for googletest (including build instructions) are
-in its interior [googletest/README.md](googletest/README.md) file.
+More detailed documentation for googletest is in its interior
+[googletest/README.md](googletest/README.md) file.
 
-## Features ##
+## Features
 
-  * An [xUnit](https://en.wikipedia.org/wiki/XUnit) test framework.
-  * Test discovery.
-  * A rich set of assertions.
-  * User-defined assertions.
-  * Death tests.
-  * Fatal and non-fatal failures.
-  * Value-parameterized tests.
-  * Type-parameterized tests.
-  * Various options for running the tests.
-  * XML test report generation.
+*   An [xUnit](https://en.wikipedia.org/wiki/XUnit) test framework.
+*   Test discovery.
+*   A rich set of assertions.
+*   User-defined assertions.
+*   Death tests.
+*   Fatal and non-fatal failures.
+*   Value-parameterized tests.
+*   Type-parameterized tests.
+*   Various options for running the tests.
+*   XML test report generation.
 
-## Platforms ##
+## Platforms
 
 Google test has been used on a variety of platforms:
 
-  * Linux
-  * Mac OS X
-  * Windows
-  * Cygwin
-  * MinGW
-  * Windows Mobile
-  * Symbian
+*   Linux
+*   Mac OS X
+*   Windows
+*   Cygwin
+*   MinGW
+*   Windows Mobile
+*   Symbian
+*   PlatformIO
 
-## Who Is Using Google Test? ##
+## Who Is Using Google Test?
 
-In addition to many internal projects at Google, Google Test is also used by
-the following notable projects:
+In addition to many internal projects at Google, Google Test is also used by the
+following notable projects:
 
-  * The [Chromium projects](http://www.chromium.org/) (behind the Chrome
-    browser and Chrome OS).
-  * The [LLVM](http://llvm.org/) compiler.
-  * [Protocol Buffers](https://github.com/google/protobuf), Google's data
+*   The [Chromium projects](http://www.chromium.org/) (behind the Chrome browser
+    and Chrome OS).
+*   The [LLVM](http://llvm.org/) compiler.
+*   [Protocol Buffers](https://github.com/google/protobuf), Google's data
     interchange format.
-  * The [OpenCV](http://opencv.org/) computer vision library.
-  * [tiny-dnn](https://github.com/tiny-dnn/tiny-dnn): header only, dependency-free deep learning framework in C++11.
+*   The [OpenCV](http://opencv.org/) computer vision library.
 
-## Related Open Source Projects ##
+## Related Open Source Projects
 
-[GTest Runner](https://github.com/nholthaus/gtest-runner) is a Qt5 based automated test-runner and Graphical User Interface with powerful features for Windows and Linux platforms.
+[GTest Runner](https://github.com/nholthaus/gtest-runner) is a Qt5 based
+automated test-runner and Graphical User Interface with powerful features for
+Windows and Linux platforms.
 
-[Google Test UI](https://github.com/ospector/gtest-gbar) is test runner that runs
-your test binary, allows you to track its progress via a progress bar, and
+[Google Test UI](https://github.com/ospector/gtest-gbar) is a test runner that
+runs your test binary, allows you to track its progress via a progress bar, and
 displays a list of test failures. Clicking on one shows failure text. Google
 Test UI is written in C#.
 
@@ -79,44 +93,40 @@ result output. If your test runner understands TAP, you may find it useful.
 [gtest-parallel](https://github.com/google/gtest-parallel) is a test runner that
 runs tests from your binary in parallel to provide significant speed-up.
 
-## Requirements ##
-
-Google Test is designed to have fairly minimal requirements to build
-and use with your projects, but there are some.  Currently, we support
-Linux, Windows, Mac OS X, and Cygwin.  We will also make our best
-effort to support other platforms (e.g. Solaris, AIX, and z/OS).
-However, since core members of the Google Test project have no access
-to these platforms, Google Test may have outstanding issues there.  If
-you notice any problems on your platform, please notify
-[googletestframework@googlegroups.com](https://groups.google.com/forum/#!forum/googletestframework). Patches for fixing them are
-even more welcome!
+[GoogleTest Adapter](https://marketplace.visualstudio.com/items?itemName=DavidSchuldenfrei.gtest-adapter)
+is a VS Code extension allowing to view Google Tests in a tree view, and
+run/debug your tests.
 
-### Linux Requirements ###
+[C++ TestMate](https://github.com/matepek/vscode-catch2-test-adapter) is a VS
+Code extension allowing to view Google Tests in a tree view, and run/debug your
+tests.
 
-These are the base requirements to build and use Google Test from a source
-package (as described below):
+[Cornichon](https://pypi.org/project/cornichon/) is a small Gherkin DSL parser
+that generates stub code for Google Test.
 
-  * GNU-compatible Make or gmake
-  * POSIX-standard shell
-  * POSIX(-2) Regular Expressions (regex.h)
-  * A C++98-standard-compliant compiler
+## Requirements
 
-### Windows Requirements ###
+Google Test is designed to have fairly minimal requirements to build and use
+with your projects, but there are some. If you notice any problems on your
+platform, please file an issue on the
+[GoogleTest GitHub Issue Tracker](https://github.com/google/googletest/issues).
 
-  * Microsoft Visual C++ 2015 or newer
+Patches for fixing them are welcome!
 
-### Cygwin Requirements ###
+### Build Requirements
 
-  * Cygwin v1.5.25-14 or newer
+These are the base requirements to build and use Google Test from a source
+package:
 
-### Mac OS X Requirements ###
+*   [Bazel](https://bazel.build/) or [CMake](https://cmake.org/). NOTE: Bazel is
+    the build system that googletest is using internally and tests against.
+    CMake is community-supported.
 
-  * Mac OS X v10.4 Tiger or newer
-  * Xcode Developer Tools
+*   A C++11-standard-compliant compiler
 
 ## Contributing change
 
-Please read the [`CONTRIBUTING.md`](CONTRIBUTING.md) for details on
-how to contribute to this project.
+Please read the [`CONTRIBUTING.md`](CONTRIBUTING.md) for details on how to
+contribute to this project.
 
 Happy testing!
diff --git a/thirdparty/GTest/googletest/WORKSPACE b/thirdparty/GTest/googletest/WORKSPACE
index 1d5d388623..1f05d21a0d 100644
--- a/thirdparty/GTest/googletest/WORKSPACE
+++ b/thirdparty/GTest/googletest/WORKSPACE
@@ -1,8 +1,30 @@
 workspace(name = "com_google_googletest")
 
-# Abseil
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+
+http_archive(
+    name = "com_google_absl",  # 2020-10-13T16:49:13Z
+    urls = [
+        "https://github.com/abseil/abseil-cpp/archive/f3f785ab59478dd0312bf1b5df65d380650bf0dc.zip"
+    ],
+    strip_prefix = "abseil-cpp-f3f785ab59478dd0312bf1b5df65d380650bf0dc",
+    sha256 = "00c3707bf9cd5eabd1ec6932cc65b97378c043f22573be3adf7d11bb7af17d06",
+)
+
+http_archive(
+    name = "rules_cc",  # 2020-10-05T06:01:24Z
+    urls = [
+        "https://github.com/bazelbuild/rules_cc/archive/f055da4ff0cb2b3c73de1fe2f094ebdfb8b3acb9.zip"
+    ],
+    strip_prefix = "rules_cc-f055da4ff0cb2b3c73de1fe2f094ebdfb8b3acb9",
+    sha256 = "35ea62c63cd71d4000efe85f9f4f17e8afb23896c37ee9510952db2e9d8fbb70",
+)
+
 http_archive(
-     name = "com_google_absl",
-     urls = ["https://github.com/abseil/abseil-cpp/archive/master.zip"],
-     strip_prefix = "abseil-cpp-master",
+    name = "rules_python",  # 2020-09-30T13:50:21Z
+    urls = [
+        "https://github.com/bazelbuild/rules_python/archive/c064f7008a30f307ea7516cf52358a653011f82b.zip",
+    ],
+    strip_prefix = "rules_python-c064f7008a30f307ea7516cf52358a653011f82b",
+    sha256 = "6e49996ad3cf45b2232b8f94ca1e3ead369c28394c51632be8d85fe826383012",
 )
diff --git a/thirdparty/GTest/googletest/appveyor.yml b/thirdparty/GTest/googletest/appveyor.yml
index 84d9fbceb7..5c419c32f9 100644
--- a/thirdparty/GTest/googletest/appveyor.yml
+++ b/thirdparty/GTest/googletest/appveyor.yml
@@ -6,27 +6,36 @@ environment:
   matrix:
     - compiler: msvc-15-seh
       generator: "Visual Studio 15 2017"
+      build_system: cmake
       APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
+      enabled_on_pr: yes
 
     - compiler: msvc-15-seh
       generator: "Visual Studio 15 2017 Win64"
+      build_system: cmake
+      APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
+      enabled_on_pr: yes
+
+    - compiler: msvc-15-seh
+      build_system: bazel
       APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
       enabled_on_pr: yes
 
     - compiler: msvc-14-seh
+      build_system: cmake
       generator: "Visual Studio 14 2015"
       enabled_on_pr: yes
 
     - compiler: msvc-14-seh
+      build_system: cmake
       generator: "Visual Studio 14 2015 Win64"
-
-    - compiler: gcc-5.3.0-posix
-      generator: "MinGW Makefiles"
-      cxx_path: 'C:\mingw-w64\i686-5.3.0-posix-dwarf-rt_v4-rev0\mingw32\bin'
+      enabled_on_pr: yes
 
     - compiler: gcc-6.3.0-posix
+      build_system: cmake
       generator: "MinGW Makefiles"
       cxx_path: 'C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\bin'
+      enabled_on_pr: yes
 
 configuration:
   - Debug
@@ -38,6 +47,8 @@ install:
 - ps: |
     Write-Output "Compiler: $env:compiler"
     Write-Output "Generator: $env:generator"
+    Write-Output "Env:Configuation: $env:configuration"
+    Write-Output "Env: $env"
     if (-not (Test-Path env:APPVEYOR_PULL_REQUEST_NUMBER)) {
       Write-Output "This is *NOT* a pull request build"
     } else {
@@ -47,20 +58,44 @@ install:
       }
     }
 
-    # git bash conflicts with MinGW makefiles
-    if ($env:generator -eq "MinGW Makefiles") {
-        $env:path = $env:path.replace("C:\Program Files\Git\usr\bin;", "")
-        if ($env:cxx_path -ne "") {
-            $env:path += ";$env:cxx_path"
+    # install Bazel
+    if ($env:build_system -eq "bazel") {
+        appveyor DownloadFile https://github.com/bazelbuild/bazel/releases/download/3.6.0/bazel-3.6.0-windows-x86_64.exe -FileName bazel.exe
+    }
+
+    if ($env:build_system -eq "cmake") {
+        # git bash conflicts with MinGW makefiles
+        if ($env:generator -eq "MinGW Makefiles") {
+            $env:path = $env:path.replace("C:\Program Files\Git\usr\bin;", "")
+            if ($env:cxx_path -ne "") {
+                $env:path += ";$env:cxx_path"
+            }
         }
     }
 
+before_build:
+- ps: |
+     $env:root=$env:APPVEYOR_BUILD_FOLDER
+     Write-Output "env:root: $env:root"
+
 build_script:
 - ps: |
     # Only enable some builds for pull requests, the AppVeyor queue is too long.
     if ((Test-Path env:APPVEYOR_PULL_REQUEST_NUMBER) -And (-not (Test-Path env:enabled_on_pr) -or $env:enabled_on_pr -ne "yes")) {
       return
+    } else {
+        # special case - build with Bazel
+        if ($env:build_system -eq "bazel") {
+            & $env:root\bazel.exe build -c opt //:gtest_samples
+            if ($LastExitCode -eq 0) { # bazel writes to StdErr and PowerShell interprets it as an error
+                $host.SetShouldExit(0)
+            } else { # a real error
+                throw "Exec: $ErrorMessage"
+            }
+            return
+        }
     }
+    # by default build with CMake
     md _build -Force | Out-Null
     cd _build
 
@@ -78,18 +113,36 @@ build_script:
         throw "Exec: $ErrorMessage"
     }
 
+
+skip_commits:
+  files:
+    - '**/*.md'
+
 test_script:
 - ps: |
     # Only enable some builds for pull requests, the AppVeyor queue is too long.
     if ((Test-Path env:APPVEYOR_PULL_REQUEST_NUMBER) -And (-not (Test-Path env:enabled_on_pr) -or $env:enabled_on_pr -ne "yes")) {
       return
     }
-    if ($env:generator -eq "MinGW Makefiles") {
-        return # No test available for MinGW
+    if ($env:build_system -eq "bazel") {
+        # special case - testing with Bazel
+        & $env:root\bazel.exe test //:gtest_samples
+        if ($LastExitCode -eq 0) { # bazel writes to StdErr and PowerShell interprets it as an error
+            $host.SetShouldExit(0)
+        } else { # a real error
+            throw "Exec: $ErrorMessage"
+        }
     }
-    & ctest -C $env:configuration --timeout 300 --output-on-failure
-    if ($LastExitCode -ne 0) {
-        throw "Exec: $ErrorMessage"
+    if ($env:build_system -eq "cmake") {
+        # built with CMake - test with CTest
+        if ($env:generator -eq "MinGW Makefiles") {
+            return # No test available for MinGW
+        }
+
+        & ctest -C $env:configuration --timeout 600 --output-on-failure
+        if ($LastExitCode -ne 0) {
+            throw "Exec: $ErrorMessage"
+        }
     }
 
 artifacts:
@@ -97,3 +150,7 @@ artifacts:
     name: logs
   - path: '_build/Testing/**/*.xml'
     name: test_results
+  - path: 'bazel-testlogs/**/test.log'
+    name: test_logs
+  - path: 'bazel-testlogs/**/test.xml'
+    name: test_results
diff --git a/thirdparty/GTest/googletest/ci/build-linux-bazel.sh b/thirdparty/GTest/googletest/ci/build-linux-bazel.sh
index 3f1c784955..cfb06a9e0a 100755
--- a/thirdparty/GTest/googletest/ci/build-linux-bazel.sh
+++ b/thirdparty/GTest/googletest/ci/build-linux-bazel.sh
@@ -31,6 +31,6 @@
 
 set -e
 
-bazel build --curses=no //...:all
+bazel version
 bazel test --curses=no //...:all
 bazel test --curses=no //...:all --define absl=1
diff --git a/thirdparty/GTest/googletest/ci/build-platformio.sh b/thirdparty/GTest/googletest/ci/build-platformio.sh
new file mode 100644
index 0000000000..1d7658d873
--- /dev/null
+++ b/thirdparty/GTest/googletest/ci/build-platformio.sh
@@ -0,0 +1,2 @@
+# run PlatformIO builds
+platformio run
diff --git a/thirdparty/GTest/googletest/ci/env-linux.sh b/thirdparty/GTest/googletest/ci/env-linux.sh
index 9086b1f986..7d2b8a8c5c 100755
--- a/thirdparty/GTest/googletest/ci/env-linux.sh
+++ b/thirdparty/GTest/googletest/ci/env-linux.sh
@@ -36,6 +36,6 @@
 # TODO() - we can check if this is being sourced using $BASH_VERSION and $BASH_SOURCE[0] != ${0}.
 
 if [ "${TRAVIS_OS_NAME}" = "linux" ]; then
-    if [ "$CXX" = "g++" ]; then export CXX="g++-4.9" CC="gcc-4.9"; fi
-    if [ "$CXX" = "clang++" ]; then export CXX="clang++-3.7" CC="clang-3.7"; fi
+    if [ "$CXX" = "g++" ]; then export CXX="g++" CC="gcc"; fi
+    if [ "$CXX" = "clang++" ]; then export CXX="clang++" CC="clang"; fi
 fi
diff --git a/thirdparty/GTest/googletest/ci/env-osx.sh b/thirdparty/GTest/googletest/ci/env-osx.sh
index 31c88357d7..9c421e1431 100755
--- a/thirdparty/GTest/googletest/ci/env-osx.sh
+++ b/thirdparty/GTest/googletest/ci/env-osx.sh
@@ -34,7 +34,14 @@
 #
 
 # TODO() - we can check if this is being sourced using $BASH_VERSION and $BASH_SOURCE[0] != ${0}.
+#
 
-if [ "${TRAVIS_OS_NAME}" = "linux" ]; then
-    if [ "$CXX" = "clang++" ]; then export CXX="clang++-3.7" CC="clang-3.7"; fi
+if [ "${TRAVIS_OS_NAME}" = "osx" ]; then
+    if [ "$CXX" = "clang++" ]; then
+        # $PATH needs to be adjusted because the llvm tap doesn't install the
+        # package to /usr/local/bin, etc, like the gcc tap does.
+        # See: https://github.com/Homebrew/legacy-homebrew/issues/29733
+        clang_version=3.9
+        export PATH="/usr/local/opt/llvm@${clang_version}/bin:$PATH";
+    fi
 fi
diff --git a/thirdparty/GTest/googletest/ci/install-linux.sh b/thirdparty/GTest/googletest/ci/install-linux.sh
index 02a1943919..f98ac7d89b 100755
--- a/thirdparty/GTest/googletest/ci/install-linux.sh
+++ b/thirdparty/GTest/googletest/ci/install-linux.sh
@@ -41,7 +41,7 @@ if [ "${TRAVIS_SUDO}" = "true" ]; then
     echo "deb [arch=amd64] http://storage.googleapis.com/bazel-apt stable jdk1.8" | \
         sudo tee /etc/apt/sources.list.d/bazel.list
     curl https://bazel.build/bazel-release.pub.gpg | sudo apt-key add -
-    sudo apt-get update && sudo apt-get install -y bazel gcc-4.9 g++-4.9 clang-3.7
+    sudo apt-get update && sudo apt-get install -y bazel gcc g++ clang
 elif [ "${CXX}" = "clang++" ]; then
     # Use ccache, assuming $HOME/bin is in the path, which is true in the Travis build environment.
     ln -sf /usr/bin/ccache $HOME/bin/${CXX};
diff --git a/thirdparty/GTest/googletest/ci/install-osx.sh b/thirdparty/GTest/googletest/ci/install-osx.sh
index 6550ff514f..cc4750829c 100755
--- a/thirdparty/GTest/googletest/ci/install-osx.sh
+++ b/thirdparty/GTest/googletest/ci/install-osx.sh
@@ -36,4 +36,5 @@ if [ "${TRAVIS_OS_NAME}" != "osx" ]; then
     exit 0
 fi
 
-brew install ccache
+brew update
+brew install ccache gcc@4.9
diff --git a/thirdparty/GTest/googletest/ci/install-platformio.sh b/thirdparty/GTest/googletest/ci/install-platformio.sh
new file mode 100644
index 0000000000..4d7860a560
--- /dev/null
+++ b/thirdparty/GTest/googletest/ci/install-platformio.sh
@@ -0,0 +1,5 @@
+# install PlatformIO
+sudo pip install -U platformio
+
+# update PlatformIO
+platformio update
diff --git a/thirdparty/GTest/googletest/ci/travis.sh b/thirdparty/GTest/googletest/ci/travis.sh
index 2dda68fdac..a24882293e 100755
--- a/thirdparty/GTest/googletest/ci/travis.sh
+++ b/thirdparty/GTest/googletest/ci/travis.sh
@@ -3,34 +3,27 @@ set -evx
 
 . ci/get-nprocessors.sh
 
-# if possible, ask for the precise number of processors,
-# otherwise take 2 processors as reasonable default; see
-# https://docs.travis-ci.com/user/speeding-up-the-build/#Makefile-optimization
-if [ -x /usr/bin/getconf ]; then
-    NPROCESSORS=$(/usr/bin/getconf _NPROCESSORS_ONLN)
-else
-    NPROCESSORS=2
-fi
-# as of 2017-09-04 Travis CI reports 32 processors, but GCC build
-# crashes if parallelized too much (maybe memory consumption problem),
-# so limit to 4 processors for the time being.
-if [ $NPROCESSORS -gt 4 ] ; then
-	echo "$0:Note: Limiting processors to use by make from $NPROCESSORS to 4."
-	NPROCESSORS=4
-fi
 # Tell make to use the processors. No preceding '-' required.
 MAKEFLAGS="j${NPROCESSORS}"
 export MAKEFLAGS
 
 env | sort
 
+# Set default values to OFF for these variables if not specified.
+: "${NO_EXCEPTION:=OFF}"
+: "${NO_RTTI:=OFF}"
+: "${COMPILER_IS_GNUCXX:=OFF}"
+
 mkdir build || true
 cd build
 cmake -Dgtest_build_samples=ON \
       -Dgtest_build_tests=ON \
       -Dgmock_build_tests=ON \
-      -DCMAKE_CXX_FLAGS=$CXX_FLAGS \
-      -DCMAKE_BUILD_TYPE=$BUILD_TYPE \
+      -Dcxx_no_exception="$NO_EXCEPTION" \
+      -Dcxx_no_rtti="$NO_RTTI" \
+      -DCMAKE_COMPILER_IS_GNUCXX="$COMPILER_IS_GNUCXX" \
+      -DCMAKE_CXX_FLAGS="$CXX_FLAGS" \
+      -DCMAKE_BUILD_TYPE="$BUILD_TYPE" \
       ..
 make
 CTEST_OUTPUT_ON_FAILURE=1 make test
diff --git a/thirdparty/GTest/googletest/configure.ac b/thirdparty/GTest/googletest/configure.ac
deleted file mode 100644
index 751b9ba8f0..0000000000
--- a/thirdparty/GTest/googletest/configure.ac
+++ /dev/null
@@ -1,16 +0,0 @@
-AC_INIT([Google C++ Mocking and Testing Frameworks],
-        [1.8.0],
-        [googlemock@googlegroups.com],
-        [googletest])
-
-# Provide various options to initialize the Autoconf and configure processes.
-AC_PREREQ([2.59])
-AC_CONFIG_SRCDIR([./README.md])
-AC_CONFIG_AUX_DIR([build-aux])
-AC_CONFIG_FILES([Makefile])
-AC_CONFIG_SUBDIRS([googletest googlemock])
-
-AM_INIT_AUTOMAKE
-
-# Output the generated files. No further autoconf macros may be used.
-AC_OUTPUT
diff --git a/thirdparty/GTest/googletest/googlemock/CHANGES b/thirdparty/GTest/googletest/googlemock/CHANGES
deleted file mode 100644
index 4328ece3d3..0000000000
--- a/thirdparty/GTest/googletest/googlemock/CHANGES
+++ /dev/null
@@ -1,126 +0,0 @@
-Changes for 1.7.0:
-
-* All new improvements in Google Test 1.7.0.
-* New feature: matchers DoubleNear(), FloatNear(),
-  NanSensitiveDoubleNear(), NanSensitiveFloatNear(),
-  UnorderedElementsAre(), UnorderedElementsAreArray(), WhenSorted(),
-  WhenSortedBy(), IsEmpty(), and SizeIs().
-* Improvement: Google Mock can now be built as a DLL.
-* Improvement: when compiled by a C++11 compiler, matchers AllOf()
-  and AnyOf() can accept an arbitrary number of matchers.
-* Improvement: when compiled by a C++11 compiler, matchers
-  ElementsAreArray() can accept an initializer list.
-* Improvement: when exceptions are enabled, a mock method with no
-  default action now throws instead crashing the test.
-* Improvement: added class testing::StringMatchResultListener to aid
-  definition of composite matchers.
-* Improvement: function return types used in MOCK_METHOD*() macros can
-  now contain unprotected commas.
-* Improvement (potentially breaking): EXPECT_THAT() and ASSERT_THAT()
-  are now more strict in ensuring that the value type and the matcher
-  type are compatible, catching potential bugs in tests.
-* Improvement: Pointee() now works on an optional.
-* Improvement: the ElementsAreArray() matcher can now take a vector or
-  iterator range as input, and makes a copy of its input elements
-  before the conversion to a Matcher.
-* Improvement: the Google Mock Generator can now generate mocks for
-  some class templates.
-* Bug fix: mock object destruction triggerred by another mock object's
-  destruction no longer hangs.
-* Improvement: Google Mock Doctor works better with newer Clang and
-  GCC now.
-* Compatibility fixes.
-* Bug/warning fixes.
-
-Changes for 1.6.0:
-
-* Compilation is much faster and uses much less memory, especially
-  when the constructor and destructor of a mock class are moved out of
-  the class body.
-* New matchers: Pointwise(), Each().
-* New actions: ReturnPointee() and ReturnRefOfCopy().
-* CMake support.
-* Project files for Visual Studio 2010.
-* AllOf() and AnyOf() can handle up-to 10 arguments now.
-* Google Mock doctor understands Clang error messages now.
-* SetArgPointee<> now accepts string literals.
-* gmock_gen.py handles storage specifier macros and template return
-  types now.
-* Compatibility fixes.
-* Bug fixes and implementation clean-ups.
-* Potentially incompatible changes: disables the harmful 'make install'
-  command in autotools.
-
-Potentially breaking changes:
-
-* The description string for MATCHER*() changes from Python-style
-  interpolation to an ordinary C++ string expression.
-* SetArgumentPointee is deprecated in favor of SetArgPointee.
-* Some non-essential project files for Visual Studio 2005 are removed.
-
-Changes for 1.5.0:
-
- * New feature: Google Mock can be safely used in multi-threaded tests
-   on platforms having pthreads.
- * New feature: function for printing a value of arbitrary type.
- * New feature: function ExplainMatchResult() for easy definition of
-   composite matchers.
- * The new matcher API lets user-defined matchers generate custom
-   explanations more directly and efficiently.
- * Better failure messages all around.
- * NotNull() and IsNull() now work with smart pointers.
- * Field() and Property() now work when the matcher argument is a pointer
-   passed by reference.
- * Regular expression matchers on all platforms.
- * Added GCC 4.0 support for Google Mock Doctor.
- * Added gmock_all_test.cc for compiling most Google Mock tests
-   in a single file.
- * Significantly cleaned up compiler warnings.
- * Bug fixes, better test coverage, and implementation clean-ups.
-
- Potentially breaking changes:
-
- * Custom matchers defined using MatcherInterface or MakePolymorphicMatcher()
-   need to be updated after upgrading to Google Mock 1.5.0; matchers defined
-   using MATCHER or MATCHER_P* aren't affected.
- * Dropped support for 'make install'.
-
-Changes for 1.4.0 (we skipped 1.2.* and 1.3.* to match the version of
-Google Test):
-
- * Works in more environments: Symbian and minGW, Visual C++ 7.1.
- * Lighter weight: comes with our own implementation of TR1 tuple (no
-   more dependency on Boost!).
- * New feature: --gmock_catch_leaked_mocks for detecting leaked mocks.
- * New feature: ACTION_TEMPLATE for defining templatized actions.
- * New feature: the .After() clause for specifying expectation order.
- * New feature: the .With() clause for specifying inter-argument
-   constraints.
- * New feature: actions ReturnArg(), ReturnNew(...), and
-   DeleteArg().
- * New feature: matchers Key(), Pair(), Args<...>(), AllArgs(), IsNull(),
-   and Contains().
- * New feature: utility class MockFunction, useful for checkpoints, etc.
- * New feature: functions Value(x, m) and SafeMatcherCast(m).
- * New feature: copying a mock object is rejected at compile time.
- * New feature: a script for fusing all Google Mock and Google Test
-   source files for easy deployment.
- * Improved the Google Mock doctor to diagnose more diseases.
- * Improved the Google Mock generator script.
- * Compatibility fixes for Mac OS X and gcc.
- * Bug fixes and implementation clean-ups.
-
-Changes for 1.1.0:
-
- * New feature: ability to use Google Mock with any testing framework.
- * New feature: macros for easily defining new matchers
- * New feature: macros for easily defining new actions.
- * New feature: more container matchers.
- * New feature: actions for accessing function arguments and throwing
-   exceptions.
- * Improved the Google Mock doctor script for diagnosing compiler errors.
- * Bug fixes and implementation clean-ups.
-
-Changes for 1.0.0:
-
- * Initial Open Source release of Google Mock
diff --git a/thirdparty/GTest/googletest/googlemock/CMakeLists.txt b/thirdparty/GTest/googletest/googlemock/CMakeLists.txt
index bac2e3bf90..188794270e 100644
--- a/thirdparty/GTest/googletest/googlemock/CMakeLists.txt
+++ b/thirdparty/GTest/googletest/googlemock/CMakeLists.txt
@@ -1,14 +1,13 @@
 ########################################################################
+# Note: CMake support is community-based. The maintainers do not use CMake
+# internally.
+#
 # CMake build script for Google Mock.
 #
 # To run the tests for Google Mock itself on Linux, use 'make test' or
 # ctest.  You can select which tests to run using 'ctest -R regex'.
 # For more options, run 'ctest --help'.
 
-# BUILD_SHARED_LIBS is a standard CMake variable, but we declare it here to
-# make it prominent in the GUI.
-option(BUILD_SHARED_LIBS "Build shared libraries (DLLs)." OFF)
-
 option(gmock_build_tests "Build all of Google Mock's own tests." OFF)
 
 # A directory to find Google Test sources.
@@ -41,7 +40,7 @@ if (CMAKE_VERSION VERSION_LESS 3.0)
   project(gmock CXX C)
 else()
   cmake_policy(SET CMP0048 NEW)
-  project(gmock VERSION 1.9.0 LANGUAGES CXX C)
+  project(gmock VERSION ${GOOGLETEST_VERSION} LANGUAGES CXX C)
 endif()
 cmake_minimum_required(VERSION 2.6.4)
 
@@ -53,7 +52,17 @@ endif()
 # targets to the current scope.  We are placing Google Test's binary
 # directory in a subdirectory of our own as VC compilation may break
 # if they are the same (the default).
-add_subdirectory("${gtest_dir}" "${gmock_BINARY_DIR}/gtest")
+add_subdirectory("${gtest_dir}" "${gmock_BINARY_DIR}/${gtest_dir}")
+
+
+# These commands only run if this is the main project
+if(CMAKE_PROJECT_NAME STREQUAL "gmock" OR CMAKE_PROJECT_NAME STREQUAL "googletest-distribution")
+  # BUILD_SHARED_LIBS is a standard CMake variable, but we declare it here to
+  # make it prominent in the GUI.
+  option(BUILD_SHARED_LIBS "Build shared libraries (DLLs)." OFF)
+else()
+  mark_as_advanced(gmock_build_tests)
+endif()
 
 # Although Google Test's CMakeLists.txt calls this function, the
 # changes there don't affect the current scope.  Therefore we have to
@@ -61,24 +70,13 @@ add_subdirectory("${gtest_dir}" "${gmock_BINARY_DIR}/gtest")
 config_compiler_and_linker()  # from ${gtest_dir}/cmake/internal_utils.cmake
 
 # Adds Google Mock's and Google Test's header directories to the search path.
-include_directories("${gmock_SOURCE_DIR}/include"
-                    "${gmock_SOURCE_DIR}"
-                    "${gtest_SOURCE_DIR}/include"
-                    # This directory is needed to build directly from Google
-                    # Test sources.
-                    "${gtest_SOURCE_DIR}")
-
-# Summary of tuple support for Microsoft Visual Studio:
-# Compiler    version(MS)  version(cmake)  Support
-# ----------  -----------  --------------  -----------------------------
-# <= VS 2010  <= 10        <= 1600         Use Google Tests's own tuple.
-# VS 2012     11           1700            std::tr1::tuple + _VARIADIC_MAX=10
-# VS 2013     12           1800            std::tr1::tuple
-# VS 2015     14           1900            std::tuple
-# VS 2017     15           >= 1910         std::tuple
-if (MSVC AND MSVC_VERSION EQUAL 1700)
-  add_definitions(/D _VARIADIC_MAX=10)
-endif()
+set(gmock_build_include_dirs
+  "${gmock_SOURCE_DIR}/include"
+  "${gmock_SOURCE_DIR}"
+  "${gtest_SOURCE_DIR}/include"
+  # This directory is needed to build directly from Google Test sources.
+  "${gtest_SOURCE_DIR}")
+include_directories(${gmock_build_include_dirs})
 
 ########################################################################
 #
@@ -101,42 +99,28 @@ if (MSVC)
               src/gmock_main.cc)
 else()
   cxx_library(gmock "${cxx_strict}" src/gmock-all.cc)
-  target_link_libraries(gmock gtest)
+  target_link_libraries(gmock PUBLIC gtest)
+  set_target_properties(gmock PROPERTIES VERSION ${GOOGLETEST_VERSION})
   cxx_library(gmock_main "${cxx_strict}" src/gmock_main.cc)
-  target_link_libraries(gmock_main gmock)
+  target_link_libraries(gmock_main PUBLIC gmock)
+  set_target_properties(gmock_main PROPERTIES VERSION ${GOOGLETEST_VERSION})
 endif()
-
 # If the CMake version supports it, attach header directory information
 # to the targets for when we are part of a parent build (ie being pulled
 # in via add_subdirectory() rather than being a standalone build).
 if (DEFINED CMAKE_VERSION AND NOT "${CMAKE_VERSION}" VERSION_LESS "2.8.11")
-  target_include_directories(gmock      SYSTEM INTERFACE "${gmock_SOURCE_DIR}/include")
-  target_include_directories(gmock_main SYSTEM INTERFACE "${gmock_SOURCE_DIR}/include")
+  target_include_directories(gmock SYSTEM INTERFACE
+    "$"
+    "$/${CMAKE_INSTALL_INCLUDEDIR}>")
+  target_include_directories(gmock_main SYSTEM INTERFACE
+    "$"
+    "$/${CMAKE_INSTALL_INCLUDEDIR}>")
 endif()
 
 ########################################################################
 #
 # Install rules
-if(INSTALL_GMOCK)
-  install(TARGETS gmock gmock_main
-    RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}"
-    LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}"
-    ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}")
-  install(DIRECTORY "${gmock_SOURCE_DIR}/include/gmock"
-    DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}")
-
-  # configure and install pkgconfig files
-  configure_file(
-    cmake/gmock.pc.in
-    "${CMAKE_BINARY_DIR}/gmock.pc"
-    @ONLY)
-  configure_file(
-    cmake/gmock_main.pc.in
-    "${CMAKE_BINARY_DIR}/gmock_main.pc"
-    @ONLY)
-  install(FILES "${CMAKE_BINARY_DIR}/gmock.pc" "${CMAKE_BINARY_DIR}/gmock_main.pc"
-    DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig")
-endif()
+install_project(gmock gmock_main)
 
 ########################################################################
 #
@@ -154,16 +138,36 @@ if (gmock_build_tests)
   # 'make test' or ctest.
   enable_testing()
 
+  if (WIN32)
+    file(GENERATE OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/$/RunTest.ps1"
+         CONTENT
+"$project_bin = \"${CMAKE_BINARY_DIR}/bin/$\"
+$env:Path = \"$project_bin;$env:Path\"
+& $args")
+  elseif (MINGW OR CYGWIN)
+    file(GENERATE OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/RunTest.ps1"
+         CONTENT
+"$project_bin = (cygpath --windows ${CMAKE_BINARY_DIR}/bin)
+$env:Path = \"$project_bin;$env:Path\"
+& $args")
+  endif()
+
+  if (MINGW OR CYGWIN)
+    if (CMAKE_VERSION VERSION_LESS "2.8.12")
+      add_compile_options("-Wa,-mbig-obj")
+    else()
+      add_definitions("-Wa,-mbig-obj")
+    endif()
+  endif()
+
   ############################################################
   # C++ tests built with standard compiler flags.
 
   cxx_test(gmock-actions_test gmock_main)
   cxx_test(gmock-cardinalities_test gmock_main)
   cxx_test(gmock_ex_test gmock_main)
+  cxx_test(gmock-function-mocker_test gmock_main)
   cxx_test(gmock-generated-actions_test gmock_main)
-  cxx_test(gmock-generated-function-mockers_test gmock_main)
-  cxx_test(gmock-generated-internal-utils_test gmock_main)
-  cxx_test(gmock-generated-matchers_test gmock_main)
   cxx_test(gmock-internal-utils_test gmock_main)
   cxx_test(gmock-matchers_test gmock_main)
   cxx_test(gmock-more-actions_test gmock_main)
@@ -191,25 +195,12 @@ if (gmock_build_tests)
     cxx_library(gmock_main_no_rtti "${cxx_no_rtti}"
       "${gtest_dir}/src/gtest-all.cc" src/gmock-all.cc src/gmock_main.cc)
 
-    if (MSVC_VERSION LESS 1600)  # 1600 is Visual Studio 2010.
-      # Visual Studio 2010, 2012, and 2013 define symbols in std::tr1 that
-      # conflict with our own definitions. Therefore using our own tuple does not
-      # work on those compilers.
-      cxx_library(gmock_main_use_own_tuple "${cxx_use_own_tuple}"
-        "${gtest_dir}/src/gtest-all.cc" src/gmock-all.cc src/gmock_main.cc)
-
-      cxx_test_with_flags(gmock_use_own_tuple_test "${cxx_use_own_tuple}"
-        gmock_main_use_own_tuple test/gmock-spec-builders_test.cc)
-    endif()
   else()
     cxx_library(gmock_main_no_exception "${cxx_no_exception}" src/gmock_main.cc)
-    target_link_libraries(gmock_main_no_exception gmock)
+    target_link_libraries(gmock_main_no_exception PUBLIC gmock)
 
     cxx_library(gmock_main_no_rtti "${cxx_no_rtti}" src/gmock_main.cc)
-    target_link_libraries(gmock_main_no_rtti gmock)
-
-    cxx_library(gmock_main_use_own_tuple "${cxx_use_own_tuple}" src/gmock_main.cc)
-    target_link_libraries(gmock_main_use_own_tuple gmock)
+    target_link_libraries(gmock_main_no_rtti PUBLIC gmock)
   endif()
   cxx_test_with_flags(gmock-more-actions_no_exception_test "${cxx_no_exception}"
     gmock_main_no_exception test/gmock-more-actions_test.cc)
diff --git a/thirdparty/GTest/googletest/googlemock/Makefile.am b/thirdparty/GTest/googletest/googlemock/Makefile.am
deleted file mode 100644
index 9adbc5163d..0000000000
--- a/thirdparty/GTest/googletest/googlemock/Makefile.am
+++ /dev/null
@@ -1,224 +0,0 @@
-# Automake file
-
-# Nonstandard package files for distribution.
-EXTRA_DIST = LICENSE
-
-# We may need to build our internally packaged gtest. If so, it will be
-# included in the 'subdirs' variable.
-SUBDIRS = $(subdirs)
-
-# This is generated by the configure script, so clean it for distribution.
-DISTCLEANFILES = scripts/gmock-config
-
-# We define the global AM_CPPFLAGS as everything we compile includes from these
-# directories.
-AM_CPPFLAGS = $(GTEST_CPPFLAGS) -I$(srcdir)/include
-
-# Modifies compiler and linker flags for pthreads compatibility.
-if HAVE_PTHREADS
-  AM_CXXFLAGS = @PTHREAD_CFLAGS@ -DGTEST_HAS_PTHREAD=1
-  AM_LIBS = @PTHREAD_LIBS@
-endif
-
-# Build rules for libraries.
-lib_LTLIBRARIES = lib/libgmock.la lib/libgmock_main.la
-
-lib_libgmock_la_SOURCES = src/gmock-all.cc
-
-pkginclude_HEADERS = \
-  include/gmock/gmock-actions.h \
-  include/gmock/gmock-cardinalities.h \
-  include/gmock/gmock-generated-actions.h \
-  include/gmock/gmock-generated-function-mockers.h \
-  include/gmock/gmock-generated-matchers.h \
-  include/gmock/gmock-generated-nice-strict.h \
-  include/gmock/gmock-matchers.h \
-  include/gmock/gmock-more-actions.h \
-  include/gmock/gmock-more-matchers.h \
-  include/gmock/gmock-spec-builders.h \
-  include/gmock/gmock.h
-
-pkginclude_internaldir = $(pkgincludedir)/internal
-pkginclude_internal_HEADERS = \
-  include/gmock/internal/gmock-generated-internal-utils.h \
-  include/gmock/internal/gmock-internal-utils.h \
-  include/gmock/internal/gmock-port.h \
-  include/gmock/internal/custom/gmock-generated-actions.h \
-  include/gmock/internal/custom/gmock-matchers.h \
-  include/gmock/internal/custom/gmock-port.h
-
-lib_libgmock_main_la_SOURCES = src/gmock_main.cc
-lib_libgmock_main_la_LIBADD = lib/libgmock.la
-
-# Build rules for tests. Automake's naming for some of these variables isn't
-# terribly obvious, so this is a brief reference:
-#
-# TESTS -- Programs run automatically by "make check"
-# check_PROGRAMS -- Programs built by "make check" but not necessarily run
-
-TESTS=
-check_PROGRAMS=
-AM_LDFLAGS = $(GTEST_LDFLAGS)
-
-# This exercises all major components of Google Mock.  It also
-# verifies that libgmock works.
-TESTS += test/gmock-spec-builders_test
-check_PROGRAMS += test/gmock-spec-builders_test
-test_gmock_spec_builders_test_SOURCES = test/gmock-spec-builders_test.cc
-test_gmock_spec_builders_test_LDADD = $(GTEST_LIBS) lib/libgmock.la
-
-# This tests using Google Mock in multiple translation units.  It also
-# verifies that libgmock_main and libgmock work.
-TESTS += test/gmock_link_test
-check_PROGRAMS += test/gmock_link_test
-test_gmock_link_test_SOURCES = \
-  test/gmock_link2_test.cc \
-  test/gmock_link_test.cc \
-  test/gmock_link_test.h
-test_gmock_link_test_LDADD = $(GTEST_LIBS) lib/libgmock_main.la  lib/libgmock.la
-
-if HAVE_PYTHON
-  # Tests that fused gmock files compile and work.
-  TESTS += test/gmock_fused_test
-  check_PROGRAMS += test/gmock_fused_test
-  test_gmock_fused_test_SOURCES = \
-    fused-src/gmock-gtest-all.cc \
-    fused-src/gmock/gmock.h \
-    fused-src/gmock_main.cc \
-    fused-src/gtest/gtest.h \
-    test/gmock_test.cc
-  test_gmock_fused_test_CPPFLAGS = -I"$(srcdir)/fused-src"
-endif
-
-# Google Mock source files that we don't compile directly.
-GMOCK_SOURCE_INGLUDES = \
-  src/gmock-cardinalities.cc \
-  src/gmock-internal-utils.cc \
-  src/gmock-matchers.cc \
-  src/gmock-spec-builders.cc \
-  src/gmock.cc
-
-EXTRA_DIST += $(GMOCK_SOURCE_INGLUDES)
-
-# C++ tests that we don't compile using autotools.
-EXTRA_DIST += \
-  test/gmock-actions_test.cc \
-  test/gmock_all_test.cc \
-  test/gmock-cardinalities_test.cc \
-  test/gmock_ex_test.cc \
-  test/gmock-generated-actions_test.cc \
-  test/gmock-generated-function-mockers_test.cc \
-  test/gmock-generated-internal-utils_test.cc \
-  test/gmock-generated-matchers_test.cc \
-  test/gmock-internal-utils_test.cc \
-  test/gmock-matchers_test.cc \
-  test/gmock-more-actions_test.cc \
-  test/gmock-nice-strict_test.cc \
-  test/gmock-port_test.cc \
-  test/gmock_stress_test.cc
-
-# Python tests, which we don't run using autotools.
-EXTRA_DIST += \
-  test/gmock_leak_test.py \
-  test/gmock_leak_test_.cc \
-  test/gmock_output_test.py \
-  test/gmock_output_test_.cc \
-  test/gmock_output_test_golden.txt \
-  test/gmock_test_utils.py
-
-# Nonstandard package files for distribution.
-EXTRA_DIST += \
-  CHANGES \
-  CONTRIBUTORS \
-  make/Makefile
-
-# Pump scripts for generating Google Mock headers.
-# TODO(chandlerc@google.com): automate the generation of *.h from *.h.pump.
-EXTRA_DIST += \
-  include/gmock/gmock-generated-actions.h.pump \
-  include/gmock/gmock-generated-function-mockers.h.pump \
-  include/gmock/gmock-generated-matchers.h.pump \
-  include/gmock/gmock-generated-nice-strict.h.pump \
-  include/gmock/internal/gmock-generated-internal-utils.h.pump \
-  include/gmock/internal/custom/gmock-generated-actions.h.pump
-
-# Script for fusing Google Mock and Google Test source files.
-EXTRA_DIST += scripts/fuse_gmock_files.py
-
-# The Google Mock Generator tool from the cppclean project.
-EXTRA_DIST += \
-  scripts/generator/LICENSE \
-  scripts/generator/README \
-  scripts/generator/README.cppclean \
-  scripts/generator/cpp/__init__.py \
-  scripts/generator/cpp/ast.py \
-  scripts/generator/cpp/gmock_class.py \
-  scripts/generator/cpp/keywords.py \
-  scripts/generator/cpp/tokenize.py \
-  scripts/generator/cpp/utils.py \
-  scripts/generator/gmock_gen.py
-
-# Script for diagnosing compiler errors in programs that use Google
-# Mock.
-EXTRA_DIST += scripts/gmock_doctor.py
-
-# CMake scripts.
-EXTRA_DIST += \
-  CMakeLists.txt
-
-# Microsoft Visual Studio 2005 projects.
-EXTRA_DIST += \
-  msvc/2005/gmock.sln \
-  msvc/2005/gmock.vcproj \
-  msvc/2005/gmock_config.vsprops \
-  msvc/2005/gmock_main.vcproj \
-  msvc/2005/gmock_test.vcproj
-
-# Microsoft Visual Studio 2010 projects.
-EXTRA_DIST += \
-  msvc/2010/gmock.sln \
-  msvc/2010/gmock.vcxproj \
-  msvc/2010/gmock_config.props \
-  msvc/2010/gmock_main.vcxproj \
-  msvc/2010/gmock_test.vcxproj
-
-if HAVE_PYTHON
-# gmock_test.cc does not really depend on files generated by the
-# fused-gmock-internal rule.  However, gmock_test.o does, and it is
-# important to include test/gmock_test.cc as part of this rule in order to
-# prevent compiling gmock_test.o until all dependent files have been
-# generated.
-$(test_gmock_fused_test_SOURCES): fused-gmock-internal
-
-# TODO(vladl@google.com): Find a way to add Google Tests's sources here.
-fused-gmock-internal: $(pkginclude_HEADERS) $(pkginclude_internal_HEADERS) \
-                      $(lib_libgmock_la_SOURCES) $(GMOCK_SOURCE_INGLUDES) \
-                      $(lib_libgmock_main_la_SOURCES) \
-                      scripts/fuse_gmock_files.py
-	mkdir -p "$(srcdir)/fused-src"
-	chmod -R u+w "$(srcdir)/fused-src"
-	rm -f "$(srcdir)/fused-src/gtest/gtest.h"
-	rm -f "$(srcdir)/fused-src/gmock/gmock.h"
-	rm -f "$(srcdir)/fused-src/gmock-gtest-all.cc"
-	"$(srcdir)/scripts/fuse_gmock_files.py" "$(srcdir)/fused-src"
-	cp -f "$(srcdir)/src/gmock_main.cc" "$(srcdir)/fused-src"
-
-maintainer-clean-local:
-	rm -rf "$(srcdir)/fused-src"
-endif
-
-# Death tests may produce core dumps in the build directory. In case
-# this happens, clean them to keep distcleancheck happy.
-CLEANFILES = core
-
-# Disables 'make install' as installing a compiled version of Google
-# Mock can lead to undefined behavior due to violation of the
-# One-Definition Rule.
-
-install-exec-local:
-	echo "'make install' is dangerous and not supported. Instead, see README for how to integrate Google Mock into your build system."
-	false
-
-install-data-local:
-	echo "'make install' is dangerous and not supported. Instead, see README for how to integrate Google Mock into your build system."
-	false
diff --git a/thirdparty/GTest/googletest/googlemock/README.md b/thirdparty/GTest/googletest/googlemock/README.md
index 1170cfab71..11414f9ca3 100644
--- a/thirdparty/GTest/googletest/googlemock/README.md
+++ b/thirdparty/GTest/googletest/googlemock/README.md
@@ -1,344 +1,44 @@
-## Google Mock ##
+# Googletest Mocking (gMock) Framework
 
-The Google C++ mocking framework.
+### Overview
 
-### Overview ###
-
-Google's framework for writing and using C++ mock classes.
-It can help you derive better designs of your system and write better tests.
+Google's framework for writing and using C++ mock classes. It can help you
+derive better designs of your system and write better tests.
 
 It is inspired by:
 
-  * [jMock](http://www.jmock.org/),
-  * [EasyMock](http://www.easymock.org/), and
-  * [Hamcrest](http://code.google.com/p/hamcrest/),
-
-and designed with C++'s specifics in mind.
-
-Google mock:
-
-  * lets you create mock classes trivially using simple macros.
-  * supports a rich set of matchers and actions.
-  * handles unordered, partially ordered, or completely ordered expectations.
-  * is extensible by users.
-
-We hope you find it useful!
-
-### Features ###
-
-  * Provides a declarative syntax for defining mocks.
-  * Can easily define partial (hybrid) mocks, which are a cross of real
-    and mock objects.
-  * Handles functions of arbitrary types and overloaded functions.
-  * Comes with a rich set of matchers for validating function arguments.
-  * Uses an intuitive syntax for controlling the behavior of a mock.
-  * Does automatic verification of expectations (no record-and-replay needed).
-  * Allows arbitrary (partial) ordering constraints on
-    function calls to be expressed,.
-  * Lets an user extend it by defining new matchers and actions.
-  * Does not use exceptions.
-  * Is easy to learn and use.
-
-Please see the project page above for more information as well as the
-mailing list for questions, discussions, and development.  There is
-also an IRC channel on OFTC (irc.oftc.net) #gtest available.  Please
-join us!
-
-Please note that code under [scripts/generator](scripts/generator/) is
-from [cppclean](http://code.google.com/p/cppclean/) and released under
-the Apache License, which is different from Google Mock's license.
-
-## Getting Started ##
-
-If you are new to the project, we suggest that you read the user
-documentation in the following order:
-
-  * Learn the [basics](../../master/googletest/docs/Primer.md) of
-    Google Test, if you choose to use Google Mock with it (recommended).
-  * Read [Google Mock for Dummies](../../master/googlemock/docs/ForDummies.md).
-  * Read the instructions below on how to build Google Mock.
-
-You can also watch Zhanyong's [talk](http://www.youtube.com/watch?v=sYpCyLI47rM) on Google Mock's usage and implementation.
-
-Once you understand the basics, check out the rest of the docs:
-
-  * [CheatSheet](../../master/googlemock/docs/CheatSheet.md) - all the commonly used stuff
-    at a glance.
-  * [CookBook](../../master/googlemock/docs/CookBook.md) - recipes for getting things done,
-    including advanced techniques.
-
-If you need help, please check the
-[KnownIssues](docs/KnownIssues.md) and
-[FrequentlyAskedQuestions](docs/FrequentlyAskedQuestions.md) before
-posting a question on the
-[discussion group](http://groups.google.com/group/googlemock).
-
-
-### Using Google Mock Without Google Test ###
-
-Google Mock is not a testing framework itself.  Instead, it needs a
-testing framework for writing tests.  Google Mock works seamlessly
-with [Google Test](https://github.com/google/googletest), but
-you can also use it with [any C++ testing framework](../../master/googlemock/docs/ForDummies.md#using-google-mock-with-any-testing-framework).
-
-### Requirements for End Users ###
-
-Google Mock is implemented on top of [Google Test](
-http://github.com/google/googletest/), and depends on it.
-You must use the bundled version of Google Test when using Google Mock.
-
-You can also easily configure Google Mock to work with another testing
-framework, although it will still need Google Test.  Please read
-["Using_Google_Mock_with_Any_Testing_Framework"](
-    ../../master/googlemock/docs/ForDummies.md#using-google-mock-with-any-testing-framework)
-for instructions.
-
-Google Mock depends on advanced C++ features and thus requires a more
-modern compiler. The following are needed to use Google Mock:
-
-#### Linux Requirements ####
-
-  * GNU-compatible Make or "gmake"
-  * POSIX-standard shell
-  * POSIX(-2) Regular Expressions (regex.h)
-  * C++98-standard-compliant compiler (e.g. GCC 3.4 or newer)
-
-#### Windows Requirements ####
-
-  * Microsoft Visual C++ 8.0 SP1 or newer
-
-#### Mac OS X Requirements ####
-
-  * Mac OS X 10.4 Tiger or newer
-  * Developer Tools Installed
-
-### Requirements for Contributors ###
-
-We welcome patches. If you plan to contribute a patch, you need to
-build Google Mock and its tests, which has further requirements:
-
-  * Automake version 1.9 or newer
-  * Autoconf version 2.59 or newer
-  * Libtool / Libtoolize
-  * Python version 2.3 or newer (for running some of the tests and
-    re-generating certain source files from templates)
-
-### Building Google Mock ###
-
-#### Using CMake ####
-
-If you have CMake available, it is recommended that you follow the
-[build instructions][gtest_cmakebuild]
-as described for Google Test. 
-
-If are using Google Mock with an
-existing CMake project, the section
-[Incorporating Into An Existing CMake Project][gtest_incorpcmake]
-may be of particular interest. 
-To make it work for Google Mock you will need to change 
-
-    target_link_libraries(example gtest_main)
-
-to 
-
-    target_link_libraries(example gmock_main)
-    
-This works because `gmock_main` library is compiled with Google Test.
-However, it does not automatically add Google Test includes.
-Therefore you will also have to change
-
-    if (CMAKE_VERSION VERSION_LESS 2.8.11)
-      include_directories("${gtest_SOURCE_DIR}/include")
-    endif()
-
-to
-
-    if (CMAKE_VERSION VERSION_LESS 2.8.11)
-      include_directories(BEFORE SYSTEM
-        "${gtest_SOURCE_DIR}/include" "${gmock_SOURCE_DIR}/include")
-    else()
-      target_include_directories(gmock_main SYSTEM BEFORE INTERFACE
-        "${gtest_SOURCE_DIR}/include" "${gmock_SOURCE_DIR}/include")
-    endif()
-
-This will addtionally mark Google Mock includes as system, which will 
-silence compiler warnings when compiling your tests using clang with 
-`-Wpedantic -Wall -Wextra -Wconversion`.
-
-
-#### Preparing to Build (Unix only) ####
-
-If you are using a Unix system and plan to use the GNU Autotools build
-system to build Google Mock (described below), you'll need to
-configure it now.
-
-To prepare the Autotools build system:
-
-    cd googlemock
-    autoreconf -fvi
-
-To build Google Mock and your tests that use it, you need to tell your
-build system where to find its headers and source files.  The exact
-way to do it depends on which build system you use, and is usually
-straightforward.
-
-This section shows how you can integrate Google Mock into your
-existing build system.
-
-Suppose you put Google Mock in directory `${GMOCK_DIR}` and Google Test
-in `${GTEST_DIR}` (the latter is `${GMOCK_DIR}/gtest` by default).  To
-build Google Mock, create a library build target (or a project as
-called by Visual Studio and Xcode) to compile
-
-    ${GTEST_DIR}/src/gtest-all.cc and ${GMOCK_DIR}/src/gmock-all.cc
-
-with
-
-    ${GTEST_DIR}/include and ${GMOCK_DIR}/include
-
-in the system header search path, and
-
-    ${GTEST_DIR} and ${GMOCK_DIR}
-
-in the normal header search path.  Assuming a Linux-like system and gcc,
-something like the following will do:
-
-    g++ -isystem ${GTEST_DIR}/include -I${GTEST_DIR} \
-        -isystem ${GMOCK_DIR}/include -I${GMOCK_DIR} \
-        -pthread -c ${GTEST_DIR}/src/gtest-all.cc
-    g++ -isystem ${GTEST_DIR}/include -I${GTEST_DIR} \
-        -isystem ${GMOCK_DIR}/include -I${GMOCK_DIR} \
-        -pthread -c ${GMOCK_DIR}/src/gmock-all.cc
-    ar -rv libgmock.a gtest-all.o gmock-all.o
-
-(We need -pthread as Google Test and Google Mock use threads.)
-
-Next, you should compile your test source file with
-${GTEST\_DIR}/include and ${GMOCK\_DIR}/include in the header search
-path, and link it with gmock and any other necessary libraries:
-
-    g++ -isystem ${GTEST_DIR}/include -isystem ${GMOCK_DIR}/include \
-        -pthread path/to/your_test.cc libgmock.a -o your_test
-
-As an example, the make/ directory contains a Makefile that you can
-use to build Google Mock on systems where GNU make is available
-(e.g. Linux, Mac OS X, and Cygwin).  It doesn't try to build Google
-Mock's own tests.  Instead, it just builds the Google Mock library and
-a sample test.  You can use it as a starting point for your own build
-script.
-
-If the default settings are correct for your environment, the
-following commands should succeed:
-
-    cd ${GMOCK_DIR}/make
-    make
-    ./gmock_test
-
-If you see errors, try to tweak the contents of
-[make/Makefile](make/Makefile) to make them go away.
-
-### Windows ###
-
-The msvc/2005 directory contains VC++ 2005 projects and the msvc/2010
-directory contains VC++ 2010 projects for building Google Mock and
-selected tests.
-
-Change to the appropriate directory and run "msbuild gmock.sln" to
-build the library and tests (or open the gmock.sln in the MSVC IDE).
-If you want to create your own project to use with Google Mock, you'll
-have to configure it to use the `gmock_config` propety sheet.  For that:
-
- * Open the Property Manager window (View | Other Windows | Property Manager)
- * Right-click on your project and select "Add Existing Property Sheet..."
- * Navigate to `gmock_config.vsprops` or `gmock_config.props` and select it.
- * In Project Properties | Configuration Properties | General | Additional
-   Include Directories, type /include.
-
-### Tweaking Google Mock ###
-
-Google Mock can be used in diverse environments.  The default
-configuration may not work (or may not work well) out of the box in
-some environments.  However, you can easily tweak Google Mock by
-defining control macros on the compiler command line.  Generally,
-these macros are named like `GTEST_XYZ` and you define them to either 1
-or 0 to enable or disable a certain feature.
-
-We list the most frequently used macros below.  For a complete list,
-see file [${GTEST\_DIR}/include/gtest/internal/gtest-port.h](
-../googletest/include/gtest/internal/gtest-port.h).
-
-### Choosing a TR1 Tuple Library ###
-
-Google Mock uses the C++ Technical Report 1 (TR1) tuple library
-heavily.  Unfortunately TR1 tuple is not yet widely available with all
-compilers.  The good news is that Google Test 1.4.0+ implements a
-subset of TR1 tuple that's enough for Google Mock's need.  Google Mock
-will automatically use that implementation when the compiler doesn't
-provide TR1 tuple.
-
-Usually you don't need to care about which tuple library Google Test
-and Google Mock use.  However, if your project already uses TR1 tuple,
-you need to tell Google Test and Google Mock to use the same TR1 tuple
-library the rest of your project uses, or the two tuple
-implementations will clash.  To do that, add
-
-    -DGTEST_USE_OWN_TR1_TUPLE=0
-
-to the compiler flags while compiling Google Test, Google Mock, and
-your tests.  If you want to force Google Test and Google Mock to use
-their own tuple library, just add
-
-    -DGTEST_USE_OWN_TR1_TUPLE=1
-
-to the compiler flags instead.
-
-If you want to use Boost's TR1 tuple library with Google Mock, please
-refer to the Boost website (http://www.boost.org/) for how to obtain
-it and set it up.
-
-### As a Shared Library (DLL) ###
-
-Google Mock is compact, so most users can build and link it as a static
-library for the simplicity.  Google Mock can be used as a DLL, but the
-same DLL must contain Google Test as well.  See
-[Google Test's README][gtest_readme]
-for instructions on how to set up necessary compiler settings.
-
-### Tweaking Google Mock ###
-
-Most of Google Test's control macros apply to Google Mock as well.
-Please see [Google Test's README][gtest_readme] for how to tweak them.
-
-### Upgrading from an Earlier Version ###
-
-We strive to keep Google Mock releases backward compatible.
-Sometimes, though, we have to make some breaking changes for the
-users' long-term benefits.  This section describes what you'll need to
-do if you are upgrading from an earlier version of Google Mock.
-
-#### Upgrading from 1.1.0 or Earlier ####
-
-You may need to explicitly enable or disable Google Test's own TR1
-tuple library.  See the instructions in section "[Choosing a TR1 Tuple
-Library](../googletest/#choosing-a-tr1-tuple-library)".
-
-#### Upgrading from 1.4.0 or Earlier ####
-
-On platforms where the pthread library is available, Google Test and
-Google Mock use it in order to be thread-safe.  For this to work, you
-may need to tweak your compiler and/or linker flags.  Please see the
-"[Multi-threaded Tests](../googletest#multi-threaded-tests
-)" section in file Google Test's README for what you may need to do.
-
-If you have custom matchers defined using `MatcherInterface` or
-`MakePolymorphicMatcher()`, you'll need to update their definitions to
-use the new matcher API (
-[monomorphic](./docs/CookBook.md#writing-new-monomorphic-matchers),
-[polymorphic](./docs/CookBook.md#writing-new-polymorphic-matchers)).
-Matchers defined using `MATCHER()` or `MATCHER_P*()` aren't affected.
-
-Happy testing!
-
-[gtest_readme]: ../googletest/README.md "googletest"
-[gtest_cmakebuild]:  ../googletest/README.md#using-cmake "Using CMake"
-[gtest_incorpcmake]: ../googletest/README.md#incorporating-into-an-existing-cmake-project "Incorporating Into An Existing CMake Project"
+*   [jMock](http://www.jmock.org/)
+*   [EasyMock](http://www.easymock.org/)
+*   [Hamcrest](http://code.google.com/p/hamcrest/)
+
+It is designed with C++'s specifics in mind.
+
+gMock:
+
+-   Provides a declarative syntax for defining mocks.
+-   Can define partial (hybrid) mocks, which are a cross of real and mock
+    objects.
+-   Handles functions of arbitrary types and overloaded functions.
+-   Comes with a rich set of matchers for validating function arguments.
+-   Uses an intuitive syntax for controlling the behavior of a mock.
+-   Does automatic verification of expectations (no record-and-replay needed).
+-   Allows arbitrary (partial) ordering constraints on function calls to be
+    expressed.
+-   Lets a user extend it by defining new matchers and actions.
+-   Does not use exceptions.
+-   Is easy to learn and use.
+
+Details and examples can be found here:
+
+*   [gMock for Dummies](docs/for_dummies.md)
+*   [Legacy gMock FAQ](docs/gmock_faq.md)
+*   [gMock Cookbook](docs/cook_book.md)
+*   [gMock Cheat Sheet](docs/cheat_sheet.md)
+
+Please note that code under scripts/generator/ is from the [cppclean
+project](http://code.google.com/p/cppclean/) and under the Apache
+License, which is different from Google Mock's license.
+
+Google Mock is a part of
+[Google Test C++ testing framework](http://github.com/google/googletest/) and a
+subject to the same requirements.
diff --git a/thirdparty/GTest/googletest/googlemock/build-aux/.keep b/thirdparty/GTest/googletest/googlemock/build-aux/.keep
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/thirdparty/GTest/googletest/googlemock/cmake/gmock.pc.in b/thirdparty/GTest/googletest/googlemock/cmake/gmock.pc.in
index c44164264a..23c67b5c88 100644
--- a/thirdparty/GTest/googletest/googlemock/cmake/gmock.pc.in
+++ b/thirdparty/GTest/googletest/googlemock/cmake/gmock.pc.in
@@ -5,5 +5,6 @@ Name: gmock
 Description: GoogleMock (without main() function)
 Version: @PROJECT_VERSION@
 URL: https://github.com/google/googletest
+Requires: gtest = @PROJECT_VERSION@
 Libs: -L${libdir} -lgmock @CMAKE_THREAD_LIBS_INIT@
-Cflags: -I${includedir} @GTEST_HAS_PTHREAD_MACRO@ @CMAKE_THREAD_LIBS_INIT@
+Cflags: -I${includedir} @GTEST_HAS_PTHREAD_MACRO@
diff --git a/thirdparty/GTest/googletest/googlemock/cmake/gmock_main.pc.in b/thirdparty/GTest/googletest/googlemock/cmake/gmock_main.pc.in
index c377dba1e5..66ffea7f44 100644
--- a/thirdparty/GTest/googletest/googlemock/cmake/gmock_main.pc.in
+++ b/thirdparty/GTest/googletest/googlemock/cmake/gmock_main.pc.in
@@ -5,5 +5,6 @@ Name: gmock_main
 Description: GoogleMock (with main() function)
 Version: @PROJECT_VERSION@
 URL: https://github.com/google/googletest
+Requires: gmock = @PROJECT_VERSION@
 Libs: -L${libdir} -lgmock_main @CMAKE_THREAD_LIBS_INIT@
-Cflags: -I${includedir} @GTEST_HAS_PTHREAD_MACRO@ @CMAKE_THREAD_LIBS_INIT@
+Cflags: -I${includedir} @GTEST_HAS_PTHREAD_MACRO@
diff --git a/thirdparty/GTest/googletest/googlemock/configure.ac b/thirdparty/GTest/googletest/googlemock/configure.ac
deleted file mode 100644
index cb5e1a6a8c..0000000000
--- a/thirdparty/GTest/googletest/googlemock/configure.ac
+++ /dev/null
@@ -1,146 +0,0 @@
-m4_include(../googletest/m4/acx_pthread.m4)
-
-AC_INIT([Google C++ Mocking Framework],
-        [1.8.0],
-        [googlemock@googlegroups.com],
-        [gmock])
-
-# Provide various options to initialize the Autoconf and configure processes.
-AC_PREREQ([2.59])
-AC_CONFIG_SRCDIR([./LICENSE])
-AC_CONFIG_AUX_DIR([build-aux])
-AC_CONFIG_HEADERS([build-aux/config.h])
-AC_CONFIG_FILES([Makefile])
-AC_CONFIG_FILES([scripts/gmock-config], [chmod +x scripts/gmock-config])
-
-# Initialize Automake with various options. We require at least v1.9, prevent
-# pedantic complaints about package files, and enable various distribution
-# targets.
-AM_INIT_AUTOMAKE([1.9 dist-bzip2 dist-zip foreign subdir-objects])
-
-# Check for programs used in building Google Test.
-AC_PROG_CC
-AC_PROG_CXX
-AC_LANG([C++])
-AC_PROG_LIBTOOL
-
-# TODO(chandlerc@google.com): Currently we aren't running the Python tests
-# against the interpreter detected by AM_PATH_PYTHON, and so we condition
-# HAVE_PYTHON by requiring "python" to be in the PATH, and that interpreter's
-# version to be >= 2.3. This will allow the scripts to use a "/usr/bin/env"
-# hashbang.
-PYTHON=  # We *do not* allow the user to specify a python interpreter
-AC_PATH_PROG([PYTHON],[python],[:])
-AS_IF([test "$PYTHON" != ":"],
-      [AM_PYTHON_CHECK_VERSION([$PYTHON],[2.3],[:],[PYTHON=":"])])
-AM_CONDITIONAL([HAVE_PYTHON],[test "$PYTHON" != ":"])
-
-# TODO(chandlerc@google.com) Check for the necessary system headers.
-
-# Configure pthreads.
-AC_ARG_WITH([pthreads],
-            [AS_HELP_STRING([--with-pthreads],
-               [use pthreads (default is yes)])],
-            [with_pthreads=$withval],
-            [with_pthreads=check])
-
-have_pthreads=no
-AS_IF([test "x$with_pthreads" != "xno"],
-      [ACX_PTHREAD(
-        [],
-        [AS_IF([test "x$with_pthreads" != "xcheck"],
-               [AC_MSG_FAILURE(
-                 [--with-pthreads was specified, but unable to be used])])])
-       have_pthreads="$acx_pthread_ok"])
-AM_CONDITIONAL([HAVE_PTHREADS],[test "x$have_pthreads" == "xyes"])
-AC_SUBST(PTHREAD_CFLAGS)
-AC_SUBST(PTHREAD_LIBS)
-
-# GoogleMock currently has hard dependencies upon GoogleTest above and beyond
-# running its own test suite, so we both provide our own version in
-# a subdirectory and provide some logic to use a custom version or a system
-# installed version.
-AC_ARG_WITH([gtest],
-            [AS_HELP_STRING([--with-gtest],
-                            [Specifies how to find the gtest package. If no
-                            arguments are given, the default behavior, a
-                            system installed gtest will be used if present,
-                            and an internal version built otherwise. If a
-                            path is provided, the gtest built or installed at
-                            that prefix will be used.])],
-            [],
-            [with_gtest=yes])
-AC_ARG_ENABLE([external-gtest],
-              [AS_HELP_STRING([--disable-external-gtest],
-                              [Disables any detection or use of a system
-                              installed or user provided gtest. Any option to
-                              '--with-gtest' is ignored. (Default is enabled.)])
-              ], [], [enable_external_gtest=yes])
-AS_IF([test "x$with_gtest" == "xno"],
-      [AC_MSG_ERROR([dnl
-Support for GoogleTest was explicitly disabled. Currently GoogleMock has a hard
-dependency upon GoogleTest to build, please provide a version, or allow
-GoogleMock to use any installed version and fall back upon its internal
-version.])])
-
-# Setup various GTEST variables. TODO(chandlerc@google.com): When these are
-# used below, they should be used such that any pre-existing values always
-# trump values we set them to, so that they can be used to selectively override
-# details of the detection process.
-AC_ARG_VAR([GTEST_CONFIG],
-           [The exact path of Google Test's 'gtest-config' script.])
-AC_ARG_VAR([GTEST_CPPFLAGS],
-           [C-like preprocessor flags for Google Test.])
-AC_ARG_VAR([GTEST_CXXFLAGS],
-           [C++ compile flags for Google Test.])
-AC_ARG_VAR([GTEST_LDFLAGS],
-           [Linker path and option flags for Google Test.])
-AC_ARG_VAR([GTEST_LIBS],
-           [Library linking flags for Google Test.])
-AC_ARG_VAR([GTEST_VERSION],
-           [The version of Google Test available.])
-HAVE_BUILT_GTEST="no"
-
-GTEST_MIN_VERSION="1.8.0"
-
-AS_IF([test "x${enable_external_gtest}" = "xyes"],
-      [# Begin filling in variables as we are able.
-      AS_IF([test "x${with_gtest}" != "xyes"],
-            [AS_IF([test -x "${with_gtest}/scripts/gtest-config"],
-                   [GTEST_CONFIG="${with_gtest}/scripts/gtest-config"],
-                   [GTEST_CONFIG="${with_gtest}/bin/gtest-config"])
-            AS_IF([test -x "${GTEST_CONFIG}"], [],
-                  [AC_MSG_ERROR([dnl
-Unable to locate either a built or installed Google Test at '${with_gtest}'.])
-                  ])])
-
-      AS_IF([test -x "${GTEST_CONFIG}"], [],
-            [AC_PATH_PROG([GTEST_CONFIG], [gtest-config])])
-      AS_IF([test -x "${GTEST_CONFIG}"],
-            [AC_MSG_CHECKING([for Google Test version >= ${GTEST_MIN_VERSION}])
-            AS_IF([${GTEST_CONFIG} --min-version=${GTEST_MIN_VERSION}],
-                  [AC_MSG_RESULT([yes])
-                  HAVE_BUILT_GTEST="yes"],
-                  [AC_MSG_RESULT([no])])])])
-
-AS_IF([test "x${HAVE_BUILT_GTEST}" = "xyes"],
-      [GTEST_CPPFLAGS=`${GTEST_CONFIG} --cppflags`
-      GTEST_CXXFLAGS=`${GTEST_CONFIG} --cxxflags`
-      GTEST_LDFLAGS=`${GTEST_CONFIG} --ldflags`
-      GTEST_LIBS=`${GTEST_CONFIG} --libs`
-      GTEST_VERSION=`${GTEST_CONFIG} --version`],
-      [
-      # GTEST_CONFIG needs to be executable both in a Makefile environment and
-      # in a shell script environment, so resolve an absolute path for it here.
-      GTEST_CONFIG="`pwd -P`/../googletest/scripts/gtest-config"
-      GTEST_CPPFLAGS='-I$(top_srcdir)/../googletest/include'
-      GTEST_CXXFLAGS='-g'
-      GTEST_LDFLAGS=''
-      GTEST_LIBS='$(top_builddir)/../googletest/lib/libgtest.la'
-      GTEST_VERSION="${GTEST_MIN_VERSION}"])
-
-# TODO(chandlerc@google.com) Check the types, structures, and other compiler
-# and architecture characteristics.
-
-# Output the generated files. No further autoconf macros may be used.
-AC_OUTPUT
diff --git a/thirdparty/GTest/googletest/googlemock/docs/CheatSheet.md b/thirdparty/GTest/googletest/googlemock/docs/CheatSheet.md
deleted file mode 100644
index f8bbbfe6d0..0000000000
--- a/thirdparty/GTest/googletest/googlemock/docs/CheatSheet.md
+++ /dev/null
@@ -1,564 +0,0 @@
-
-
-# Defining a Mock Class #
-
-## Mocking a Normal Class ##
-
-Given
-```
-class Foo {
-  ...
-  virtual ~Foo();
-  virtual int GetSize() const = 0;
-  virtual string Describe(const char* name) = 0;
-  virtual string Describe(int type) = 0;
-  virtual bool Process(Bar elem, int count) = 0;
-};
-```
-(note that `~Foo()` **must** be virtual) we can define its mock as
-```
-#include "gmock/gmock.h"
-
-class MockFoo : public Foo {
-  MOCK_CONST_METHOD0(GetSize, int());
-  MOCK_METHOD1(Describe, string(const char* name));
-  MOCK_METHOD1(Describe, string(int type));
-  MOCK_METHOD2(Process, bool(Bar elem, int count));
-};
-```
-
-To create a "nice" mock object which ignores all uninteresting calls,
-or a "strict" mock object, which treats them as failures:
-```
-NiceMock nice_foo;     // The type is a subclass of MockFoo.
-StrictMock strict_foo; // The type is a subclass of MockFoo.
-```
-
-## Mocking a Class Template ##
-
-To mock
-```
-template 
-class StackInterface {
- public:
-  ...
-  virtual ~StackInterface();
-  virtual int GetSize() const = 0;
-  virtual void Push(const Elem& x) = 0;
-};
-```
-(note that `~StackInterface()` **must** be virtual) just append `_T` to the `MOCK_*` macros:
-```
-template 
-class MockStack : public StackInterface {
- public:
-  ...
-  MOCK_CONST_METHOD0_T(GetSize, int());
-  MOCK_METHOD1_T(Push, void(const Elem& x));
-};
-```
-
-## Specifying Calling Conventions for Mock Functions ##
-
-If your mock function doesn't use the default calling convention, you
-can specify it by appending `_WITH_CALLTYPE` to any of the macros
-described in the previous two sections and supplying the calling
-convention as the first argument to the macro. For example,
-```
-  MOCK_METHOD1_WITH_CALLTYPE(STDMETHODCALLTYPE, Foo, bool(int n));
-  MOCK_CONST_METHOD2_WITH_CALLTYPE(STDMETHODCALLTYPE, Bar, int(double x, double y));
-```
-where `STDMETHODCALLTYPE` is defined by `` on Windows.
-
-# Using Mocks in Tests #
-
-The typical flow is:
-  1. Import the Google Mock names you need to use. All Google Mock names are in the `testing` namespace unless they are macros or otherwise noted.
-  1. Create the mock objects.
-  1. Optionally, set the default actions of the mock objects.
-  1. Set your expectations on the mock objects (How will they be called? What wil they do?).
-  1. Exercise code that uses the mock objects; if necessary, check the result using [Google Test](../../googletest/) assertions.
-  1. When a mock objects is destructed, Google Mock automatically verifies that all expectations on it have been satisfied.
-
-Here is an example:
-```
-using ::testing::Return;                            // #1
-
-TEST(BarTest, DoesThis) {
-  MockFoo foo;                                    // #2
-
-  ON_CALL(foo, GetSize())                         // #3
-      .WillByDefault(Return(1));
-  // ... other default actions ...
-
-  EXPECT_CALL(foo, Describe(5))                   // #4
-      .Times(3)
-      .WillRepeatedly(Return("Category 5"));
-  // ... other expectations ...
-
-  EXPECT_EQ("good", MyProductionFunction(&foo));  // #5
-}                                                 // #6
-```
-
-# Setting Default Actions #
-
-Google Mock has a **built-in default action** for any function that
-returns `void`, `bool`, a numeric value, or a pointer.
-
-To customize the default action for functions with return type `T` globally:
-```
-using ::testing::DefaultValue;
-
-// Sets the default value to be returned. T must be CopyConstructible.
-DefaultValue::Set(value);
-// Sets a factory. Will be invoked on demand. T must be MoveConstructible.
-//   T MakeT();
-DefaultValue::SetFactory(&MakeT);
-// ... use the mocks ...
-// Resets the default value.
-DefaultValue::Clear();
-```
-
-To customize the default action for a particular method, use `ON_CALL()`:
-```
-ON_CALL(mock_object, method(matchers))
-    .With(multi_argument_matcher)  ?
-    .WillByDefault(action);
-```
-
-# Setting Expectations #
-
-`EXPECT_CALL()` sets **expectations** on a mock method (How will it be
-called? What will it do?):
-```
-EXPECT_CALL(mock_object, method(matchers))
-    .With(multi_argument_matcher)  ?
-    .Times(cardinality)            ?
-    .InSequence(sequences)         *
-    .After(expectations)           *
-    .WillOnce(action)              *
-    .WillRepeatedly(action)        ?
-    .RetiresOnSaturation();        ?
-```
-
-If `Times()` is omitted, the cardinality is assumed to be:
-
-  * `Times(1)` when there is neither `WillOnce()` nor `WillRepeatedly()`;
-  * `Times(n)` when there are `n WillOnce()`s but no `WillRepeatedly()`, where `n` >= 1; or
-  * `Times(AtLeast(n))` when there are `n WillOnce()`s and a `WillRepeatedly()`, where `n` >= 0.
-
-A method with no `EXPECT_CALL()` is free to be invoked _any number of times_, and the default action will be taken each time.
-
-# Matchers #
-
-A **matcher** matches a _single_ argument.  You can use it inside
-`ON_CALL()` or `EXPECT_CALL()`, or use it to validate a value
-directly:
-
-| `EXPECT_THAT(value, matcher)` | Asserts that `value` matches `matcher`. |
-|:------------------------------|:----------------------------------------|
-| `ASSERT_THAT(value, matcher)` | The same as `EXPECT_THAT(value, matcher)`, except that it generates a **fatal** failure. |
-
-Built-in matchers (where `argument` is the function argument) are
-divided into several categories:
-
-## Wildcard ##
-|`_`|`argument` can be any value of the correct type.|
-|:--|:-----------------------------------------------|
-|`A()` or `An()`|`argument` can be any value of type `type`.     |
-
-## Generic Comparison ##
-
-|`Eq(value)` or `value`|`argument == value`|
-|:---------------------|:------------------|
-|`Ge(value)`           |`argument >= value`|
-|`Gt(value)`           |`argument > value` |
-|`Le(value)`           |`argument <= value`|
-|`Lt(value)`           |`argument < value` |
-|`Ne(value)`           |`argument != value`|
-|`IsNull()`            |`argument` is a `NULL` pointer (raw or smart).|
-|`NotNull()`           |`argument` is a non-null pointer (raw or smart).|
-|`VariantWith(m)`   |`argument` is `variant<>` that holds the alternative of
-type T with a value matching `m`.|
-|`Ref(variable)`       |`argument` is a reference to `variable`.|
-|`TypedEq(value)`|`argument` has type `type` and is equal to `value`. You may need to use this instead of `Eq(value)` when the mock function is overloaded.|
-
-Except `Ref()`, these matchers make a _copy_ of `value` in case it's
-modified or destructed later. If the compiler complains that `value`
-doesn't have a public copy constructor, try wrap it in `ByRef()`,
-e.g. `Eq(ByRef(non_copyable_value))`. If you do that, make sure
-`non_copyable_value` is not changed afterwards, or the meaning of your
-matcher will be changed.
-
-## Floating-Point Matchers ##
-
-|`DoubleEq(a_double)`|`argument` is a `double` value approximately equal to `a_double`, treating two NaNs as unequal.|
-|:-------------------|:----------------------------------------------------------------------------------------------|
-|`FloatEq(a_float)`  |`argument` is a `float` value approximately equal to `a_float`, treating two NaNs as unequal.  |
-|`NanSensitiveDoubleEq(a_double)`|`argument` is a `double` value approximately equal to `a_double`, treating two NaNs as equal.  |
-|`NanSensitiveFloatEq(a_float)`|`argument` is a `float` value approximately equal to `a_float`, treating two NaNs as equal.    |
-
-The above matchers use ULP-based comparison (the same as used in
-[Google Test](../../googletest/)). They
-automatically pick a reasonable error bound based on the absolute
-value of the expected value.  `DoubleEq()` and `FloatEq()` conform to
-the IEEE standard, which requires comparing two NaNs for equality to
-return false. The `NanSensitive*` version instead treats two NaNs as
-equal, which is often what a user wants.
-
-|`DoubleNear(a_double, max_abs_error)`|`argument` is a `double` value close to `a_double` (absolute error <= `max_abs_error`), treating two NaNs as unequal.|
-|:------------------------------------|:--------------------------------------------------------------------------------------------------------------------|
-|`FloatNear(a_float, max_abs_error)`  |`argument` is a `float` value close to `a_float` (absolute error <= `max_abs_error`), treating two NaNs as unequal.  |
-|`NanSensitiveDoubleNear(a_double, max_abs_error)`|`argument` is a `double` value close to `a_double` (absolute error <= `max_abs_error`), treating two NaNs as equal.  |
-|`NanSensitiveFloatNear(a_float, max_abs_error)`|`argument` is a `float` value close to `a_float` (absolute error <= `max_abs_error`), treating two NaNs as equal.    |
-
-## String Matchers ##
-
-The `argument` can be either a C string or a C++ string object:
-
-|`ContainsRegex(string)`|`argument` matches the given regular expression.|
-|:----------------------|:-----------------------------------------------|
-|`EndsWith(suffix)`     |`argument` ends with string `suffix`.           |
-|`HasSubstr(string)`    |`argument` contains `string` as a sub-string.   |
-|`MatchesRegex(string)` |`argument` matches the given regular expression with the match starting at the first character and ending at the last character.|
-|`StartsWith(prefix)`   |`argument` starts with string `prefix`.         |
-|`StrCaseEq(string)`    |`argument` is equal to `string`, ignoring case. |
-|`StrCaseNe(string)`    |`argument` is not equal to `string`, ignoring case.|
-|`StrEq(string)`        |`argument` is equal to `string`.                |
-|`StrNe(string)`        |`argument` is not equal to `string`.            |
-
-`ContainsRegex()` and `MatchesRegex()` use the regular expression
-syntax defined
-[here](../../googletest/docs/AdvancedGuide.md#regular-expression-syntax).
-`StrCaseEq()`, `StrCaseNe()`, `StrEq()`, and `StrNe()` work for wide
-strings as well.
-
-## Container Matchers ##
-
-Most STL-style containers support `==`, so you can use
-`Eq(expected_container)` or simply `expected_container` to match a
-container exactly.   If you want to write the elements in-line,
-match them more flexibly, or get more informative messages, you can use:
-
-| `ContainerEq(container)` | The same as `Eq(container)` except that the failure message also includes which elements are in one container but not the other. |
-|:-------------------------|:---------------------------------------------------------------------------------------------------------------------------------|
-| `Contains(e)`            | `argument` contains an element that matches `e`, which can be either a value or a matcher.                                       |
-| `Each(e)`                | `argument` is a container where _every_ element matches `e`, which can be either a value or a matcher.                           |
-| `ElementsAre(e0, e1, ..., en)` | `argument` has `n + 1` elements, where the i-th element matches `ei`, which can be a value or a matcher. 0 to 10 arguments are allowed. |
-| `ElementsAreArray({ e0, e1, ..., en })`, `ElementsAreArray(array)`, or `ElementsAreArray(array, count)` | The same as `ElementsAre()` except that the expected element values/matchers come from an initializer list, STL-style container, or C-style array. |
-| `IsEmpty()`              | `argument` is an empty container (`container.empty()`).                                                                          |
-| `Pointwise(m, container)` | `argument` contains the same number of elements as in `container`, and for all i, (the i-th element in `argument`, the i-th element in `container`) match `m`, which is a matcher on 2-tuples. E.g. `Pointwise(Le(), upper_bounds)` verifies that each element in `argument` doesn't exceed the corresponding element in `upper_bounds`. See more detail below. |
-| `SizeIs(m)`              | `argument` is a container whose size matches `m`. E.g. `SizeIs(2)` or `SizeIs(Lt(2))`.                                           |
-| `UnorderedElementsAre(e0, e1, ..., en)` | `argument` has `n + 1` elements, and under some permutation each element matches an `ei` (for a different `i`), which can be a value or a matcher. 0 to 10 arguments are allowed. |
-| `UnorderedElementsAreArray({ e0, e1, ..., en })`, `UnorderedElementsAreArray(array)`, or `UnorderedElementsAreArray(array, count)` | The same as `UnorderedElementsAre()` except that the expected element values/matchers come from an initializer list, STL-style container, or C-style array. |
-| `WhenSorted(m)`          | When `argument` is sorted using the `<` operator, it matches container matcher `m`. E.g. `WhenSorted(ElementsAre(1, 2, 3))` verifies that `argument` contains elements `1`, `2`, and `3`, ignoring order. |
-| `WhenSortedBy(comparator, m)` | The same as `WhenSorted(m)`, except that the given comparator instead of `<` is used to sort `argument`. E.g. `WhenSortedBy(std::greater(), ElementsAre(3, 2, 1))`. |
-
-Notes:
-
-  * These matchers can also match:
-    1. a native array passed by reference (e.g. in `Foo(const int (&a)[5])`), and
-    1. an array passed as a pointer and a count (e.g. in `Bar(const T* buffer, int len)` -- see [Multi-argument Matchers](#Multiargument_Matchers.md)).
-  * The array being matched may be multi-dimensional (i.e. its elements can be arrays).
-  * `m` in `Pointwise(m, ...)` should be a matcher for `::testing::tuple` where `T` and `U` are the element type of the actual container and the expected container, respectively. For example, to compare two `Foo` containers where `Foo` doesn't support `operator==` but has an `Equals()` method, one might write:
-
-```
-using ::testing::get;
-MATCHER(FooEq, "") {
-  return get<0>(arg).Equals(get<1>(arg));
-}
-...
-EXPECT_THAT(actual_foos, Pointwise(FooEq(), expected_foos));
-```
-
-## Member Matchers ##
-
-|`Field(&class::field, m)`|`argument.field` (or `argument->field` when `argument` is a plain pointer) matches matcher `m`, where `argument` is an object of type _class_.|
-|:------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------|
-|`Key(e)`                 |`argument.first` matches `e`, which can be either a value or a matcher. E.g. `Contains(Key(Le(5)))` can verify that a `map` contains a key `<= 5`.|
-|`Pair(m1, m2)`           |`argument` is an `std::pair` whose `first` field matches `m1` and `second` field matches `m2`.                                                |
-|`Property(&class::property, m)`|`argument.property()` (or `argument->property()` when `argument` is a plain pointer) matches matcher `m`, where `argument` is an object of type _class_.|
-
-## Matching the Result of a Function or Functor ##
-
-|`ResultOf(f, m)`|`f(argument)` matches matcher `m`, where `f` is a function or functor.|
-|:---------------|:---------------------------------------------------------------------|
-
-## Pointer Matchers ##
-
-|`Pointee(m)`|`argument` (either a smart pointer or a raw pointer) points to a value that matches matcher `m`.|
-|:-----------|:-----------------------------------------------------------------------------------------------|
-|`WhenDynamicCastTo(m)`| when `argument` is passed through `dynamic_cast()`, it matches matcher `m`.                 |
-
-## Multiargument Matchers ##
-
-Technically, all matchers match a _single_ value. A "multi-argument"
-matcher is just one that matches a _tuple_. The following matchers can
-be used to match a tuple `(x, y)`:
-
-|`Eq()`|`x == y`|
-|:-----|:-------|
-|`Ge()`|`x >= y`|
-|`Gt()`|`x > y` |
-|`Le()`|`x <= y`|
-|`Lt()`|`x < y` |
-|`Ne()`|`x != y`|
-
-You can use the following selectors to pick a subset of the arguments
-(or reorder them) to participate in the matching:
-
-|`AllArgs(m)`|Equivalent to `m`. Useful as syntactic sugar in `.With(AllArgs(m))`.|
-|:-----------|:-------------------------------------------------------------------|
-|`Args(m)`|The tuple of the `k` selected (using 0-based indices) arguments matches `m`, e.g. `Args<1, 2>(Eq())`.|
-
-## Composite Matchers ##
-
-You can make a matcher from one or more other matchers:
-
-|`AllOf(m1, m2, ..., mn)`|`argument` matches all of the matchers `m1` to `mn`.|
-|:-----------------------|:---------------------------------------------------|
-|`AnyOf(m1, m2, ..., mn)`|`argument` matches at least one of the matchers `m1` to `mn`.|
-|`Not(m)`                |`argument` doesn't match matcher `m`.               |
-
-## Adapters for Matchers ##
-
-|`MatcherCast(m)`|casts matcher `m` to type `Matcher`.|
-|:------------------|:--------------------------------------|
-|`SafeMatcherCast(m)`| [safely casts](CookBook.md#casting-matchers) matcher `m` to type `Matcher`. |
-|`Truly(predicate)` |`predicate(argument)` returns something considered by C++ to be true, where `predicate` is a function or functor.|
-
-## Matchers as Predicates ##
-
-|`Matches(m)(value)`|evaluates to `true` if `value` matches `m`. You can use `Matches(m)` alone as a unary functor.|
-|:------------------|:---------------------------------------------------------------------------------------------|
-|`ExplainMatchResult(m, value, result_listener)`|evaluates to `true` if `value` matches `m`, explaining the result to `result_listener`.       |
-|`Value(value, m)`  |evaluates to `true` if `value` matches `m`.                                                   |
-
-## Defining Matchers ##
-
-| `MATCHER(IsEven, "") { return (arg % 2) == 0; }` | Defines a matcher `IsEven()` to match an even number. |
-|:-------------------------------------------------|:------------------------------------------------------|
-| `MATCHER_P(IsDivisibleBy, n, "") { *result_listener << "where the remainder is " << (arg % n); return (arg % n) == 0; }` | Defines a macher `IsDivisibleBy(n)` to match a number divisible by `n`. |
-| `MATCHER_P2(IsBetween, a, b, std::string(negation ? "isn't" : "is") + " between " + PrintToString(a) + " and " + PrintToString(b)) { return a <= arg && arg <= b; }` | Defines a matcher `IsBetween(a, b)` to match a value in the range [`a`, `b`]. |
-
-**Notes:**
-
-  1. The `MATCHER*` macros cannot be used inside a function or class.
-  1. The matcher body must be _purely functional_ (i.e. it cannot have any side effect, and the result must not depend on anything other than the value being matched and the matcher parameters).
-  1. You can use `PrintToString(x)` to convert a value `x` of any type to a string.
-
-## Matchers as Test Assertions ##
-
-|`ASSERT_THAT(expression, m)`|Generates a [fatal failure](../../googletest/docs/Primer.md#assertions) if the value of `expression` doesn't match matcher `m`.|
-|:---------------------------|:----------------------------------------------------------------------------------------------------------------------------------------------|
-|`EXPECT_THAT(expression, m)`|Generates a non-fatal failure if the value of `expression` doesn't match matcher `m`.                                                          |
-
-# Actions #
-
-**Actions** specify what a mock function should do when invoked.
-
-## Returning a Value ##
-
-|`Return()`|Return from a `void` mock function.|
-|:---------|:----------------------------------|
-|`Return(value)`|Return `value`. If the type of `value` is different to the mock function's return type, `value` is converted to the latter type at the time the expectation is set, not when the action is executed.|
-|`ReturnArg()`|Return the `N`-th (0-based) argument.|
-|`ReturnNew(a1, ..., ak)`|Return `new T(a1, ..., ak)`; a different object is created each time.|
-|`ReturnNull()`|Return a null pointer.             |
-|`ReturnPointee(ptr)`|Return the value pointed to by `ptr`.|
-|`ReturnRef(variable)`|Return a reference to `variable`.  |
-|`ReturnRefOfCopy(value)`|Return a reference to a copy of `value`; the copy lives as long as the action.|
-
-## Side Effects ##
-
-|`Assign(&variable, value)`|Assign `value` to variable.|
-|:-------------------------|:--------------------------|
-| `DeleteArg()`         | Delete the `N`-th (0-based) argument, which must be a pointer. |
-| `SaveArg(pointer)`    | Save the `N`-th (0-based) argument to `*pointer`. |
-| `SaveArgPointee(pointer)` | Save the value pointed to by the `N`-th (0-based) argument to `*pointer`. |
-| `SetArgReferee(value)` |	Assign value to the variable referenced by the `N`-th (0-based) argument. |
-|`SetArgPointee(value)` |Assign `value` to the variable pointed by the `N`-th (0-based) argument.|
-|`SetArgumentPointee(value)`|Same as `SetArgPointee(value)`. Deprecated. Will be removed in v1.7.0.|
-|`SetArrayArgument(first, last)`|Copies the elements in source range [`first`, `last`) to the array pointed to by the `N`-th (0-based) argument, which can be either a pointer or an iterator. The action does not take ownership of the elements in the source range.|
-|`SetErrnoAndReturn(error, value)`|Set `errno` to `error` and return `value`.|
-|`Throw(exception)`        |Throws the given exception, which can be any copyable value. Available since v1.1.0.|
-
-## Using a Function or a Functor as an Action ##
-
-|`Invoke(f)`|Invoke `f` with the arguments passed to the mock function, where `f` can be a global/static function or a functor.|
-|:----------|:-----------------------------------------------------------------------------------------------------------------|
-|`Invoke(object_pointer, &class::method)`|Invoke the {method on the object with the arguments passed to the mock function.                                  |
-|`InvokeWithoutArgs(f)`|Invoke `f`, which can be a global/static function or a functor. `f` must take no arguments.                       |
-|`InvokeWithoutArgs(object_pointer, &class::method)`|Invoke the method on the object, which takes no arguments.                                                        |
-|`InvokeArgument(arg1, arg2, ..., argk)`|Invoke the mock function's `N`-th (0-based) argument, which must be a function or a functor, with the `k` arguments.|
-
-The return value of the invoked function is used as the return value
-of the action.
-
-When defining a function or functor to be used with `Invoke*()`, you can declare any unused parameters as `Unused`:
-```
-  double Distance(Unused, double x, double y) { return sqrt(x*x + y*y); }
-  ...
-  EXPECT_CALL(mock, Foo("Hi", _, _)).WillOnce(Invoke(Distance));
-```
-
-In `InvokeArgument(...)`, if an argument needs to be passed by reference, wrap it inside `ByRef()`. For example,
-```
-  InvokeArgument<2>(5, string("Hi"), ByRef(foo))
-```
-calls the mock function's #2 argument, passing to it `5` and `string("Hi")` by value, and `foo` by reference.
-
-## Default Action ##
-
-|`DoDefault()`|Do the default action (specified by `ON_CALL()` or the built-in one).|
-|:------------|:--------------------------------------------------------------------|
-
-**Note:** due to technical reasons, `DoDefault()` cannot be used inside  a composite action - trying to do so will result in a run-time error.
-
-## Composite Actions ##
-
-|`DoAll(a1, a2, ..., an)`|Do all actions `a1` to `an` and return the result of `an` in each invocation. The first `n - 1` sub-actions must return void. |
-|:-----------------------|:-----------------------------------------------------------------------------------------------------------------------------|
-|`IgnoreResult(a)`       |Perform action `a` and ignore its result. `a` must not return void.                                                           |
-|`WithArg(a)`         |Pass the `N`-th (0-based) argument of the mock function to action `a` and perform it.                                         |
-|`WithArgs(a)`|Pass the selected (0-based) arguments of the mock function to action `a` and perform it.                                      |
-|`WithoutArgs(a)`        |Perform action `a` without any arguments.                                                                                     |
-
-## Defining Actions ##
-
-| `ACTION(Sum) { return arg0 + arg1; }` | Defines an action `Sum()` to return the sum of the mock function's argument #0 and #1. |
-|:--------------------------------------|:---------------------------------------------------------------------------------------|
-| `ACTION_P(Plus, n) { return arg0 + n; }` | Defines an action `Plus(n)` to return the sum of the mock function's argument #0 and `n`. |
-| `ACTION_Pk(Foo, p1, ..., pk) { statements; }` | Defines a parameterized action `Foo(p1, ..., pk)` to execute the given `statements`.   |
-
-The `ACTION*` macros cannot be used inside a function or class.
-
-# Cardinalities #
-
-These are used in `Times()` to specify how many times a mock function will be called:
-
-|`AnyNumber()`|The function can be called any number of times.|
-|:------------|:----------------------------------------------|
-|`AtLeast(n)` |The call is expected at least `n` times.       |
-|`AtMost(n)`  |The call is expected at most `n` times.        |
-|`Between(m, n)`|The call is expected between `m` and `n` (inclusive) times.|
-|`Exactly(n) or n`|The call is expected exactly `n` times. In particular, the call should never happen when `n` is 0.|
-
-# Expectation Order #
-
-By default, the expectations can be matched in _any_ order.  If some
-or all expectations must be matched in a given order, there are two
-ways to specify it.  They can be used either independently or
-together.
-
-## The After Clause ##
-
-```
-using ::testing::Expectation;
-...
-Expectation init_x = EXPECT_CALL(foo, InitX());
-Expectation init_y = EXPECT_CALL(foo, InitY());
-EXPECT_CALL(foo, Bar())
-    .After(init_x, init_y);
-```
-says that `Bar()` can be called only after both `InitX()` and
-`InitY()` have been called.
-
-If you don't know how many pre-requisites an expectation has when you
-write it, you can use an `ExpectationSet` to collect them:
-
-```
-using ::testing::ExpectationSet;
-...
-ExpectationSet all_inits;
-for (int i = 0; i < element_count; i++) {
-  all_inits += EXPECT_CALL(foo, InitElement(i));
-}
-EXPECT_CALL(foo, Bar())
-    .After(all_inits);
-```
-says that `Bar()` can be called only after all elements have been
-initialized (but we don't care about which elements get initialized
-before the others).
-
-Modifying an `ExpectationSet` after using it in an `.After()` doesn't
-affect the meaning of the `.After()`.
-
-## Sequences ##
-
-When you have a long chain of sequential expectations, it's easier to
-specify the order using **sequences**, which don't require you to given
-each expectation in the chain a different name.  All expected

-calls in the same sequence must occur in the order they are
-specified.
-
-```
-using ::testing::Sequence;
-Sequence s1, s2;
-...
-EXPECT_CALL(foo, Reset())
-    .InSequence(s1, s2)
-    .WillOnce(Return(true));
-EXPECT_CALL(foo, GetSize())
-    .InSequence(s1)
-    .WillOnce(Return(1));
-EXPECT_CALL(foo, Describe(A()))
-    .InSequence(s2)
-    .WillOnce(Return("dummy"));
-```
-says that `Reset()` must be called before _both_ `GetSize()` _and_
-`Describe()`, and the latter two can occur in any order.
-
-To put many expectations in a sequence conveniently:
-```
-using ::testing::InSequence;
-{
-  InSequence dummy;
-
-  EXPECT_CALL(...)...;
-  EXPECT_CALL(...)...;
-  ...
-  EXPECT_CALL(...)...;
-}
-```
-says that all expected calls in the scope of `dummy` must occur in
-strict order. The name `dummy` is irrelevant.)
-
-# Verifying and Resetting a Mock #
-
-Google Mock will verify the expectations on a mock object when it is destructed, or you can do it earlier:
-```
-using ::testing::Mock;
-...
-// Verifies and removes the expectations on mock_obj;
-// returns true iff successful.
-Mock::VerifyAndClearExpectations(&mock_obj);
-...
-// Verifies and removes the expectations on mock_obj;
-// also removes the default actions set by ON_CALL();
-// returns true iff successful.
-Mock::VerifyAndClear(&mock_obj);
-```
-
-You can also tell Google Mock that a mock object can be leaked and doesn't
-need to be verified:
-```
-Mock::AllowLeak(&mock_obj);
-```
-
-# Mock Classes #
-
-Google Mock defines a convenient mock class template
-```
-class MockFunction {
- public:
-  MOCK_METHODn(Call, R(A1, ..., An));
-};
-```
-See this [recipe](CookBook.md#using-check-points) for one application of it.
-
-# Flags #
-
-| `--gmock_catch_leaked_mocks=0` | Don't report leaked mock objects as failures. |
-|:-------------------------------|:----------------------------------------------|
-| `--gmock_verbose=LEVEL`        | Sets the default verbosity level (`info`, `warning`, or `error`) of Google Mock messages. |
diff --git a/thirdparty/GTest/googletest/googlemock/docs/CookBook.md b/thirdparty/GTest/googletest/googlemock/docs/CookBook.md
deleted file mode 100644
index c2565f1e18..0000000000
--- a/thirdparty/GTest/googletest/googlemock/docs/CookBook.md
+++ /dev/null
@@ -1,3679 +0,0 @@
-
-
-You can find recipes for using Google Mock here. If you haven't yet,
-please read the [ForDummies](ForDummies.md) document first to make sure you understand
-the basics.
-
-**Note:** Google Mock lives in the `testing` name space. For
-readability, it is recommended to write `using ::testing::Foo;` once in
-your file before using the name `Foo` defined by Google Mock. We omit
-such `using` statements in this page for brevity, but you should do it
-in your own code.
-
-# Creating Mock Classes #
-
-## Mocking Private or Protected Methods ##
-
-You must always put a mock method definition (`MOCK_METHOD*`) in a
-`public:` section of the mock class, regardless of the method being
-mocked being `public`, `protected`, or `private` in the base class.
-This allows `ON_CALL` and `EXPECT_CALL` to reference the mock function
-from outside of the mock class.  (Yes, C++ allows a subclass to specify
-a different access level than the base class on a virtual function.)
-Example:
-
-```
-class Foo {
- public:
-  ...
-  virtual bool Transform(Gadget* g) = 0;
-
- protected:
-  virtual void Resume();
-
- private:
-  virtual int GetTimeOut();
-};
-
-class MockFoo : public Foo {
- public:
-  ...
-  MOCK_METHOD1(Transform, bool(Gadget* g));
-
-  // The following must be in the public section, even though the
-  // methods are protected or private in the base class.
-  MOCK_METHOD0(Resume, void());
-  MOCK_METHOD0(GetTimeOut, int());
-};
-```
-
-## Mocking Overloaded Methods ##
-
-You can mock overloaded functions as usual. No special attention is required:
-
-```
-class Foo {
-  ...
-
-  // Must be virtual as we'll inherit from Foo.
-  virtual ~Foo();
-
-  // Overloaded on the types and/or numbers of arguments.
-  virtual int Add(Element x);
-  virtual int Add(int times, Element x);
-
-  // Overloaded on the const-ness of this object.
-  virtual Bar& GetBar();
-  virtual const Bar& GetBar() const;
-};
-
-class MockFoo : public Foo {
-  ...
-  MOCK_METHOD1(Add, int(Element x));
-  MOCK_METHOD2(Add, int(int times, Element x);
-
-  MOCK_METHOD0(GetBar, Bar&());
-  MOCK_CONST_METHOD0(GetBar, const Bar&());
-};
-```
-
-**Note:** if you don't mock all versions of the overloaded method, the
-compiler will give you a warning about some methods in the base class
-being hidden. To fix that, use `using` to bring them in scope:
-
-```
-class MockFoo : public Foo {
-  ...
-  using Foo::Add;
-  MOCK_METHOD1(Add, int(Element x));
-  // We don't want to mock int Add(int times, Element x);
-  ...
-};
-```
-
-## Mocking Class Templates ##
-
-To mock a class template, append `_T` to the `MOCK_*` macros:
-
-```
-template 
-class StackInterface {
-  ...
-  // Must be virtual as we'll inherit from StackInterface.
-  virtual ~StackInterface();
-
-  virtual int GetSize() const = 0;
-  virtual void Push(const Elem& x) = 0;
-};
-
-template 
-class MockStack : public StackInterface {
-  ...
-  MOCK_CONST_METHOD0_T(GetSize, int());
-  MOCK_METHOD1_T(Push, void(const Elem& x));
-};
-```
-
-## Mocking Nonvirtual Methods ##
-
-Google Mock can mock non-virtual functions to be used in what we call _hi-perf
-dependency injection_.
-
-In this case, instead of sharing a common base class with the real
-class, your mock class will be _unrelated_ to the real class, but
-contain methods with the same signatures.  The syntax for mocking
-non-virtual methods is the _same_ as mocking virtual methods:
-
-```
-// A simple packet stream class.  None of its members is virtual.
-class ConcretePacketStream {
- public:
-  void AppendPacket(Packet* new_packet);
-  const Packet* GetPacket(size_t packet_number) const;
-  size_t NumberOfPackets() const;
-  ...
-};
-
-// A mock packet stream class.  It inherits from no other, but defines
-// GetPacket() and NumberOfPackets().
-class MockPacketStream {
- public:
-  MOCK_CONST_METHOD1(GetPacket, const Packet*(size_t packet_number));
-  MOCK_CONST_METHOD0(NumberOfPackets, size_t());
-  ...
-};
-```
-
-Note that the mock class doesn't define `AppendPacket()`, unlike the
-real class. That's fine as long as the test doesn't need to call it.
-
-Next, you need a way to say that you want to use
-`ConcretePacketStream` in production code and to use `MockPacketStream`
-in tests.  Since the functions are not virtual and the two classes are
-unrelated, you must specify your choice at _compile time_ (as opposed
-to run time).
-
-One way to do it is to templatize your code that needs to use a packet
-stream.  More specifically, you will give your code a template type
-argument for the type of the packet stream.  In production, you will
-instantiate your template with `ConcretePacketStream` as the type
-argument.  In tests, you will instantiate the same template with
-`MockPacketStream`.  For example, you may write:
-
-```
-template 
-void CreateConnection(PacketStream* stream) { ... }
-
-template 
-class PacketReader {
- public:
-  void ReadPackets(PacketStream* stream, size_t packet_num);
-};
-```
-
-Then you can use `CreateConnection()` and
-`PacketReader` in production code, and use
-`CreateConnection()` and
-`PacketReader` in tests.
-
-```
-  MockPacketStream mock_stream;
-  EXPECT_CALL(mock_stream, ...)...;
-  .. set more expectations on mock_stream ...
-  PacketReader reader(&mock_stream);
-  ... exercise reader ...
-```
-
-## Mocking Free Functions ##
-
-It's possible to use Google Mock to mock a free function (i.e. a
-C-style function or a static method).  You just need to rewrite your
-code to use an interface (abstract class).
-
-Instead of calling a free function (say, `OpenFile`) directly,
-introduce an interface for it and have a concrete subclass that calls
-the free function:
-
-```
-class FileInterface {
- public:
-  ...
-  virtual bool Open(const char* path, const char* mode) = 0;
-};
-
-class File : public FileInterface {
- public:
-  ...
-  virtual bool Open(const char* path, const char* mode) {
-    return OpenFile(path, mode);
-  }
-};
-```
-
-Your code should talk to `FileInterface` to open a file.  Now it's
-easy to mock out the function.
-
-This may seem much hassle, but in practice you often have multiple
-related functions that you can put in the same interface, so the
-per-function syntactic overhead will be much lower.
-
-If you are concerned about the performance overhead incurred by
-virtual functions, and profiling confirms your concern, you can
-combine this with the recipe for [mocking non-virtual methods](#mocking-nonvirtual-methods).
-
-## The Nice, the Strict, and the Naggy ##
-
-If a mock method has no `EXPECT_CALL` spec but is called, Google Mock
-will print a warning about the "uninteresting call". The rationale is:
-
-  * New methods may be added to an interface after a test is written. We shouldn't fail a test just because a method it doesn't know about is called.
-  * However, this may also mean there's a bug in the test, so Google Mock shouldn't be silent either. If the user believes these calls are harmless, they can add an `EXPECT_CALL()` to suppress the warning.
-
-However, sometimes you may want to suppress all "uninteresting call"
-warnings, while sometimes you may want the opposite, i.e. to treat all
-of them as errors. Google Mock lets you make the decision on a
-per-mock-object basis.
-
-Suppose your test uses a mock class `MockFoo`:
-
-```
-TEST(...) {
-  MockFoo mock_foo;
-  EXPECT_CALL(mock_foo, DoThis());
-  ... code that uses mock_foo ...
-}
-```
-
-If a method of `mock_foo` other than `DoThis()` is called, it will be
-reported by Google Mock as a warning. However, if you rewrite your
-test to use `NiceMock` instead, the warning will be gone,
-resulting in a cleaner test output:
-
-```
-using ::testing::NiceMock;
-
-TEST(...) {
-  NiceMock mock_foo;
-  EXPECT_CALL(mock_foo, DoThis());
-  ... code that uses mock_foo ...
-}
-```
-
-`NiceMock` is a subclass of `MockFoo`, so it can be used
-wherever `MockFoo` is accepted.
-
-It also works if `MockFoo`'s constructor takes some arguments, as
-`NiceMock` "inherits" `MockFoo`'s constructors:
-
-```
-using ::testing::NiceMock;
-
-TEST(...) {
-  NiceMock mock_foo(5, "hi");  // Calls MockFoo(5, "hi").
-  EXPECT_CALL(mock_foo, DoThis());
-  ... code that uses mock_foo ...
-}
-```
-
-The usage of `StrictMock` is similar, except that it makes all
-uninteresting calls failures:
-
-```
-using ::testing::StrictMock;
-
-TEST(...) {
-  StrictMock mock_foo;
-  EXPECT_CALL(mock_foo, DoThis());
-  ... code that uses mock_foo ...
-
-  // The test will fail if a method of mock_foo other than DoThis()
-  // is called.
-}
-```
-
-There are some caveats though (I don't like them just as much as the
-next guy, but sadly they are side effects of C++'s limitations):
-
-  1. `NiceMock` and `StrictMock` only work for mock methods defined using the `MOCK_METHOD*` family of macros **directly** in the `MockFoo` class. If a mock method is defined in a **base class** of `MockFoo`, the "nice" or "strict" modifier may not affect it, depending on the compiler. In particular, nesting `NiceMock` and `StrictMock` (e.g. `NiceMock >`) is **not** supported.
-  1. The constructors of the base mock (`MockFoo`) cannot have arguments passed by non-const reference, which happens to be banned by the [Google C++ style guide](https://google.github.io/styleguide/cppguide.html).
-  1. During the constructor or destructor of `MockFoo`, the mock object is _not_ nice or strict.  This may cause surprises if the constructor or destructor calls a mock method on `this` object. (This behavior, however, is consistent with C++'s general rule: if a constructor or destructor calls a virtual method of `this` object, that method is treated as non-virtual.  In other words, to the base class's constructor or destructor, `this` object behaves like an instance of the base class, not the derived class.  This rule is required for safety.  Otherwise a base constructor may use members of a derived class before they are initialized, or a base destructor may use members of a derived class after they have been destroyed.)
-
-Finally, you should be **very cautious** about when to use naggy or strict mocks, as they tend to make tests more brittle and harder to maintain. When you refactor your code without changing its externally visible behavior, ideally you should't need to update any tests. If your code interacts with a naggy mock, however, you may start to get spammed with warnings as the result of your change. Worse, if your code interacts with a strict mock, your tests may start to fail and you'll be forced to fix them. Our general recommendation is to use nice mocks (not yet the default) most of the time, use naggy mocks (the current default) when developing or debugging tests, and use strict mocks only as the last resort.
-
-## Simplifying the Interface without Breaking Existing Code ##
-
-Sometimes a method has a long list of arguments that is mostly
-uninteresting. For example,
-
-```
-class LogSink {
- public:
-  ...
-  virtual void send(LogSeverity severity, const char* full_filename,
-                    const char* base_filename, int line,
-                    const struct tm* tm_time,
-                    const char* message, size_t message_len) = 0;
-};
-```
-
-This method's argument list is lengthy and hard to work with (let's
-say that the `message` argument is not even 0-terminated). If we mock
-it as is, using the mock will be awkward. If, however, we try to
-simplify this interface, we'll need to fix all clients depending on
-it, which is often infeasible.
-
-The trick is to re-dispatch the method in the mock class:
-
-```
-class ScopedMockLog : public LogSink {
- public:
-  ...
-  virtual void send(LogSeverity severity, const char* full_filename,
-                    const char* base_filename, int line, const tm* tm_time,
-                    const char* message, size_t message_len) {
-    // We are only interested in the log severity, full file name, and
-    // log message.
-    Log(severity, full_filename, std::string(message, message_len));
-  }
-
-  // Implements the mock method:
-  //
-  //   void Log(LogSeverity severity,
-  //            const string& file_path,
-  //            const string& message);
-  MOCK_METHOD3(Log, void(LogSeverity severity, const string& file_path,
-                         const string& message));
-};
-```
-
-By defining a new mock method with a trimmed argument list, we make
-the mock class much more user-friendly.
-
-## Alternative to Mocking Concrete Classes ##
-
-Often you may find yourself using classes that don't implement
-interfaces. In order to test your code that uses such a class (let's
-call it `Concrete`), you may be tempted to make the methods of
-`Concrete` virtual and then mock it.
-
-Try not to do that.
-
-Making a non-virtual function virtual is a big decision. It creates an
-extension point where subclasses can tweak your class' behavior. This
-weakens your control on the class because now it's harder to maintain
-the class' invariants. You should make a function virtual only when
-there is a valid reason for a subclass to override it.
-
-Mocking concrete classes directly is problematic as it creates a tight
-coupling between the class and the tests - any small change in the
-class may invalidate your tests and make test maintenance a pain.
-
-To avoid such problems, many programmers have been practicing "coding
-to interfaces": instead of talking to the `Concrete` class, your code
-would define an interface and talk to it. Then you implement that
-interface as an adaptor on top of `Concrete`. In tests, you can easily
-mock that interface to observe how your code is doing.
-
-This technique incurs some overhead:
-
-  * You pay the cost of virtual function calls (usually not a problem).
-  * There is more abstraction for the programmers to learn.
-
-However, it can also bring significant benefits in addition to better
-testability:
-
-  * `Concrete`'s API may not fit your problem domain very well, as you may not be the only client it tries to serve. By designing your own interface, you have a chance to tailor it to your need - you may add higher-level functionalities, rename stuff, etc instead of just trimming the class. This allows you to write your code (user of the interface) in a more natural way, which means it will be more readable, more maintainable, and you'll be more productive.
-  * If `Concrete`'s implementation ever has to change, you don't have to rewrite everywhere it is used. Instead, you can absorb the change in your implementation of the interface, and your other code and tests will be insulated from this change.
-
-Some people worry that if everyone is practicing this technique, they
-will end up writing lots of redundant code. This concern is totally
-understandable. However, there are two reasons why it may not be the
-case:
-
-  * Different projects may need to use `Concrete` in different ways, so the best interfaces for them will be different. Therefore, each of them will have its own domain-specific interface on top of `Concrete`, and they will not be the same code.
-  * If enough projects want to use the same interface, they can always share it, just like they have been sharing `Concrete`. You can check in the interface and the adaptor somewhere near `Concrete` (perhaps in a `contrib` sub-directory) and let many projects use it.
-
-You need to weigh the pros and cons carefully for your particular
-problem, but I'd like to assure you that the Java community has been
-practicing this for a long time and it's a proven effective technique
-applicable in a wide variety of situations. :-)
-
-## Delegating Calls to a Fake ##
-
-Some times you have a non-trivial fake implementation of an
-interface. For example:
-
-```
-class Foo {
- public:
-  virtual ~Foo() {}
-  virtual char DoThis(int n) = 0;
-  virtual void DoThat(const char* s, int* p) = 0;
-};
-
-class FakeFoo : public Foo {
- public:
-  virtual char DoThis(int n) {
-    return (n > 0) ? '+' :
-        (n < 0) ? '-' : '0';
-  }
-
-  virtual void DoThat(const char* s, int* p) {
-    *p = strlen(s);
-  }
-};
-```
-
-Now you want to mock this interface such that you can set expectations
-on it. However, you also want to use `FakeFoo` for the default
-behavior, as duplicating it in the mock object is, well, a lot of
-work.
-
-When you define the mock class using Google Mock, you can have it
-delegate its default action to a fake class you already have, using
-this pattern:
-
-```
-using ::testing::_;
-using ::testing::Invoke;
-
-class MockFoo : public Foo {
- public:
-  // Normal mock method definitions using Google Mock.
-  MOCK_METHOD1(DoThis, char(int n));
-  MOCK_METHOD2(DoThat, void(const char* s, int* p));
-
-  // Delegates the default actions of the methods to a FakeFoo object.
-  // This must be called *before* the custom ON_CALL() statements.
-  void DelegateToFake() {
-    ON_CALL(*this, DoThis(_))
-        .WillByDefault(Invoke(&fake_, &FakeFoo::DoThis));
-    ON_CALL(*this, DoThat(_, _))
-        .WillByDefault(Invoke(&fake_, &FakeFoo::DoThat));
-  }
- private:
-  FakeFoo fake_;  // Keeps an instance of the fake in the mock.
-};
-```
-
-With that, you can use `MockFoo` in your tests as usual. Just remember
-that if you don't explicitly set an action in an `ON_CALL()` or
-`EXPECT_CALL()`, the fake will be called upon to do it:
-
-```
-using ::testing::_;
-
-TEST(AbcTest, Xyz) {
-  MockFoo foo;
-  foo.DelegateToFake(); // Enables the fake for delegation.
-
-  // Put your ON_CALL(foo, ...)s here, if any.
-
-  // No action specified, meaning to use the default action.
-  EXPECT_CALL(foo, DoThis(5));
-  EXPECT_CALL(foo, DoThat(_, _));
-
-  int n = 0;
-  EXPECT_EQ('+', foo.DoThis(5));  // FakeFoo::DoThis() is invoked.
-  foo.DoThat("Hi", &n);           // FakeFoo::DoThat() is invoked.
-  EXPECT_EQ(2, n);
-}
-```
-
-**Some tips:**
-
-  * If you want, you can still override the default action by providing your own `ON_CALL()` or using `.WillOnce()` / `.WillRepeatedly()` in `EXPECT_CALL()`.
-  * In `DelegateToFake()`, you only need to delegate the methods whose fake implementation you intend to use.
-  * The general technique discussed here works for overloaded methods, but you'll need to tell the compiler which version you mean. To disambiguate a mock function (the one you specify inside the parentheses of `ON_CALL()`), see the "Selecting Between Overloaded Functions" section on this page; to disambiguate a fake function (the one you place inside `Invoke()`), use a `static_cast` to specify the function's type. For instance, if class `Foo` has methods `char DoThis(int n)` and `bool DoThis(double x) const`, and you want to invoke the latter, you need to write `Invoke(&fake_, static_cast(&FakeFoo::DoThis))` instead of `Invoke(&fake_, &FakeFoo::DoThis)` (The strange-looking thing inside the angled brackets of `static_cast` is the type of a function pointer to the second `DoThis()` method.).
-  * Having to mix a mock and a fake is often a sign of something gone wrong. Perhaps you haven't got used to the interaction-based way of testing yet. Or perhaps your interface is taking on too many roles and should be split up. Therefore, **don't abuse this**. We would only recommend to do it as an intermediate step when you are refactoring your code.
-
-Regarding the tip on mixing a mock and a fake, here's an example on
-why it may be a bad sign: Suppose you have a class `System` for
-low-level system operations. In particular, it does file and I/O
-operations. And suppose you want to test how your code uses `System`
-to do I/O, and you just want the file operations to work normally. If
-you mock out the entire `System` class, you'll have to provide a fake
-implementation for the file operation part, which suggests that
-`System` is taking on too many roles.
-
-Instead, you can define a `FileOps` interface and an `IOOps` interface
-and split `System`'s functionalities into the two. Then you can mock
-`IOOps` without mocking `FileOps`.
-
-## Delegating Calls to a Real Object ##
-
-When using testing doubles (mocks, fakes, stubs, and etc), sometimes
-their behaviors will differ from those of the real objects. This
-difference could be either intentional (as in simulating an error such
-that you can test the error handling code) or unintentional. If your
-mocks have different behaviors than the real objects by mistake, you
-could end up with code that passes the tests but fails in production.
-
-You can use the _delegating-to-real_ technique to ensure that your
-mock has the same behavior as the real object while retaining the
-ability to validate calls. This technique is very similar to the
-delegating-to-fake technique, the difference being that we use a real
-object instead of a fake. Here's an example:
-
-```
-using ::testing::_;
-using ::testing::AtLeast;
-using ::testing::Invoke;
-
-class MockFoo : public Foo {
- public:
-  MockFoo() {
-    // By default, all calls are delegated to the real object.
-    ON_CALL(*this, DoThis())
-        .WillByDefault(Invoke(&real_, &Foo::DoThis));
-    ON_CALL(*this, DoThat(_))
-        .WillByDefault(Invoke(&real_, &Foo::DoThat));
-    ...
-  }
-  MOCK_METHOD0(DoThis, ...);
-  MOCK_METHOD1(DoThat, ...);
-  ...
- private:
-  Foo real_;
-};
-...
-
-  MockFoo mock;
-
-  EXPECT_CALL(mock, DoThis())
-      .Times(3);
-  EXPECT_CALL(mock, DoThat("Hi"))
-      .Times(AtLeast(1));
-  ... use mock in test ...
-```
-
-With this, Google Mock will verify that your code made the right calls
-(with the right arguments, in the right order, called the right number
-of times, etc), and a real object will answer the calls (so the
-behavior will be the same as in production). This gives you the best
-of both worlds.
-
-## Delegating Calls to a Parent Class ##
-
-Ideally, you should code to interfaces, whose methods are all pure
-virtual. In reality, sometimes you do need to mock a virtual method
-that is not pure (i.e, it already has an implementation). For example:
-
-```
-class Foo {
- public:
-  virtual ~Foo();
-
-  virtual void Pure(int n) = 0;
-  virtual int Concrete(const char* str) { ... }
-};
-
-class MockFoo : public Foo {
- public:
-  // Mocking a pure method.
-  MOCK_METHOD1(Pure, void(int n));
-  // Mocking a concrete method.  Foo::Concrete() is shadowed.
-  MOCK_METHOD1(Concrete, int(const char* str));
-};
-```
-
-Sometimes you may want to call `Foo::Concrete()` instead of
-`MockFoo::Concrete()`. Perhaps you want to do it as part of a stub
-action, or perhaps your test doesn't need to mock `Concrete()` at all
-(but it would be oh-so painful to have to define a new mock class
-whenever you don't need to mock one of its methods).
-
-The trick is to leave a back door in your mock class for accessing the
-real methods in the base class:
-
-```
-class MockFoo : public Foo {
- public:
-  // Mocking a pure method.
-  MOCK_METHOD1(Pure, void(int n));
-  // Mocking a concrete method.  Foo::Concrete() is shadowed.
-  MOCK_METHOD1(Concrete, int(const char* str));
-
-  // Use this to call Concrete() defined in Foo.
-  int FooConcrete(const char* str) { return Foo::Concrete(str); }
-};
-```
-
-Now, you can call `Foo::Concrete()` inside an action by:
-
-```
-using ::testing::_;
-using ::testing::Invoke;
-...
-  EXPECT_CALL(foo, Concrete(_))
-      .WillOnce(Invoke(&foo, &MockFoo::FooConcrete));
-```
-
-or tell the mock object that you don't want to mock `Concrete()`:
-
-```
-using ::testing::Invoke;
-...
-  ON_CALL(foo, Concrete(_))
-      .WillByDefault(Invoke(&foo, &MockFoo::FooConcrete));
-```
-
-(Why don't we just write `Invoke(&foo, &Foo::Concrete)`? If you do
-that, `MockFoo::Concrete()` will be called (and cause an infinite
-recursion) since `Foo::Concrete()` is virtual. That's just how C++
-works.)
-
-# Using Matchers #
-
-## Matching Argument Values Exactly ##
-
-You can specify exactly which arguments a mock method is expecting:
-
-```
-using ::testing::Return;
-...
-  EXPECT_CALL(foo, DoThis(5))
-      .WillOnce(Return('a'));
-  EXPECT_CALL(foo, DoThat("Hello", bar));
-```
-
-## Using Simple Matchers ##
-
-You can use matchers to match arguments that have a certain property:
-
-```
-using ::testing::Ge;
-using ::testing::NotNull;
-using ::testing::Return;
-...
-  EXPECT_CALL(foo, DoThis(Ge(5)))  // The argument must be >= 5.
-      .WillOnce(Return('a'));
-  EXPECT_CALL(foo, DoThat("Hello", NotNull()));
-  // The second argument must not be NULL.
-```
-
-A frequently used matcher is `_`, which matches anything:
-
-```
-using ::testing::_;
-using ::testing::NotNull;
-...
-  EXPECT_CALL(foo, DoThat(_, NotNull()));
-```
-
-## Combining Matchers ##
-
-You can build complex matchers from existing ones using `AllOf()`,
-`AnyOf()`, and `Not()`:
-
-```
-using ::testing::AllOf;
-using ::testing::Gt;
-using ::testing::HasSubstr;
-using ::testing::Ne;
-using ::testing::Not;
-...
-  // The argument must be > 5 and != 10.
-  EXPECT_CALL(foo, DoThis(AllOf(Gt(5),
-                                Ne(10))));
-
-  // The first argument must not contain sub-string "blah".
-  EXPECT_CALL(foo, DoThat(Not(HasSubstr("blah")),
-                          NULL));
-```
-
-## Casting Matchers ##
-
-Google Mock matchers are statically typed, meaning that the compiler
-can catch your mistake if you use a matcher of the wrong type (for
-example, if you use `Eq(5)` to match a `string` argument). Good for
-you!
-
-Sometimes, however, you know what you're doing and want the compiler
-to give you some slack. One example is that you have a matcher for
-`long` and the argument you want to match is `int`. While the two
-types aren't exactly the same, there is nothing really wrong with
-using a `Matcher` to match an `int` - after all, we can first
-convert the `int` argument to a `long` before giving it to the
-matcher.
-
-To support this need, Google Mock gives you the
-`SafeMatcherCast(m)` function. It casts a matcher `m` to type
-`Matcher`. To ensure safety, Google Mock checks that (let `U` be the
-type `m` accepts):
-
-  1. Type `T` can be implicitly cast to type `U`;
-  1. When both `T` and `U` are built-in arithmetic types (`bool`, integers, and floating-point numbers), the conversion from `T` to `U` is not lossy (in other words, any value representable by `T` can also be represented by `U`); and
-  1. When `U` is a reference, `T` must also be a reference (as the underlying matcher may be interested in the address of the `U` value).
-
-The code won't compile if any of these conditions aren't met.
-
-Here's one example:
-
-```
-using ::testing::SafeMatcherCast;
-
-// A base class and a child class.
-class Base { ... };
-class Derived : public Base { ... };
-
-class MockFoo : public Foo {
- public:
-  MOCK_METHOD1(DoThis, void(Derived* derived));
-};
-...
-
-  MockFoo foo;
-  // m is a Matcher we got from somewhere.
-  EXPECT_CALL(foo, DoThis(SafeMatcherCast(m)));
-```
-
-If you find `SafeMatcherCast(m)` too limiting, you can use a similar
-function `MatcherCast(m)`. The difference is that `MatcherCast` works
-as long as you can `static_cast` type `T` to type `U`.
-
-`MatcherCast` essentially lets you bypass C++'s type system
-(`static_cast` isn't always safe as it could throw away information,
-for example), so be careful not to misuse/abuse it.
-
-## Selecting Between Overloaded Functions ##
-
-If you expect an overloaded function to be called, the compiler may
-need some help on which overloaded version it is.
-
-To disambiguate functions overloaded on the const-ness of this object,
-use the `Const()` argument wrapper.
-
-```
-using ::testing::ReturnRef;
-
-class MockFoo : public Foo {
-  ...
-  MOCK_METHOD0(GetBar, Bar&());
-  MOCK_CONST_METHOD0(GetBar, const Bar&());
-};
-...
-
-  MockFoo foo;
-  Bar bar1, bar2;
-  EXPECT_CALL(foo, GetBar())         // The non-const GetBar().
-      .WillOnce(ReturnRef(bar1));
-  EXPECT_CALL(Const(foo), GetBar())  // The const GetBar().
-      .WillOnce(ReturnRef(bar2));
-```
-
-(`Const()` is defined by Google Mock and returns a `const` reference
-to its argument.)
-
-To disambiguate overloaded functions with the same number of arguments
-but different argument types, you may need to specify the exact type
-of a matcher, either by wrapping your matcher in `Matcher()`, or
-using a matcher whose type is fixed (`TypedEq`, `An()`,
-etc):
-
-```
-using ::testing::An;
-using ::testing::Lt;
-using ::testing::Matcher;
-using ::testing::TypedEq;
-
-class MockPrinter : public Printer {
- public:
-  MOCK_METHOD1(Print, void(int n));
-  MOCK_METHOD1(Print, void(char c));
-};
-
-TEST(PrinterTest, Print) {
-  MockPrinter printer;
-
-  EXPECT_CALL(printer, Print(An()));            // void Print(int);
-  EXPECT_CALL(printer, Print(Matcher(Lt(5))));  // void Print(int);
-  EXPECT_CALL(printer, Print(TypedEq('a')));   // void Print(char);
-
-  printer.Print(3);
-  printer.Print(6);
-  printer.Print('a');
-}
-```
-
-## Performing Different Actions Based on the Arguments ##
-
-When a mock method is called, the _last_ matching expectation that's
-still active will be selected (think "newer overrides older"). So, you
-can make a method do different things depending on its argument values
-like this:
-
-```
-using ::testing::_;
-using ::testing::Lt;
-using ::testing::Return;
-...
-  // The default case.
-  EXPECT_CALL(foo, DoThis(_))
-      .WillRepeatedly(Return('b'));
-
-  // The more specific case.
-  EXPECT_CALL(foo, DoThis(Lt(5)))
-      .WillRepeatedly(Return('a'));
-```
-
-Now, if `foo.DoThis()` is called with a value less than 5, `'a'` will
-be returned; otherwise `'b'` will be returned.
-
-## Matching Multiple Arguments as a Whole ##
-
-Sometimes it's not enough to match the arguments individually. For
-example, we may want to say that the first argument must be less than
-the second argument. The `With()` clause allows us to match
-all arguments of a mock function as a whole. For example,
-
-```
-using ::testing::_;
-using ::testing::Lt;
-using ::testing::Ne;
-...
-  EXPECT_CALL(foo, InRange(Ne(0), _))
-      .With(Lt());
-```
-
-says that the first argument of `InRange()` must not be 0, and must be
-less than the second argument.
-
-The expression inside `With()` must be a matcher of type
-`Matcher< ::testing::tuple >`, where `A1`, ..., `An` are the
-types of the function arguments.
-
-You can also write `AllArgs(m)` instead of `m` inside `.With()`. The
-two forms are equivalent, but `.With(AllArgs(Lt()))` is more readable
-than `.With(Lt())`.
-
-You can use `Args(m)` to match the `n` selected arguments
-(as a tuple) against `m`. For example,
-
-```
-using ::testing::_;
-using ::testing::AllOf;
-using ::testing::Args;
-using ::testing::Lt;
-...
-  EXPECT_CALL(foo, Blah(_, _, _))
-      .With(AllOf(Args<0, 1>(Lt()), Args<1, 2>(Lt())));
-```
-
-says that `Blah()` will be called with arguments `x`, `y`, and `z` where
-`x < y < z`.
-
-As a convenience and example, Google Mock provides some matchers for
-2-tuples, including the `Lt()` matcher above. See the [CheatSheet](CheatSheet.md) for
-the complete list.
-
-Note that if you want to pass the arguments to a predicate of your own
-(e.g. `.With(Args<0, 1>(Truly(&MyPredicate)))`), that predicate MUST be
-written to take a `::testing::tuple` as its argument; Google Mock will pass the `n` selected arguments as _one_ single tuple to the predicate.
-
-## Using Matchers as Predicates ##
-
-Have you noticed that a matcher is just a fancy predicate that also
-knows how to describe itself? Many existing algorithms take predicates
-as arguments (e.g. those defined in STL's `` header), and
-it would be a shame if Google Mock matchers are not allowed to
-participate.
-
-Luckily, you can use a matcher where a unary predicate functor is
-expected by wrapping it inside the `Matches()` function. For example,
-
-```
-#include 
-#include 
-
-std::vector v;
-...
-// How many elements in v are >= 10?
-const int count = count_if(v.begin(), v.end(), Matches(Ge(10)));
-```
-
-Since you can build complex matchers from simpler ones easily using
-Google Mock, this gives you a way to conveniently construct composite
-predicates (doing the same using STL's `` header is just
-painful). For example, here's a predicate that's satisfied by any
-number that is >= 0, <= 100, and != 50:
-
-```
-Matches(AllOf(Ge(0), Le(100), Ne(50)))
-```
-
-## Using Matchers in Google Test Assertions ##
-
-Since matchers are basically predicates that also know how to describe
-themselves, there is a way to take advantage of them in
-[Google Test](../../googletest/) assertions. It's
-called `ASSERT_THAT` and `EXPECT_THAT`:
-
-```
-  ASSERT_THAT(value, matcher);  // Asserts that value matches matcher.
-  EXPECT_THAT(value, matcher);  // The non-fatal version.
-```
-
-For example, in a Google Test test you can write:
-
-```
-#include "gmock/gmock.h"
-
-using ::testing::AllOf;
-using ::testing::Ge;
-using ::testing::Le;
-using ::testing::MatchesRegex;
-using ::testing::StartsWith;
-...
-
-  EXPECT_THAT(Foo(), StartsWith("Hello"));
-  EXPECT_THAT(Bar(), MatchesRegex("Line \\d+"));
-  ASSERT_THAT(Baz(), AllOf(Ge(5), Le(10)));
-```
-
-which (as you can probably guess) executes `Foo()`, `Bar()`, and
-`Baz()`, and verifies that:
-
-  * `Foo()` returns a string that starts with `"Hello"`.
-  * `Bar()` returns a string that matches regular expression `"Line \\d+"`.
-  * `Baz()` returns a number in the range [5, 10].
-
-The nice thing about these macros is that _they read like
-English_. They generate informative messages too. For example, if the
-first `EXPECT_THAT()` above fails, the message will be something like:
-
-```
-Value of: Foo()
-  Actual: "Hi, world!"
-Expected: starts with "Hello"
-```
-
-**Credit:** The idea of `(ASSERT|EXPECT)_THAT` was stolen from the
-[Hamcrest](https://github.com/hamcrest/) project, which adds
-`assertThat()` to JUnit.
-
-## Using Predicates as Matchers ##
-
-Google Mock provides a built-in set of matchers. In case you find them
-lacking, you can use an arbitray unary predicate function or functor
-as a matcher - as long as the predicate accepts a value of the type
-you want. You do this by wrapping the predicate inside the `Truly()`
-function, for example:
-
-```
-using ::testing::Truly;
-
-int IsEven(int n) { return (n % 2) == 0 ? 1 : 0; }
-...
-
-  // Bar() must be called with an even number.
-  EXPECT_CALL(foo, Bar(Truly(IsEven)));
-```
-
-Note that the predicate function / functor doesn't have to return
-`bool`. It works as long as the return value can be used as the
-condition in statement `if (condition) ...`.
-
-## Matching Arguments that Are Not Copyable ##
-
-When you do an `EXPECT_CALL(mock_obj, Foo(bar))`, Google Mock saves
-away a copy of `bar`. When `Foo()` is called later, Google Mock
-compares the argument to `Foo()` with the saved copy of `bar`. This
-way, you don't need to worry about `bar` being modified or destroyed
-after the `EXPECT_CALL()` is executed. The same is true when you use
-matchers like `Eq(bar)`, `Le(bar)`, and so on.
-
-But what if `bar` cannot be copied (i.e. has no copy constructor)? You
-could define your own matcher function and use it with `Truly()`, as
-the previous couple of recipes have shown. Or, you may be able to get
-away from it if you can guarantee that `bar` won't be changed after
-the `EXPECT_CALL()` is executed. Just tell Google Mock that it should
-save a reference to `bar`, instead of a copy of it. Here's how:
-
-```
-using ::testing::Eq;
-using ::testing::ByRef;
-using ::testing::Lt;
-...
-  // Expects that Foo()'s argument == bar.
-  EXPECT_CALL(mock_obj, Foo(Eq(ByRef(bar))));
-
-  // Expects that Foo()'s argument < bar.
-  EXPECT_CALL(mock_obj, Foo(Lt(ByRef(bar))));
-```
-
-Remember: if you do this, don't change `bar` after the
-`EXPECT_CALL()`, or the result is undefined.
-
-## Validating a Member of an Object ##
-
-Often a mock function takes a reference to object as an argument. When
-matching the argument, you may not want to compare the entire object
-against a fixed object, as that may be over-specification. Instead,
-you may need to validate a certain member variable or the result of a
-certain getter method of the object. You can do this with `Field()`
-and `Property()`. More specifically,
-
-```
-Field(&Foo::bar, m)
-```
-
-is a matcher that matches a `Foo` object whose `bar` member variable
-satisfies matcher `m`.
-
-```
-Property(&Foo::baz, m)
-```
-
-is a matcher that matches a `Foo` object whose `baz()` method returns
-a value that satisfies matcher `m`.
-
-For example:
-
-| Expression                   | Description                        |
-|:-----------------------------|:-----------------------------------|
-| `Field(&Foo::number, Ge(3))` | Matches `x` where `x.number >= 3`. |
-| `Property(&Foo::name, StartsWith("John "))` | Matches `x` where `x.name()` starts with `"John "`. |
-
-Note that in `Property(&Foo::baz, ...)`, method `baz()` must take no
-argument and be declared as `const`.
-
-BTW, `Field()` and `Property()` can also match plain pointers to
-objects. For instance,
-
-```
-Field(&Foo::number, Ge(3))
-```
-
-matches a plain pointer `p` where `p->number >= 3`. If `p` is `NULL`,
-the match will always fail regardless of the inner matcher.
-
-What if you want to validate more than one members at the same time?
-Remember that there is `AllOf()`.
-
-## Validating the Value Pointed to by a Pointer Argument ##
-
-C++ functions often take pointers as arguments. You can use matchers
-like `IsNull()`, `NotNull()`, and other comparison matchers to match a
-pointer, but what if you want to make sure the value _pointed to_ by
-the pointer, instead of the pointer itself, has a certain property?
-Well, you can use the `Pointee(m)` matcher.
-
-`Pointee(m)` matches a pointer iff `m` matches the value the pointer
-points to. For example:
-
-```
-using ::testing::Ge;
-using ::testing::Pointee;
-...
-  EXPECT_CALL(foo, Bar(Pointee(Ge(3))));
-```
-
-expects `foo.Bar()` to be called with a pointer that points to a value
-greater than or equal to 3.
-
-One nice thing about `Pointee()` is that it treats a `NULL` pointer as
-a match failure, so you can write `Pointee(m)` instead of
-
-```
-  AllOf(NotNull(), Pointee(m))
-```
-
-without worrying that a `NULL` pointer will crash your test.
-
-Also, did we tell you that `Pointee()` works with both raw pointers
-**and** smart pointers (`linked_ptr`, `shared_ptr`, `scoped_ptr`, and
-etc)?
-
-What if you have a pointer to pointer? You guessed it - you can use
-nested `Pointee()` to probe deeper inside the value. For example,
-`Pointee(Pointee(Lt(3)))` matches a pointer that points to a pointer
-that points to a number less than 3 (what a mouthful...).
-
-## Testing a Certain Property of an Object ##
-
-Sometimes you want to specify that an object argument has a certain
-property, but there is no existing matcher that does this. If you want
-good error messages, you should define a matcher. If you want to do it
-quick and dirty, you could get away with writing an ordinary function.
-
-Let's say you have a mock function that takes an object of type `Foo`,
-which has an `int bar()` method and an `int baz()` method, and you
-want to constrain that the argument's `bar()` value plus its `baz()`
-value is a given number. Here's how you can define a matcher to do it:
-
-```
-using ::testing::MatcherInterface;
-using ::testing::MatchResultListener;
-
-class BarPlusBazEqMatcher : public MatcherInterface {
- public:
-  explicit BarPlusBazEqMatcher(int expected_sum)
-      : expected_sum_(expected_sum) {}
-
-  virtual bool MatchAndExplain(const Foo& foo,
-                               MatchResultListener* listener) const {
-    return (foo.bar() + foo.baz()) == expected_sum_;
-  }
-
-  virtual void DescribeTo(::std::ostream* os) const {
-    *os << "bar() + baz() equals " << expected_sum_;
-  }
-
-  virtual void DescribeNegationTo(::std::ostream* os) const {
-    *os << "bar() + baz() does not equal " << expected_sum_;
-  }
- private:
-  const int expected_sum_;
-};
-
-inline Matcher BarPlusBazEq(int expected_sum) {
-  return MakeMatcher(new BarPlusBazEqMatcher(expected_sum));
-}
-
-...
-
-  EXPECT_CALL(..., DoThis(BarPlusBazEq(5)))...;
-```
-
-## Matching Containers ##
-
-Sometimes an STL container (e.g. list, vector, map, ...) is passed to
-a mock function and you may want to validate it. Since most STL
-containers support the `==` operator, you can write
-`Eq(expected_container)` or simply `expected_container` to match a
-container exactly.
-
-Sometimes, though, you may want to be more flexible (for example, the
-first element must be an exact match, but the second element can be
-any positive number, and so on). Also, containers used in tests often
-have a small number of elements, and having to define the expected
-container out-of-line is a bit of a hassle.
-
-You can use the `ElementsAre()` or `UnorderedElementsAre()` matcher in
-such cases:
-
-```
-using ::testing::_;
-using ::testing::ElementsAre;
-using ::testing::Gt;
-...
-
-  MOCK_METHOD1(Foo, void(const vector& numbers));
-...
-
-  EXPECT_CALL(mock, Foo(ElementsAre(1, Gt(0), _, 5)));
-```
-
-The above matcher says that the container must have 4 elements, which
-must be 1, greater than 0, anything, and 5 respectively.
-
-If you instead write:
-
-```
-using ::testing::_;
-using ::testing::Gt;
-using ::testing::UnorderedElementsAre;
-...
-
-  MOCK_METHOD1(Foo, void(const vector& numbers));
-...
-
-  EXPECT_CALL(mock, Foo(UnorderedElementsAre(1, Gt(0), _, 5)));
-```
-
-It means that the container must have 4 elements, which under some
-permutation must be 1, greater than 0, anything, and 5 respectively.
-
-`ElementsAre()` and `UnorderedElementsAre()` are overloaded to take 0
-to 10 arguments. If more are needed, you can place them in a C-style
-array and use `ElementsAreArray()` or `UnorderedElementsAreArray()`
-instead:
-
-```
-using ::testing::ElementsAreArray;
-...
-
-  // ElementsAreArray accepts an array of element values.
-  const int expected_vector1[] = { 1, 5, 2, 4, ... };
-  EXPECT_CALL(mock, Foo(ElementsAreArray(expected_vector1)));
-
-  // Or, an array of element matchers.
-  Matcher expected_vector2 = { 1, Gt(2), _, 3, ... };
-  EXPECT_CALL(mock, Foo(ElementsAreArray(expected_vector2)));
-```
-
-In case the array needs to be dynamically created (and therefore the
-array size cannot be inferred by the compiler), you can give
-`ElementsAreArray()` an additional argument to specify the array size:
-
-```
-using ::testing::ElementsAreArray;
-...
-  int* const expected_vector3 = new int[count];
-  ... fill expected_vector3 with values ...
-  EXPECT_CALL(mock, Foo(ElementsAreArray(expected_vector3, count)));
-```
-
-**Tips:**
-
-  * `ElementsAre*()` can be used to match _any_ container that implements the STL iterator pattern (i.e. it has a `const_iterator` type and supports `begin()/end()`), not just the ones defined in STL. It will even work with container types yet to be written - as long as they follows the above pattern.
-  * You can use nested `ElementsAre*()` to match nested (multi-dimensional) containers.
-  * If the container is passed by pointer instead of by reference, just write `Pointee(ElementsAre*(...))`.
-  * The order of elements _matters_ for `ElementsAre*()`. Therefore don't use it with containers whose element order is undefined (e.g. `hash_map`).
-
-## Sharing Matchers ##
-
-Under the hood, a Google Mock matcher object consists of a pointer to
-a ref-counted implementation object. Copying matchers is allowed and
-very efficient, as only the pointer is copied. When the last matcher
-that references the implementation object dies, the implementation
-object will be deleted.
-
-Therefore, if you have some complex matcher that you want to use again
-and again, there is no need to build it every time. Just assign it to a
-matcher variable and use that variable repeatedly! For example,
-
-```
-  Matcher in_range = AllOf(Gt(5), Le(10));
-  ... use in_range as a matcher in multiple EXPECT_CALLs ...
-```
-
-# Setting Expectations #
-
-## Knowing When to Expect ##
-
-`ON_CALL` is likely the single most under-utilized construct in Google Mock.
-
-There are basically two constructs for defining the behavior of a mock object: `ON_CALL` and `EXPECT_CALL`. The difference? `ON_CALL` defines what happens when a mock method is called, but _doesn't imply any expectation on the method being called._ `EXPECT_CALL` not only defines the behavior, but also sets an expectation that _the method will be called with the given arguments, for the given number of times_ (and _in the given order_ when you specify the order too).
-
-Since `EXPECT_CALL` does more, isn't it better than `ON_CALL`? Not really. Every `EXPECT_CALL` adds a constraint on the behavior of the code under test. Having more constraints than necessary is _baaad_ - even worse than not having enough constraints.
-
-This may be counter-intuitive. How could tests that verify more be worse than tests that verify less? Isn't verification the whole point of tests?
-
-The answer, lies in _what_ a test should verify. **A good test verifies the contract of the code.** If a test over-specifies, it doesn't leave enough freedom to the implementation. As a result, changing the implementation without breaking the contract (e.g. refactoring and optimization), which should be perfectly fine to do, can break such tests. Then you have to spend time fixing them, only to see them broken again the next time the implementation is changed.
-
-Keep in mind that one doesn't have to verify more than one property in one test. In fact, **it's a good style to verify only one thing in one test.** If you do that, a bug will likely break only one or two tests instead of dozens (which case would you rather debug?). If you are also in the habit of giving tests descriptive names that tell what they verify, you can often easily guess what's wrong just from the test log itself.
-
-So use `ON_CALL` by default, and only use `EXPECT_CALL` when you actually intend to verify that the call is made. For example, you may have a bunch of `ON_CALL`s in your test fixture to set the common mock behavior shared by all tests in the same group, and write (scarcely) different `EXPECT_CALL`s in different `TEST_F`s to verify different aspects of the code's behavior. Compared with the style where each `TEST` has many `EXPECT_CALL`s, this leads to tests that are more resilient to implementational changes (and thus less likely to require maintenance) and makes the intent of the tests more obvious (so they are easier to maintain when you do need to maintain them).
-
-If you are bothered by the "Uninteresting mock function call" message printed when a mock method without an `EXPECT_CALL` is called, you may use a `NiceMock` instead to suppress all such messages for the mock object, or suppress the message for specific methods by adding `EXPECT_CALL(...).Times(AnyNumber())`. DO NOT suppress it by blindly adding an `EXPECT_CALL(...)`, or you'll have a test that's a pain to maintain.
-
-## Ignoring Uninteresting Calls ##
-
-If you are not interested in how a mock method is called, just don't
-say anything about it. In this case, if the method is ever called,
-Google Mock will perform its default action to allow the test program
-to continue. If you are not happy with the default action taken by
-Google Mock, you can override it using `DefaultValue::Set()`
-(described later in this document) or `ON_CALL()`.
-
-Please note that once you expressed interest in a particular mock
-method (via `EXPECT_CALL()`), all invocations to it must match some
-expectation. If this function is called but the arguments don't match
-any `EXPECT_CALL()` statement, it will be an error.
-
-## Disallowing Unexpected Calls ##
-
-If a mock method shouldn't be called at all, explicitly say so:
-
-```
-using ::testing::_;
-...
-  EXPECT_CALL(foo, Bar(_))
-      .Times(0);
-```
-
-If some calls to the method are allowed, but the rest are not, just
-list all the expected calls:
-
-```
-using ::testing::AnyNumber;
-using ::testing::Gt;
-...
-  EXPECT_CALL(foo, Bar(5));
-  EXPECT_CALL(foo, Bar(Gt(10)))
-      .Times(AnyNumber());
-```
-
-A call to `foo.Bar()` that doesn't match any of the `EXPECT_CALL()`
-statements will be an error.
-
-## Understanding Uninteresting vs Unexpected Calls ##
-
-_Uninteresting_ calls and _unexpected_ calls are different concepts in Google Mock. _Very_ different.
-
-A call `x.Y(...)` is **uninteresting** if there's _not even a single_ `EXPECT_CALL(x, Y(...))` set. In other words, the test isn't interested in the `x.Y()` method at all, as evident in that the test doesn't care to say anything about it.
-
-A call `x.Y(...)` is **unexpected** if there are some `EXPECT_CALL(x, Y(...))s` set, but none of them matches the call. Put another way, the test is interested in the `x.Y()` method (therefore it _explicitly_ sets some `EXPECT_CALL` to verify how it's called); however, the verification fails as the test doesn't expect this particular call to happen.
-
-**An unexpected call is always an error,** as the code under test doesn't behave the way the test expects it to behave.
-
-**By default, an uninteresting call is not an error,** as it violates no constraint specified by the test. (Google Mock's philosophy is that saying nothing means there is no constraint.) However, it leads to a warning, as it _might_ indicate a problem (e.g. the test author might have forgotten to specify a constraint).
-
-In Google Mock, `NiceMock` and `StrictMock` can be used to make a mock class "nice" or "strict". How does this affect uninteresting calls and unexpected calls?
-
-A **nice mock** suppresses uninteresting call warnings. It is less chatty than the default mock, but otherwise is the same. If a test fails with a default mock, it will also fail using a nice mock instead. And vice versa. Don't expect making a mock nice to change the test's result.
-
-A **strict mock** turns uninteresting call warnings into errors. So making a mock strict may change the test's result.
-
-Let's look at an example:
-
-```
-TEST(...) {
-  NiceMock mock_registry;
-  EXPECT_CALL(mock_registry, GetDomainOwner("google.com"))
-          .WillRepeatedly(Return("Larry Page"));
-
-  // Use mock_registry in code under test.
-  ... &mock_registry ...
-}
-```
-
-The sole `EXPECT_CALL` here says that all calls to `GetDomainOwner()` must have `"google.com"` as the argument. If `GetDomainOwner("yahoo.com")` is called, it will be an unexpected call, and thus an error. Having a nice mock doesn't change the severity of an unexpected call.
-
-So how do we tell Google Mock that `GetDomainOwner()` can be called with some other arguments as well? The standard technique is to add a "catch all" `EXPECT_CALL`:
-
-```
-  EXPECT_CALL(mock_registry, GetDomainOwner(_))
-        .Times(AnyNumber());  // catches all other calls to this method.
-  EXPECT_CALL(mock_registry, GetDomainOwner("google.com"))
-        .WillRepeatedly(Return("Larry Page"));
-```
-
-Remember that `_` is the wildcard matcher that matches anything. With this, if `GetDomainOwner("google.com")` is called, it will do what the second `EXPECT_CALL` says; if it is called with a different argument, it will do what the first `EXPECT_CALL` says.
-
-Note that the order of the two `EXPECT_CALLs` is important, as a newer `EXPECT_CALL` takes precedence over an older one.
-
-For more on uninteresting calls, nice mocks, and strict mocks, read ["The Nice, the Strict, and the Naggy"](#the-nice-the-strict-and-the-naggy).
-
-## Expecting Ordered Calls ##
-
-Although an `EXPECT_CALL()` statement defined earlier takes precedence
-when Google Mock tries to match a function call with an expectation,
-by default calls don't have to happen in the order `EXPECT_CALL()`
-statements are written. For example, if the arguments match the
-matchers in the third `EXPECT_CALL()`, but not those in the first two,
-then the third expectation will be used.
-
-If you would rather have all calls occur in the order of the
-expectations, put the `EXPECT_CALL()` statements in a block where you
-define a variable of type `InSequence`:
-
-```
-  using ::testing::_;
-  using ::testing::InSequence;
-
-  {
-    InSequence s;
-
-    EXPECT_CALL(foo, DoThis(5));
-    EXPECT_CALL(bar, DoThat(_))
-        .Times(2);
-    EXPECT_CALL(foo, DoThis(6));
-  }
-```
-
-In this example, we expect a call to `foo.DoThis(5)`, followed by two
-calls to `bar.DoThat()` where the argument can be anything, which are
-in turn followed by a call to `foo.DoThis(6)`. If a call occurred
-out-of-order, Google Mock will report an error.
-
-## Expecting Partially Ordered Calls ##
-
-Sometimes requiring everything to occur in a predetermined order can
-lead to brittle tests. For example, we may care about `A` occurring
-before both `B` and `C`, but aren't interested in the relative order
-of `B` and `C`. In this case, the test should reflect our real intent,
-instead of being overly constraining.
-
-Google Mock allows you to impose an arbitrary DAG (directed acyclic
-graph) on the calls. One way to express the DAG is to use the
-[After](CheatSheet.md#the-after-clause) clause of `EXPECT_CALL`.
-
-Another way is via the `InSequence()` clause (not the same as the
-`InSequence` class), which we borrowed from jMock 2. It's less
-flexible than `After()`, but more convenient when you have long chains
-of sequential calls, as it doesn't require you to come up with
-different names for the expectations in the chains.  Here's how it
-works:
-
-If we view `EXPECT_CALL()` statements as nodes in a graph, and add an
-edge from node A to node B wherever A must occur before B, we can get
-a DAG. We use the term "sequence" to mean a directed path in this
-DAG. Now, if we decompose the DAG into sequences, we just need to know
-which sequences each `EXPECT_CALL()` belongs to in order to be able to
-reconstruct the original DAG.
-
-So, to specify the partial order on the expectations we need to do two
-things: first to define some `Sequence` objects, and then for each
-`EXPECT_CALL()` say which `Sequence` objects it is part
-of. Expectations in the same sequence must occur in the order they are
-written. For example,
-
-```
-  using ::testing::Sequence;
-
-  Sequence s1, s2;
-
-  EXPECT_CALL(foo, A())
-      .InSequence(s1, s2);
-  EXPECT_CALL(bar, B())
-      .InSequence(s1);
-  EXPECT_CALL(bar, C())
-      .InSequence(s2);
-  EXPECT_CALL(foo, D())
-      .InSequence(s2);
-```
-
-specifies the following DAG (where `s1` is `A -> B`, and `s2` is `A ->
-C -> D`):
-
-```
-       +---> B
-       |
-  A ---|
-       |
-       +---> C ---> D
-```
-
-This means that A must occur before B and C, and C must occur before
-D. There's no restriction about the order other than these.
-
-## Controlling When an Expectation Retires ##
-
-When a mock method is called, Google Mock only consider expectations
-that are still active. An expectation is active when created, and
-becomes inactive (aka _retires_) when a call that has to occur later
-has occurred. For example, in
-
-```
-  using ::testing::_;
-  using ::testing::Sequence;
-
-  Sequence s1, s2;
-
-  EXPECT_CALL(log, Log(WARNING, _, "File too large."))     // #1
-      .Times(AnyNumber())
-      .InSequence(s1, s2);
-  EXPECT_CALL(log, Log(WARNING, _, "Data set is empty."))  // #2
-      .InSequence(s1);
-  EXPECT_CALL(log, Log(WARNING, _, "User not found."))     // #3
-      .InSequence(s2);
-```
-
-as soon as either #2 or #3 is matched, #1 will retire. If a warning
-`"File too large."` is logged after this, it will be an error.
-
-Note that an expectation doesn't retire automatically when it's
-saturated. For example,
-
-```
-using ::testing::_;
-...
-  EXPECT_CALL(log, Log(WARNING, _, _));                  // #1
-  EXPECT_CALL(log, Log(WARNING, _, "File too large."));  // #2
-```
-
-says that there will be exactly one warning with the message `"File
-too large."`. If the second warning contains this message too, #2 will
-match again and result in an upper-bound-violated error.
-
-If this is not what you want, you can ask an expectation to retire as
-soon as it becomes saturated:
-
-```
-using ::testing::_;
-...
-  EXPECT_CALL(log, Log(WARNING, _, _));                 // #1
-  EXPECT_CALL(log, Log(WARNING, _, "File too large."))  // #2
-      .RetiresOnSaturation();
-```
-
-Here #2 can be used only once, so if you have two warnings with the
-message `"File too large."`, the first will match #2 and the second
-will match #1 - there will be no error.
-
-# Using Actions #
-
-## Returning References from Mock Methods ##
-
-If a mock function's return type is a reference, you need to use
-`ReturnRef()` instead of `Return()` to return a result:
-
-```
-using ::testing::ReturnRef;
-
-class MockFoo : public Foo {
- public:
-  MOCK_METHOD0(GetBar, Bar&());
-};
-...
-
-  MockFoo foo;
-  Bar bar;
-  EXPECT_CALL(foo, GetBar())
-      .WillOnce(ReturnRef(bar));
-```
-
-## Returning Live Values from Mock Methods ##
-
-The `Return(x)` action saves a copy of `x` when the action is
-_created_, and always returns the same value whenever it's
-executed. Sometimes you may want to instead return the _live_ value of
-`x` (i.e. its value at the time when the action is _executed_.).
-
-If the mock function's return type is a reference, you can do it using
-`ReturnRef(x)`, as shown in the previous recipe ("Returning References
-from Mock Methods"). However, Google Mock doesn't let you use
-`ReturnRef()` in a mock function whose return type is not a reference,
-as doing that usually indicates a user error. So, what shall you do?
-
-You may be tempted to try `ByRef()`:
-
-```
-using testing::ByRef;
-using testing::Return;
-
-class MockFoo : public Foo {
- public:
-  MOCK_METHOD0(GetValue, int());
-};
-...
-  int x = 0;
-  MockFoo foo;
-  EXPECT_CALL(foo, GetValue())
-      .WillRepeatedly(Return(ByRef(x)));
-  x = 42;
-  EXPECT_EQ(42, foo.GetValue());
-```
-
-Unfortunately, it doesn't work here. The above code will fail with error:
-
-```
-Value of: foo.GetValue()
-  Actual: 0
-Expected: 42
-```
-
-The reason is that `Return(value)` converts `value` to the actual
-return type of the mock function at the time when the action is
-_created_, not when it is _executed_. (This behavior was chosen for
-the action to be safe when `value` is a proxy object that references
-some temporary objects.) As a result, `ByRef(x)` is converted to an
-`int` value (instead of a `const int&`) when the expectation is set,
-and `Return(ByRef(x))` will always return 0.
-
-`ReturnPointee(pointer)` was provided to solve this problem
-specifically. It returns the value pointed to by `pointer` at the time
-the action is _executed_:
-
-```
-using testing::ReturnPointee;
-...
-  int x = 0;
-  MockFoo foo;
-  EXPECT_CALL(foo, GetValue())
-      .WillRepeatedly(ReturnPointee(&x));  // Note the & here.
-  x = 42;
-  EXPECT_EQ(42, foo.GetValue());  // This will succeed now.
-```
-
-## Combining Actions ##
-
-Want to do more than one thing when a function is called? That's
-fine. `DoAll()` allow you to do sequence of actions every time. Only
-the return value of the last action in the sequence will be used.
-
-```
-using ::testing::DoAll;
-
-class MockFoo : public Foo {
- public:
-  MOCK_METHOD1(Bar, bool(int n));
-};
-...
-
-  EXPECT_CALL(foo, Bar(_))
-      .WillOnce(DoAll(action_1,
-                      action_2,
-                      ...
-                      action_n));
-```
-
-## Mocking Side Effects ##
-
-Sometimes a method exhibits its effect not via returning a value but
-via side effects. For example, it may change some global state or
-modify an output argument. To mock side effects, in general you can
-define your own action by implementing `::testing::ActionInterface`.
-
-If all you need to do is to change an output argument, the built-in
-`SetArgPointee()` action is convenient:
-
-```
-using ::testing::SetArgPointee;
-
-class MockMutator : public Mutator {
- public:
-  MOCK_METHOD2(Mutate, void(bool mutate, int* value));
-  ...
-};
-...
-
-  MockMutator mutator;
-  EXPECT_CALL(mutator, Mutate(true, _))
-      .WillOnce(SetArgPointee<1>(5));
-```
-
-In this example, when `mutator.Mutate()` is called, we will assign 5
-to the `int` variable pointed to by argument #1
-(0-based).
-
-`SetArgPointee()` conveniently makes an internal copy of the
-value you pass to it, removing the need to keep the value in scope and
-alive. The implication however is that the value must have a copy
-constructor and assignment operator.
-
-If the mock method also needs to return a value as well, you can chain
-`SetArgPointee()` with `Return()` using `DoAll()`:
-
-```
-using ::testing::_;
-using ::testing::Return;
-using ::testing::SetArgPointee;
-
-class MockMutator : public Mutator {
- public:
-  ...
-  MOCK_METHOD1(MutateInt, bool(int* value));
-};
-...
-
-  MockMutator mutator;
-  EXPECT_CALL(mutator, MutateInt(_))
-      .WillOnce(DoAll(SetArgPointee<0>(5),
-                      Return(true)));
-```
-
-If the output argument is an array, use the
-`SetArrayArgument(first, last)` action instead. It copies the
-elements in source range `[first, last)` to the array pointed to by
-the `N`-th (0-based) argument:
-
-```
-using ::testing::NotNull;
-using ::testing::SetArrayArgument;
-
-class MockArrayMutator : public ArrayMutator {
- public:
-  MOCK_METHOD2(Mutate, void(int* values, int num_values));
-  ...
-};
-...
-
-  MockArrayMutator mutator;
-  int values[5] = { 1, 2, 3, 4, 5 };
-  EXPECT_CALL(mutator, Mutate(NotNull(), 5))
-      .WillOnce(SetArrayArgument<0>(values, values + 5));
-```
-
-This also works when the argument is an output iterator:
-
-```
-using ::testing::_;
-using ::testing::SetArrayArgument;
-
-class MockRolodex : public Rolodex {
- public:
-  MOCK_METHOD1(GetNames, void(std::back_insert_iterator >));
-  ...
-};
-...
-
-  MockRolodex rolodex;
-  vector names;
-  names.push_back("George");
-  names.push_back("John");
-  names.push_back("Thomas");
-  EXPECT_CALL(rolodex, GetNames(_))
-      .WillOnce(SetArrayArgument<0>(names.begin(), names.end()));
-```
-
-## Changing a Mock Object's Behavior Based on the State ##
-
-If you expect a call to change the behavior of a mock object, you can use `::testing::InSequence` to specify different behaviors before and after the call:
-
-```
-using ::testing::InSequence;
-using ::testing::Return;
-
-...
-  {
-    InSequence seq;
-    EXPECT_CALL(my_mock, IsDirty())
-        .WillRepeatedly(Return(true));
-    EXPECT_CALL(my_mock, Flush());
-    EXPECT_CALL(my_mock, IsDirty())
-        .WillRepeatedly(Return(false));
-  }
-  my_mock.FlushIfDirty();
-```
-
-This makes `my_mock.IsDirty()` return `true` before `my_mock.Flush()` is called and return `false` afterwards.
-
-If the behavior change is more complex, you can store the effects in a variable and make a mock method get its return value from that variable:
-
-```
-using ::testing::_;
-using ::testing::SaveArg;
-using ::testing::Return;
-
-ACTION_P(ReturnPointee, p) { return *p; }
-...
-  int previous_value = 0;
-  EXPECT_CALL(my_mock, GetPrevValue())
-      .WillRepeatedly(ReturnPointee(&previous_value));
-  EXPECT_CALL(my_mock, UpdateValue(_))
-      .WillRepeatedly(SaveArg<0>(&previous_value));
-  my_mock.DoSomethingToUpdateValue();
-```
-
-Here `my_mock.GetPrevValue()` will always return the argument of the last `UpdateValue()` call.
-
-## Setting the Default Value for a Return Type ##
-
-If a mock method's return type is a built-in C++ type or pointer, by
-default it will return 0 when invoked. Also, in C++ 11 and above, a mock
-method whose return type has a default constructor will return a default-constructed
-value by default.  You only need to specify an
-action if this default value doesn't work for you.
-
-Sometimes, you may want to change this default value, or you may want
-to specify a default value for types Google Mock doesn't know
-about. You can do this using the `::testing::DefaultValue` class
-template:
-
-```
-class MockFoo : public Foo {
- public:
-  MOCK_METHOD0(CalculateBar, Bar());
-};
-...
-
-  Bar default_bar;
-  // Sets the default return value for type Bar.
-  DefaultValue::Set(default_bar);
-
-  MockFoo foo;
-
-  // We don't need to specify an action here, as the default
-  // return value works for us.
-  EXPECT_CALL(foo, CalculateBar());
-
-  foo.CalculateBar();  // This should return default_bar.
-
-  // Unsets the default return value.
-  DefaultValue::Clear();
-```
-
-Please note that changing the default value for a type can make you
-tests hard to understand. We recommend you to use this feature
-judiciously. For example, you may want to make sure the `Set()` and
-`Clear()` calls are right next to the code that uses your mock.
-
-## Setting the Default Actions for a Mock Method ##
-
-You've learned how to change the default value of a given
-type. However, this may be too coarse for your purpose: perhaps you
-have two mock methods with the same return type and you want them to
-have different behaviors. The `ON_CALL()` macro allows you to
-customize your mock's behavior at the method level:
-
-```
-using ::testing::_;
-using ::testing::AnyNumber;
-using ::testing::Gt;
-using ::testing::Return;
-...
-  ON_CALL(foo, Sign(_))
-      .WillByDefault(Return(-1));
-  ON_CALL(foo, Sign(0))
-      .WillByDefault(Return(0));
-  ON_CALL(foo, Sign(Gt(0)))
-      .WillByDefault(Return(1));
-
-  EXPECT_CALL(foo, Sign(_))
-      .Times(AnyNumber());
-
-  foo.Sign(5);   // This should return 1.
-  foo.Sign(-9);  // This should return -1.
-  foo.Sign(0);   // This should return 0.
-```
-
-As you may have guessed, when there are more than one `ON_CALL()`
-statements, the news order take precedence over the older ones. In
-other words, the **last** one that matches the function arguments will
-be used. This matching order allows you to set up the common behavior
-in a mock object's constructor or the test fixture's set-up phase and
-specialize the mock's behavior later.
-
-## Using Functions/Methods/Functors as Actions ##
-
-If the built-in actions don't suit you, you can easily use an existing
-function, method, or functor as an action:
-
-```
-using ::testing::_;
-using ::testing::Invoke;
-
-class MockFoo : public Foo {
- public:
-  MOCK_METHOD2(Sum, int(int x, int y));
-  MOCK_METHOD1(ComplexJob, bool(int x));
-};
-
-int CalculateSum(int x, int y) { return x + y; }
-
-class Helper {
- public:
-  bool ComplexJob(int x);
-};
-...
-
-  MockFoo foo;
-  Helper helper;
-  EXPECT_CALL(foo, Sum(_, _))
-      .WillOnce(Invoke(CalculateSum));
-  EXPECT_CALL(foo, ComplexJob(_))
-      .WillOnce(Invoke(&helper, &Helper::ComplexJob));
-
-  foo.Sum(5, 6);       // Invokes CalculateSum(5, 6).
-  foo.ComplexJob(10);  // Invokes helper.ComplexJob(10);
-```
-
-The only requirement is that the type of the function, etc must be
-_compatible_ with the signature of the mock function, meaning that the
-latter's arguments can be implicitly converted to the corresponding
-arguments of the former, and the former's return type can be
-implicitly converted to that of the latter. So, you can invoke
-something whose type is _not_ exactly the same as the mock function,
-as long as it's safe to do so - nice, huh?
-
-## Invoking a Function/Method/Functor Without Arguments ##
-
-`Invoke()` is very useful for doing actions that are more complex. It
-passes the mock function's arguments to the function or functor being
-invoked such that the callee has the full context of the call to work
-with. If the invoked function is not interested in some or all of the
-arguments, it can simply ignore them.
-
-Yet, a common pattern is that a test author wants to invoke a function
-without the arguments of the mock function. `Invoke()` allows her to
-do that using a wrapper function that throws away the arguments before
-invoking an underlining nullary function. Needless to say, this can be
-tedious and obscures the intent of the test.
-
-`InvokeWithoutArgs()` solves this problem. It's like `Invoke()` except
-that it doesn't pass the mock function's arguments to the
-callee. Here's an example:
-
-```
-using ::testing::_;
-using ::testing::InvokeWithoutArgs;
-
-class MockFoo : public Foo {
- public:
-  MOCK_METHOD1(ComplexJob, bool(int n));
-};
-
-bool Job1() { ... }
-...
-
-  MockFoo foo;
-  EXPECT_CALL(foo, ComplexJob(_))
-      .WillOnce(InvokeWithoutArgs(Job1));
-
-  foo.ComplexJob(10);  // Invokes Job1().
-```
-
-## Invoking an Argument of the Mock Function ##
-
-Sometimes a mock function will receive a function pointer or a functor
-(in other words, a "callable") as an argument, e.g.
-
-```
-class MockFoo : public Foo {
- public:
-  MOCK_METHOD2(DoThis, bool(int n, bool (*fp)(int)));
-};
-```
-
-and you may want to invoke this callable argument:
-
-```
-using ::testing::_;
-...
-  MockFoo foo;
-  EXPECT_CALL(foo, DoThis(_, _))
-      .WillOnce(...);
-  // Will execute (*fp)(5), where fp is the
-  // second argument DoThis() receives.
-```
-
-Arghh, you need to refer to a mock function argument but your version
-of C++ has no lambdas, so you have to define your own action. :-(
-Or do you really?
-
-Well, Google Mock has an action to solve _exactly_ this problem:
-
-```
-  InvokeArgument(arg_1, arg_2, ..., arg_m)
-```
-
-will invoke the `N`-th (0-based) argument the mock function receives,
-with `arg_1`, `arg_2`, ..., and `arg_m`. No matter if the argument is
-a function pointer or a functor, Google Mock handles them both.
-
-With that, you could write:
-
-```
-using ::testing::_;
-using ::testing::InvokeArgument;
-...
-  EXPECT_CALL(foo, DoThis(_, _))
-      .WillOnce(InvokeArgument<1>(5));
-  // Will execute (*fp)(5), where fp is the
-  // second argument DoThis() receives.
-```
-
-What if the callable takes an argument by reference? No problem - just
-wrap it inside `ByRef()`:
-
-```
-...
-  MOCK_METHOD1(Bar, bool(bool (*fp)(int, const Helper&)));
-...
-using ::testing::_;
-using ::testing::ByRef;
-using ::testing::InvokeArgument;
-...
-
-  MockFoo foo;
-  Helper helper;
-  ...
-  EXPECT_CALL(foo, Bar(_))
-      .WillOnce(InvokeArgument<0>(5, ByRef(helper)));
-  // ByRef(helper) guarantees that a reference to helper, not a copy of it,
-  // will be passed to the callable.
-```
-
-What if the callable takes an argument by reference and we do **not**
-wrap the argument in `ByRef()`? Then `InvokeArgument()` will _make a
-copy_ of the argument, and pass a _reference to the copy_, instead of
-a reference to the original value, to the callable. This is especially
-handy when the argument is a temporary value:
-
-```
-...
-  MOCK_METHOD1(DoThat, bool(bool (*f)(const double& x, const string& s)));
-...
-using ::testing::_;
-using ::testing::InvokeArgument;
-...
-
-  MockFoo foo;
-  ...
-  EXPECT_CALL(foo, DoThat(_))
-      .WillOnce(InvokeArgument<0>(5.0, string("Hi")));
-  // Will execute (*f)(5.0, string("Hi")), where f is the function pointer
-  // DoThat() receives.  Note that the values 5.0 and string("Hi") are
-  // temporary and dead once the EXPECT_CALL() statement finishes.  Yet
-  // it's fine to perform this action later, since a copy of the values
-  // are kept inside the InvokeArgument action.
-```
-
-## Ignoring an Action's Result ##
-
-Sometimes you have an action that returns _something_, but you need an
-action that returns `void` (perhaps you want to use it in a mock
-function that returns `void`, or perhaps it needs to be used in
-`DoAll()` and it's not the last in the list). `IgnoreResult()` lets
-you do that. For example:
-
-```
-using ::testing::_;
-using ::testing::Invoke;
-using ::testing::Return;
-
-int Process(const MyData& data);
-string DoSomething();
-
-class MockFoo : public Foo {
- public:
-  MOCK_METHOD1(Abc, void(const MyData& data));
-  MOCK_METHOD0(Xyz, bool());
-};
-...
-
-  MockFoo foo;
-  EXPECT_CALL(foo, Abc(_))
-  // .WillOnce(Invoke(Process));
-  // The above line won't compile as Process() returns int but Abc() needs
-  // to return void.
-      .WillOnce(IgnoreResult(Invoke(Process)));
-
-  EXPECT_CALL(foo, Xyz())
-      .WillOnce(DoAll(IgnoreResult(Invoke(DoSomething)),
-      // Ignores the string DoSomething() returns.
-                      Return(true)));
-```
-
-Note that you **cannot** use `IgnoreResult()` on an action that already
-returns `void`. Doing so will lead to ugly compiler errors.
-
-## Selecting an Action's Arguments ##
-
-Say you have a mock function `Foo()` that takes seven arguments, and
-you have a custom action that you want to invoke when `Foo()` is
-called. Trouble is, the custom action only wants three arguments:
-
-```
-using ::testing::_;
-using ::testing::Invoke;
-...
-  MOCK_METHOD7(Foo, bool(bool visible, const string& name, int x, int y,
-                         const map, double>& weight,
-                         double min_weight, double max_wight));
-...
-
-bool IsVisibleInQuadrant1(bool visible, int x, int y) {
-  return visible && x >= 0 && y >= 0;
-}
-...
-
-  EXPECT_CALL(mock, Foo(_, _, _, _, _, _, _))
-      .WillOnce(Invoke(IsVisibleInQuadrant1));  // Uh, won't compile. :-(
-```
-
-To please the compiler God, you can to define an "adaptor" that has
-the same signature as `Foo()` and calls the custom action with the
-right arguments:
-
-```
-using ::testing::_;
-using ::testing::Invoke;
-
-bool MyIsVisibleInQuadrant1(bool visible, const string& name, int x, int y,
-                            const map, double>& weight,
-                            double min_weight, double max_wight) {
-  return IsVisibleInQuadrant1(visible, x, y);
-}
-...
-
-  EXPECT_CALL(mock, Foo(_, _, _, _, _, _, _))
-      .WillOnce(Invoke(MyIsVisibleInQuadrant1));  // Now it works.
-```
-
-But isn't this awkward?
-
-Google Mock provides a generic _action adaptor_, so you can spend your
-time minding more important business than writing your own
-adaptors. Here's the syntax:
-
-```
-  WithArgs(action)
-```
-
-creates an action that passes the arguments of the mock function at
-the given indices (0-based) to the inner `action` and performs
-it. Using `WithArgs`, our original example can be written as:
-
-```
-using ::testing::_;
-using ::testing::Invoke;
-using ::testing::WithArgs;
-...
-  EXPECT_CALL(mock, Foo(_, _, _, _, _, _, _))
-      .WillOnce(WithArgs<0, 2, 3>(Invoke(IsVisibleInQuadrant1)));
-      // No need to define your own adaptor.
-```
-
-For better readability, Google Mock also gives you:
-
-  * `WithoutArgs(action)` when the inner `action` takes _no_ argument, and
-  * `WithArg(action)` (no `s` after `Arg`) when the inner `action` takes _one_ argument.
-
-As you may have realized, `InvokeWithoutArgs(...)` is just syntactic
-sugar for `WithoutArgs(Invoke(...))`.
-
-Here are more tips:
-
-  * The inner action used in `WithArgs` and friends does not have to be `Invoke()` -- it can be anything.
-  * You can repeat an argument in the argument list if necessary, e.g. `WithArgs<2, 3, 3, 5>(...)`.
-  * You can change the order of the arguments, e.g. `WithArgs<3, 2, 1>(...)`.
-  * The types of the selected arguments do _not_ have to match the signature of the inner action exactly. It works as long as they can be implicitly converted to the corresponding arguments of the inner action. For example, if the 4-th argument of the mock function is an `int` and `my_action` takes a `double`, `WithArg<4>(my_action)` will work.
-
-## Ignoring Arguments in Action Functions ##
-
-The selecting-an-action's-arguments recipe showed us one way to make a
-mock function and an action with incompatible argument lists fit
-together. The downside is that wrapping the action in
-`WithArgs<...>()` can get tedious for people writing the tests.
-
-If you are defining a function, method, or functor to be used with
-`Invoke*()`, and you are not interested in some of its arguments, an
-alternative to `WithArgs` is to declare the uninteresting arguments as
-`Unused`. This makes the definition less cluttered and less fragile in
-case the types of the uninteresting arguments change. It could also
-increase the chance the action function can be reused. For example,
-given
-
-```
-  MOCK_METHOD3(Foo, double(const string& label, double x, double y));
-  MOCK_METHOD3(Bar, double(int index, double x, double y));
-```
-
-instead of
-
-```
-using ::testing::_;
-using ::testing::Invoke;
-
-double DistanceToOriginWithLabel(const string& label, double x, double y) {
-  return sqrt(x*x + y*y);
-}
-
-double DistanceToOriginWithIndex(int index, double x, double y) {
-  return sqrt(x*x + y*y);
-}
-...
-
-  EXEPCT_CALL(mock, Foo("abc", _, _))
-      .WillOnce(Invoke(DistanceToOriginWithLabel));
-  EXEPCT_CALL(mock, Bar(5, _, _))
-      .WillOnce(Invoke(DistanceToOriginWithIndex));
-```
-
-you could write
-
-```
-using ::testing::_;
-using ::testing::Invoke;
-using ::testing::Unused;
-
-double DistanceToOrigin(Unused, double x, double y) {
-  return sqrt(x*x + y*y);
-}
-...
-
-  EXEPCT_CALL(mock, Foo("abc", _, _))
-      .WillOnce(Invoke(DistanceToOrigin));
-  EXEPCT_CALL(mock, Bar(5, _, _))
-      .WillOnce(Invoke(DistanceToOrigin));
-```
-
-## Sharing Actions ##
-
-Just like matchers, a Google Mock action object consists of a pointer
-to a ref-counted implementation object. Therefore copying actions is
-also allowed and very efficient. When the last action that references
-the implementation object dies, the implementation object will be
-deleted.
-
-If you have some complex action that you want to use again and again,
-you may not have to build it from scratch every time. If the action
-doesn't have an internal state (i.e. if it always does the same thing
-no matter how many times it has been called), you can assign it to an
-action variable and use that variable repeatedly. For example:
-
-```
-  Action set_flag = DoAll(SetArgPointee<0>(5),
-                                      Return(true));
-  ... use set_flag in .WillOnce() and .WillRepeatedly() ...
-```
-
-However, if the action has its own state, you may be surprised if you
-share the action object. Suppose you have an action factory
-`IncrementCounter(init)` which creates an action that increments and
-returns a counter whose initial value is `init`, using two actions
-created from the same expression and using a shared action will
-exihibit different behaviors. Example:
-
-```
-  EXPECT_CALL(foo, DoThis())
-      .WillRepeatedly(IncrementCounter(0));
-  EXPECT_CALL(foo, DoThat())
-      .WillRepeatedly(IncrementCounter(0));
-  foo.DoThis();  // Returns 1.
-  foo.DoThis();  // Returns 2.
-  foo.DoThat();  // Returns 1 - Blah() uses a different
-                 // counter than Bar()'s.
-```
-
-versus
-
-```
-  Action increment = IncrementCounter(0);
-
-  EXPECT_CALL(foo, DoThis())
-      .WillRepeatedly(increment);
-  EXPECT_CALL(foo, DoThat())
-      .WillRepeatedly(increment);
-  foo.DoThis();  // Returns 1.
-  foo.DoThis();  // Returns 2.
-  foo.DoThat();  // Returns 3 - the counter is shared.
-```
-
-# Misc Recipes on Using Google Mock #
-
-## Mocking Methods That Use Move-Only Types ##
-
-C++11 introduced move-only types.  A move-only-typed value can be moved from one object to another, but cannot be copied.  `std::unique_ptr` is probably the most commonly used move-only type.
-
-Mocking a method that takes and/or returns move-only types presents some challenges, but nothing insurmountable.  This recipe shows you how you can do it.
-
-Let’s say we are working on a fictional project that lets one post and share snippets called “buzzes”.  Your code uses these types:
-
-```
-enum class AccessLevel { kInternal, kPublic };
-
-class Buzz {
- public:
-  explicit Buzz(AccessLevel access) { … }
-  ...
-};
-
-class Buzzer {
- public:
-  virtual ~Buzzer() {}
-  virtual std::unique_ptr MakeBuzz(const std::string& text) = 0;
-  virtual bool ShareBuzz(std::unique_ptr buzz, Time timestamp) = 0;
-  ...
-};
-```
-
-A `Buzz` object represents a snippet being posted.  A class that implements the `Buzzer` interface is capable of creating and sharing `Buzz`.  Methods in `Buzzer` may return a `unique_ptr` or take a `unique_ptr`.  Now we need to mock `Buzzer` in our tests.
-
-To mock a method that returns a move-only type, you just use the familiar `MOCK_METHOD` syntax as usual:
-
-```
-class MockBuzzer : public Buzzer {
- public:
-  MOCK_METHOD1(MakeBuzz, std::unique_ptr(const std::string& text));
-  …
-};
-```
-
-However, if you attempt to use the same `MOCK_METHOD` pattern to mock a method that takes a move-only parameter, you’ll get a compiler error currently:
-
-```
-  // Does NOT compile!
-  MOCK_METHOD2(ShareBuzz, bool(std::unique_ptr buzz, Time timestamp));
-```
-
-While it’s highly desirable to make this syntax just work, it’s not trivial and the work hasn’t been done yet.  Fortunately, there is a trick you can apply today to get something that works nearly as well as this.
-
-The trick, is to delegate the `ShareBuzz()` method to a mock method (let’s call it `DoShareBuzz()`) that does not take move-only parameters:
-
-```
-class MockBuzzer : public Buzzer {
- public:
-  MOCK_METHOD1(MakeBuzz, std::unique_ptr(const std::string& text));
-  MOCK_METHOD2(DoShareBuzz, bool(Buzz* buzz, Time timestamp));
-  bool ShareBuzz(std::unique_ptr buzz, Time timestamp) {
-    return DoShareBuzz(buzz.get(), timestamp);
-  }
-};
-```
-
-Note that there's no need to define or declare `DoShareBuzz()` in a base class.  You only need to define it as a `MOCK_METHOD` in the mock class.
-
-Now that we have the mock class defined, we can use it in tests.  In the following code examples, we assume that we have defined a `MockBuzzer` object named `mock_buzzer_`:
-
-```
-  MockBuzzer mock_buzzer_;
-```
-
-First let’s see how we can set expectations on the `MakeBuzz()` method, which returns a `unique_ptr`.
-
-As usual, if you set an expectation without an action (i.e. the `.WillOnce()` or `.WillRepeated()` clause), when that expectation fires, the default action for that method will be taken.  Since `unique_ptr<>` has a default constructor that returns a null `unique_ptr`, that’s what you’ll get if you don’t specify an action:
-
-```
-  // Use the default action.
-  EXPECT_CALL(mock_buzzer_, MakeBuzz("hello"));
-
-  // Triggers the previous EXPECT_CALL.
-  EXPECT_EQ(nullptr, mock_buzzer_.MakeBuzz("hello"));
-```
-
-If you are not happy with the default action, you can tweak it.  Depending on what you need, you may either tweak the default action for a specific (mock object, mock method) combination using `ON_CALL()`, or you may tweak the default action for all mock methods that return a specific type.  The usage of `ON_CALL()` is similar to `EXPECT_CALL()`, so we’ll skip it and just explain how to do the latter (tweaking the default action for a specific return type).  You do this via the `DefaultValue<>::SetFactory()` and `DefaultValue<>::Clear()` API:
-
-```
-  // Sets the default action for return type std::unique_ptr to
-  // creating a new Buzz every time.
-  DefaultValue>::SetFactory(
-      [] { return MakeUnique(AccessLevel::kInternal); });
-
-  // When this fires, the default action of MakeBuzz() will run, which
-  // will return a new Buzz object.
-  EXPECT_CALL(mock_buzzer_, MakeBuzz("hello")).Times(AnyNumber());
-
-  auto buzz1 = mock_buzzer_.MakeBuzz("hello");
-  auto buzz2 = mock_buzzer_.MakeBuzz("hello");
-  EXPECT_NE(nullptr, buzz1);
-  EXPECT_NE(nullptr, buzz2);
-  EXPECT_NE(buzz1, buzz2);
-
-  // Resets the default action for return type std::unique_ptr,
-  // to avoid interfere with other tests.
-  DefaultValue>::Clear();
-```
-
-What if you want the method to do something other than the default action?  If you just need to return a pre-defined move-only value, you can use the `Return(ByMove(...))` action:
-
-```
-  // When this fires, the unique_ptr<> specified by ByMove(...) will
-  // be returned.
-  EXPECT_CALL(mock_buzzer_, MakeBuzz("world"))
-      .WillOnce(Return(ByMove(MakeUnique(AccessLevel::kInternal))));
-
-  EXPECT_NE(nullptr, mock_buzzer_.MakeBuzz("world"));
-```
-
-Note that `ByMove()` is essential here - if you drop it, the code won’t compile.
-
-Quiz time!  What do you think will happen if a `Return(ByMove(...))` action is performed more than once (e.g. you write `….WillRepeatedly(Return(ByMove(...)));`)?  Come think of it, after the first time the action runs, the source value will be consumed (since it’s a move-only value), so the next time around, there’s no value to move from -- you’ll get a run-time error that `Return(ByMove(...))` can only be run once.
-
-If you need your mock method to do more than just moving a pre-defined value, remember that you can always use `Invoke()` to call a lambda or a callable object, which can do pretty much anything you want:
-
-```
-  EXPECT_CALL(mock_buzzer_, MakeBuzz("x"))
-      .WillRepeatedly(Invoke([](const std::string& text) {
-        return std::make_unique(AccessLevel::kInternal);
-      }));
-
-  EXPECT_NE(nullptr, mock_buzzer_.MakeBuzz("x"));
-  EXPECT_NE(nullptr, mock_buzzer_.MakeBuzz("x"));
-```
-
-Every time this `EXPECT_CALL` fires, a new `unique_ptr` will be created and returned.  You cannot do this with `Return(ByMove(...))`.
-
-Now there’s one topic we haven’t covered: how do you set expectations on `ShareBuzz()`, which takes a move-only-typed parameter?  The answer is you don’t.  Instead, you set expectations on the `DoShareBuzz()` mock method (remember that we defined a `MOCK_METHOD` for `DoShareBuzz()`, not `ShareBuzz()`):
-
-```
-  EXPECT_CALL(mock_buzzer_, DoShareBuzz(NotNull(), _));
-
-  // When one calls ShareBuzz() on the MockBuzzer like this, the call is
-  // forwarded to DoShareBuzz(), which is mocked.  Therefore this statement
-  // will trigger the above EXPECT_CALL.
-  mock_buzzer_.ShareBuzz(MakeUnique(AccessLevel::kInternal),
-                         ::base::Now());
-```
-
-Some of you may have spotted one problem with this approach: the `DoShareBuzz()` mock method differs from the real `ShareBuzz()` method in that it cannot take ownership of the buzz parameter - `ShareBuzz()` will always delete buzz after `DoShareBuzz()` returns.  What if you need to save the buzz object somewhere for later use when `ShareBuzz()` is called?  Indeed, you'd be stuck.
-
-Another problem with the `DoShareBuzz()` we had is that it can surprise people reading or maintaining the test, as one would expect that `DoShareBuzz()` has (logically) the same contract as `ShareBuzz()`.
-
-Fortunately, these problems can be fixed with a bit more code.  Let's try to get it right this time:
-
-```
-class MockBuzzer : public Buzzer {
- public:
-  MockBuzzer() {
-    // Since DoShareBuzz(buzz, time) is supposed to take ownership of
-    // buzz, define a default behavior for DoShareBuzz(buzz, time) to
-    // delete buzz.
-    ON_CALL(*this, DoShareBuzz(_, _))
-        .WillByDefault(Invoke([](Buzz* buzz, Time timestamp) {
-          delete buzz;
-          return true;
-        }));
-  }
-
-  MOCK_METHOD1(MakeBuzz, std::unique_ptr(const std::string& text));
-
-  // Takes ownership of buzz.
-  MOCK_METHOD2(DoShareBuzz, bool(Buzz* buzz, Time timestamp));
-  bool ShareBuzz(std::unique_ptr buzz, Time timestamp) {
-    return DoShareBuzz(buzz.release(), timestamp);
-  }
-};
-```
-
-Now, the mock `DoShareBuzz()` method is free to save the buzz argument for later use if this is what you want:
-
-```
-  std::unique_ptr intercepted_buzz;
-  EXPECT_CALL(mock_buzzer_, DoShareBuzz(NotNull(), _))
-      .WillOnce(Invoke([&intercepted_buzz](Buzz* buzz, Time timestamp) {
-        // Save buzz in intercepted_buzz for analysis later.
-        intercepted_buzz.reset(buzz);
-        return false;
-      }));
-
-  mock_buzzer_.ShareBuzz(std::make_unique(AccessLevel::kInternal),
-                         Now());
-  EXPECT_NE(nullptr, intercepted_buzz);
-```
-
-Using the tricks covered in this recipe, you are now able to mock methods that take and/or return move-only types.  Put your newly-acquired power to good use - when you design a new API, you can now feel comfortable using `unique_ptrs` as appropriate, without fearing that doing so will compromise your tests.
-
-## Making the Compilation Faster ##
-
-Believe it or not, the _vast majority_ of the time spent on compiling
-a mock class is in generating its constructor and destructor, as they
-perform non-trivial tasks (e.g. verification of the
-expectations). What's more, mock methods with different signatures
-have different types and thus their constructors/destructors need to
-be generated by the compiler separately. As a result, if you mock many
-different types of methods, compiling your mock class can get really
-slow.
-
-If you are experiencing slow compilation, you can move the definition
-of your mock class' constructor and destructor out of the class body
-and into a `.cpp` file. This way, even if you `#include` your mock
-class in N files, the compiler only needs to generate its constructor
-and destructor once, resulting in a much faster compilation.
-
-Let's illustrate the idea using an example. Here's the definition of a
-mock class before applying this recipe:
-
-```
-// File mock_foo.h.
-...
-class MockFoo : public Foo {
- public:
-  // Since we don't declare the constructor or the destructor,
-  // the compiler will generate them in every translation unit
-  // where this mock class is used.
-
-  MOCK_METHOD0(DoThis, int());
-  MOCK_METHOD1(DoThat, bool(const char* str));
-  ... more mock methods ...
-};
-```
-
-After the change, it would look like:
-
-```
-// File mock_foo.h.
-...
-class MockFoo : public Foo {
- public:
-  // The constructor and destructor are declared, but not defined, here.
-  MockFoo();
-  virtual ~MockFoo();
-
-  MOCK_METHOD0(DoThis, int());
-  MOCK_METHOD1(DoThat, bool(const char* str));
-  ... more mock methods ...
-};
-```
-and
-```
-// File mock_foo.cpp.
-#include "path/to/mock_foo.h"
-
-// The definitions may appear trivial, but the functions actually do a
-// lot of things through the constructors/destructors of the member
-// variables used to implement the mock methods.
-MockFoo::MockFoo() {}
-MockFoo::~MockFoo() {}
-```
-
-## Forcing a Verification ##
-
-When it's being destroyed, your friendly mock object will automatically
-verify that all expectations on it have been satisfied, and will
-generate [Google Test](../../googletest/) failures
-if not. This is convenient as it leaves you with one less thing to
-worry about. That is, unless you are not sure if your mock object will
-be destroyed.
-
-How could it be that your mock object won't eventually be destroyed?
-Well, it might be created on the heap and owned by the code you are
-testing. Suppose there's a bug in that code and it doesn't delete the
-mock object properly - you could end up with a passing test when
-there's actually a bug.
-
-Using a heap checker is a good idea and can alleviate the concern, but
-its implementation may not be 100% reliable. So, sometimes you do want
-to _force_ Google Mock to verify a mock object before it is
-(hopefully) destructed. You can do this with
-`Mock::VerifyAndClearExpectations(&mock_object)`:
-
-```
-TEST(MyServerTest, ProcessesRequest) {
-  using ::testing::Mock;
-
-  MockFoo* const foo = new MockFoo;
-  EXPECT_CALL(*foo, ...)...;
-  // ... other expectations ...
-
-  // server now owns foo.
-  MyServer server(foo);
-  server.ProcessRequest(...);
-
-  // In case that server's destructor will forget to delete foo,
-  // this will verify the expectations anyway.
-  Mock::VerifyAndClearExpectations(foo);
-}  // server is destroyed when it goes out of scope here.
-```
-
-**Tip:** The `Mock::VerifyAndClearExpectations()` function returns a
-`bool` to indicate whether the verification was successful (`true` for
-yes), so you can wrap that function call inside a `ASSERT_TRUE()` if
-there is no point going further when the verification has failed.
-
-## Using Check Points ##
-
-Sometimes you may want to "reset" a mock object at various check
-points in your test: at each check point, you verify that all existing
-expectations on the mock object have been satisfied, and then you set
-some new expectations on it as if it's newly created. This allows you
-to work with a mock object in "phases" whose sizes are each
-manageable.
-
-One such scenario is that in your test's `SetUp()` function, you may
-want to put the object you are testing into a certain state, with the
-help from a mock object. Once in the desired state, you want to clear
-all expectations on the mock, such that in the `TEST_F` body you can
-set fresh expectations on it.
-
-As you may have figured out, the `Mock::VerifyAndClearExpectations()`
-function we saw in the previous recipe can help you here. Or, if you
-are using `ON_CALL()` to set default actions on the mock object and
-want to clear the default actions as well, use
-`Mock::VerifyAndClear(&mock_object)` instead. This function does what
-`Mock::VerifyAndClearExpectations(&mock_object)` does and returns the
-same `bool`, **plus** it clears the `ON_CALL()` statements on
-`mock_object` too.
-
-Another trick you can use to achieve the same effect is to put the
-expectations in sequences and insert calls to a dummy "check-point"
-function at specific places. Then you can verify that the mock
-function calls do happen at the right time. For example, if you are
-exercising code:
-
-```
-Foo(1);
-Foo(2);
-Foo(3);
-```
-
-and want to verify that `Foo(1)` and `Foo(3)` both invoke
-`mock.Bar("a")`, but `Foo(2)` doesn't invoke anything. You can write:
-
-```
-using ::testing::MockFunction;
-
-TEST(FooTest, InvokesBarCorrectly) {
-  MyMock mock;
-  // Class MockFunction has exactly one mock method.  It is named
-  // Call() and has type F.
-  MockFunction check;
-  {
-    InSequence s;
-
-    EXPECT_CALL(mock, Bar("a"));
-    EXPECT_CALL(check, Call("1"));
-    EXPECT_CALL(check, Call("2"));
-    EXPECT_CALL(mock, Bar("a"));
-  }
-  Foo(1);
-  check.Call("1");
-  Foo(2);
-  check.Call("2");
-  Foo(3);
-}
-```
-
-The expectation spec says that the first `Bar("a")` must happen before
-check point "1", the second `Bar("a")` must happen after check point "2",
-and nothing should happen between the two check points. The explicit
-check points make it easy to tell which `Bar("a")` is called by which
-call to `Foo()`.
-
-## Mocking Destructors ##
-
-Sometimes you want to make sure a mock object is destructed at the
-right time, e.g. after `bar->A()` is called but before `bar->B()` is
-called. We already know that you can specify constraints on the order
-of mock function calls, so all we need to do is to mock the destructor
-of the mock function.
-
-This sounds simple, except for one problem: a destructor is a special
-function with special syntax and special semantics, and the
-`MOCK_METHOD0` macro doesn't work for it:
-
-```
-  MOCK_METHOD0(~MockFoo, void());  // Won't compile!
-```
-
-The good news is that you can use a simple pattern to achieve the same
-effect. First, add a mock function `Die()` to your mock class and call
-it in the destructor, like this:
-
-```
-class MockFoo : public Foo {
-  ...
-  // Add the following two lines to the mock class.
-  MOCK_METHOD0(Die, void());
-  virtual ~MockFoo() { Die(); }
-};
-```
-
-(If the name `Die()` clashes with an existing symbol, choose another
-name.) Now, we have translated the problem of testing when a `MockFoo`
-object dies to testing when its `Die()` method is called:
-
-```
-  MockFoo* foo = new MockFoo;
-  MockBar* bar = new MockBar;
-  ...
-  {
-    InSequence s;
-
-    // Expects *foo to die after bar->A() and before bar->B().
-    EXPECT_CALL(*bar, A());
-    EXPECT_CALL(*foo, Die());
-    EXPECT_CALL(*bar, B());
-  }
-```
-
-And that's that.
-
-## Using Google Mock and Threads ##
-
-**IMPORTANT NOTE:** What we describe in this recipe is **ONLY** true on
-platforms where Google Mock is thread-safe. Currently these are only
-platforms that support the pthreads library (this includes Linux and Mac).
-To make it thread-safe on other platforms we only need to implement
-some synchronization operations in `"gtest/internal/gtest-port.h"`.
-
-In a **unit** test, it's best if you could isolate and test a piece of
-code in a single-threaded context. That avoids race conditions and
-dead locks, and makes debugging your test much easier.
-
-Yet many programs are multi-threaded, and sometimes to test something
-we need to pound on it from more than one thread. Google Mock works
-for this purpose too.
-
-Remember the steps for using a mock:
-
-  1. Create a mock object `foo`.
-  1. Set its default actions and expectations using `ON_CALL()` and `EXPECT_CALL()`.
-  1. The code under test calls methods of `foo`.
-  1. Optionally, verify and reset the mock.
-  1. Destroy the mock yourself, or let the code under test destroy it. The destructor will automatically verify it.
-
-If you follow the following simple rules, your mocks and threads can
-live happily together:
-
-  * Execute your _test code_ (as opposed to the code being tested) in _one_ thread. This makes your test easy to follow.
-  * Obviously, you can do step #1 without locking.
-  * When doing step #2 and #5, make sure no other thread is accessing `foo`. Obvious too, huh?
-  * #3 and #4 can be done either in one thread or in multiple threads - anyway you want. Google Mock takes care of the locking, so you don't have to do any - unless required by your test logic.
-
-If you violate the rules (for example, if you set expectations on a
-mock while another thread is calling its methods), you get undefined
-behavior. That's not fun, so don't do it.
-
-Google Mock guarantees that the action for a mock function is done in
-the same thread that called the mock function. For example, in
-
-```
-  EXPECT_CALL(mock, Foo(1))
-      .WillOnce(action1);
-  EXPECT_CALL(mock, Foo(2))
-      .WillOnce(action2);
-```
-
-if `Foo(1)` is called in thread 1 and `Foo(2)` is called in thread 2,
-Google Mock will execute `action1` in thread 1 and `action2` in thread
-2.
-
-Google Mock does _not_ impose a sequence on actions performed in
-different threads (doing so may create deadlocks as the actions may
-need to cooperate). This means that the execution of `action1` and
-`action2` in the above example _may_ interleave. If this is a problem,
-you should add proper synchronization logic to `action1` and `action2`
-to make the test thread-safe.
-
-
-Also, remember that `DefaultValue` is a global resource that
-potentially affects _all_ living mock objects in your
-program. Naturally, you won't want to mess with it from multiple
-threads or when there still are mocks in action.
-
-## Controlling How Much Information Google Mock Prints ##
-
-When Google Mock sees something that has the potential of being an
-error (e.g. a mock function with no expectation is called, a.k.a. an
-uninteresting call, which is allowed but perhaps you forgot to
-explicitly ban the call), it prints some warning messages, including
-the arguments of the function and the return value. Hopefully this
-will remind you to take a look and see if there is indeed a problem.
-
-Sometimes you are confident that your tests are correct and may not
-appreciate such friendly messages. Some other times, you are debugging
-your tests or learning about the behavior of the code you are testing,
-and wish you could observe every mock call that happens (including
-argument values and the return value). Clearly, one size doesn't fit
-all.
-
-You can control how much Google Mock tells you using the
-`--gmock_verbose=LEVEL` command-line flag, where `LEVEL` is a string
-with three possible values:
-
-  * `info`: Google Mock will print all informational messages, warnings, and errors (most verbose). At this setting, Google Mock will also log any calls to the `ON_CALL/EXPECT_CALL` macros.
-  * `warning`: Google Mock will print both warnings and errors (less verbose). This is the default.
-  * `error`: Google Mock will print errors only (least verbose).
-
-Alternatively, you can adjust the value of that flag from within your
-tests like so:
-
-```
-  ::testing::FLAGS_gmock_verbose = "error";
-```
-
-Now, judiciously use the right flag to enable Google Mock serve you better!
-
-## Gaining Super Vision into Mock Calls ##
-
-You have a test using Google Mock. It fails: Google Mock tells you
-that some expectations aren't satisfied. However, you aren't sure why:
-Is there a typo somewhere in the matchers? Did you mess up the order
-of the `EXPECT_CALL`s? Or is the code under test doing something
-wrong?  How can you find out the cause?
-
-Won't it be nice if you have X-ray vision and can actually see the
-trace of all `EXPECT_CALL`s and mock method calls as they are made?
-For each call, would you like to see its actual argument values and
-which `EXPECT_CALL` Google Mock thinks it matches?
-
-You can unlock this power by running your test with the
-`--gmock_verbose=info` flag. For example, given the test program:
-
-```
-using testing::_;
-using testing::HasSubstr;
-using testing::Return;
-
-class MockFoo {
- public:
-  MOCK_METHOD2(F, void(const string& x, const string& y));
-};
-
-TEST(Foo, Bar) {
-  MockFoo mock;
-  EXPECT_CALL(mock, F(_, _)).WillRepeatedly(Return());
-  EXPECT_CALL(mock, F("a", "b"));
-  EXPECT_CALL(mock, F("c", HasSubstr("d")));
-
-  mock.F("a", "good");
-  mock.F("a", "b");
-}
-```
-
-if you run it with `--gmock_verbose=info`, you will see this output:
-
-```
-[ RUN      ] Foo.Bar
-
-foo_test.cc:14: EXPECT_CALL(mock, F(_, _)) invoked
-foo_test.cc:15: EXPECT_CALL(mock, F("a", "b")) invoked
-foo_test.cc:16: EXPECT_CALL(mock, F("c", HasSubstr("d"))) invoked
-foo_test.cc:14: Mock function call matches EXPECT_CALL(mock, F(_, _))...
-    Function call: F(@0x7fff7c8dad40"a", @0x7fff7c8dad10"good")
-foo_test.cc:15: Mock function call matches EXPECT_CALL(mock, F("a", "b"))...
-    Function call: F(@0x7fff7c8dada0"a", @0x7fff7c8dad70"b")
-foo_test.cc:16: Failure
-Actual function call count doesn't match EXPECT_CALL(mock, F("c", HasSubstr("d")))...
-         Expected: to be called once
-           Actual: never called - unsatisfied and active
-[  FAILED  ] Foo.Bar
-```
-
-Suppose the bug is that the `"c"` in the third `EXPECT_CALL` is a typo
-and should actually be `"a"`. With the above message, you should see
-that the actual `F("a", "good")` call is matched by the first
-`EXPECT_CALL`, not the third as you thought. From that it should be
-obvious that the third `EXPECT_CALL` is written wrong. Case solved.
-
-## Running Tests in Emacs ##
-
-If you build and run your tests in Emacs, the source file locations of
-Google Mock and [Google Test](../../googletest/)
-errors will be highlighted. Just press `` on one of them and
-you'll be taken to the offending line. Or, you can just type `C-x ``
-to jump to the next error.
-
-To make it even easier, you can add the following lines to your
-`~/.emacs` file:
-
-```
-(global-set-key "\M-m"   'compile)  ; m is for make
-(global-set-key [M-down] 'next-error)
-(global-set-key [M-up]   '(lambda () (interactive) (next-error -1)))
-```
-
-Then you can type `M-m` to start a build, or `M-up`/`M-down` to move
-back and forth between errors.
-
-## Fusing Google Mock Source Files ##
-
-Google Mock's implementation consists of dozens of files (excluding
-its own tests).  Sometimes you may want them to be packaged up in
-fewer files instead, such that you can easily copy them to a new
-machine and start hacking there.  For this we provide an experimental
-Python script `fuse_gmock_files.py` in the `scripts/` directory
-(starting with release 1.2.0).  Assuming you have Python 2.4 or above
-installed on your machine, just go to that directory and run
-```
-python fuse_gmock_files.py OUTPUT_DIR
-```
-
-and you should see an `OUTPUT_DIR` directory being created with files
-`gtest/gtest.h`, `gmock/gmock.h`, and `gmock-gtest-all.cc` in it.
-These three files contain everything you need to use Google Mock (and
-Google Test).  Just copy them to anywhere you want and you are ready
-to write tests and use mocks.  You can use the
-[scrpts/test/Makefile](../scripts/test/Makefile) file as an example on how to compile your tests
-against them.
-
-# Extending Google Mock #
-
-## Writing New Matchers Quickly ##
-
-The `MATCHER*` family of macros can be used to define custom matchers
-easily.  The syntax:
-
-```
-MATCHER(name, description_string_expression) { statements; }
-```
-
-will define a matcher with the given name that executes the
-statements, which must return a `bool` to indicate if the match
-succeeds.  Inside the statements, you can refer to the value being
-matched by `arg`, and refer to its type by `arg_type`.
-
-The description string is a `string`-typed expression that documents
-what the matcher does, and is used to generate the failure message
-when the match fails.  It can (and should) reference the special
-`bool` variable `negation`, and should evaluate to the description of
-the matcher when `negation` is `false`, or that of the matcher's
-negation when `negation` is `true`.
-
-For convenience, we allow the description string to be empty (`""`),
-in which case Google Mock will use the sequence of words in the
-matcher name as the description.
-
-For example:
-```
-MATCHER(IsDivisibleBy7, "") { return (arg % 7) == 0; }
-```
-allows you to write
-```
-  // Expects mock_foo.Bar(n) to be called where n is divisible by 7.
-  EXPECT_CALL(mock_foo, Bar(IsDivisibleBy7()));
-```
-or,
-```
-using ::testing::Not;
-...
-  EXPECT_THAT(some_expression, IsDivisibleBy7());
-  EXPECT_THAT(some_other_expression, Not(IsDivisibleBy7()));
-```
-If the above assertions fail, they will print something like:
-```
-  Value of: some_expression
-  Expected: is divisible by 7
-    Actual: 27
-...
-  Value of: some_other_expression
-  Expected: not (is divisible by 7)
-    Actual: 21
-```
-where the descriptions `"is divisible by 7"` and `"not (is divisible
-by 7)"` are automatically calculated from the matcher name
-`IsDivisibleBy7`.
-
-As you may have noticed, the auto-generated descriptions (especially
-those for the negation) may not be so great. You can always override
-them with a string expression of your own:
-```
-MATCHER(IsDivisibleBy7, std::string(negation ? "isn't" : "is") +
-                        " divisible by 7") {
-  return (arg % 7) == 0;
-}
-```
-
-Optionally, you can stream additional information to a hidden argument
-named `result_listener` to explain the match result. For example, a
-better definition of `IsDivisibleBy7` is:
-```
-MATCHER(IsDivisibleBy7, "") {
-  if ((arg % 7) == 0)
-    return true;
-
-  *result_listener << "the remainder is " << (arg % 7);
-  return false;
-}
-```
-
-With this definition, the above assertion will give a better message:
-```
-  Value of: some_expression
-  Expected: is divisible by 7
-    Actual: 27 (the remainder is 6)
-```
-
-You should let `MatchAndExplain()` print _any additional information_
-that can help a user understand the match result. Note that it should
-explain why the match succeeds in case of a success (unless it's
-obvious) - this is useful when the matcher is used inside
-`Not()`. There is no need to print the argument value itself, as
-Google Mock already prints it for you.
-
-**Notes:**
-
-  1. The type of the value being matched (`arg_type`) is determined by the context in which you use the matcher and is supplied to you by the compiler, so you don't need to worry about declaring it (nor can you).  This allows the matcher to be polymorphic.  For example, `IsDivisibleBy7()` can be used to match any type where the value of `(arg % 7) == 0` can be implicitly converted to a `bool`.  In the `Bar(IsDivisibleBy7())` example above, if method `Bar()` takes an `int`, `arg_type` will be `int`; if it takes an `unsigned long`, `arg_type` will be `unsigned long`; and so on.
-  1. Google Mock doesn't guarantee when or how many times a matcher will be invoked. Therefore the matcher logic must be _purely functional_ (i.e. it cannot have any side effect, and the result must not depend on anything other than the value being matched and the matcher parameters). This requirement must be satisfied no matter how you define the matcher (e.g. using one of the methods described in the following recipes). In particular, a matcher can never call a mock function, as that will affect the state of the mock object and Google Mock.
-
-## Writing New Parameterized Matchers Quickly ##
-
-Sometimes you'll want to define a matcher that has parameters.  For that you
-can use the macro:
-```
-MATCHER_P(name, param_name, description_string) { statements; }
-```
-where the description string can be either `""` or a string expression
-that references `negation` and `param_name`.
-
-For example:
-```
-MATCHER_P(HasAbsoluteValue, value, "") { return abs(arg) == value; }
-```
-will allow you to write:
-```
-  EXPECT_THAT(Blah("a"), HasAbsoluteValue(n));
-```
-which may lead to this message (assuming `n` is 10):
-```
-  Value of: Blah("a")
-  Expected: has absolute value 10
-    Actual: -9
-```
-
-Note that both the matcher description and its parameter are
-printed, making the message human-friendly.
-
-In the matcher definition body, you can write `foo_type` to
-reference the type of a parameter named `foo`.  For example, in the
-body of `MATCHER_P(HasAbsoluteValue, value)` above, you can write
-`value_type` to refer to the type of `value`.
-
-Google Mock also provides `MATCHER_P2`, `MATCHER_P3`, ..., up to
-`MATCHER_P10` to support multi-parameter matchers:
-```
-MATCHER_Pk(name, param_1, ..., param_k, description_string) { statements; }
-```
-
-Please note that the custom description string is for a particular
-**instance** of the matcher, where the parameters have been bound to
-actual values.  Therefore usually you'll want the parameter values to
-be part of the description.  Google Mock lets you do that by
-referencing the matcher parameters in the description string
-expression.
-
-For example,
-```
-  using ::testing::PrintToString;
-  MATCHER_P2(InClosedRange, low, hi,
-             std::string(negation ? "isn't" : "is") + " in range [" +
-             PrintToString(low) + ", " + PrintToString(hi) + "]") {
-    return low <= arg && arg <= hi;
-  }
-  ...
-  EXPECT_THAT(3, InClosedRange(4, 6));
-```
-would generate a failure that contains the message:
-```
-  Expected: is in range [4, 6]
-```
-
-If you specify `""` as the description, the failure message will
-contain the sequence of words in the matcher name followed by the
-parameter values printed as a tuple.  For example,
-```
-  MATCHER_P2(InClosedRange, low, hi, "") { ... }
-  ...
-  EXPECT_THAT(3, InClosedRange(4, 6));
-```
-would generate a failure that contains the text:
-```
-  Expected: in closed range (4, 6)
-```
-
-For the purpose of typing, you can view
-```
-MATCHER_Pk(Foo, p1, ..., pk, description_string) { ... }
-```
-as shorthand for
-```
-template 
-FooMatcherPk
-Foo(p1_type p1, ..., pk_type pk) { ... }
-```
-
-When you write `Foo(v1, ..., vk)`, the compiler infers the types of
-the parameters `v1`, ..., and `vk` for you.  If you are not happy with
-the result of the type inference, you can specify the types by
-explicitly instantiating the template, as in `Foo(5, false)`.
-As said earlier, you don't get to (or need to) specify
-`arg_type` as that's determined by the context in which the matcher
-is used.
-
-You can assign the result of expression `Foo(p1, ..., pk)` to a
-variable of type `FooMatcherPk`.  This can be
-useful when composing matchers.  Matchers that don't have a parameter
-or have only one parameter have special types: you can assign `Foo()`
-to a `FooMatcher`-typed variable, and assign `Foo(p)` to a
-`FooMatcherP`-typed variable.
-
-While you can instantiate a matcher template with reference types,
-passing the parameters by pointer usually makes your code more
-readable.  If, however, you still want to pass a parameter by
-reference, be aware that in the failure message generated by the
-matcher you will see the value of the referenced object but not its
-address.
-
-You can overload matchers with different numbers of parameters:
-```
-MATCHER_P(Blah, a, description_string_1) { ... }
-MATCHER_P2(Blah, a, b, description_string_2) { ... }
-```
-
-While it's tempting to always use the `MATCHER*` macros when defining
-a new matcher, you should also consider implementing
-`MatcherInterface` or using `MakePolymorphicMatcher()` instead (see
-the recipes that follow), especially if you need to use the matcher a
-lot.  While these approaches require more work, they give you more
-control on the types of the value being matched and the matcher
-parameters, which in general leads to better compiler error messages
-that pay off in the long run.  They also allow overloading matchers
-based on parameter types (as opposed to just based on the number of
-parameters).
-
-## Writing New Monomorphic Matchers ##
-
-A matcher of argument type `T` implements
-`::testing::MatcherInterface` and does two things: it tests whether a
-value of type `T` matches the matcher, and can describe what kind of
-values it matches. The latter ability is used for generating readable
-error messages when expectations are violated.
-
-The interface looks like this:
-
-```
-class MatchResultListener {
- public:
-  ...
-  // Streams x to the underlying ostream; does nothing if the ostream
-  // is NULL.
-  template 
-  MatchResultListener& operator<<(const T& x);
-
-  // Returns the underlying ostream.
-  ::std::ostream* stream();
-};
-
-template 
-class MatcherInterface {
- public:
-  virtual ~MatcherInterface();
-
-  // Returns true iff the matcher matches x; also explains the match
-  // result to 'listener'.
-  virtual bool MatchAndExplain(T x, MatchResultListener* listener) const = 0;
-
-  // Describes this matcher to an ostream.
-  virtual void DescribeTo(::std::ostream* os) const = 0;
-
-  // Describes the negation of this matcher to an ostream.
-  virtual void DescribeNegationTo(::std::ostream* os) const;
-};
-```
-
-If you need a custom matcher but `Truly()` is not a good option (for
-example, you may not be happy with the way `Truly(predicate)`
-describes itself, or you may want your matcher to be polymorphic as
-`Eq(value)` is), you can define a matcher to do whatever you want in
-two steps: first implement the matcher interface, and then define a
-factory function to create a matcher instance. The second step is not
-strictly needed but it makes the syntax of using the matcher nicer.
-
-For example, you can define a matcher to test whether an `int` is
-divisible by 7 and then use it like this:
-```
-using ::testing::MakeMatcher;
-using ::testing::Matcher;
-using ::testing::MatcherInterface;
-using ::testing::MatchResultListener;
-
-class DivisibleBy7Matcher : public MatcherInterface {
- public:
-  virtual bool MatchAndExplain(int n, MatchResultListener* listener) const {
-    return (n % 7) == 0;
-  }
-
-  virtual void DescribeTo(::std::ostream* os) const {
-    *os << "is divisible by 7";
-  }
-
-  virtual void DescribeNegationTo(::std::ostream* os) const {
-    *os << "is not divisible by 7";
-  }
-};
-
-inline Matcher DivisibleBy7() {
-  return MakeMatcher(new DivisibleBy7Matcher);
-}
-...
-
-  EXPECT_CALL(foo, Bar(DivisibleBy7()));
-```
-
-You may improve the matcher message by streaming additional
-information to the `listener` argument in `MatchAndExplain()`:
-
-```
-class DivisibleBy7Matcher : public MatcherInterface {
- public:
-  virtual bool MatchAndExplain(int n,
-                               MatchResultListener* listener) const {
-    const int remainder = n % 7;
-    if (remainder != 0) {
-      *listener << "the remainder is " << remainder;
-    }
-    return remainder == 0;
-  }
-  ...
-};
-```
-
-Then, `EXPECT_THAT(x, DivisibleBy7());` may general a message like this:
-```
-Value of: x
-Expected: is divisible by 7
-  Actual: 23 (the remainder is 2)
-```
-
-## Writing New Polymorphic Matchers ##
-
-You've learned how to write your own matchers in the previous
-recipe. Just one problem: a matcher created using `MakeMatcher()` only
-works for one particular type of arguments. If you want a
-_polymorphic_ matcher that works with arguments of several types (for
-instance, `Eq(x)` can be used to match a `value` as long as `value` ==
-`x` compiles -- `value` and `x` don't have to share the same type),
-you can learn the trick from `"gmock/gmock-matchers.h"` but it's a bit
-involved.
-
-Fortunately, most of the time you can define a polymorphic matcher
-easily with the help of `MakePolymorphicMatcher()`. Here's how you can
-define `NotNull()` as an example:
-
-```
-using ::testing::MakePolymorphicMatcher;
-using ::testing::MatchResultListener;
-using ::testing::NotNull;
-using ::testing::PolymorphicMatcher;
-
-class NotNullMatcher {
- public:
-  // To implement a polymorphic matcher, first define a COPYABLE class
-  // that has three members MatchAndExplain(), DescribeTo(), and
-  // DescribeNegationTo(), like the following.
-
-  // In this example, we want to use NotNull() with any pointer, so
-  // MatchAndExplain() accepts a pointer of any type as its first argument.
-  // In general, you can define MatchAndExplain() as an ordinary method or
-  // a method template, or even overload it.
-  template 
-  bool MatchAndExplain(T* p,
-                       MatchResultListener* /* listener */) const {
-    return p != NULL;
-  }
-
-  // Describes the property of a value matching this matcher.
-  void DescribeTo(::std::ostream* os) const { *os << "is not NULL"; }
-
-  // Describes the property of a value NOT matching this matcher.
-  void DescribeNegationTo(::std::ostream* os) const { *os << "is NULL"; }
-};
-
-// To construct a polymorphic matcher, pass an instance of the class
-// to MakePolymorphicMatcher().  Note the return type.
-inline PolymorphicMatcher NotNull() {
-  return MakePolymorphicMatcher(NotNullMatcher());
-}
-...
-
-  EXPECT_CALL(foo, Bar(NotNull()));  // The argument must be a non-NULL pointer.
-```
-
-**Note:** Your polymorphic matcher class does **not** need to inherit from
-`MatcherInterface` or any other class, and its methods do **not** need
-to be virtual.
-
-Like in a monomorphic matcher, you may explain the match result by
-streaming additional information to the `listener` argument in
-`MatchAndExplain()`.
-
-## Writing New Cardinalities ##
-
-A cardinality is used in `Times()` to tell Google Mock how many times
-you expect a call to occur. It doesn't have to be exact. For example,
-you can say `AtLeast(5)` or `Between(2, 4)`.
-
-If the built-in set of cardinalities doesn't suit you, you are free to
-define your own by implementing the following interface (in namespace
-`testing`):
-
-```
-class CardinalityInterface {
- public:
-  virtual ~CardinalityInterface();
-
-  // Returns true iff call_count calls will satisfy this cardinality.
-  virtual bool IsSatisfiedByCallCount(int call_count) const = 0;
-
-  // Returns true iff call_count calls will saturate this cardinality.
-  virtual bool IsSaturatedByCallCount(int call_count) const = 0;
-
-  // Describes self to an ostream.
-  virtual void DescribeTo(::std::ostream* os) const = 0;
-};
-```
-
-For example, to specify that a call must occur even number of times,
-you can write
-
-```
-using ::testing::Cardinality;
-using ::testing::CardinalityInterface;
-using ::testing::MakeCardinality;
-
-class EvenNumberCardinality : public CardinalityInterface {
- public:
-  virtual bool IsSatisfiedByCallCount(int call_count) const {
-    return (call_count % 2) == 0;
-  }
-
-  virtual bool IsSaturatedByCallCount(int call_count) const {
-    return false;
-  }
-
-  virtual void DescribeTo(::std::ostream* os) const {
-    *os << "called even number of times";
-  }
-};
-
-Cardinality EvenNumber() {
-  return MakeCardinality(new EvenNumberCardinality);
-}
-...
-
-  EXPECT_CALL(foo, Bar(3))
-      .Times(EvenNumber());
-```
-
-## Writing New Actions Quickly ##
-
-If the built-in actions don't work for you, and you find it
-inconvenient to use `Invoke()`, you can use a macro from the `ACTION*`
-family to quickly define a new action that can be used in your code as
-if it's a built-in action.
-
-By writing
-```
-ACTION(name) { statements; }
-```
-in a namespace scope (i.e. not inside a class or function), you will
-define an action with the given name that executes the statements.
-The value returned by `statements` will be used as the return value of
-the action.  Inside the statements, you can refer to the K-th
-(0-based) argument of the mock function as `argK`.  For example:
-```
-ACTION(IncrementArg1) { return ++(*arg1); }
-```
-allows you to write
-```
-... WillOnce(IncrementArg1());
-```
-
-Note that you don't need to specify the types of the mock function
-arguments.  Rest assured that your code is type-safe though:
-you'll get a compiler error if `*arg1` doesn't support the `++`
-operator, or if the type of `++(*arg1)` isn't compatible with the mock
-function's return type.
-
-Another example:
-```
-ACTION(Foo) {
-  (*arg2)(5);
-  Blah();
-  *arg1 = 0;
-  return arg0;
-}
-```
-defines an action `Foo()` that invokes argument #2 (a function pointer)
-with 5, calls function `Blah()`, sets the value pointed to by argument
-#1 to 0, and returns argument #0.
-
-For more convenience and flexibility, you can also use the following
-pre-defined symbols in the body of `ACTION`:
-
-| `argK_type` | The type of the K-th (0-based) argument of the mock function |
-|:------------|:-------------------------------------------------------------|
-| `args`      | All arguments of the mock function as a tuple                |
-| `args_type` | The type of all arguments of the mock function as a tuple    |
-| `return_type` | The return type of the mock function                         |
-| `function_type` | The type of the mock function                                |
-
-For example, when using an `ACTION` as a stub action for mock function:
-```
-int DoSomething(bool flag, int* ptr);
-```
-we have:
-
-| **Pre-defined Symbol** | **Is Bound To** |
-|:-----------------------|:----------------|
-| `arg0`                 | the value of `flag` |
-| `arg0_type`            | the type `bool` |
-| `arg1`                 | the value of `ptr` |
-| `arg1_type`            | the type `int*` |
-| `args`                 | the tuple `(flag, ptr)` |
-| `args_type`            | the type `::testing::tuple` |
-| `return_type`          | the type `int`  |
-| `function_type`        | the type `int(bool, int*)` |
-
-## Writing New Parameterized Actions Quickly ##
-
-Sometimes you'll want to parameterize an action you define.  For that
-we have another macro
-```
-ACTION_P(name, param) { statements; }
-```
-
-For example,
-```
-ACTION_P(Add, n) { return arg0 + n; }
-```
-will allow you to write
-```
-// Returns argument #0 + 5.
-... WillOnce(Add(5));
-```
-
-For convenience, we use the term _arguments_ for the values used to
-invoke the mock function, and the term _parameters_ for the values
-used to instantiate an action.
-
-Note that you don't need to provide the type of the parameter either.
-Suppose the parameter is named `param`, you can also use the
-Google-Mock-defined symbol `param_type` to refer to the type of the
-parameter as inferred by the compiler.  For example, in the body of
-`ACTION_P(Add, n)` above, you can write `n_type` for the type of `n`.
-
-Google Mock also provides `ACTION_P2`, `ACTION_P3`, and etc to support
-multi-parameter actions.  For example,
-```
-ACTION_P2(ReturnDistanceTo, x, y) {
-  double dx = arg0 - x;
-  double dy = arg1 - y;
-  return sqrt(dx*dx + dy*dy);
-}
-```
-lets you write
-```
-... WillOnce(ReturnDistanceTo(5.0, 26.5));
-```
-
-You can view `ACTION` as a degenerated parameterized action where the
-number of parameters is 0.
-
-You can also easily define actions overloaded on the number of parameters:
-```
-ACTION_P(Plus, a) { ... }
-ACTION_P2(Plus, a, b) { ... }
-```
-
-## Restricting the Type of an Argument or Parameter in an ACTION ##
-
-For maximum brevity and reusability, the `ACTION*` macros don't ask
-you to provide the types of the mock function arguments and the action
-parameters.  Instead, we let the compiler infer the types for us.
-
-Sometimes, however, we may want to be more explicit about the types.
-There are several tricks to do that.  For example:
-```
-ACTION(Foo) {
-  // Makes sure arg0 can be converted to int.
-  int n = arg0;
-  ... use n instead of arg0 here ...
-}
-
-ACTION_P(Bar, param) {
-  // Makes sure the type of arg1 is const char*.
-  ::testing::StaticAssertTypeEq();
-
-  // Makes sure param can be converted to bool.
-  bool flag = param;
-}
-```
-where `StaticAssertTypeEq` is a compile-time assertion in Google Test
-that verifies two types are the same.
-
-## Writing New Action Templates Quickly ##
-
-Sometimes you want to give an action explicit template parameters that
-cannot be inferred from its value parameters.  `ACTION_TEMPLATE()`
-supports that and can be viewed as an extension to `ACTION()` and
-`ACTION_P*()`.
-
-The syntax:
-```
-ACTION_TEMPLATE(ActionName,
-                HAS_m_TEMPLATE_PARAMS(kind1, name1, ..., kind_m, name_m),
-                AND_n_VALUE_PARAMS(p1, ..., p_n)) { statements; }
-```
-
-defines an action template that takes _m_ explicit template parameters
-and _n_ value parameters, where _m_ is between 1 and 10, and _n_ is
-between 0 and 10.  `name_i` is the name of the i-th template
-parameter, and `kind_i` specifies whether it's a `typename`, an
-integral constant, or a template.  `p_i` is the name of the i-th value
-parameter.
-
-Example:
-```
-// DuplicateArg(output) converts the k-th argument of the mock
-// function to type T and copies it to *output.
-ACTION_TEMPLATE(DuplicateArg,
-                // Note the comma between int and k:
-                HAS_2_TEMPLATE_PARAMS(int, k, typename, T),
-                AND_1_VALUE_PARAMS(output)) {
-  *output = T(::testing::get(args));
-}
-```
-
-To create an instance of an action template, write:
-```
-  ActionName(v1, ..., v_n)
-```
-where the `t`s are the template arguments and the
-`v`s are the value arguments.  The value argument
-types are inferred by the compiler.  For example:
-```
-using ::testing::_;
-...
-  int n;
-  EXPECT_CALL(mock, Foo(_, _))
-      .WillOnce(DuplicateArg<1, unsigned char>(&n));
-```
-
-If you want to explicitly specify the value argument types, you can
-provide additional template arguments:
-```
-  ActionName(v1, ..., v_n)
-```
-where `u_i` is the desired type of `v_i`.
-
-`ACTION_TEMPLATE` and `ACTION`/`ACTION_P*` can be overloaded on the
-number of value parameters, but not on the number of template
-parameters.  Without the restriction, the meaning of the following is
-unclear:
-
-```
-  OverloadedAction(x);
-```
-
-Are we using a single-template-parameter action where `bool` refers to
-the type of `x`, or a two-template-parameter action where the compiler
-is asked to infer the type of `x`?
-
-## Using the ACTION Object's Type ##
-
-If you are writing a function that returns an `ACTION` object, you'll
-need to know its type.  The type depends on the macro used to define
-the action and the parameter types.  The rule is relatively simple:
-
-| **Given Definition** | **Expression** | **Has Type** |
-|:---------------------|:---------------|:-------------|
-| `ACTION(Foo)`        | `Foo()`        | `FooAction`  |
-| `ACTION_TEMPLATE(Foo, HAS_m_TEMPLATE_PARAMS(...), AND_0_VALUE_PARAMS())` |	`Foo()` | `FooAction` |
-| `ACTION_P(Bar, param)` | `Bar(int_value)` | `BarActionP` |
-| `ACTION_TEMPLATE(Bar, HAS_m_TEMPLATE_PARAMS(...), AND_1_VALUE_PARAMS(p1))` | `Bar(int_value)` | `FooActionP` |
-| `ACTION_P2(Baz, p1, p2)` | `Baz(bool_value, int_value)` | `BazActionP2` |
-| `ACTION_TEMPLATE(Baz, HAS_m_TEMPLATE_PARAMS(...), AND_2_VALUE_PARAMS(p1, p2))`| `Baz(bool_value, int_value)` | `FooActionP2` |
-| ...                  | ...            | ...          |
-
-Note that we have to pick different suffixes (`Action`, `ActionP`,
-`ActionP2`, and etc) for actions with different numbers of value
-parameters, or the action definitions cannot be overloaded on the
-number of them.
-
-## Writing New Monomorphic Actions ##
-
-While the `ACTION*` macros are very convenient, sometimes they are
-inappropriate.  For example, despite the tricks shown in the previous
-recipes, they don't let you directly specify the types of the mock
-function arguments and the action parameters, which in general leads
-to unoptimized compiler error messages that can baffle unfamiliar
-users.  They also don't allow overloading actions based on parameter
-types without jumping through some hoops.
-
-An alternative to the `ACTION*` macros is to implement
-`::testing::ActionInterface`, where `F` is the type of the mock
-function in which the action will be used. For example:
-
-```
-template class ActionInterface {
- public:
-  virtual ~ActionInterface();
-
-  // Performs the action.  Result is the return type of function type
-  // F, and ArgumentTuple is the tuple of arguments of F.
-  //
-  // For example, if F is int(bool, const string&), then Result would
-  // be int, and ArgumentTuple would be ::testing::tuple.
-  virtual Result Perform(const ArgumentTuple& args) = 0;
-};
-
-using ::testing::_;
-using ::testing::Action;
-using ::testing::ActionInterface;
-using ::testing::MakeAction;
-
-typedef int IncrementMethod(int*);
-
-class IncrementArgumentAction : public ActionInterface {
- public:
-  virtual int Perform(const ::testing::tuple& args) {
-    int* p = ::testing::get<0>(args);  // Grabs the first argument.
-    return *p++;
-  }
-};
-
-Action IncrementArgument() {
-  return MakeAction(new IncrementArgumentAction);
-}
-...
-
-  EXPECT_CALL(foo, Baz(_))
-      .WillOnce(IncrementArgument());
-
-  int n = 5;
-  foo.Baz(&n);  // Should return 5 and change n to 6.
-```
-
-## Writing New Polymorphic Actions ##
-
-The previous recipe showed you how to define your own action. This is
-all good, except that you need to know the type of the function in
-which the action will be used. Sometimes that can be a problem. For
-example, if you want to use the action in functions with _different_
-types (e.g. like `Return()` and `SetArgPointee()`).
-
-If an action can be used in several types of mock functions, we say
-it's _polymorphic_. The `MakePolymorphicAction()` function template
-makes it easy to define such an action:
-
-```
-namespace testing {
-
-template 
-PolymorphicAction MakePolymorphicAction(const Impl& impl);
-
-}  // namespace testing
-```
-
-As an example, let's define an action that returns the second argument
-in the mock function's argument list. The first step is to define an
-implementation class:
-
-```
-class ReturnSecondArgumentAction {
- public:
-  template 
-  Result Perform(const ArgumentTuple& args) const {
-    // To get the i-th (0-based) argument, use ::testing::get(args).
-    return ::testing::get<1>(args);
-  }
-};
-```
-
-This implementation class does _not_ need to inherit from any
-particular class. What matters is that it must have a `Perform()`
-method template. This method template takes the mock function's
-arguments as a tuple in a **single** argument, and returns the result of
-the action. It can be either `const` or not, but must be invokable
-with exactly one template argument, which is the result type. In other
-words, you must be able to call `Perform(args)` where `R` is the
-mock function's return type and `args` is its arguments in a tuple.
-
-Next, we use `MakePolymorphicAction()` to turn an instance of the
-implementation class into the polymorphic action we need. It will be
-convenient to have a wrapper for this:
-
-```
-using ::testing::MakePolymorphicAction;
-using ::testing::PolymorphicAction;
-
-PolymorphicAction ReturnSecondArgument() {
-  return MakePolymorphicAction(ReturnSecondArgumentAction());
-}
-```
-
-Now, you can use this polymorphic action the same way you use the
-built-in ones:
-
-```
-using ::testing::_;
-
-class MockFoo : public Foo {
- public:
-  MOCK_METHOD2(DoThis, int(bool flag, int n));
-  MOCK_METHOD3(DoThat, string(int x, const char* str1, const char* str2));
-};
-...
-
-  MockFoo foo;
-  EXPECT_CALL(foo, DoThis(_, _))
-      .WillOnce(ReturnSecondArgument());
-  EXPECT_CALL(foo, DoThat(_, _, _))
-      .WillOnce(ReturnSecondArgument());
-  ...
-  foo.DoThis(true, 5);         // Will return 5.
-  foo.DoThat(1, "Hi", "Bye");  // Will return "Hi".
-```
-
-## Teaching Google Mock How to Print Your Values ##
-
-When an uninteresting or unexpected call occurs, Google Mock prints the
-argument values and the stack trace to help you debug.  Assertion
-macros like `EXPECT_THAT` and `EXPECT_EQ` also print the values in
-question when the assertion fails.  Google Mock and Google Test do this using
-Google Test's user-extensible value printer.
-
-This printer knows how to print built-in C++ types, native arrays, STL
-containers, and any type that supports the `<<` operator.  For other
-types, it prints the raw bytes in the value and hopes that you the
-user can figure it out.
-[Google Test's advanced guide](../../googletest/docs/AdvancedGuide.md#teaching-google-test-how-to-print-your-values)
-explains how to extend the printer to do a better job at
-printing your particular type than to dump the bytes.
diff --git a/thirdparty/GTest/googletest/googlemock/docs/DesignDoc.md b/thirdparty/GTest/googletest/googlemock/docs/DesignDoc.md
deleted file mode 100644
index 3f515c3b6d..0000000000
--- a/thirdparty/GTest/googletest/googlemock/docs/DesignDoc.md
+++ /dev/null
@@ -1,280 +0,0 @@
-This page discusses the design of new Google Mock features.
-
-
-
-# Macros for Defining Actions #
-
-## Problem ##
-
-Due to the lack of closures in C++, it currently requires some
-non-trivial effort to define a custom action in Google Mock.  For
-example, suppose you want to "increment the value pointed to by the
-second argument of the mock function and return it", you could write:
-
-```
-int IncrementArg1(Unused, int* p, Unused) {
-  return ++(*p);
-}
-
-... WillOnce(Invoke(IncrementArg1));
-```
-
-There are several things unsatisfactory about this approach:
-
-  * Even though the action only cares about the second argument of the mock function, its definition needs to list other arguments as dummies.  This is tedious.
-  * The defined action is usable only in mock functions that takes exactly 3 arguments - an unnecessary restriction.
-  * To use the action, one has to say `Invoke(IncrementArg1)`, which isn't as nice as `IncrementArg1()`.
-
-The latter two problems can be overcome using `MakePolymorphicAction()`,
-but it requires much more boilerplate code:
-
-```
-class IncrementArg1Action {
- public:
-  template 
-  Result Perform(const ArgumentTuple& args) const {
-    return ++(*tr1::get<1>(args));
-  }
-};
-
-PolymorphicAction IncrementArg1() {
-  return MakePolymorphicAction(IncrementArg1Action());
-}
-
-... WillOnce(IncrementArg1());
-```
-
-Our goal is to allow defining custom actions with the least amount of
-boiler-plate C++ requires.
-
-## Solution ##
-
-We propose to introduce a new macro:
-```
-ACTION(name) { statements; }
-```
-
-Using this in a namespace scope will define an action with the given
-name that executes the statements.  Inside the statements, you can
-refer to the K-th (0-based) argument of the mock function as `argK`.
-For example:
-```
-ACTION(IncrementArg1) { return ++(*arg1); }
-```
-allows you to write
-```
-... WillOnce(IncrementArg1());
-```
-
-Note that you don't need to specify the types of the mock function
-arguments, as brevity is a top design goal here.  Rest assured that
-your code is still type-safe though: you'll get a compiler error if
-`*arg1` doesn't support the `++` operator, or if the type of
-`++(*arg1)` isn't compatible with the mock function's return type.
-
-Another example:
-```
-ACTION(Foo) {
-  (*arg2)(5);
-  Blah();
-  *arg1 = 0;
-  return arg0;
-}
-```
-defines an action `Foo()` that invokes argument #2 (a function pointer)
-with 5, calls function `Blah()`, sets the value pointed to by argument
-#1 to 0, and returns argument #0.
-
-For more convenience and flexibility, you can also use the following
-pre-defined symbols in the body of `ACTION`:
-
-| `argK_type` | The type of the K-th (0-based) argument of the mock function |
-|:------------|:-------------------------------------------------------------|
-| `args`      | All arguments of the mock function as a tuple                |
-| `args_type` | The type of all arguments of the mock function as a tuple    |
-| `return_type` | The return type of the mock function                         |
-| `function_type` | The type of the mock function                                |
-
-For example, when using an `ACTION` as a stub action for mock function:
-```
-int DoSomething(bool flag, int* ptr);
-```
-we have:
-| **Pre-defined Symbol** | **Is Bound To** |
-|:-----------------------|:----------------|
-| `arg0`                 | the value of `flag` |
-| `arg0_type`            | the type `bool` |
-| `arg1`                 | the value of `ptr` |
-| `arg1_type`            | the type `int*` |
-| `args`                 | the tuple `(flag, ptr)` |
-| `args_type`            | the type `std::tr1::tuple` |
-| `return_type`          | the type `int`  |
-| `function_type`        | the type `int(bool, int*)` |
-
-## Parameterized actions ##
-
-Sometimes you'll want to parameterize the action.   For that we propose
-another macro
-```
-ACTION_P(name, param) { statements; }
-```
-
-For example,
-```
-ACTION_P(Add, n) { return arg0 + n; }
-```
-will allow you to write
-```
-// Returns argument #0 + 5.
-... WillOnce(Add(5));
-```
-
-For convenience, we use the term _arguments_ for the values used to
-invoke the mock function, and the term _parameters_ for the values
-used to instantiate an action.
-
-Note that you don't need to provide the type of the parameter either.
-Suppose the parameter is named `param`, you can also use the
-Google-Mock-defined symbol `param_type` to refer to the type of the
-parameter as inferred by the compiler.
-
-We will also provide `ACTION_P2`, `ACTION_P3`, and etc to support
-multi-parameter actions.  For example,
-```
-ACTION_P2(ReturnDistanceTo, x, y) {
-  double dx = arg0 - x;
-  double dy = arg1 - y;
-  return sqrt(dx*dx + dy*dy);
-}
-```
-lets you write
-```
-... WillOnce(ReturnDistanceTo(5.0, 26.5));
-```
-
-You can view `ACTION` as a degenerated parameterized action where the
-number of parameters is 0.
-
-## Advanced Usages ##
-
-### Overloading Actions ###
-
-You can easily define actions overloaded on the number of parameters:
-```
-ACTION_P(Plus, a) { ... }
-ACTION_P2(Plus, a, b) { ... }
-```
-
-### Restricting the Type of an Argument or Parameter ###
-
-For maximum brevity and reusability, the `ACTION*` macros don't let
-you specify the types of the mock function arguments and the action
-parameters.  Instead, we let the compiler infer the types for us.
-
-Sometimes, however, we may want to be more explicit about the types.
-There are several tricks to do that.  For example:
-```
-ACTION(Foo) {
-  // Makes sure arg0 can be converted to int.
-  int n = arg0;
-  ... use n instead of arg0 here ...
-}
-
-ACTION_P(Bar, param) {
-  // Makes sure the type of arg1 is const char*.
-  ::testing::StaticAssertTypeEq();
-
-  // Makes sure param can be converted to bool.
-  bool flag = param;
-}
-```
-where `StaticAssertTypeEq` is a compile-time assertion we plan to add to
-Google Test (the name is chosen to match `static_assert` in C++0x).
-
-### Using the ACTION Object's Type ###
-
-If you are writing a function that returns an `ACTION` object, you'll
-need to know its type.  The type depends on the macro used to define
-the action and the parameter types.  The rule is relatively simple:
-| **Given Definition** | **Expression** | **Has Type** |
-|:---------------------|:---------------|:-------------|
-| `ACTION(Foo)`        | `Foo()`        | `FooAction`  |
-| `ACTION_P(Bar, param)` | `Bar(int_value)` | `BarActionP` |
-| `ACTION_P2(Baz, p1, p2)` | `Baz(bool_value, int_value)` | `BazActionP2` |
-| ...                  | ...            | ...          |
-
-Note that we have to pick different suffixes (`Action`, `ActionP`,
-`ActionP2`, and etc) for actions with different numbers of parameters,
-or the action definitions cannot be overloaded on the number of
-parameters.
-
-## When to Use ##
-
-While the new macros are very convenient, please also consider other
-means of implementing actions (e.g. via `ActionInterface` or
-`MakePolymorphicAction()`), especially if you need to use the defined
-action a lot.  While the other approaches require more work, they give
-you more control on the types of the mock function arguments and the
-action parameters, which in general leads to better compiler error
-messages that pay off in the long run.  They also allow overloading
-actions based on parameter types, as opposed to just the number of
-parameters.
-
-## Related Work ##
-
-As you may have realized, the `ACTION*` macros resemble closures (also
-known as lambda expressions or anonymous functions).  Indeed, both of
-them seek to lower the syntactic overhead for defining a function.
-
-C++0x will support lambdas, but they are not part of C++ right now.
-Some non-standard libraries (most notably BLL or Boost Lambda Library)
-try to alleviate this problem.  However, they are not a good choice
-for defining actions as:
-
-  * They are non-standard and not widely installed.  Google Mock only depends on standard libraries and `tr1::tuple`, which is part of the new C++ standard and comes with gcc 4+.  We want to keep it that way.
-  * They are not trivial to learn.
-  * They will become obsolete when C++0x's lambda feature is widely supported.  We don't want to make our users use a dying library.
-  * Since they are based on operators, they are rather ad hoc: you cannot use statements, and you cannot pass the lambda arguments to a function, for example.
-  * They have subtle semantics that easily confuses new users.  For example, in expression `_1++ + foo++`, `foo` will be incremented only once where the expression is evaluated, while `_1` will be incremented every time the unnamed function is invoked.  This is far from intuitive.
-
-`ACTION*` avoid all these problems.
-
-## Future Improvements ##
-
-There may be a need for composing `ACTION*` definitions (i.e. invoking
-another `ACTION` inside the definition of one `ACTION*`).  We are not
-sure we want it yet, as one can get a similar effect by putting
-`ACTION` definitions in function templates and composing the function
-templates.  We'll revisit this based on user feedback.
-
-The reason we don't allow `ACTION*()` inside a function body is that
-the current C++ standard doesn't allow function-local types to be used
-to instantiate templates.  The upcoming C++0x standard will lift this
-restriction.  Once this feature is widely supported by compilers, we
-can revisit the implementation and add support for using `ACTION*()`
-inside a function.
-
-C++0x will also support lambda expressions.  When they become
-available, we may want to support using lambdas as actions.
-
-# Macros for Defining Matchers #
-
-Once the macros for defining actions are implemented, we plan to do
-the same for matchers:
-
-```
-MATCHER(name) { statements; }
-```
-
-where you can refer to the value being matched as `arg`.  For example,
-given:
-
-```
-MATCHER(IsPositive) { return arg > 0; }
-```
-
-you can use `IsPositive()` as a matcher that matches a value iff it is
-greater than 0.
-
-We will also add `MATCHER_P`, `MATCHER_P2`, and etc for parameterized
-matchers.
\ No newline at end of file
diff --git a/thirdparty/GTest/googletest/googlemock/docs/Documentation.md b/thirdparty/GTest/googletest/googlemock/docs/Documentation.md
deleted file mode 100644
index 16083e7047..0000000000
--- a/thirdparty/GTest/googletest/googlemock/docs/Documentation.md
+++ /dev/null
@@ -1,15 +0,0 @@
-This page lists all documentation markdown files for Google Mock **(the
-current git version)**
--- **if you use a former version of Google Mock, please read the
-documentation for that specific version instead (e.g. by checking out
-the respective git branch/tag).**
-
-  * [ForDummies](ForDummies.md) -- start here if you are new to Google Mock.
-  * [CheatSheet](CheatSheet.md) -- a quick reference.
-  * [CookBook](CookBook.md) -- recipes for doing various tasks using Google Mock.
-  * [FrequentlyAskedQuestions](FrequentlyAskedQuestions.md) -- check here before asking a question on the mailing list.
-
-To contribute code to Google Mock, read:
-
-  * [CONTRIBUTING](../CONTRIBUTING.md) -- read this _before_ writing your first patch.
-  * [Pump Manual](../../googletest/docs/PumpManual.md) -- how we generate some of Google Mock's source files.
diff --git a/thirdparty/GTest/googletest/googlemock/docs/ForDummies.md b/thirdparty/GTest/googletest/googlemock/docs/ForDummies.md
deleted file mode 100644
index 769105698c..0000000000
--- a/thirdparty/GTest/googletest/googlemock/docs/ForDummies.md
+++ /dev/null
@@ -1,447 +0,0 @@
-
-
-(**Note:** If you get compiler errors that you don't understand, be sure to consult [Google Mock Doctor](FrequentlyAskedQuestions.md#how-am-i-supposed-to-make-sense-of-these-horrible-template-errors).)
-
-# What Is Google C++ Mocking Framework? #
-When you write a prototype or test, often it's not feasible or wise to rely on real objects entirely. A **mock object** implements the same interface as a real object (so it can be used as one), but lets you specify at run time how it will be used and what it should do (which methods will be called? in which order? how many times? with what arguments? what will they return? etc).
-
-**Note:** It is easy to confuse the term _fake objects_ with mock objects. Fakes and mocks actually mean very different things in the Test-Driven Development (TDD) community:
-
-  * **Fake** objects have working implementations, but usually take some shortcut (perhaps to make the operations less expensive), which makes them not suitable for production. An in-memory file system would be an example of a fake.
-  * **Mocks** are objects pre-programmed with _expectations_, which form a specification of the calls they are expected to receive.
-
-If all this seems too abstract for you, don't worry - the most important thing to remember is that a mock allows you to check the _interaction_ between itself and code that uses it. The difference between fakes and mocks will become much clearer once you start to use mocks.
-
-**Google C++ Mocking Framework** (or **Google Mock** for short) is a library (sometimes we also call it a "framework" to make it sound cool) for creating mock classes and using them. It does to C++ what [jMock](http://www.jmock.org/) and [EasyMock](http://www.easymock.org/) do to Java.
-
-Using Google Mock involves three basic steps:
-
-  1. Use some simple macros to describe the interface you want to mock, and they will expand to the implementation of your mock class;
-  1. Create some mock objects and specify its expectations and behavior using an intuitive syntax;
-  1. Exercise code that uses the mock objects. Google Mock will catch any violation of the expectations as soon as it arises.
-
-# Why Google Mock? #
-While mock objects help you remove unnecessary dependencies in tests and make them fast and reliable, using mocks manually in C++ is _hard_:
-
-  * Someone has to implement the mocks. The job is usually tedious and error-prone. No wonder people go great distances to avoid it.
-  * The quality of those manually written mocks is a bit, uh, unpredictable. You may see some really polished ones, but you may also see some that were hacked up in a hurry and have all sorts of ad-hoc restrictions.
-  * The knowledge you gained from using one mock doesn't transfer to the next.
-
-In contrast, Java and Python programmers have some fine mock frameworks, which automate the creation of mocks. As a result, mocking is a proven effective technique and widely adopted practice in those communities. Having the right tool absolutely makes the difference.
-
-Google Mock was built to help C++ programmers. It was inspired by [jMock](http://www.jmock.org/) and [EasyMock](http://www.easymock.org/), but designed with C++'s specifics in mind. It is your friend if any of the following problems is bothering you:
-
-  * You are stuck with a sub-optimal design and wish you had done more prototyping before it was too late, but prototyping in C++ is by no means "rapid".
-  * Your tests are slow as they depend on too many libraries or use expensive resources (e.g. a database).
-  * Your tests are brittle as some resources they use are unreliable (e.g. the network).
-  * You want to test how your code handles a failure (e.g. a file checksum error), but it's not easy to cause one.
-  * You need to make sure that your module interacts with other modules in the right way, but it's hard to observe the interaction; therefore you resort to observing the side effects at the end of the action, which is awkward at best.
-  * You want to "mock out" your dependencies, except that they don't have mock implementations yet; and, frankly, you aren't thrilled by some of those hand-written mocks.
-
-We encourage you to use Google Mock as:
-
-  * a _design_ tool, for it lets you experiment with your interface design early and often. More iterations lead to better designs!
-  * a _testing_ tool to cut your tests' outbound dependencies and probe the interaction between your module and its collaborators.
-
-# Getting Started #
-Using Google Mock is easy! Inside your C++ source file, just `#include` `"gtest/gtest.h"` and `"gmock/gmock.h"`, and you are ready to go.
-
-# A Case for Mock Turtles #
-Let's look at an example. Suppose you are developing a graphics program that relies on a LOGO-like API for drawing. How would you test that it does the right thing? Well, you can run it and compare the screen with a golden screen snapshot, but let's admit it: tests like this are expensive to run and fragile (What if you just upgraded to a shiny new graphics card that has better anti-aliasing? Suddenly you have to update all your golden images.). It would be too painful if all your tests are like this. Fortunately, you learned about Dependency Injection and know the right thing to do: instead of having your application talk to the drawing API directly, wrap the API in an interface (say, `Turtle`) and code to that interface:
-
-```
-class Turtle {
-  ...
-  virtual ~Turtle() {}
-  virtual void PenUp() = 0;
-  virtual void PenDown() = 0;
-  virtual void Forward(int distance) = 0;
-  virtual void Turn(int degrees) = 0;
-  virtual void GoTo(int x, int y) = 0;
-  virtual int GetX() const = 0;
-  virtual int GetY() const = 0;
-};
-```
-
-(Note that the destructor of `Turtle` **must** be virtual, as is the case for **all** classes you intend to inherit from - otherwise the destructor of the derived class will not be called when you delete an object through a base pointer, and you'll get corrupted program states like memory leaks.)
-
-You can control whether the turtle's movement will leave a trace using `PenUp()` and `PenDown()`, and control its movement using `Forward()`, `Turn()`, and `GoTo()`. Finally, `GetX()` and `GetY()` tell you the current position of the turtle.
-
-Your program will normally use a real implementation of this interface. In tests, you can use a mock implementation instead. This allows you to easily check what drawing primitives your program is calling, with what arguments, and in which order. Tests written this way are much more robust (they won't break because your new machine does anti-aliasing differently), easier to read and maintain (the intent of a test is expressed in the code, not in some binary images), and run _much, much faster_.
-
-# Writing the Mock Class #
-If you are lucky, the mocks you need to use have already been implemented by some nice people. If, however, you find yourself in the position to write a mock class, relax - Google Mock turns this task into a fun game! (Well, almost.)
-
-## How to Define It ##
-Using the `Turtle` interface as example, here are the simple steps you need to follow:
-
-  1. Derive a class `MockTurtle` from `Turtle`.
-  1. Take a _virtual_ function of `Turtle` (while it's possible to [mock non-virtual methods using templates](CookBook.md#mocking-nonvirtual-methods), it's much more involved). Count how many arguments it has.
-  1. In the `public:` section of the child class, write `MOCK_METHODn();` (or `MOCK_CONST_METHODn();` if you are mocking a `const` method), where `n` is the number of the arguments; if you counted wrong, shame on you, and a compiler error will tell you so.
-  1. Now comes the fun part: you take the function signature, cut-and-paste the _function name_ as the _first_ argument to the macro, and leave what's left as the _second_ argument (in case you're curious, this is the _type of the function_).
-  1. Repeat until all virtual functions you want to mock are done.
-
-After the process, you should have something like:
-
-```
-#include "gmock/gmock.h"  // Brings in Google Mock.
-class MockTurtle : public Turtle {
- public:
-  ...
-  MOCK_METHOD0(PenUp, void());
-  MOCK_METHOD0(PenDown, void());
-  MOCK_METHOD1(Forward, void(int distance));
-  MOCK_METHOD1(Turn, void(int degrees));
-  MOCK_METHOD2(GoTo, void(int x, int y));
-  MOCK_CONST_METHOD0(GetX, int());
-  MOCK_CONST_METHOD0(GetY, int());
-};
-```
-
-You don't need to define these mock methods somewhere else - the `MOCK_METHOD*` macros will generate the definitions for you. It's that simple! Once you get the hang of it, you can pump out mock classes faster than your source-control system can handle your check-ins.
-
-**Tip:** If even this is too much work for you, you'll find the
-`gmock_gen.py` tool in Google Mock's `scripts/generator/` directory (courtesy of the [cppclean](http://code.google.com/p/cppclean/) project) useful.  This command-line
-tool requires that you have Python 2.4 installed.  You give it a C++ file and the name of an abstract class defined in it,
-and it will print the definition of the mock class for you.  Due to the
-complexity of the C++ language, this script may not always work, but
-it can be quite handy when it does.  For more details, read the [user documentation](../scripts/generator/README).
-
-## Where to Put It ##
-When you define a mock class, you need to decide where to put its definition. Some people put it in a `*_test.cc`. This is fine when the interface being mocked (say, `Foo`) is owned by the same person or team. Otherwise, when the owner of `Foo` changes it, your test could break. (You can't really expect `Foo`'s maintainer to fix every test that uses `Foo`, can you?)
-
-So, the rule of thumb is: if you need to mock `Foo` and it's owned by others, define the mock class in `Foo`'s package (better, in a `testing` sub-package such that you can clearly separate production code and testing utilities), and put it in a `mock_foo.h`. Then everyone can reference `mock_foo.h` from their tests. If `Foo` ever changes, there is only one copy of `MockFoo` to change, and only tests that depend on the changed methods need to be fixed.
-
-Another way to do it: you can introduce a thin layer `FooAdaptor` on top of `Foo` and code to this new interface. Since you own `FooAdaptor`, you can absorb changes in `Foo` much more easily. While this is more work initially, carefully choosing the adaptor interface can make your code easier to write and more readable (a net win in the long run), as you can choose `FooAdaptor` to fit your specific domain much better than `Foo` does.
-
-# Using Mocks in Tests #
-Once you have a mock class, using it is easy. The typical work flow is:
-
-  1. Import the Google Mock names from the `testing` namespace such that you can use them unqualified (You only have to do it once per file. Remember that namespaces are a good idea and good for your health.).
-  1. Create some mock objects.
-  1. Specify your expectations on them (How many times will a method be called? With what arguments? What should it do? etc.).
-  1. Exercise some code that uses the mocks; optionally, check the result using Google Test assertions. If a mock method is called more than expected or with wrong arguments, you'll get an error immediately.
-  1. When a mock is destructed, Google Mock will automatically check whether all expectations on it have been satisfied.
-
-Here's an example:
-
-```
-#include "path/to/mock-turtle.h"
-#include "gmock/gmock.h"
-#include "gtest/gtest.h"
-using ::testing::AtLeast;                     // #1
-
-TEST(PainterTest, CanDrawSomething) {
-  MockTurtle turtle;                          // #2
-  EXPECT_CALL(turtle, PenDown())              // #3
-      .Times(AtLeast(1));
-
-  Painter painter(&turtle);                   // #4
-
-  EXPECT_TRUE(painter.DrawCircle(0, 0, 10));
-}                                             // #5
-
-int main(int argc, char** argv) {
-  // The following line must be executed to initialize Google Mock
-  // (and Google Test) before running the tests.
-  ::testing::InitGoogleMock(&argc, argv);
-  return RUN_ALL_TESTS();
-}
-```
-
-As you might have guessed, this test checks that `PenDown()` is called at least once. If the `painter` object didn't call this method, your test will fail with a message like this:
-
-```
-path/to/my_test.cc:119: Failure
-Actual function call count doesn't match this expectation:
-Actually: never called;
-Expected: called at least once.
-```
-
-**Tip 1:** If you run the test from an Emacs buffer, you can hit `` on the line number displayed in the error message to jump right to the failed expectation.
-
-**Tip 2:** If your mock objects are never deleted, the final verification won't happen. Therefore it's a good idea to use a heap leak checker in your tests when you allocate mocks on the heap.
-
-**Important note:** Google Mock requires expectations to be set **before** the mock functions are called, otherwise the behavior is **undefined**. In particular, you mustn't interleave `EXPECT_CALL()`s and calls to the mock functions.
-
-This means `EXPECT_CALL()` should be read as expecting that a call will occur _in the future_, not that a call has occurred. Why does Google Mock work like that? Well, specifying the expectation beforehand allows Google Mock to report a violation as soon as it arises, when the context (stack trace, etc) is still available. This makes debugging much easier.
-
-Admittedly, this test is contrived and doesn't do much. You can easily achieve the same effect without using Google Mock. However, as we shall reveal soon, Google Mock allows you to do _much more_ with the mocks.
-
-## Using Google Mock with Any Testing Framework ##
-If you want to use something other than Google Test (e.g. [CppUnit](http://sourceforge.net/projects/cppunit/) or
-[CxxTest](http://cxxtest.tigris.org/)) as your testing framework, just change the `main()` function in the previous section to:
-```
-int main(int argc, char** argv) {
-  // The following line causes Google Mock to throw an exception on failure,
-  // which will be interpreted by your testing framework as a test failure.
-  ::testing::GTEST_FLAG(throw_on_failure) = true;
-  ::testing::InitGoogleMock(&argc, argv);
-  ... whatever your testing framework requires ...
-}
-```
-
-This approach has a catch: it makes Google Mock throw an exception
-from a mock object's destructor sometimes.  With some compilers, this
-sometimes causes the test program to crash.  You'll still be able to
-notice that the test has failed, but it's not a graceful failure.
-
-A better solution is to use Google Test's
-[event listener API](../../googletest/docs/AdvancedGuide.md#extending-google-test-by-handling-test-events)
-to report a test failure to your testing framework properly.  You'll need to
-implement the `OnTestPartResult()` method of the event listener interface, but it
-should be straightforward.
-
-If this turns out to be too much work, we suggest that you stick with
-Google Test, which works with Google Mock seamlessly (in fact, it is
-technically part of Google Mock.).  If there is a reason that you
-cannot use Google Test, please let us know.
-
-# Setting Expectations #
-The key to using a mock object successfully is to set the _right expectations_ on it. If you set the expectations too strict, your test will fail as the result of unrelated changes. If you set them too loose, bugs can slip through. You want to do it just right such that your test can catch exactly the kind of bugs you intend it to catch. Google Mock provides the necessary means for you to do it "just right."
-
-## General Syntax ##
-In Google Mock we use the `EXPECT_CALL()` macro to set an expectation on a mock method. The general syntax is:
-
-```
-EXPECT_CALL(mock_object, method(matchers))
-    .Times(cardinality)
-    .WillOnce(action)
-    .WillRepeatedly(action);
-```
-
-The macro has two arguments: first the mock object, and then the method and its arguments. Note that the two are separated by a comma (`,`), not a period (`.`). (Why using a comma? The answer is that it was necessary for technical reasons.)
-
-The macro can be followed by some optional _clauses_ that provide more information about the expectation. We'll discuss how each clause works in the coming sections.
-
-This syntax is designed to make an expectation read like English. For example, you can probably guess that
-
-```
-using ::testing::Return;
-...
-EXPECT_CALL(turtle, GetX())
-    .Times(5)
-    .WillOnce(Return(100))
-    .WillOnce(Return(150))
-    .WillRepeatedly(Return(200));
-```
-
-says that the `turtle` object's `GetX()` method will be called five times, it will return 100 the first time, 150 the second time, and then 200 every time. Some people like to call this style of syntax a Domain-Specific Language (DSL).
-
-**Note:** Why do we use a macro to do this? It serves two purposes: first it makes expectations easily identifiable (either by `grep` or by a human reader), and second it allows Google Mock to include the source file location of a failed expectation in messages, making debugging easier.
-
-## Matchers: What Arguments Do We Expect? ##
-When a mock function takes arguments, we must specify what arguments we are expecting; for example:
-
-```
-// Expects the turtle to move forward by 100 units.
-EXPECT_CALL(turtle, Forward(100));
-```
-
-Sometimes you may not want to be too specific (Remember that talk about tests being too rigid? Over specification leads to brittle tests and obscures the intent of tests. Therefore we encourage you to specify only what's necessary - no more, no less.). If you care to check that `Forward()` will be called but aren't interested in its actual argument, write `_` as the argument, which means "anything goes":
-
-```
-using ::testing::_;
-...
-// Expects the turtle to move forward.
-EXPECT_CALL(turtle, Forward(_));
-```
-
-`_` is an instance of what we call **matchers**. A matcher is like a predicate and can test whether an argument is what we'd expect. You can use a matcher inside `EXPECT_CALL()` wherever a function argument is expected.
-
-A list of built-in matchers can be found in the [CheatSheet](CheatSheet.md). For example, here's the `Ge` (greater than or equal) matcher:
-
-```
-using ::testing::Ge;
-...
-EXPECT_CALL(turtle, Forward(Ge(100)));
-```
-
-This checks that the turtle will be told to go forward by at least 100 units.
-
-## Cardinalities: How Many Times Will It Be Called? ##
-The first clause we can specify following an `EXPECT_CALL()` is `Times()`. We call its argument a **cardinality** as it tells _how many times_ the call should occur. It allows us to repeat an expectation many times without actually writing it as many times. More importantly, a cardinality can be "fuzzy", just like a matcher can be. This allows a user to express the intent of a test exactly.
-
-An interesting special case is when we say `Times(0)`. You may have guessed - it means that the function shouldn't be called with the given arguments at all, and Google Mock will report a Google Test failure whenever the function is (wrongfully) called.
-
-We've seen `AtLeast(n)` as an example of fuzzy cardinalities earlier. For the list of built-in cardinalities you can use, see the [CheatSheet](CheatSheet.md).
-
-The `Times()` clause can be omitted. **If you omit `Times()`, Google Mock will infer the cardinality for you.** The rules are easy to remember:
-
-  * If **neither** `WillOnce()` **nor** `WillRepeatedly()` is in the `EXPECT_CALL()`, the inferred cardinality is `Times(1)`.
-  * If there are `n WillOnce()`'s but **no** `WillRepeatedly()`, where `n` >= 1, the cardinality is `Times(n)`.
-  * If there are `n WillOnce()`'s and **one** `WillRepeatedly()`, where `n` >= 0, the cardinality is `Times(AtLeast(n))`.
-
-**Quick quiz:** what do you think will happen if a function is expected to be called twice but actually called four times?
-
-## Actions: What Should It Do? ##
-Remember that a mock object doesn't really have a working implementation? We as users have to tell it what to do when a method is invoked. This is easy in Google Mock.
-
-First, if the return type of a mock function is a built-in type or a pointer, the function has a **default action** (a `void` function will just return, a `bool` function will return `false`, and other functions will return 0). In addition, in C++ 11 and above, a mock function whose return type is default-constructible (i.e. has a default constructor) has a default action of returning a default-constructed value.  If you don't say anything, this behavior will be used.
-
-Second, if a mock function doesn't have a default action, or the default action doesn't suit you, you can specify the action to be taken each time the expectation matches using a series of `WillOnce()` clauses followed by an optional `WillRepeatedly()`. For example,
-
-```
-using ::testing::Return;
-...
-EXPECT_CALL(turtle, GetX())
-    .WillOnce(Return(100))
-    .WillOnce(Return(200))
-    .WillOnce(Return(300));
-```
-
-This says that `turtle.GetX()` will be called _exactly three times_ (Google Mock inferred this from how many `WillOnce()` clauses we've written, since we didn't explicitly write `Times()`), and will return 100, 200, and 300 respectively.
-
-```
-using ::testing::Return;
-...
-EXPECT_CALL(turtle, GetY())
-    .WillOnce(Return(100))
-    .WillOnce(Return(200))
-    .WillRepeatedly(Return(300));
-```
-
-says that `turtle.GetY()` will be called _at least twice_ (Google Mock knows this as we've written two `WillOnce()` clauses and a `WillRepeatedly()` while having no explicit `Times()`), will return 100 the first time, 200 the second time, and 300 from the third time on.
-
-Of course, if you explicitly write a `Times()`, Google Mock will not try to infer the cardinality itself. What if the number you specified is larger than there are `WillOnce()` clauses? Well, after all `WillOnce()`s are used up, Google Mock will do the _default_ action for the function every time (unless, of course, you have a `WillRepeatedly()`.).
-
-What can we do inside `WillOnce()` besides `Return()`? You can return a reference using `ReturnRef(variable)`, or invoke a pre-defined function, among [others](CheatSheet.md#actions).
-
-**Important note:** The `EXPECT_CALL()` statement evaluates the action clause only once, even though the action may be performed many times. Therefore you must be careful about side effects. The following may not do what you want:
-
-```
-int n = 100;
-EXPECT_CALL(turtle, GetX())
-.Times(4)
-.WillRepeatedly(Return(n++));
-```
-
-Instead of returning 100, 101, 102, ..., consecutively, this mock function will always return 100 as `n++` is only evaluated once. Similarly, `Return(new Foo)` will create a new `Foo` object when the `EXPECT_CALL()` is executed, and will return the same pointer every time. If you want the side effect to happen every time, you need to define a custom action, which we'll teach in the [CookBook](CookBook.md).
-
-Time for another quiz! What do you think the following means?
-
-```
-using ::testing::Return;
-...
-EXPECT_CALL(turtle, GetY())
-.Times(4)
-.WillOnce(Return(100));
-```
-
-Obviously `turtle.GetY()` is expected to be called four times. But if you think it will return 100 every time, think twice! Remember that one `WillOnce()` clause will be consumed each time the function is invoked and the default action will be taken afterwards. So the right answer is that `turtle.GetY()` will return 100 the first time, but **return 0 from the second time on**, as returning 0 is the default action for `int` functions.
-
-## Using Multiple Expectations ##
-So far we've only shown examples where you have a single expectation. More realistically, you're going to specify expectations on multiple mock methods, which may be from multiple mock objects.
-
-By default, when a mock method is invoked, Google Mock will search the expectations in the **reverse order** they are defined, and stop when an active expectation that matches the arguments is found (you can think of it as "newer rules override older ones."). If the matching expectation cannot take any more calls, you will get an upper-bound-violated failure. Here's an example:
-
-```
-using ::testing::_;
-...
-EXPECT_CALL(turtle, Forward(_));  // #1
-EXPECT_CALL(turtle, Forward(10))  // #2
-    .Times(2);
-```
-
-If `Forward(10)` is called three times in a row, the third time it will be an error, as the last matching expectation (#2) has been saturated. If, however, the third `Forward(10)` call is replaced by `Forward(20)`, then it would be OK, as now #1 will be the matching expectation.
-
-**Side note:** Why does Google Mock search for a match in the _reverse_ order of the expectations? The reason is that this allows a user to set up the default expectations in a mock object's constructor or the test fixture's set-up phase and then customize the mock by writing more specific expectations in the test body. So, if you have two expectations on the same method, you want to put the one with more specific matchers **after** the other, or the more specific rule would be shadowed by the more general one that comes after it.
-
-## Ordered vs Unordered Calls ##
-By default, an expectation can match a call even though an earlier expectation hasn't been satisfied. In other words, the calls don't have to occur in the order the expectations are specified.
-
-Sometimes, you may want all the expected calls to occur in a strict order. To say this in Google Mock is easy:
-
-```
-using ::testing::InSequence;
-...
-TEST(FooTest, DrawsLineSegment) {
-  ...
-  {
-    InSequence dummy;
-
-    EXPECT_CALL(turtle, PenDown());
-    EXPECT_CALL(turtle, Forward(100));
-    EXPECT_CALL(turtle, PenUp());
-  }
-  Foo();
-}
-```
-
-By creating an object of type `InSequence`, all expectations in its scope are put into a _sequence_ and have to occur _sequentially_. Since we are just relying on the constructor and destructor of this object to do the actual work, its name is really irrelevant.
-
-In this example, we test that `Foo()` calls the three expected functions in the order as written. If a call is made out-of-order, it will be an error.
-
-(What if you care about the relative order of some of the calls, but not all of them? Can you specify an arbitrary partial order? The answer is ... yes! If you are impatient, the details can be found in the [CookBook](CookBook.md#expecting-partially-ordered-calls).)
-
-## All Expectations Are Sticky (Unless Said Otherwise) ##
-Now let's do a quick quiz to see how well you can use this mock stuff already. How would you test that the turtle is asked to go to the origin _exactly twice_ (you want to ignore any other instructions it receives)?
-
-After you've come up with your answer, take a look at ours and compare notes (solve it yourself first - don't cheat!):
-
-```
-using ::testing::_;
-...
-EXPECT_CALL(turtle, GoTo(_, _))  // #1
-    .Times(AnyNumber());
-EXPECT_CALL(turtle, GoTo(0, 0))  // #2
-    .Times(2);
-```
-
-Suppose `turtle.GoTo(0, 0)` is called three times. In the third time, Google Mock will see that the arguments match expectation #2 (remember that we always pick the last matching expectation). Now, since we said that there should be only two such calls, Google Mock will report an error immediately. This is basically what we've told you in the "Using Multiple Expectations" section above.
-
-This example shows that **expectations in Google Mock are "sticky" by default**, in the sense that they remain active even after we have reached their invocation upper bounds. This is an important rule to remember, as it affects the meaning of the spec, and is **different** to how it's done in many other mocking frameworks (Why'd we do that? Because we think our rule makes the common cases easier to express and understand.).
-
-Simple? Let's see if you've really understood it: what does the following code say?
-
-```
-using ::testing::Return;
-...
-for (int i = n; i > 0; i--) {
-  EXPECT_CALL(turtle, GetX())
-      .WillOnce(Return(10*i));
-}
-```
-
-If you think it says that `turtle.GetX()` will be called `n` times and will return 10, 20, 30, ..., consecutively, think twice! The problem is that, as we said, expectations are sticky. So, the second time `turtle.GetX()` is called, the last (latest) `EXPECT_CALL()` statement will match, and will immediately lead to an "upper bound exceeded" error - this piece of code is not very useful!
-
-One correct way of saying that `turtle.GetX()` will return 10, 20, 30, ..., is to explicitly say that the expectations are _not_ sticky. In other words, they should _retire_ as soon as they are saturated:
-
-```
-using ::testing::Return;
-...
-for (int i = n; i > 0; i--) {
-  EXPECT_CALL(turtle, GetX())
-    .WillOnce(Return(10*i))
-    .RetiresOnSaturation();
-}
-```
-
-And, there's a better way to do it: in this case, we expect the calls to occur in a specific order, and we line up the actions to match the order. Since the order is important here, we should make it explicit using a sequence:
-
-```
-using ::testing::InSequence;
-using ::testing::Return;
-...
-{
-  InSequence s;
-
-  for (int i = 1; i <= n; i++) {
-    EXPECT_CALL(turtle, GetX())
-        .WillOnce(Return(10*i))
-        .RetiresOnSaturation();
-  }
-}
-```
-
-By the way, the other situation where an expectation may _not_ be sticky is when it's in a sequence - as soon as another expectation that comes after it in the sequence has been used, it automatically retires (and will never be used to match any call).
-
-## Uninteresting Calls ##
-A mock object may have many methods, and not all of them are that interesting. For example, in some tests we may not care about how many times `GetX()` and `GetY()` get called.
-
-In Google Mock, if you are not interested in a method, just don't say anything about it. If a call to this method occurs, you'll see a warning in the test output, but it won't be a failure.
-
-# What Now? #
-Congratulations! You've learned enough about Google Mock to start using it. Now, you might want to join the [googlemock](http://groups.google.com/group/googlemock) discussion group and actually write some tests using Google Mock - it will be fun. Hey, it may even be addictive - you've been warned.
-
-Then, if you feel like increasing your mock quotient, you should move on to the [CookBook](CookBook.md). You can learn many advanced features of Google Mock there -- and advance your level of enjoyment and testing bliss.
diff --git a/thirdparty/GTest/googletest/googlemock/docs/FrequentlyAskedQuestions.md b/thirdparty/GTest/googletest/googlemock/docs/FrequentlyAskedQuestions.md
deleted file mode 100644
index ccaa3d7a87..0000000000
--- a/thirdparty/GTest/googletest/googlemock/docs/FrequentlyAskedQuestions.md
+++ /dev/null
@@ -1,628 +0,0 @@
-
-
-Please send your questions to the
-[googlemock](http://groups.google.com/group/googlemock) discussion
-group. If you need help with compiler errors, make sure you have
-tried [Google Mock Doctor](#How_am_I_supposed_to_make_sense_of_these_horrible_template_error.md) first.
-
-## When I call a method on my mock object, the method for the real object is invoked instead.  What's the problem? ##
-
-In order for a method to be mocked, it must be _virtual_, unless you use the [high-perf dependency injection technique](CookBook.md#mocking-nonvirtual-methods).
-
-## I wrote some matchers.  After I upgraded to a new version of Google Mock, they no longer compile.  What's going on? ##
-
-After version 1.4.0 of Google Mock was released, we had an idea on how
-to make it easier to write matchers that can generate informative
-messages efficiently.  We experimented with this idea and liked what
-we saw.  Therefore we decided to implement it.
-
-Unfortunately, this means that if you have defined your own matchers
-by implementing `MatcherInterface` or using `MakePolymorphicMatcher()`,
-your definitions will no longer compile.  Matchers defined using the
-`MATCHER*` family of macros are not affected.
-
-Sorry for the hassle if your matchers are affected.  We believe it's
-in everyone's long-term interest to make this change sooner than
-later.  Fortunately, it's usually not hard to migrate an existing
-matcher to the new API.  Here's what you need to do:
-
-If you wrote your matcher like this:
-```
-// Old matcher definition that doesn't work with the latest
-// Google Mock.
-using ::testing::MatcherInterface;
-...
-class MyWonderfulMatcher : public MatcherInterface {
- public:
-  ...
-  virtual bool Matches(MyType value) const {
-    // Returns true if value matches.
-    return value.GetFoo() > 5;
-  }
-  ...
-};
-```
-
-you'll need to change it to:
-```
-// New matcher definition that works with the latest Google Mock.
-using ::testing::MatcherInterface;
-using ::testing::MatchResultListener;
-...
-class MyWonderfulMatcher : public MatcherInterface {
- public:
-  ...
-  virtual bool MatchAndExplain(MyType value,
-                               MatchResultListener* listener) const {
-    // Returns true if value matches.
-    return value.GetFoo() > 5;
-  }
-  ...
-};
-```
-(i.e. rename `Matches()` to `MatchAndExplain()` and give it a second
-argument of type `MatchResultListener*`.)
-
-If you were also using `ExplainMatchResultTo()` to improve the matcher
-message:
-```
-// Old matcher definition that doesn't work with the lastest
-// Google Mock.
-using ::testing::MatcherInterface;
-...
-class MyWonderfulMatcher : public MatcherInterface {
- public:
-  ...
-  virtual bool Matches(MyType value) const {
-    // Returns true if value matches.
-    return value.GetFoo() > 5;
-  }
-
-  virtual void ExplainMatchResultTo(MyType value,
-                                    ::std::ostream* os) const {
-    // Prints some helpful information to os to help
-    // a user understand why value matches (or doesn't match).
-    *os << "the Foo property is " << value.GetFoo();
-  }
-  ...
-};
-```
-
-you should move the logic of `ExplainMatchResultTo()` into
-`MatchAndExplain()`, using the `MatchResultListener` argument where
-the `::std::ostream` was used:
-```
-// New matcher definition that works with the latest Google Mock.
-using ::testing::MatcherInterface;
-using ::testing::MatchResultListener;
-...
-class MyWonderfulMatcher : public MatcherInterface {
- public:
-  ...
-  virtual bool MatchAndExplain(MyType value,
-                               MatchResultListener* listener) const {
-    // Returns true if value matches.
-    *listener << "the Foo property is " << value.GetFoo();
-    return value.GetFoo() > 5;
-  }
-  ...
-};
-```
-
-If your matcher is defined using `MakePolymorphicMatcher()`:
-```
-// Old matcher definition that doesn't work with the latest
-// Google Mock.
-using ::testing::MakePolymorphicMatcher;
-...
-class MyGreatMatcher {
- public:
-  ...
-  bool Matches(MyType value) const {
-    // Returns true if value matches.
-    return value.GetBar() < 42;
-  }
-  ...
-};
-... MakePolymorphicMatcher(MyGreatMatcher()) ...
-```
-
-you should rename the `Matches()` method to `MatchAndExplain()` and
-add a `MatchResultListener*` argument (the same as what you need to do
-for matchers defined by implementing `MatcherInterface`):
-```
-// New matcher definition that works with the latest Google Mock.
-using ::testing::MakePolymorphicMatcher;
-using ::testing::MatchResultListener;
-...
-class MyGreatMatcher {
- public:
-  ...
-  bool MatchAndExplain(MyType value,
-                       MatchResultListener* listener) const {
-    // Returns true if value matches.
-    return value.GetBar() < 42;
-  }
-  ...
-};
-... MakePolymorphicMatcher(MyGreatMatcher()) ...
-```
-
-If your polymorphic matcher uses `ExplainMatchResultTo()` for better
-failure messages:
-```
-// Old matcher definition that doesn't work with the latest
-// Google Mock.
-using ::testing::MakePolymorphicMatcher;
-...
-class MyGreatMatcher {
- public:
-  ...
-  bool Matches(MyType value) const {
-    // Returns true if value matches.
-    return value.GetBar() < 42;
-  }
-  ...
-};
-void ExplainMatchResultTo(const MyGreatMatcher& matcher,
-                          MyType value,
-                          ::std::ostream* os) {
-  // Prints some helpful information to os to help
-  // a user understand why value matches (or doesn't match).
-  *os << "the Bar property is " << value.GetBar();
-}
-... MakePolymorphicMatcher(MyGreatMatcher()) ...
-```
-
-you'll need to move the logic inside `ExplainMatchResultTo()` to
-`MatchAndExplain()`:
-```
-// New matcher definition that works with the latest Google Mock.
-using ::testing::MakePolymorphicMatcher;
-using ::testing::MatchResultListener;
-...
-class MyGreatMatcher {
- public:
-  ...
-  bool MatchAndExplain(MyType value,
-                       MatchResultListener* listener) const {
-    // Returns true if value matches.
-    *listener << "the Bar property is " << value.GetBar();
-    return value.GetBar() < 42;
-  }
-  ...
-};
-... MakePolymorphicMatcher(MyGreatMatcher()) ...
-```
-
-For more information, you can read these
-[two](CookBook.md#writing-new-monomorphic-matchers)
-[recipes](CookBook.md#writing-new-polymorphic-matchers)
-from the cookbook.  As always, you
-are welcome to post questions on `googlemock@googlegroups.com` if you
-need any help.
-
-## When using Google Mock, do I have to use Google Test as the testing framework?  I have my favorite testing framework and don't want to switch. ##
-
-Google Mock works out of the box with Google Test.  However, it's easy
-to configure it to work with any testing framework of your choice.
-[Here](ForDummies.md#using-google-mock-with-any-testing-framework) is how.
-
-## How am I supposed to make sense of these horrible template errors? ##
-
-If you are confused by the compiler errors gcc threw at you,
-try consulting the _Google Mock Doctor_ tool first.  What it does is to
-scan stdin for gcc error messages, and spit out diagnoses on the
-problems (we call them diseases) your code has.
-
-To "install", run command:
-```
-alias gmd='/scripts/gmock_doctor.py'
-```
-
-To use it, do:
-```
-  2>&1 | gmd
-```
-
-For example:
-```
-make my_test 2>&1 | gmd
-```
-
-Or you can run `gmd` and copy-n-paste gcc's error messages to it.
-
-## Can I mock a variadic function? ##
-
-You cannot mock a variadic function (i.e. a function taking ellipsis
-(`...`) arguments) directly in Google Mock.
-
-The problem is that in general, there is _no way_ for a mock object to
-know how many arguments are passed to the variadic method, and what
-the arguments' types are.  Only the _author of the base class_ knows
-the protocol, and we cannot look into their head.
-
-Therefore, to mock such a function, the _user_ must teach the mock
-object how to figure out the number of arguments and their types.  One
-way to do it is to provide overloaded versions of the function.
-
-Ellipsis arguments are inherited from C and not really a C++ feature.
-They are unsafe to use and don't work with arguments that have
-constructors or destructors.  Therefore we recommend to avoid them in
-C++ as much as possible.
-
-## MSVC gives me warning C4301 or C4373 when I define a mock method with a const parameter.  Why? ##
-
-If you compile this using Microsoft Visual C++ 2005 SP1:
-```
-class Foo {
-  ...
-  virtual void Bar(const int i) = 0;
-};
-
-class MockFoo : public Foo {
-  ...
-  MOCK_METHOD1(Bar, void(const int i));
-};
-```
-You may get the following warning:
-```
-warning C4301: 'MockFoo::Bar': overriding virtual function only differs from 'Foo::Bar' by const/volatile qualifier
-```
-
-This is a MSVC bug.  The same code compiles fine with gcc ,for
-example.  If you use Visual C++ 2008 SP1, you would get the warning:
-```
-warning C4373: 'MockFoo::Bar': virtual function overrides 'Foo::Bar', previous versions of the compiler did not override when parameters only differed by const/volatile qualifiers
-```
-
-In C++, if you _declare_ a function with a `const` parameter, the
-`const` modifier is _ignored_.  Therefore, the `Foo` base class above
-is equivalent to:
-```
-class Foo {
-  ...
-  virtual void Bar(int i) = 0;  // int or const int?  Makes no difference.
-};
-```
-
-In fact, you can _declare_ Bar() with an `int` parameter, and _define_
-it with a `const int` parameter.  The compiler will still match them
-up.
-
-Since making a parameter `const` is meaningless in the method
-_declaration_, we recommend to remove it in both `Foo` and `MockFoo`.
-That should workaround the VC bug.
-
-Note that we are talking about the _top-level_ `const` modifier here.
-If the function parameter is passed by pointer or reference, declaring
-the _pointee_ or _referee_ as `const` is still meaningful.  For
-example, the following two declarations are _not_ equivalent:
-```
-void Bar(int* p);        // Neither p nor *p is const.
-void Bar(const int* p);  // p is not const, but *p is.
-```
-
-## I have a huge mock class, and Microsoft Visual C++ runs out of memory when compiling it.  What can I do? ##
-
-We've noticed that when the `/clr` compiler flag is used, Visual C++
-uses 5~6 times as much memory when compiling a mock class.  We suggest
-to avoid `/clr` when compiling native C++ mocks.
-
-## I can't figure out why Google Mock thinks my expectations are not satisfied.  What should I do? ##
-
-You might want to run your test with
-`--gmock_verbose=info`.  This flag lets Google Mock print a trace
-of every mock function call it receives.  By studying the trace,
-you'll gain insights on why the expectations you set are not met.
-
-## How can I assert that a function is NEVER called? ##
-
-```
-EXPECT_CALL(foo, Bar(_))
-    .Times(0);
-```
-
-## I have a failed test where Google Mock tells me TWICE that a particular expectation is not satisfied.  Isn't this redundant? ##
-
-When Google Mock detects a failure, it prints relevant information
-(the mock function arguments, the state of relevant expectations, and
-etc) to help the user debug.  If another failure is detected, Google
-Mock will do the same, including printing the state of relevant
-expectations.
-
-Sometimes an expectation's state didn't change between two failures,
-and you'll see the same description of the state twice.  They are
-however _not_ redundant, as they refer to _different points in time_.
-The fact they are the same _is_ interesting information.
-
-## I get a heap check failure when using a mock object, but using a real object is fine.  What can be wrong? ##
-
-Does the class (hopefully a pure interface) you are mocking have a
-virtual destructor?
-
-Whenever you derive from a base class, make sure its destructor is
-virtual.  Otherwise Bad Things will happen.  Consider the following
-code:
-
-```
-class Base {
- public:
-  // Not virtual, but should be.
-  ~Base() { ... }
-  ...
-};
-
-class Derived : public Base {
- public:
-  ...
- private:
-  std::string value_;
-};
-
-...
-  Base* p = new Derived;
-  ...
-  delete p;  // Surprise! ~Base() will be called, but ~Derived() will not
-             // - value_ is leaked.
-```
-
-By changing `~Base()` to virtual, `~Derived()` will be correctly
-called when `delete p` is executed, and the heap checker
-will be happy.
-
-## The "newer expectations override older ones" rule makes writing expectations awkward.  Why does Google Mock do that? ##
-
-When people complain about this, often they are referring to code like:
-
-```
-// foo.Bar() should be called twice, return 1 the first time, and return
-// 2 the second time.  However, I have to write the expectations in the
-// reverse order.  This sucks big time!!!
-EXPECT_CALL(foo, Bar())
-    .WillOnce(Return(2))
-    .RetiresOnSaturation();
-EXPECT_CALL(foo, Bar())
-    .WillOnce(Return(1))
-    .RetiresOnSaturation();
-```
-
-The problem is that they didn't pick the **best** way to express the test's
-intent.
-
-By default, expectations don't have to be matched in _any_ particular
-order.  If you want them to match in a certain order, you need to be
-explicit.  This is Google Mock's (and jMock's) fundamental philosophy: it's
-easy to accidentally over-specify your tests, and we want to make it
-harder to do so.
-
-There are two better ways to write the test spec.  You could either
-put the expectations in sequence:
-
-```
-// foo.Bar() should be called twice, return 1 the first time, and return
-// 2 the second time.  Using a sequence, we can write the expectations
-// in their natural order.
-{
-  InSequence s;
-  EXPECT_CALL(foo, Bar())
-      .WillOnce(Return(1))
-      .RetiresOnSaturation();
-  EXPECT_CALL(foo, Bar())
-      .WillOnce(Return(2))
-      .RetiresOnSaturation();
-}
-```
-
-or you can put the sequence of actions in the same expectation:
-
-```
-// foo.Bar() should be called twice, return 1 the first time, and return
-// 2 the second time.
-EXPECT_CALL(foo, Bar())
-    .WillOnce(Return(1))
-    .WillOnce(Return(2))
-    .RetiresOnSaturation();
-```
-
-Back to the original questions: why does Google Mock search the
-expectations (and `ON_CALL`s) from back to front?  Because this
-allows a user to set up a mock's behavior for the common case early
-(e.g. in the mock's constructor or the test fixture's set-up phase)
-and customize it with more specific rules later.  If Google Mock
-searches from front to back, this very useful pattern won't be
-possible.
-
-## Google Mock prints a warning when a function without EXPECT\_CALL is called, even if I have set its behavior using ON\_CALL.  Would it be reasonable not to show the warning in this case? ##
-
-When choosing between being neat and being safe, we lean toward the
-latter.  So the answer is that we think it's better to show the
-warning.
-
-Often people write `ON_CALL`s in the mock object's
-constructor or `SetUp()`, as the default behavior rarely changes from
-test to test.  Then in the test body they set the expectations, which
-are often different for each test.  Having an `ON_CALL` in the set-up
-part of a test doesn't mean that the calls are expected.  If there's
-no `EXPECT_CALL` and the method is called, it's possibly an error.  If
-we quietly let the call go through without notifying the user, bugs
-may creep in unnoticed.
-
-If, however, you are sure that the calls are OK, you can write
-
-```
-EXPECT_CALL(foo, Bar(_))
-    .WillRepeatedly(...);
-```
-
-instead of
-
-```
-ON_CALL(foo, Bar(_))
-    .WillByDefault(...);
-```
-
-This tells Google Mock that you do expect the calls and no warning should be
-printed.
-
-Also, you can control the verbosity using the `--gmock_verbose` flag.
-If you find the output too noisy when debugging, just choose a less
-verbose level.
-
-## How can I delete the mock function's argument in an action? ##
-
-If you find yourself needing to perform some action that's not
-supported by Google Mock directly, remember that you can define your own
-actions using
-[MakeAction()](CookBook.md#writing-new-actions) or
-[MakePolymorphicAction()](CookBook.md#writing_new_polymorphic_actions),
-or you can write a stub function and invoke it using
-[Invoke()](CookBook.md#using-functions_methods_functors).
-
-## MOCK\_METHODn()'s second argument looks funny.  Why don't you use the MOCK\_METHODn(Method, return\_type, arg\_1, ..., arg\_n) syntax? ##
-
-What?!  I think it's beautiful. :-)
-
-While which syntax looks more natural is a subjective matter to some
-extent, Google Mock's syntax was chosen for several practical advantages it
-has.
-
-Try to mock a function that takes a map as an argument:
-```
-virtual int GetSize(const map& m);
-```
-
-Using the proposed syntax, it would be:
-```
-MOCK_METHOD1(GetSize, int, const map& m);
-```
-
-Guess what?  You'll get a compiler error as the compiler thinks that
-`const map& m` are **two**, not one, arguments. To work
-around this you can use `typedef` to give the map type a name, but
-that gets in the way of your work.  Google Mock's syntax avoids this
-problem as the function's argument types are protected inside a pair
-of parentheses:
-```
-// This compiles fine.
-MOCK_METHOD1(GetSize, int(const map& m));
-```
-
-You still need a `typedef` if the return type contains an unprotected
-comma, but that's much rarer.
-
-Other advantages include:
-  1. `MOCK_METHOD1(Foo, int, bool)` can leave a reader wonder whether the method returns `int` or `bool`, while there won't be such confusion using Google Mock's syntax.
-  1. The way Google Mock describes a function type is nothing new, although many people may not be familiar with it.  The same syntax was used in C, and the `function` library in `tr1` uses this syntax extensively.  Since `tr1` will become a part of the new version of STL, we feel very comfortable to be consistent with it.
-  1. The function type syntax is also used in other parts of Google Mock's API (e.g. the action interface) in order to make the implementation tractable. A user needs to learn it anyway in order to utilize Google Mock's more advanced features.  We'd as well stick to the same syntax in `MOCK_METHOD*`!
-
-## My code calls a static/global function.  Can I mock it? ##
-
-You can, but you need to make some changes.
-
-In general, if you find yourself needing to mock a static function,
-it's a sign that your modules are too tightly coupled (and less
-flexible, less reusable, less testable, etc).  You are probably better
-off defining a small interface and call the function through that
-interface, which then can be easily mocked.  It's a bit of work
-initially, but usually pays for itself quickly.
-
-This Google Testing Blog
-[post](http://googletesting.blogspot.com/2008/06/defeat-static-cling.html)
-says it excellently.  Check it out.
-
-## My mock object needs to do complex stuff.  It's a lot of pain to specify the actions.  Google Mock sucks! ##
-
-I know it's not a question, but you get an answer for free any way. :-)
-
-With Google Mock, you can create mocks in C++ easily.  And people might be
-tempted to use them everywhere. Sometimes they work great, and
-sometimes you may find them, well, a pain to use. So, what's wrong in
-the latter case?
-
-When you write a test without using mocks, you exercise the code and
-assert that it returns the correct value or that the system is in an
-expected state.  This is sometimes called "state-based testing".
-
-Mocks are great for what some call "interaction-based" testing:
-instead of checking the system state at the very end, mock objects
-verify that they are invoked the right way and report an error as soon
-as it arises, giving you a handle on the precise context in which the
-error was triggered.  This is often more effective and economical to
-do than state-based testing.
-
-If you are doing state-based testing and using a test double just to
-simulate the real object, you are probably better off using a fake.
-Using a mock in this case causes pain, as it's not a strong point for
-mocks to perform complex actions.  If you experience this and think
-that mocks suck, you are just not using the right tool for your
-problem. Or, you might be trying to solve the wrong problem. :-)
-
-## I got a warning "Uninteresting function call encountered - default action taken.."  Should I panic? ##
-
-By all means, NO!  It's just an FYI.
-
-What it means is that you have a mock function, you haven't set any
-expectations on it (by Google Mock's rule this means that you are not
-interested in calls to this function and therefore it can be called
-any number of times), and it is called.  That's OK - you didn't say
-it's not OK to call the function!
-
-What if you actually meant to disallow this function to be called, but
-forgot to write `EXPECT_CALL(foo, Bar()).Times(0)`?  While
-one can argue that it's the user's fault, Google Mock tries to be nice and
-prints you a note.
-
-So, when you see the message and believe that there shouldn't be any
-uninteresting calls, you should investigate what's going on.  To make
-your life easier, Google Mock prints the function name and arguments
-when an uninteresting call is encountered.
-
-## I want to define a custom action.  Should I use Invoke() or implement the action interface? ##
-
-Either way is fine - you want to choose the one that's more convenient
-for your circumstance.
-
-Usually, if your action is for a particular function type, defining it
-using `Invoke()` should be easier; if your action can be used in
-functions of different types (e.g. if you are defining
-`Return(value)`), `MakePolymorphicAction()` is
-easiest.  Sometimes you want precise control on what types of
-functions the action can be used in, and implementing
-`ActionInterface` is the way to go here. See the implementation of
-`Return()` in `include/gmock/gmock-actions.h` for an example.
-
-## I'm using the set-argument-pointee action, and the compiler complains about "conflicting return type specified".  What does it mean? ##
-
-You got this error as Google Mock has no idea what value it should return
-when the mock method is called.  `SetArgPointee()` says what the
-side effect is, but doesn't say what the return value should be.  You
-need `DoAll()` to chain a `SetArgPointee()` with a `Return()`.
-
-See this [recipe](CookBook.md#mocking_side_effects) for more details and an example.
-
-
-## My question is not in your FAQ! ##
-
-If you cannot find the answer to your question in this FAQ, there are
-some other resources you can use:
-
-  1. read other [documentation](Documentation.md),
-  1. search the mailing list [archive](http://groups.google.com/group/googlemock/topics),
-  1. ask it on [googlemock@googlegroups.com](mailto:googlemock@googlegroups.com) and someone will answer it (to prevent spam, we require you to join the [discussion group](http://groups.google.com/group/googlemock) before you can post.).
-
-Please note that creating an issue in the
-[issue tracker](https://github.com/google/googletest/issues) is _not_
-a good way to get your answer, as it is monitored infrequently by a
-very small number of people.
-
-When asking a question, it's helpful to provide as much of the
-following information as possible (people cannot help you if there's
-not enough information in your question):
-
-  * the version (or the revision number if you check out from SVN directly) of Google Mock you use (Google Mock is under active development, so it's possible that your problem has been solved in a later version),
-  * your operating system,
-  * the name and version of your compiler,
-  * the complete command line flags you give to your compiler,
-  * the complete compiler error messages (if the question is about compilation),
-  * the _actual_ code (ideally, a minimal but complete program) that has the problem you encounter.
diff --git a/thirdparty/GTest/googletest/googlemock/docs/KnownIssues.md b/thirdparty/GTest/googletest/googlemock/docs/KnownIssues.md
deleted file mode 100644
index adadf5144b..0000000000
--- a/thirdparty/GTest/googletest/googlemock/docs/KnownIssues.md
+++ /dev/null
@@ -1,19 +0,0 @@
-As any non-trivial software system, Google Mock has some known limitations and problems.  We are working on improving it, and welcome your help!  The follow is a list of issues we know about.
-
-
-
-## README contains outdated information on Google Mock's compatibility with other testing frameworks ##
-
-The `README` file in release 1.1.0 still says that Google Mock only works with Google Test.  Actually, you can configure Google Mock to work with any testing framework you choose.
-
-## Tests failing on machines using Power PC CPUs (e.g. some Macs) ##
-
-`gmock_output_test` and `gmock-printers_test` are known to fail with Power PC CPUs.  This is due to portability issues with these tests, and is not an indication of problems in Google Mock itself.  You can safely ignore them.
-
-## Failed to resolve libgtest.so.0 in tests when built against installed Google Test ##
-
-This only applies if you manually built and installed Google Test, and then built a Google Mock against it (either explicitly, or because gtest-config was in your path post-install). In this situation, Libtool has a known issue with certain systems' ldconfig setup:
-
-http://article.gmane.org/gmane.comp.sysutils.automake.general/9025
-
-This requires a manual run of "sudo ldconfig" after the "sudo make install" for Google Test before any binaries which link against it can be executed. This isn't a bug in our install, but we should at least have documented it or hacked a work-around into our install. We should have one of these solutions in our next release.
\ No newline at end of file
diff --git a/thirdparty/GTest/googletest/googlemock/docs/cheat_sheet.md b/thirdparty/GTest/googletest/googlemock/docs/cheat_sheet.md
new file mode 100644
index 0000000000..e6cffd0cfa
--- /dev/null
+++ b/thirdparty/GTest/googletest/googlemock/docs/cheat_sheet.md
@@ -0,0 +1,784 @@
+# gMock Cheat Sheet
+
+
+
+
+
+
+
+## Defining a Mock Class
+
+### Mocking a Normal Class {#MockClass}
+
+Given
+
+```cpp
+class Foo {
+  ...
+  virtual ~Foo();
+  virtual int GetSize() const = 0;
+  virtual string Describe(const char* name) = 0;
+  virtual string Describe(int type) = 0;
+  virtual bool Process(Bar elem, int count) = 0;
+};
+```
+
+(note that `~Foo()` **must** be virtual) we can define its mock as
+
+```cpp
+#include "gmock/gmock.h"
+
+class MockFoo : public Foo {
+  ...
+  MOCK_METHOD(int, GetSize, (), (const, override));
+  MOCK_METHOD(string, Describe, (const char* name), (override));
+  MOCK_METHOD(string, Describe, (int type), (override));
+  MOCK_METHOD(bool, Process, (Bar elem, int count), (override));
+};
+```
+
+To create a "nice" mock, which ignores all uninteresting calls, a "naggy" mock,
+which warns on all uninteresting calls, or a "strict" mock, which treats them as
+failures:
+
+```cpp
+using ::testing::NiceMock;
+using ::testing::NaggyMock;
+using ::testing::StrictMock;
+
+NiceMock nice_foo;      // The type is a subclass of MockFoo.
+NaggyMock naggy_foo;    // The type is a subclass of MockFoo.
+StrictMock strict_foo;  // The type is a subclass of MockFoo.
+```
+
+**Note:** A mock object is currently naggy by default. We may make it nice by
+default in the future.
+
+### Mocking a Class Template {#MockTemplate}
+
+Class templates can be mocked just like any class.
+
+To mock
+
+```cpp
+template 
+class StackInterface {
+  ...
+  virtual ~StackInterface();
+  virtual int GetSize() const = 0;
+  virtual void Push(const Elem& x) = 0;
+};
+```
+
+(note that all member functions that are mocked, including `~StackInterface()`
+**must** be virtual).
+
+```cpp
+template 
+class MockStack : public StackInterface {
+  ...
+  MOCK_METHOD(int, GetSize, (), (const, override));
+  MOCK_METHOD(void, Push, (const Elem& x), (override));
+};
+```
+
+### Specifying Calling Conventions for Mock Functions
+
+If your mock function doesn't use the default calling convention, you can
+specify it by adding `Calltype(convention)` to `MOCK_METHOD`'s 4th parameter.
+For example,
+
+```cpp
+  MOCK_METHOD(bool, Foo, (int n), (Calltype(STDMETHODCALLTYPE)));
+  MOCK_METHOD(int, Bar, (double x, double y),
+              (const, Calltype(STDMETHODCALLTYPE)));
+```
+
+where `STDMETHODCALLTYPE` is defined by `` on Windows.
+
+## Using Mocks in Tests {#UsingMocks}
+
+The typical work flow is:
+
+1.  Import the gMock names you need to use. All gMock symbols are in the
+    `testing` namespace unless they are macros or otherwise noted.
+2.  Create the mock objects.
+3.  Optionally, set the default actions of the mock objects.
+4.  Set your expectations on the mock objects (How will they be called? What
+    will they do?).
+5.  Exercise code that uses the mock objects; if necessary, check the result
+    using googletest assertions.
+6.  When a mock object is destructed, gMock automatically verifies that all
+    expectations on it have been satisfied.
+
+Here's an example:
+
+```cpp
+using ::testing::Return;                          // #1
+
+TEST(BarTest, DoesThis) {
+  MockFoo foo;                                    // #2
+
+  ON_CALL(foo, GetSize())                         // #3
+      .WillByDefault(Return(1));
+  // ... other default actions ...
+
+  EXPECT_CALL(foo, Describe(5))                   // #4
+      .Times(3)
+      .WillRepeatedly(Return("Category 5"));
+  // ... other expectations ...
+
+  EXPECT_EQ("good", MyProductionFunction(&foo));  // #5
+}                                                 // #6
+```
+
+## Setting Default Actions {#OnCall}
+
+gMock has a **built-in default action** for any function that returns `void`,
+`bool`, a numeric value, or a pointer. In C++11, it will additionally returns
+the default-constructed value, if one exists for the given type.
+
+To customize the default action for functions with return type *`T`*:
+
+```cpp
+using ::testing::DefaultValue;
+
+// Sets the default value to be returned. T must be CopyConstructible.
+DefaultValue::Set(value);
+// Sets a factory. Will be invoked on demand. T must be MoveConstructible.
+//  T MakeT();
+DefaultValue::SetFactory(&MakeT);
+// ... use the mocks ...
+// Resets the default value.
+DefaultValue::Clear();
+```
+
+Example usage:
+
+```cpp
+  // Sets the default action for return type std::unique_ptr to
+  // creating a new Buzz every time.
+  DefaultValue>::SetFactory(
+      [] { return MakeUnique(AccessLevel::kInternal); });
+
+  // When this fires, the default action of MakeBuzz() will run, which
+  // will return a new Buzz object.
+  EXPECT_CALL(mock_buzzer_, MakeBuzz("hello")).Times(AnyNumber());
+
+  auto buzz1 = mock_buzzer_.MakeBuzz("hello");
+  auto buzz2 = mock_buzzer_.MakeBuzz("hello");
+  EXPECT_NE(nullptr, buzz1);
+  EXPECT_NE(nullptr, buzz2);
+  EXPECT_NE(buzz1, buzz2);
+
+  // Resets the default action for return type std::unique_ptr,
+  // to avoid interfere with other tests.
+  DefaultValue>::Clear();
+```
+
+To customize the default action for a particular method of a specific mock
+object, use `ON_CALL()`. `ON_CALL()` has a similar syntax to `EXPECT_CALL()`,
+but it is used for setting default behaviors (when you do not require that the
+mock method is called). See [here](cook_book.md#UseOnCall) for a more detailed
+discussion.
+
+```cpp
+ON_CALL(mock-object, method(matchers))
+    .With(multi-argument-matcher)   ?
+    .WillByDefault(action);
+```
+
+## Setting Expectations {#ExpectCall}
+
+`EXPECT_CALL()` sets **expectations** on a mock method (How will it be called?
+What will it do?):
+
+```cpp
+EXPECT_CALL(mock-object, method (matchers)?)
+     .With(multi-argument-matcher)  ?
+     .Times(cardinality)            ?
+     .InSequence(sequences)         *
+     .After(expectations)           *
+     .WillOnce(action)              *
+     .WillRepeatedly(action)        ?
+     .RetiresOnSaturation();        ?
+```
+
+For each item above, `?` means it can be used at most once, while `*` means it
+can be used any number of times.
+
+In order to pass, `EXPECT_CALL` must be used before the calls are actually made.
+
+The `(matchers)` is a comma-separated list of matchers that correspond to each
+of the arguments of `method`, and sets the expectation only for calls of
+`method` that matches all of the matchers.
+
+If `(matchers)` is omitted, the expectation is the same as if the matchers were
+set to anything matchers (for example, `(_, _, _, _)` for a four-arg method).
+
+If `Times()` is omitted, the cardinality is assumed to be:
+
+*   `Times(1)` when there is neither `WillOnce()` nor `WillRepeatedly()`;
+*   `Times(n)` when there are `n` `WillOnce()`s but no `WillRepeatedly()`, where
+    `n` >= 1; or
+*   `Times(AtLeast(n))` when there are `n` `WillOnce()`s and a
+    `WillRepeatedly()`, where `n` >= 0.
+
+A method with no `EXPECT_CALL()` is free to be invoked *any number of times*,
+and the default action will be taken each time.
+
+## Matchers {#MatcherList}
+
+
+
+A **matcher** matches a *single* argument. You can use it inside `ON_CALL()` or
+`EXPECT_CALL()`, or use it to validate a value directly using two macros:
+
+
+| Macro                                | Description                           |
+| :----------------------------------- | :------------------------------------ |
+| `EXPECT_THAT(actual_value, matcher)` | Asserts that `actual_value` matches `matcher`. |
+| `ASSERT_THAT(actual_value, matcher)` | The same as `EXPECT_THAT(actual_value, matcher)`, except that it generates a **fatal** failure. |
+
+
+**Note:** Although equality matching via `EXPECT_THAT(actual_value,
+expected_value)` is supported, prefer to make the comparison explicit via
+`EXPECT_THAT(actual_value, Eq(expected_value))` or `EXPECT_EQ(actual_value,
+expected_value)`.
+
+Built-in matchers (where `argument` is the function argument, e.g.
+`actual_value` in the example above, or when used in the context of
+`EXPECT_CALL(mock_object, method(matchers))`, the arguments of `method`) are
+divided into several categories:
+
+### Wildcard
+
+Matcher                     | Description
+:-------------------------- | :-----------------------------------------------
+`_`                         | `argument` can be any value of the correct type.
+`A()` or `An()` | `argument` can be any value of type `type`.
+
+### Generic Comparison
+
+
+| Matcher                | Description                                         |
+| :--------------------- | :-------------------------------------------------- |
+| `Eq(value)` or `value` | `argument == value`                                 |
+| `Ge(value)`            | `argument >= value`                                 |
+| `Gt(value)`            | `argument > value`                                  |
+| `Le(value)`            | `argument <= value`                                 |
+| `Lt(value)`            | `argument < value`                                  |
+| `Ne(value)`            | `argument != value`                                 |
+| `IsFalse()`            | `argument` evaluates to `false` in a Boolean context. |
+| `IsTrue()`             | `argument` evaluates to `true` in a Boolean context. |
+| `IsNull()`             | `argument` is a `NULL` pointer (raw or smart).      |
+| `NotNull()`            | `argument` is a non-null pointer (raw or smart).    |
+| `Optional(m)`          | `argument` is `optional<>` that contains a value matching `m`. (For testing whether an `optional<>` is set, check for equality with `nullopt`. You may need to use `Eq(nullopt)` if the inner type doesn't have `==`.)|
+| `VariantWith(m)`    | `argument` is `variant<>` that holds the alternative of type T with a value matching `m`. |
+| `Ref(variable)`        | `argument` is a reference to `variable`.            |
+| `TypedEq(value)` | `argument` has type `type` and is equal to `value`. You may need to use this instead of `Eq(value)` when the mock function is overloaded. |
+
+
+Except `Ref()`, these matchers make a *copy* of `value` in case it's modified or
+destructed later. If the compiler complains that `value` doesn't have a public
+copy constructor, try wrap it in `std::ref()`, e.g.
+`Eq(std::ref(non_copyable_value))`. If you do that, make sure
+`non_copyable_value` is not changed afterwards, or the meaning of your matcher
+will be changed.
+
+`IsTrue` and `IsFalse` are useful when you need to use a matcher, or for types
+that can be explicitly converted to Boolean, but are not implicitly converted to
+Boolean. In other cases, you can use the basic
+[`EXPECT_TRUE` and `EXPECT_FALSE`](../../googletest/docs/primer#basic-assertions)
+assertions.
+
+### Floating-Point Matchers {#FpMatchers}
+
+
+| Matcher                          | Description                        |
+| :------------------------------- | :--------------------------------- |
+| `DoubleEq(a_double)`             | `argument` is a `double` value approximately equal to `a_double`, treating two NaNs as unequal. |
+| `FloatEq(a_float)`               | `argument` is a `float` value approximately equal to `a_float`, treating two NaNs as unequal. |
+| `NanSensitiveDoubleEq(a_double)` | `argument` is a `double` value approximately equal to `a_double`, treating two NaNs as equal. |
+| `NanSensitiveFloatEq(a_float)`   | `argument` is a `float` value approximately equal to `a_float`, treating two NaNs as equal. |
+| `IsNan()`   | `argument` is any floating-point type with a NaN value. |
+
+
+The above matchers use ULP-based comparison (the same as used in googletest).
+They automatically pick a reasonable error bound based on the absolute value of
+the expected value. `DoubleEq()` and `FloatEq()` conform to the IEEE standard,
+which requires comparing two NaNs for equality to return false. The
+`NanSensitive*` version instead treats two NaNs as equal, which is often what a
+user wants.
+
+
+| Matcher                                           | Description              |
+| :------------------------------------------------ | :----------------------- |
+| `DoubleNear(a_double, max_abs_error)`             | `argument` is a `double` value close to `a_double` (absolute error <= `max_abs_error`), treating two NaNs as unequal. |
+| `FloatNear(a_float, max_abs_error)`               | `argument` is a `float` value close to `a_float` (absolute error <= `max_abs_error`), treating two NaNs as unequal. |
+| `NanSensitiveDoubleNear(a_double, max_abs_error)` | `argument` is a `double` value close to `a_double` (absolute error <= `max_abs_error`), treating two NaNs as equal. |
+| `NanSensitiveFloatNear(a_float, max_abs_error)`   | `argument` is a `float` value close to `a_float` (absolute error <= `max_abs_error`), treating two NaNs as equal. |
+
+
+### String Matchers
+
+The `argument` can be either a C string or a C++ string object:
+
+
+| Matcher                 | Description                                        |
+| :---------------------- | :------------------------------------------------- |
+| `ContainsRegex(string)` | `argument` matches the given regular expression.   |
+| `EndsWith(suffix)`      | `argument` ends with string `suffix`.              |
+| `HasSubstr(string)`     | `argument` contains `string` as a sub-string.      |
+| `MatchesRegex(string)`  | `argument` matches the given regular expression with the match starting at the first character and ending at the last character. |
+| `StartsWith(prefix)`    | `argument` starts with string `prefix`.            |
+| `StrCaseEq(string)`     | `argument` is equal to `string`, ignoring case.    |
+| `StrCaseNe(string)`     | `argument` is not equal to `string`, ignoring case. |
+| `StrEq(string)`         | `argument` is equal to `string`.                   |
+| `StrNe(string)`         | `argument` is not equal to `string`.               |
+
+
+`ContainsRegex()` and `MatchesRegex()` take ownership of the `RE` object. They
+use the regular expression syntax defined
+[here](../../googletest/docs/advanced.md#regular-expression-syntax). All of
+these matchers, except `ContainsRegex()` and `MatchesRegex()` work for wide
+strings as well.
+
+### Container Matchers
+
+Most STL-style containers support `==`, so you can use `Eq(expected_container)`
+or simply `expected_container` to match a container exactly. If you want to
+write the elements in-line, match them more flexibly, or get more informative
+messages, you can use:
+
+
+| Matcher                                   | Description                      |
+| :---------------------------------------- | :------------------------------- |
+| `BeginEndDistanceIs(m)` | `argument` is a container whose `begin()` and `end()` iterators are separated by a number of increments matching `m`. E.g. `BeginEndDistanceIs(2)` or `BeginEndDistanceIs(Lt(2))`. For containers that define a `size()` method, `SizeIs(m)` may be more efficient. |
+| `ContainerEq(container)` | The same as `Eq(container)` except that the failure message also includes which elements are in one container but not the other. |
+| `Contains(e)` | `argument` contains an element that matches `e`, which can be either a value or a matcher. |
+| `Each(e)` | `argument` is a container where *every* element matches `e`, which can be either a value or a matcher. |
+| `ElementsAre(e0, e1, ..., en)` | `argument` has `n + 1` elements, where the *i*-th element matches `ei`, which can be a value or a matcher. |
+| `ElementsAreArray({e0, e1, ..., en})`, `ElementsAreArray(a_container)`, `ElementsAreArray(begin, end)`, `ElementsAreArray(array)`, or `ElementsAreArray(array, count)` | The same as `ElementsAre()` except that the expected element values/matchers come from an initializer list, STL-style container, iterator range, or C-style array. |
+| `IsEmpty()` | `argument` is an empty container (`container.empty()`). |
+| `IsSubsetOf({e0, e1, ..., en})`, `IsSubsetOf(a_container)`, `IsSubsetOf(begin, end)`, `IsSubsetOf(array)`, or `IsSubsetOf(array, count)` | `argument` matches `UnorderedElementsAre(x0, x1, ..., xk)` for some subset `{x0, x1, ..., xk}` of the expected matchers. |
+| `IsSupersetOf({e0, e1, ..., en})`, `IsSupersetOf(a_container)`, `IsSupersetOf(begin, end)`, `IsSupersetOf(array)`, or `IsSupersetOf(array, count)` | Some subset of `argument` matches `UnorderedElementsAre(`expected matchers`)`. |
+| `Pointwise(m, container)`, `Pointwise(m, {e0, e1, ..., en})` | `argument` contains the same number of elements as in `container`, and for all i, (the i-th element in `argument`, the i-th element in `container`) match `m`, which is a matcher on 2-tuples. E.g. `Pointwise(Le(), upper_bounds)` verifies that each element in `argument` doesn't exceed the corresponding element in `upper_bounds`. See more detail below. |
+| `SizeIs(m)` | `argument` is a container whose size matches `m`. E.g. `SizeIs(2)` or `SizeIs(Lt(2))`. |
+| `UnorderedElementsAre(e0, e1, ..., en)` | `argument` has `n + 1` elements, and under *some* permutation of the elements, each element matches an `ei` (for a different `i`), which can be a value or a matcher. |
+| `UnorderedElementsAreArray({e0, e1, ..., en})`, `UnorderedElementsAreArray(a_container)`, `UnorderedElementsAreArray(begin, end)`, `UnorderedElementsAreArray(array)`, or `UnorderedElementsAreArray(array, count)` | The same as `UnorderedElementsAre()` except that the expected element values/matchers come from an initializer list, STL-style container, iterator range, or C-style array. |
+| `UnorderedPointwise(m, container)`, `UnorderedPointwise(m, {e0, e1, ..., en})` | Like `Pointwise(m, container)`, but ignores the order of elements. |
+| `WhenSorted(m)` | When `argument` is sorted using the `<` operator, it matches container matcher `m`. E.g. `WhenSorted(ElementsAre(1, 2, 3))` verifies that `argument` contains elements 1, 2, and 3, ignoring order. |
+| `WhenSortedBy(comparator, m)` | The same as `WhenSorted(m)`, except that the given comparator instead of `<` is used to sort `argument`. E.g. `WhenSortedBy(std::greater(), ElementsAre(3, 2, 1))`. |
+
+
+**Notes:**
+
+*   These matchers can also match:
+    1.  a native array passed by reference (e.g. in `Foo(const int (&a)[5])`),
+        and
+    2.  an array passed as a pointer and a count (e.g. in `Bar(const T* buffer,
+        int len)` -- see [Multi-argument Matchers](#MultiArgMatchers)).
+*   The array being matched may be multi-dimensional (i.e. its elements can be
+    arrays).
+*   `m` in `Pointwise(m, ...)` should be a matcher for `::std::tuple`
+    where `T` and `U` are the element type of the actual container and the
+    expected container, respectively. For example, to compare two `Foo`
+    containers where `Foo` doesn't support `operator==`, one might write:
+
+    ```cpp
+    using ::std::get;
+    MATCHER(FooEq, "") {
+      return std::get<0>(arg).Equals(std::get<1>(arg));
+    }
+    ...
+    EXPECT_THAT(actual_foos, Pointwise(FooEq(), expected_foos));
+    ```
+
+### Member Matchers
+
+
+| Matcher                         | Description                                |
+| :------------------------------ | :----------------------------------------- |
+| `Field(&class::field, m)`       | `argument.field` (or `argument->field` when `argument` is a plain pointer) matches matcher `m`, where `argument` is an object of type _class_. |
+| `Key(e)`                        | `argument.first` matches `e`, which can be either a value or a matcher. E.g. `Contains(Key(Le(5)))` can verify that a `map` contains a key `<= 5`. |
+| `Pair(m1, m2)`                  | `argument` is an `std::pair` whose `first` field matches `m1` and `second` field matches `m2`. |
+| `FieldsAre(m...)`                   | `argument` is a compatible object where each field matches piecewise with `m...`. A compatible object is any that supports the `std::tuple_size`+`get(obj)` protocol. In C++17 and up this also supports types compatible with structured bindings, like aggregates. |
+| `Property(&class::property, m)` | `argument.property()` (or `argument->property()` when `argument` is a plain pointer) matches matcher `m`, where `argument` is an object of type _class_. |
+
+
+### Matching the Result of a Function, Functor, or Callback
+
+
+| Matcher          | Description                                       |
+| :--------------- | :------------------------------------------------ |
+| `ResultOf(f, m)` | `f(argument)` matches matcher `m`, where `f` is a function or functor. |
+
+
+### Pointer Matchers
+
+
+| Matcher                   | Description                                     |
+| :------------------------ | :---------------------------------------------- |
+| `Pointee(m)`              | `argument` (either a smart pointer or a raw pointer) points to a value that matches matcher `m`. |
+| `WhenDynamicCastTo(m)` | when `argument` is passed through `dynamic_cast()`, it matches matcher `m`. |
+
+
+
+
+
+
+### Multi-argument Matchers {#MultiArgMatchers}
+
+Technically, all matchers match a *single* value. A "multi-argument" matcher is
+just one that matches a *tuple*. The following matchers can be used to match a
+tuple `(x, y)`:
+
+Matcher | Description
+:------ | :----------
+`Eq()`  | `x == y`
+`Ge()`  | `x >= y`
+`Gt()`  | `x > y`
+`Le()`  | `x <= y`
+`Lt()`  | `x < y`
+`Ne()`  | `x != y`
+
+You can use the following selectors to pick a subset of the arguments (or
+reorder them) to participate in the matching:
+
+
+| Matcher                    | Description                                     |
+| :------------------------- | :---------------------------------------------- |
+| `AllArgs(m)`               | Equivalent to `m`. Useful as syntactic sugar in `.With(AllArgs(m))`. |
+| `Args(m)` | The tuple of the `k` selected (using 0-based indices) arguments matches `m`, e.g. `Args<1, 2>(Eq())`. |
+
+
+### Composite Matchers
+
+You can make a matcher from one or more other matchers:
+
+
+| Matcher                          | Description                             |
+| :------------------------------- | :-------------------------------------- |
+| `AllOf(m1, m2, ..., mn)` | `argument` matches all of the matchers `m1` to `mn`. |
+| `AllOfArray({m0, m1, ..., mn})`, `AllOfArray(a_container)`, `AllOfArray(begin, end)`, `AllOfArray(array)`, or `AllOfArray(array, count)` | The same as `AllOf()` except that the matchers come from an initializer list, STL-style container, iterator range, or C-style array. |
+| `AnyOf(m1, m2, ..., mn)` | `argument` matches at least one of the matchers `m1` to `mn`. |
+| `AnyOfArray({m0, m1, ..., mn})`, `AnyOfArray(a_container)`, `AnyOfArray(begin, end)`, `AnyOfArray(array)`, or `AnyOfArray(array, count)` | The same as `AnyOf()` except that the matchers come from an initializer list, STL-style container, iterator range, or C-style array. |
+| `Not(m)` | `argument` doesn't match matcher `m`. |
+
+
+
+
+### Adapters for Matchers
+
+
+| Matcher                 | Description                           |
+| :---------------------- | :------------------------------------ |
+| `MatcherCast(m)`     | casts matcher `m` to type `Matcher`. |
+| `SafeMatcherCast(m)` | [safely casts](cook_book.md#casting-matchers) matcher `m` to type `Matcher`. |
+| `Truly(predicate)`      | `predicate(argument)` returns something considered by C++ to be true, where `predicate` is a function or functor. |
+
+
+`AddressSatisfies(callback)` and `Truly(callback)` take ownership of `callback`,
+which must be a permanent callback.
+
+### Using Matchers as Predicates {#MatchersAsPredicatesCheat}
+
+
+| Matcher                       | Description                                 |
+| :---------------------------- | :------------------------------------------ |
+| `Matches(m)(value)` | evaluates to `true` if `value` matches `m`. You can use `Matches(m)` alone as a unary functor. |
+| `ExplainMatchResult(m, value, result_listener)` | evaluates to `true` if `value` matches `m`, explaining the result to `result_listener`. |
+| `Value(value, m)` | evaluates to `true` if `value` matches `m`. |
+
+
+### Defining Matchers
+
+
+| Matcher                              | Description                           |
+| :----------------------------------- | :------------------------------------ |
+| `MATCHER(IsEven, "") { return (arg % 2) == 0; }` | Defines a matcher `IsEven()` to match an even number. |
+| `MATCHER_P(IsDivisibleBy, n, "") { *result_listener << "where the remainder is " << (arg % n); return (arg % n) == 0; }` | Defines a matcher `IsDivisibleBy(n)` to match a number divisible by `n`. |
+| `MATCHER_P2(IsBetween, a, b, absl::StrCat(negation ? "isn't" : "is", " between ", PrintToString(a), " and ", PrintToString(b))) { return a <= arg && arg <= b; }` | Defines a matcher `IsBetween(a, b)` to match a value in the range [`a`, `b`]. |
+
+
+**Notes:**
+
+1.  The `MATCHER*` macros cannot be used inside a function or class.
+2.  The matcher body must be *purely functional* (i.e. it cannot have any side
+    effect, and the result must not depend on anything other than the value
+    being matched and the matcher parameters).
+3.  You can use `PrintToString(x)` to convert a value `x` of any type to a
+    string.
+
+## Actions {#ActionList}
+
+**Actions** specify what a mock function should do when invoked.
+
+### Returning a Value
+
+
+|                                   |                                               |
+| :-------------------------------- | :-------------------------------------------- |
+| `Return()`                        | Return from a `void` mock function.           |
+| `Return(value)`                   | Return `value`. If the type of `value` is     different to the mock function's return type, `value` is converted to the latter type at the time the expectation is set, not when the action is executed. |
+| `ReturnArg()`                  | Return the `N`-th (0-based) argument.         |
+| `ReturnNew(a1, ..., ak)`       | Return `new T(a1, ..., ak)`; a different      object is created each time. |
+| `ReturnNull()`                    | Return a null pointer.                        |
+| `ReturnPointee(ptr)`              | Return the value pointed to by `ptr`.         |
+| `ReturnRef(variable)`             | Return a reference to `variable`.             |
+| `ReturnRefOfCopy(value)`          | Return a reference to a copy of `value`; the  copy lives as long as the action. |
+| `ReturnRoundRobin({a1, ..., ak})` | Each call will return the next `ai` in the list, starting at the beginning when the end of the list is reached. |
+
+
+### Side Effects
+
+
+|                                    |                                         |
+| :--------------------------------- | :-------------------------------------- |
+| `Assign(&variable, value)` | Assign `value` to variable. |
+| `DeleteArg()` | Delete the `N`-th (0-based) argument, which must be a pointer. |
+| `SaveArg(pointer)` | Save the `N`-th (0-based) argument to `*pointer`. |
+| `SaveArgPointee(pointer)` | Save the value pointed to by the `N`-th (0-based) argument to `*pointer`. |
+| `SetArgReferee(value)` | Assign value to the variable referenced by the `N`-th (0-based) argument. |
+| `SetArgPointee(value)` | Assign `value` to the variable pointed by the `N`-th (0-based) argument. |
+| `SetArgumentPointee(value)` | Same as `SetArgPointee(value)`. Deprecated. Will be removed in v1.7.0. |
+| `SetArrayArgument(first, last)` | Copies the elements in source range [`first`, `last`) to the array pointed to by the `N`-th (0-based) argument, which can be either a pointer or an iterator. The action does not take ownership of the elements in the source range. |
+| `SetErrnoAndReturn(error, value)` | Set `errno` to `error` and return `value`. |
+| `Throw(exception)` | Throws the given exception, which can be any copyable value. Available since v1.1.0. |
+
+
+### Using a Function, Functor, or Lambda as an Action
+
+In the following, by "callable" we mean a free function, `std::function`,
+functor, or lambda.
+
+
+|                                     |                                        |
+| :---------------------------------- | :------------------------------------- |
+| `f` | Invoke f with the arguments passed to the mock function, where f is a callable. |
+| `Invoke(f)` | Invoke `f` with the arguments passed to the mock function, where `f` can be a global/static function or a functor. |
+| `Invoke(object_pointer, &class::method)` | Invoke the method on the object with the arguments passed to the mock function. |
+| `InvokeWithoutArgs(f)` | Invoke `f`, which can be a global/static function or a functor. `f` must take no arguments. |
+| `InvokeWithoutArgs(object_pointer, &class::method)` | Invoke the method on the object, which takes no arguments. |
+| `InvokeArgument(arg1, arg2, ..., argk)` | Invoke the mock function's `N`-th (0-based) argument, which must be a function or a functor, with the `k` arguments. |
+
+
+The return value of the invoked function is used as the return value of the
+action.
+
+When defining a callable to be used with `Invoke*()`, you can declare any unused
+parameters as `Unused`:
+
+```cpp
+using ::testing::Invoke;
+double Distance(Unused, double x, double y) { return sqrt(x*x + y*y); }
+...
+EXPECT_CALL(mock, Foo("Hi", _, _)).WillOnce(Invoke(Distance));
+```
+
+`Invoke(callback)` and `InvokeWithoutArgs(callback)` take ownership of
+`callback`, which must be permanent. The type of `callback` must be a base
+callback type instead of a derived one, e.g.
+
+```cpp
+  BlockingClosure* done = new BlockingClosure;
+  ... Invoke(done) ...;  // This won't compile!
+
+  Closure* done2 = new BlockingClosure;
+  ... Invoke(done2) ...;  // This works.
+```
+
+In `InvokeArgument(...)`, if an argument needs to be passed by reference,
+wrap it inside `std::ref()`. For example,
+
+```cpp
+using ::testing::InvokeArgument;
+...
+InvokeArgument<2>(5, string("Hi"), std::ref(foo))
+```
+
+calls the mock function's #2 argument, passing to it `5` and `string("Hi")` by
+value, and `foo` by reference.
+
+### Default Action
+
+
+| Matcher       | Description                                            |
+| :------------ | :----------------------------------------------------- |
+| `DoDefault()` | Do the default action (specified by `ON_CALL()` or the built-in one). |
+
+
+**Note:** due to technical reasons, `DoDefault()` cannot be used inside a
+composite action - trying to do so will result in a run-time error.
+
+
+
+### Composite Actions
+
+
+|                                |                                             |
+| :----------------------------- | :------------------------------------------ |
+| `DoAll(a1, a2, ..., an)`       | Do all actions `a1` to `an` and return the result of `an` in each invocation. The first `n - 1` sub-actions must return void and will receive a  readonly view of the arguments. |
+| `IgnoreResult(a)`              | Perform action `a` and ignore its result. `a` must not return void. |
+| `WithArg(a)`                | Pass the `N`-th (0-based) argument of the mock function to action `a` and perform it. |
+| `WithArgs(a)` | Pass the selected (0-based) arguments of the mock function to action `a` and perform it. |
+| `WithoutArgs(a)`               | Perform action `a` without any arguments. |
+
+
+### Defining Actions
+
+
+|                                    |                                         |
+| :--------------------------------- | :-------------------------------------- |
+| `ACTION(Sum) { return arg0 + arg1; }` | Defines an action `Sum()` to return the sum of the mock function's argument #0 and #1. |
+| `ACTION_P(Plus, n) { return arg0 + n; }` | Defines an action `Plus(n)` to return the sum of the mock function's argument #0 and `n`. |
+| `ACTION_Pk(Foo, p1, ..., pk) { statements; }` | Defines a parameterized action `Foo(p1, ..., pk)` to execute the given `statements`. |
+
+
+The `ACTION*` macros cannot be used inside a function or class.
+
+## Cardinalities {#CardinalityList}
+
+These are used in `Times()` to specify how many times a mock function will be
+called:
+
+
+|                   |                                                        |
+| :---------------- | :----------------------------------------------------- |
+| `AnyNumber()`     | The function can be called any number of times.        |
+| `AtLeast(n)`      | The call is expected at least `n` times.               |
+| `AtMost(n)`       | The call is expected at most `n` times.                |
+| `Between(m, n)`   | The call is expected between `m` and `n` (inclusive) times. |
+| `Exactly(n) or n` | The call is expected exactly `n` times. In particular, the call should never happen when `n` is 0. |
+
+
+## Expectation Order
+
+By default, the expectations can be matched in *any* order. If some or all
+expectations must be matched in a given order, there are two ways to specify it.
+They can be used either independently or together.
+
+### The After Clause {#AfterClause}
+
+```cpp
+using ::testing::Expectation;
+...
+Expectation init_x = EXPECT_CALL(foo, InitX());
+Expectation init_y = EXPECT_CALL(foo, InitY());
+EXPECT_CALL(foo, Bar())
+     .After(init_x, init_y);
+```
+
+says that `Bar()` can be called only after both `InitX()` and `InitY()` have
+been called.
+
+If you don't know how many pre-requisites an expectation has when you write it,
+you can use an `ExpectationSet` to collect them:
+
+```cpp
+using ::testing::ExpectationSet;
+...
+ExpectationSet all_inits;
+for (int i = 0; i < element_count; i++) {
+  all_inits += EXPECT_CALL(foo, InitElement(i));
+}
+EXPECT_CALL(foo, Bar())
+     .After(all_inits);
+```
+
+says that `Bar()` can be called only after all elements have been initialized
+(but we don't care about which elements get initialized before the others).
+
+Modifying an `ExpectationSet` after using it in an `.After()` doesn't affect the
+meaning of the `.After()`.
+
+### Sequences {#UsingSequences}
+
+When you have a long chain of sequential expectations, it's easier to specify
+the order using **sequences**, which don't require you to given each expectation
+in the chain a different name. *All expected calls* in the same sequence must
+occur in the order they are specified.
+
+```cpp
+using ::testing::Return;
+using ::testing::Sequence;
+Sequence s1, s2;
+...
+EXPECT_CALL(foo, Reset())
+    .InSequence(s1, s2)
+    .WillOnce(Return(true));
+EXPECT_CALL(foo, GetSize())
+    .InSequence(s1)
+    .WillOnce(Return(1));
+EXPECT_CALL(foo, Describe(A()))
+    .InSequence(s2)
+    .WillOnce(Return("dummy"));
+```
+
+says that `Reset()` must be called before *both* `GetSize()` *and* `Describe()`,
+and the latter two can occur in any order.
+
+To put many expectations in a sequence conveniently:
+
+```cpp
+using ::testing::InSequence;
+{
+  InSequence seq;
+
+  EXPECT_CALL(...)...;
+  EXPECT_CALL(...)...;
+  ...
+  EXPECT_CALL(...)...;
+}
+```
+
+says that all expected calls in the scope of `seq` must occur in strict order.
+The name `seq` is irrelevant.
+
+## Verifying and Resetting a Mock
+
+gMock will verify the expectations on a mock object when it is destructed, or
+you can do it earlier:
+
+```cpp
+using ::testing::Mock;
+...
+// Verifies and removes the expectations on mock_obj;
+// returns true if and only if successful.
+Mock::VerifyAndClearExpectations(&mock_obj);
+...
+// Verifies and removes the expectations on mock_obj;
+// also removes the default actions set by ON_CALL();
+// returns true if and only if successful.
+Mock::VerifyAndClear(&mock_obj);
+```
+
+You can also tell gMock that a mock object can be leaked and doesn't need to be
+verified:
+
+```cpp
+Mock::AllowLeak(&mock_obj);
+```
+
+## Mock Classes
+
+gMock defines a convenient mock class template
+
+```cpp
+class MockFunction {
+ public:
+  MOCK_METHOD(R, Call, (A1, ..., An));
+};
+```
+
+See this [recipe](cook_book.md#using-check-points) for one application of it.
+
+## Flags
+
+
+| Flag                           | Description                               |
+| :----------------------------- | :---------------------------------------- |
+| `--gmock_catch_leaked_mocks=0` | Don't report leaked mock objects as failures. |
+| `--gmock_verbose=LEVEL` | Sets the default verbosity level (`info`, `warning`, or `error`) of Google Mock messages. |
+
diff --git a/thirdparty/GTest/googletest/googlemock/docs/community_created_documentation.md b/thirdparty/GTest/googletest/googlemock/docs/community_created_documentation.md
new file mode 100644
index 0000000000..dfd87f7a61
--- /dev/null
+++ b/thirdparty/GTest/googletest/googlemock/docs/community_created_documentation.md
@@ -0,0 +1,9 @@
+# Community-Created Documentation
+
+go/gunit-community-created-docs
+
+The following is a list, in no particular order, of links to documentation
+created by the Googletest community.
+
+*   [Googlemock Insights](https://github.com/ElectricRCAircraftGuy/eRCaGuy_dotfiles/blob/master/googletest/insights.md),
+    by [ElectricRCAircraftGuy](https://github.com/ElectricRCAircraftGuy)
diff --git a/thirdparty/GTest/googletest/googlemock/docs/cook_book.md b/thirdparty/GTest/googletest/googlemock/docs/cook_book.md
new file mode 100644
index 0000000000..817d5cabec
--- /dev/null
+++ b/thirdparty/GTest/googletest/googlemock/docs/cook_book.md
@@ -0,0 +1,4273 @@
+# gMock Cookbook
+
+
+
+You can find recipes for using gMock here. If you haven't yet, please read
+[the dummy guide](for_dummies.md) first to make sure you understand the basics.
+
+**Note:** gMock lives in the `testing` name space. For readability, it is
+recommended to write `using ::testing::Foo;` once in your file before using the
+name `Foo` defined by gMock. We omit such `using` statements in this section for
+brevity, but you should do it in your own code.
+
+
+
+## Creating Mock Classes
+
+Mock classes are defined as normal classes, using the `MOCK_METHOD` macro to
+generate mocked methods. The macro gets 3 or 4 parameters:
+
+```cpp
+class MyMock {
+ public:
+  MOCK_METHOD(ReturnType, MethodName, (Args...));
+  MOCK_METHOD(ReturnType, MethodName, (Args...), (Specs...));
+};
+```
+
+The first 3 parameters are simply the method declaration, split into 3 parts.
+The 4th parameter accepts a closed list of qualifiers, which affect the
+generated method:
+
+*   **`const`** - Makes the mocked method a `const` method. Required if
+    overriding a `const` method.
+*   **`override`** - Marks the method with `override`. Recommended if overriding
+    a `virtual` method.
+*   **`noexcept`** - Marks the method with `noexcept`. Required if overriding a
+    `noexcept` method.
+*   **`Calltype(...)`** - Sets the call type for the method (e.g. to
+    `STDMETHODCALLTYPE`), useful in Windows.
+
+### Dealing with unprotected commas
+
+Unprotected commas, i.e. commas which are not surrounded by parentheses, prevent
+`MOCK_METHOD` from parsing its arguments correctly:
+
+```cpp {.bad}
+class MockFoo {
+ public:
+  MOCK_METHOD(std::pair, GetPair, ());  // Won't compile!
+  MOCK_METHOD(bool, CheckMap, (std::map, bool));  // Won't compile!
+};
+```
+
+Solution 1 - wrap with parentheses:
+
+```cpp {.good}
+class MockFoo {
+ public:
+  MOCK_METHOD((std::pair), GetPair, ());
+  MOCK_METHOD(bool, CheckMap, ((std::map), bool));
+};
+```
+
+Note that wrapping a return or argument type with parentheses is, in general,
+invalid C++. `MOCK_METHOD` removes the parentheses.
+
+Solution 2 - define an alias:
+
+```cpp {.good}
+class MockFoo {
+ public:
+  using BoolAndInt = std::pair;
+  MOCK_METHOD(BoolAndInt, GetPair, ());
+  using MapIntDouble = std::map;
+  MOCK_METHOD(bool, CheckMap, (MapIntDouble, bool));
+};
+```
+
+### Mocking Private or Protected Methods
+
+You must always put a mock method definition (`MOCK_METHOD`) in a `public:`
+section of the mock class, regardless of the method being mocked being `public`,
+`protected`, or `private` in the base class. This allows `ON_CALL` and
+`EXPECT_CALL` to reference the mock function from outside of the mock class.
+(Yes, C++ allows a subclass to change the access level of a virtual function in
+the base class.) Example:
+
+```cpp
+class Foo {
+ public:
+  ...
+  virtual bool Transform(Gadget* g) = 0;
+
+ protected:
+  virtual void Resume();
+
+ private:
+  virtual int GetTimeOut();
+};
+
+class MockFoo : public Foo {
+ public:
+  ...
+  MOCK_METHOD(bool, Transform, (Gadget* g), (override));
+
+  // The following must be in the public section, even though the
+  // methods are protected or private in the base class.
+  MOCK_METHOD(void, Resume, (), (override));
+  MOCK_METHOD(int, GetTimeOut, (), (override));
+};
+```
+
+### Mocking Overloaded Methods
+
+You can mock overloaded functions as usual. No special attention is required:
+
+```cpp
+class Foo {
+  ...
+
+  // Must be virtual as we'll inherit from Foo.
+  virtual ~Foo();
+
+  // Overloaded on the types and/or numbers of arguments.
+  virtual int Add(Element x);
+  virtual int Add(int times, Element x);
+
+  // Overloaded on the const-ness of this object.
+  virtual Bar& GetBar();
+  virtual const Bar& GetBar() const;
+};
+
+class MockFoo : public Foo {
+  ...
+  MOCK_METHOD(int, Add, (Element x), (override));
+  MOCK_METHOD(int, Add, (int times, Element x), (override));
+
+  MOCK_METHOD(Bar&, GetBar, (), (override));
+  MOCK_METHOD(const Bar&, GetBar, (), (const, override));
+};
+```
+
+**Note:** if you don't mock all versions of the overloaded method, the compiler
+will give you a warning about some methods in the base class being hidden. To
+fix that, use `using` to bring them in scope:
+
+```cpp
+class MockFoo : public Foo {
+  ...
+  using Foo::Add;
+  MOCK_METHOD(int, Add, (Element x), (override));
+  // We don't want to mock int Add(int times, Element x);
+  ...
+};
+```
+
+### Mocking Class Templates
+
+You can mock class templates just like any class.
+
+```cpp
+template 
+class StackInterface {
+  ...
+  // Must be virtual as we'll inherit from StackInterface.
+  virtual ~StackInterface();
+
+  virtual int GetSize() const = 0;
+  virtual void Push(const Elem& x) = 0;
+};
+
+template 
+class MockStack : public StackInterface {
+  ...
+  MOCK_METHOD(int, GetSize, (), (override));
+  MOCK_METHOD(void, Push, (const Elem& x), (override));
+};
+```
+
+### Mocking Non-virtual Methods {#MockingNonVirtualMethods}
+
+gMock can mock non-virtual functions to be used in Hi-perf dependency
+injection.
+
+In this case, instead of sharing a common base class with the real class, your
+mock class will be *unrelated* to the real class, but contain methods with the
+same signatures. The syntax for mocking non-virtual methods is the *same* as
+mocking virtual methods (just don't add `override`):
+
+```cpp
+// A simple packet stream class.  None of its members is virtual.
+class ConcretePacketStream {
+ public:
+  void AppendPacket(Packet* new_packet);
+  const Packet* GetPacket(size_t packet_number) const;
+  size_t NumberOfPackets() const;
+  ...
+};
+
+// A mock packet stream class.  It inherits from no other, but defines
+// GetPacket() and NumberOfPackets().
+class MockPacketStream {
+ public:
+  MOCK_METHOD(const Packet*, GetPacket, (size_t packet_number), (const));
+  MOCK_METHOD(size_t, NumberOfPackets, (), (const));
+  ...
+};
+```
+
+Note that the mock class doesn't define `AppendPacket()`, unlike the real class.
+That's fine as long as the test doesn't need to call it.
+
+Next, you need a way to say that you want to use `ConcretePacketStream` in
+production code, and use `MockPacketStream` in tests. Since the functions are
+not virtual and the two classes are unrelated, you must specify your choice at
+*compile time* (as opposed to run time).
+
+One way to do it is to templatize your code that needs to use a packet stream.
+More specifically, you will give your code a template type argument for the type
+of the packet stream. In production, you will instantiate your template with
+`ConcretePacketStream` as the type argument. In tests, you will instantiate the
+same template with `MockPacketStream`. For example, you may write:
+
+```cpp
+template 
+void CreateConnection(PacketStream* stream) { ... }
+
+template 
+class PacketReader {
+ public:
+  void ReadPackets(PacketStream* stream, size_t packet_num);
+};
+```
+
+Then you can use `CreateConnection()` and
+`PacketReader` in production code, and use
+`CreateConnection()` and `PacketReader` in
+tests.
+
+```cpp
+  MockPacketStream mock_stream;
+  EXPECT_CALL(mock_stream, ...)...;
+  .. set more expectations on mock_stream ...
+  PacketReader reader(&mock_stream);
+  ... exercise reader ...
+```
+
+### Mocking Free Functions
+
+It's possible to use gMock to mock a free function (i.e. a C-style function or a
+static method). You just need to rewrite your code to use an interface (abstract
+class).
+
+Instead of calling a free function (say, `OpenFile`) directly, introduce an
+interface for it and have a concrete subclass that calls the free function:
+
+```cpp
+class FileInterface {
+ public:
+  ...
+  virtual bool Open(const char* path, const char* mode) = 0;
+};
+
+class File : public FileInterface {
+ public:
+  ...
+  virtual bool Open(const char* path, const char* mode) {
+     return OpenFile(path, mode);
+  }
+};
+```
+
+Your code should talk to `FileInterface` to open a file. Now it's easy to mock
+out the function.
+
+This may seem like a lot of hassle, but in practice you often have multiple
+related functions that you can put in the same interface, so the per-function
+syntactic overhead will be much lower.
+
+If you are concerned about the performance overhead incurred by virtual
+functions, and profiling confirms your concern, you can combine this with the
+recipe for [mocking non-virtual methods](#MockingNonVirtualMethods).
+
+### Old-Style `MOCK_METHODn` Macros
+
+Before the generic `MOCK_METHOD` macro
+[was introduced in 2018](https://github.com/google/googletest/commit/c5f08bf91944ce1b19bcf414fa1760e69d20afc2),
+mocks where created using a family of macros collectively called `MOCK_METHODn`.
+These macros are still supported, though migration to the new `MOCK_METHOD` is
+recommended.
+
+The macros in the `MOCK_METHODn` family differ from `MOCK_METHOD`:
+
+*   The general structure is `MOCK_METHODn(MethodName, ReturnType(Args))`,
+    instead of `MOCK_METHOD(ReturnType, MethodName, (Args))`.
+*   The number `n` must equal the number of arguments.
+*   When mocking a const method, one must use `MOCK_CONST_METHODn`.
+*   When mocking a class template, the macro name must be suffixed with `_T`.
+*   In order to specify the call type, the macro name must be suffixed with
+    `_WITH_CALLTYPE`, and the call type is the first macro argument.
+
+Old macros and their new equivalents:
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
  Simple 
  Old   `MOCK_METHOD1(Foo, bool(int))`  
  New   `MOCK_METHOD(bool, Foo, (int))`  
  Const Method 
 
  Old  
+`MOCK_CONST_METHOD1(Foo, bool(int))`  
 
  New  
+`MOCK_METHOD(bool, Foo, (int), (const))`  
  Method in a Class Template 
 
  Old  `MOCK_METHOD1_T(Foo, bool(int))`  
 
  New  
+`MOCK_METHOD(bool, Foo, (int))`  
  Const Method in a Class Template 
 
  Old
+  `MOCK_CONST_METHOD1_T(Foo, bool(int))`  
 
  New
+  `MOCK_METHOD(bool, Foo, (int), (const))`  
  Method with Call Type 
 
  Old  
+`MOCK_METHOD1_WITH_CALLTYPE(STDMETHODCALLTYPE, Foo, bool(int))`  
 
 New   `MOCK_METHOD(bool, Foo, (int),
+(Calltype(STDMETHODCALLTYPE)))`  
  Const Method with Call Type 
 
  Old `MOCK_CONST_METHOD1_WITH_CALLTYPE(STDMETHODCALLTYPE, Foo, bool(int))` 
 
  New   `MOCK_METHOD(bool, Foo, (int), (const,
+Calltype(STDMETHODCALLTYPE)))`  
  Method with Call Type in a Class Template 
 
 Old   `MOCK_METHOD1_T_WITH_CALLTYPE(STDMETHODCALLTYPE, Foo,
+bool(int))`  
 
  New   `MOCK_METHOD(bool, Foo, (int),
+(Calltype(STDMETHODCALLTYPE)))`  
  Const Method with Call Type in a Class Template 
  Old   `MOCK_CONST_METHOD1_T_WITH_CALLTYPE(STDMETHODCALLTYPE,
+Foo, bool(int))`  
 
  New   `MOCK_METHOD(bool, Foo,
+(int), (const, Calltype(STDMETHODCALLTYPE)))`  

+
+### The Nice, the Strict, and the Naggy {#NiceStrictNaggy}
+
+If a mock method has no `EXPECT_CALL` spec but is called, we say that it's an
+"uninteresting call", and the default action (which can be specified using
+`ON_CALL()`) of the method will be taken. Currently, an uninteresting call will
+also by default cause gMock to print a warning. (In the future, we might remove
+this warning by default.)
+
+However, sometimes you may want to ignore these uninteresting calls, and
+sometimes you may want to treat them as errors. gMock lets you make the decision
+on a per-mock-object basis.
+
+Suppose your test uses a mock class `MockFoo`:
+
+```cpp
+TEST(...) {
+  MockFoo mock_foo;
+  EXPECT_CALL(mock_foo, DoThis());
+  ... code that uses mock_foo ...
+}
+```
+
+If a method of `mock_foo` other than `DoThis()` is called, you will get a
+warning. However, if you rewrite your test to use `NiceMock` instead,
+you can suppress the warning:
+
+```cpp
+using ::testing::NiceMock;
+
+TEST(...) {
+  NiceMock mock_foo;
+  EXPECT_CALL(mock_foo, DoThis());
+  ... code that uses mock_foo ...
+}
+```
+
+`NiceMock` is a subclass of `MockFoo`, so it can be used wherever
+`MockFoo` is accepted.
+
+It also works if `MockFoo`'s constructor takes some arguments, as
+`NiceMock` "inherits" `MockFoo`'s constructors:
+
+```cpp
+using ::testing::NiceMock;
+
+TEST(...) {
+  NiceMock mock_foo(5, "hi");  // Calls MockFoo(5, "hi").
+  EXPECT_CALL(mock_foo, DoThis());
+  ... code that uses mock_foo ...
+}
+```
+
+The usage of `StrictMock` is similar, except that it makes all uninteresting
+calls failures:
+
+```cpp
+using ::testing::StrictMock;
+
+TEST(...) {
+  StrictMock mock_foo;
+  EXPECT_CALL(mock_foo, DoThis());
+  ... code that uses mock_foo ...
+
+  // The test will fail if a method of mock_foo other than DoThis()
+  // is called.
+}
+```
+
+NOTE: `NiceMock` and `StrictMock` only affects *uninteresting* calls (calls of
+*methods* with no expectations); they do not affect *unexpected* calls (calls of
+methods with expectations, but they don't match). See
+[Understanding Uninteresting vs Unexpected Calls](#uninteresting-vs-unexpected).
+
+There are some caveats though (sadly they are side effects of C++'s
+limitations):
+
+1.  `NiceMock` and `StrictMock` only work for mock methods
+    defined using the `MOCK_METHOD` macro **directly** in the `MockFoo` class.
+    If a mock method is defined in a **base class** of `MockFoo`, the "nice" or
+    "strict" modifier may not affect it, depending on the compiler. In
+    particular, nesting `NiceMock` and `StrictMock` (e.g.
+    `NiceMock >`) is **not** supported.
+2.  `NiceMock` and `StrictMock` may not work correctly if the
+    destructor of `MockFoo` is not virtual. We would like to fix this, but it
+    requires cleaning up existing tests.
+3.  During the constructor or destructor of `MockFoo`, the mock object is *not*
+    nice or strict. This may cause surprises if the constructor or destructor
+    calls a mock method on `this` object. (This behavior, however, is consistent
+    with C++'s general rule: if a constructor or destructor calls a virtual
+    method of `this` object, that method is treated as non-virtual. In other
+    words, to the base class's constructor or destructor, `this` object behaves
+    like an instance of the base class, not the derived class. This rule is
+    required for safety. Otherwise a base constructor may use members of a
+    derived class before they are initialized, or a base destructor may use
+    members of a derived class after they have been destroyed.)
+
+Finally, you should be **very cautious** about when to use naggy or strict
+mocks, as they tend to make tests more brittle and harder to maintain. When you
+refactor your code without changing its externally visible behavior, ideally you
+shouldn't need to update any tests. If your code interacts with a naggy mock,
+however, you may start to get spammed with warnings as the result of your
+change. Worse, if your code interacts with a strict mock, your tests may start
+to fail and you'll be forced to fix them. Our general recommendation is to use
+nice mocks (not yet the default) most of the time, use naggy mocks (the current
+default) when developing or debugging tests, and use strict mocks only as the
+last resort.
+
+### Simplifying the Interface without Breaking Existing Code {#SimplerInterfaces}
+
+Sometimes a method has a long list of arguments that is mostly uninteresting.
+For example:
+
+```cpp
+class LogSink {
+ public:
+  ...
+  virtual void send(LogSeverity severity, const char* full_filename,
+                    const char* base_filename, int line,
+                    const struct tm* tm_time,
+                    const char* message, size_t message_len) = 0;
+};
+```
+
+This method's argument list is lengthy and hard to work with (the `message`
+argument is not even 0-terminated). If we mock it as is, using the mock will be
+awkward. If, however, we try to simplify this interface, we'll need to fix all
+clients depending on it, which is often infeasible.
+
+The trick is to redispatch the method in the mock class:
+
+```cpp
+class ScopedMockLog : public LogSink {
+ public:
+  ...
+  virtual void send(LogSeverity severity, const char* full_filename,
+                    const char* base_filename, int line, const tm* tm_time,
+                    const char* message, size_t message_len) {
+    // We are only interested in the log severity, full file name, and
+    // log message.
+    Log(severity, full_filename, std::string(message, message_len));
+  }
+
+  // Implements the mock method:
+  //
+  //   void Log(LogSeverity severity,
+  //            const string& file_path,
+  //            const string& message);
+  MOCK_METHOD(void, Log,
+              (LogSeverity severity, const string& file_path,
+               const string& message));
+};
+```
+
+By defining a new mock method with a trimmed argument list, we make the mock
+class more user-friendly.
+
+This technique may also be applied to make overloaded methods more amenable to
+mocking. For example, when overloads have been used to implement default
+arguments:
+
+```cpp
+class MockTurtleFactory : public TurtleFactory {
+ public:
+  Turtle* MakeTurtle(int length, int weight) override { ... }
+  Turtle* MakeTurtle(int length, int weight, int speed) override { ... }
+
+  // the above methods delegate to this one:
+  MOCK_METHOD(Turtle*, DoMakeTurtle, ());
+};
+```
+
+This allows tests that don't care which overload was invoked to avoid specifying
+argument matchers:
+
+```cpp
+ON_CALL(factory, DoMakeTurtle)
+    .WillByDefault(MakeMockTurtle());
+```
+
+### Alternative to Mocking Concrete Classes
+
+Often you may find yourself using classes that don't implement interfaces. In
+order to test your code that uses such a class (let's call it `Concrete`), you
+may be tempted to make the methods of `Concrete` virtual and then mock it.
+
+Try not to do that.
+
+Making a non-virtual function virtual is a big decision. It creates an extension
+point where subclasses can tweak your class' behavior. This weakens your control
+on the class because now it's harder to maintain the class invariants. You
+should make a function virtual only when there is a valid reason for a subclass
+to override it.
+
+Mocking concrete classes directly is problematic as it creates a tight coupling
+between the class and the tests - any small change in the class may invalidate
+your tests and make test maintenance a pain.
+
+To avoid such problems, many programmers have been practicing "coding to
+interfaces": instead of talking to the `Concrete` class, your code would define
+an interface and talk to it. Then you implement that interface as an adaptor on
+top of `Concrete`. In tests, you can easily mock that interface to observe how
+your code is doing.
+
+This technique incurs some overhead:
+
+*   You pay the cost of virtual function calls (usually not a problem).
+*   There is more abstraction for the programmers to learn.
+
+However, it can also bring significant benefits in addition to better
+testability:
+
+*   `Concrete`'s API may not fit your problem domain very well, as you may not
+    be the only client it tries to serve. By designing your own interface, you
+    have a chance to tailor it to your need - you may add higher-level
+    functionalities, rename stuff, etc instead of just trimming the class. This
+    allows you to write your code (user of the interface) in a more natural way,
+    which means it will be more readable, more maintainable, and you'll be more
+    productive.
+*   If `Concrete`'s implementation ever has to change, you don't have to rewrite
+    everywhere it is used. Instead, you can absorb the change in your
+    implementation of the interface, and your other code and tests will be
+    insulated from this change.
+
+Some people worry that if everyone is practicing this technique, they will end
+up writing lots of redundant code. This concern is totally understandable.
+However, there are two reasons why it may not be the case:
+
+*   Different projects may need to use `Concrete` in different ways, so the best
+    interfaces for them will be different. Therefore, each of them will have its
+    own domain-specific interface on top of `Concrete`, and they will not be the
+    same code.
+*   If enough projects want to use the same interface, they can always share it,
+    just like they have been sharing `Concrete`. You can check in the interface
+    and the adaptor somewhere near `Concrete` (perhaps in a `contrib`
+    sub-directory) and let many projects use it.
+
+You need to weigh the pros and cons carefully for your particular problem, but
+I'd like to assure you that the Java community has been practicing this for a
+long time and it's a proven effective technique applicable in a wide variety of
+situations. :-)
+
+### Delegating Calls to a Fake {#DelegatingToFake}
+
+Some times you have a non-trivial fake implementation of an interface. For
+example:
+
+```cpp
+class Foo {
+ public:
+  virtual ~Foo() {}
+  virtual char DoThis(int n) = 0;
+  virtual void DoThat(const char* s, int* p) = 0;
+};
+
+class FakeFoo : public Foo {
+ public:
+  char DoThis(int n) override {
+    return (n > 0) ? '+' :
+           (n < 0) ? '-' : '0';
+  }
+
+  void DoThat(const char* s, int* p) override {
+    *p = strlen(s);
+  }
+};
+```
+
+Now you want to mock this interface such that you can set expectations on it.
+However, you also want to use `FakeFoo` for the default behavior, as duplicating
+it in the mock object is, well, a lot of work.
+
+When you define the mock class using gMock, you can have it delegate its default
+action to a fake class you already have, using this pattern:
+
+```cpp
+class MockFoo : public Foo {
+ public:
+  // Normal mock method definitions using gMock.
+  MOCK_METHOD(char, DoThis, (int n), (override));
+  MOCK_METHOD(void, DoThat, (const char* s, int* p), (override));
+
+  // Delegates the default actions of the methods to a FakeFoo object.
+  // This must be called *before* the custom ON_CALL() statements.
+  void DelegateToFake() {
+    ON_CALL(*this, DoThis).WillByDefault([this](int n) {
+      return fake_.DoThis(n);
+    });
+    ON_CALL(*this, DoThat).WillByDefault([this](const char* s, int* p) {
+      fake_.DoThat(s, p);
+    });
+  }
+
+ private:
+  FakeFoo fake_;  // Keeps an instance of the fake in the mock.
+};
+```
+
+With that, you can use `MockFoo` in your tests as usual. Just remember that if
+you don't explicitly set an action in an `ON_CALL()` or `EXPECT_CALL()`, the
+fake will be called upon to do it.:
+
+```cpp
+using ::testing::_;
+
+TEST(AbcTest, Xyz) {
+  MockFoo foo;
+
+  foo.DelegateToFake();  // Enables the fake for delegation.
+
+  // Put your ON_CALL(foo, ...)s here, if any.
+
+  // No action specified, meaning to use the default action.
+  EXPECT_CALL(foo, DoThis(5));
+  EXPECT_CALL(foo, DoThat(_, _));
+
+  int n = 0;
+  EXPECT_EQ('+', foo.DoThis(5));  // FakeFoo::DoThis() is invoked.
+  foo.DoThat("Hi", &n);  // FakeFoo::DoThat() is invoked.
+  EXPECT_EQ(2, n);
+}
+```
+
+**Some tips:**
+
+*   If you want, you can still override the default action by providing your own
+    `ON_CALL()` or using `.WillOnce()` / `.WillRepeatedly()` in `EXPECT_CALL()`.
+*   In `DelegateToFake()`, you only need to delegate the methods whose fake
+    implementation you intend to use.
+
+*   The general technique discussed here works for overloaded methods, but
+    you'll need to tell the compiler which version you mean. To disambiguate a
+    mock function (the one you specify inside the parentheses of `ON_CALL()`),
+    use [this technique](#SelectOverload); to disambiguate a fake function (the
+    one you place inside `Invoke()`), use a `static_cast` to specify the
+    function's type. For instance, if class `Foo` has methods `char DoThis(int
+    n)` and `bool DoThis(double x) const`, and you want to invoke the latter,
+    you need to write `Invoke(&fake_, static_cast(&FakeFoo::DoThis))` instead of `Invoke(&fake_, &FakeFoo::DoThis)`
+    (The strange-looking thing inside the angled brackets of `static_cast` is
+    the type of a function pointer to the second `DoThis()` method.).
+
+*   Having to mix a mock and a fake is often a sign of something gone wrong.
+    Perhaps you haven't got used to the interaction-based way of testing yet. Or
+    perhaps your interface is taking on too many roles and should be split up.
+    Therefore, **don't abuse this**. We would only recommend to do it as an
+    intermediate step when you are refactoring your code.
+
+Regarding the tip on mixing a mock and a fake, here's an example on why it may
+be a bad sign: Suppose you have a class `System` for low-level system
+operations. In particular, it does file and I/O operations. And suppose you want
+to test how your code uses `System` to do I/O, and you just want the file
+operations to work normally. If you mock out the entire `System` class, you'll
+have to provide a fake implementation for the file operation part, which
+suggests that `System` is taking on too many roles.
+
+Instead, you can define a `FileOps` interface and an `IOOps` interface and split
+`System`'s functionalities into the two. Then you can mock `IOOps` without
+mocking `FileOps`.
+
+### Delegating Calls to a Real Object
+
+When using testing doubles (mocks, fakes, stubs, and etc), sometimes their
+behaviors will differ from those of the real objects. This difference could be
+either intentional (as in simulating an error such that you can test the error
+handling code) or unintentional. If your mocks have different behaviors than the
+real objects by mistake, you could end up with code that passes the tests but
+fails in production.
+
+You can use the *delegating-to-real* technique to ensure that your mock has the
+same behavior as the real object while retaining the ability to validate calls.
+This technique is very similar to the [delegating-to-fake](#DelegatingToFake)
+technique, the difference being that we use a real object instead of a fake.
+Here's an example:
+
+```cpp
+using ::testing::AtLeast;
+
+class MockFoo : public Foo {
+ public:
+  MockFoo() {
+    // By default, all calls are delegated to the real object.
+    ON_CALL(*this, DoThis).WillByDefault([this](int n) {
+      return real_.DoThis(n);
+    });
+    ON_CALL(*this, DoThat).WillByDefault([this](const char* s, int* p) {
+      real_.DoThat(s, p);
+    });
+    ...
+  }
+  MOCK_METHOD(char, DoThis, ...);
+  MOCK_METHOD(void, DoThat, ...);
+  ...
+ private:
+  Foo real_;
+};
+
+...
+  MockFoo mock;
+  EXPECT_CALL(mock, DoThis())
+      .Times(3);
+  EXPECT_CALL(mock, DoThat("Hi"))
+      .Times(AtLeast(1));
+  ... use mock in test ...
+```
+
+With this, gMock will verify that your code made the right calls (with the right
+arguments, in the right order, called the right number of times, etc), and a
+real object will answer the calls (so the behavior will be the same as in
+production). This gives you the best of both worlds.
+
+### Delegating Calls to a Parent Class
+
+Ideally, you should code to interfaces, whose methods are all pure virtual. In
+reality, sometimes you do need to mock a virtual method that is not pure (i.e,
+it already has an implementation). For example:
+
+```cpp
+class Foo {
+ public:
+  virtual ~Foo();
+
+  virtual void Pure(int n) = 0;
+  virtual int Concrete(const char* str) { ... }
+};
+
+class MockFoo : public Foo {
+ public:
+  // Mocking a pure method.
+  MOCK_METHOD(void, Pure, (int n), (override));
+  // Mocking a concrete method.  Foo::Concrete() is shadowed.
+  MOCK_METHOD(int, Concrete, (const char* str), (override));
+};
+```
+
+Sometimes you may want to call `Foo::Concrete()` instead of
+`MockFoo::Concrete()`. Perhaps you want to do it as part of a stub action, or
+perhaps your test doesn't need to mock `Concrete()` at all (but it would be
+oh-so painful to have to define a new mock class whenever you don't need to mock
+one of its methods).
+
+You can call `Foo::Concrete()` inside an action by:
+
+```cpp
+...
+  EXPECT_CALL(foo, Concrete).WillOnce([&foo](const char* str) {
+    return foo.Foo::Concrete(str);
+  });
+```
+
+or tell the mock object that you don't want to mock `Concrete()`:
+
+```cpp
+...
+  ON_CALL(foo, Concrete).WillByDefault([&foo](const char* str) {
+    return foo.Foo::Concrete(str);
+  });
+```
+
+(Why don't we just write `{ return foo.Concrete(str); }`? If you do that,
+`MockFoo::Concrete()` will be called (and cause an infinite recursion) since
+`Foo::Concrete()` is virtual. That's just how C++ works.)
+
+## Using Matchers
+
+### Matching Argument Values Exactly
+
+You can specify exactly which arguments a mock method is expecting:
+
+```cpp
+using ::testing::Return;
+...
+  EXPECT_CALL(foo, DoThis(5))
+      .WillOnce(Return('a'));
+  EXPECT_CALL(foo, DoThat("Hello", bar));
+```
+
+### Using Simple Matchers
+
+You can use matchers to match arguments that have a certain property:
+
+```cpp
+using ::testing::NotNull;
+using ::testing::Return;
+...
+  EXPECT_CALL(foo, DoThis(Ge(5)))  // The argument must be >= 5.
+      .WillOnce(Return('a'));
+  EXPECT_CALL(foo, DoThat("Hello", NotNull()));
+      // The second argument must not be NULL.
+```
+
+A frequently used matcher is `_`, which matches anything:
+
+```cpp
+  EXPECT_CALL(foo, DoThat(_, NotNull()));
+```
+
+
+### Combining Matchers {#CombiningMatchers}
+
+You can build complex matchers from existing ones using `AllOf()`,
+`AllOfArray()`, `AnyOf()`, `AnyOfArray()` and `Not()`:
+
+```cpp
+using ::testing::AllOf;
+using ::testing::Gt;
+using ::testing::HasSubstr;
+using ::testing::Ne;
+using ::testing::Not;
+...
+  // The argument must be > 5 and != 10.
+  EXPECT_CALL(foo, DoThis(AllOf(Gt(5),
+                                Ne(10))));
+
+  // The first argument must not contain sub-string "blah".
+  EXPECT_CALL(foo, DoThat(Not(HasSubstr("blah")),
+                          NULL));
+```
+
+Matchers are function objects, and parametrized matchers can be composed just
+like any other function. However because their types can be long and rarely
+provide meaningful information, it can be easier to express them with C++14
+generic lambdas to avoid specifying types. For example,
+
+```cpp
+using ::testing::Contains;
+using ::testing::Property;
+
+inline constexpr auto HasFoo = [](const auto& f) {
+  return Property(&MyClass::foo, Contains(f));
+};
+...
+  EXPECT_THAT(x, HasFoo("blah"));
+```
+
+### Casting Matchers {#SafeMatcherCast}
+
+gMock matchers are statically typed, meaning that the compiler can catch your
+mistake if you use a matcher of the wrong type (for example, if you use `Eq(5)`
+to match a `string` argument). Good for you!
+
+Sometimes, however, you know what you're doing and want the compiler to give you
+some slack. One example is that you have a matcher for `long` and the argument
+you want to match is `int`. While the two types aren't exactly the same, there
+is nothing really wrong with using a `Matcher` to match an `int` - after
+all, we can first convert the `int` argument to a `long` losslessly before
+giving it to the matcher.
+
+To support this need, gMock gives you the `SafeMatcherCast(m)` function. It
+casts a matcher `m` to type `Matcher`. To ensure safety, gMock checks that
+(let `U` be the type `m` accepts :
+
+1.  Type `T` can be *implicitly* cast to type `U`;
+2.  When both `T` and `U` are built-in arithmetic types (`bool`, integers, and
+    floating-point numbers), the conversion from `T` to `U` is not lossy (in
+    other words, any value representable by `T` can also be represented by `U`);
+    and
+3.  When `U` is a reference, `T` must also be a reference (as the underlying
+    matcher may be interested in the address of the `U` value).
+
+The code won't compile if any of these conditions isn't met.
+
+Here's one example:
+
+```cpp
+using ::testing::SafeMatcherCast;
+
+// A base class and a child class.
+class Base { ... };
+class Derived : public Base { ... };
+
+class MockFoo : public Foo {
+ public:
+  MOCK_METHOD(void, DoThis, (Derived* derived), (override));
+};
+
+...
+  MockFoo foo;
+  // m is a Matcher we got from somewhere.
+  EXPECT_CALL(foo, DoThis(SafeMatcherCast(m)));
+```
+
+If you find `SafeMatcherCast(m)` too limiting, you can use a similar function
+`MatcherCast(m)`. The difference is that `MatcherCast` works as long as you
+can `static_cast` type `T` to type `U`.
+
+`MatcherCast` essentially lets you bypass C++'s type system (`static_cast` isn't
+always safe as it could throw away information, for example), so be careful not
+to misuse/abuse it.
+
+### Selecting Between Overloaded Functions {#SelectOverload}
+
+If you expect an overloaded function to be called, the compiler may need some
+help on which overloaded version it is.
+
+To disambiguate functions overloaded on the const-ness of this object, use the
+`Const()` argument wrapper.
+
+```cpp
+using ::testing::ReturnRef;
+
+class MockFoo : public Foo {
+  ...
+  MOCK_METHOD(Bar&, GetBar, (), (override));
+  MOCK_METHOD(const Bar&, GetBar, (), (const, override));
+};
+
+...
+  MockFoo foo;
+  Bar bar1, bar2;
+  EXPECT_CALL(foo, GetBar())         // The non-const GetBar().
+      .WillOnce(ReturnRef(bar1));
+  EXPECT_CALL(Const(foo), GetBar())  // The const GetBar().
+      .WillOnce(ReturnRef(bar2));
+```
+
+(`Const()` is defined by gMock and returns a `const` reference to its argument.)
+
+To disambiguate overloaded functions with the same number of arguments but
+different argument types, you may need to specify the exact type of a matcher,
+either by wrapping your matcher in `Matcher()`, or using a matcher whose
+type is fixed (`TypedEq`, `An()`, etc):
+
+```cpp
+using ::testing::An;
+using ::testing::Matcher;
+using ::testing::TypedEq;
+
+class MockPrinter : public Printer {
+ public:
+  MOCK_METHOD(void, Print, (int n), (override));
+  MOCK_METHOD(void, Print, (char c), (override));
+};
+
+TEST(PrinterTest, Print) {
+  MockPrinter printer;
+
+  EXPECT_CALL(printer, Print(An()));            // void Print(int);
+  EXPECT_CALL(printer, Print(Matcher(Lt(5))));  // void Print(int);
+  EXPECT_CALL(printer, Print(TypedEq('a')));   // void Print(char);
+
+  printer.Print(3);
+  printer.Print(6);
+  printer.Print('a');
+}
+```
+
+### Performing Different Actions Based on the Arguments
+
+When a mock method is called, the *last* matching expectation that's still
+active will be selected (think "newer overrides older"). So, you can make a
+method do different things depending on its argument values like this:
+
+```cpp
+using ::testing::_;
+using ::testing::Lt;
+using ::testing::Return;
+...
+  // The default case.
+  EXPECT_CALL(foo, DoThis(_))
+      .WillRepeatedly(Return('b'));
+  // The more specific case.
+  EXPECT_CALL(foo, DoThis(Lt(5)))
+      .WillRepeatedly(Return('a'));
+```
+
+Now, if `foo.DoThis()` is called with a value less than 5, `'a'` will be
+returned; otherwise `'b'` will be returned.
+
+### Matching Multiple Arguments as a Whole
+
+Sometimes it's not enough to match the arguments individually. For example, we
+may want to say that the first argument must be less than the second argument.
+The `With()` clause allows us to match all arguments of a mock function as a
+whole. For example,
+
+```cpp
+using ::testing::_;
+using ::testing::Ne;
+using ::testing::Lt;
+...
+  EXPECT_CALL(foo, InRange(Ne(0), _))
+      .With(Lt());
+```
+
+says that the first argument of `InRange()` must not be 0, and must be less than
+the second argument.
+
+The expression inside `With()` must be a matcher of type `Matcher>`, where `A1`, ..., `An` are the types of the function arguments.
+
+You can also write `AllArgs(m)` instead of `m` inside `.With()`. The two forms
+are equivalent, but `.With(AllArgs(Lt()))` is more readable than `.With(Lt())`.
+
+You can use `Args(m)` to match the `n` selected arguments (as a
+tuple) against `m`. For example,
+
+```cpp
+using ::testing::_;
+using ::testing::AllOf;
+using ::testing::Args;
+using ::testing::Lt;
+...
+  EXPECT_CALL(foo, Blah)
+      .With(AllOf(Args<0, 1>(Lt()), Args<1, 2>(Lt())));
+```
+
+says that `Blah` will be called with arguments `x`, `y`, and `z` where `x < y <
+z`. Note that in this example, it wasn't necessary specify the positional
+matchers.
+
+As a convenience and example, gMock provides some matchers for 2-tuples,
+including the `Lt()` matcher above. See [here](#MultiArgMatchers) for the
+complete list.
+
+Note that if you want to pass the arguments to a predicate of your own (e.g.
+`.With(Args<0, 1>(Truly(&MyPredicate)))`), that predicate MUST be written to
+take a `std::tuple` as its argument; gMock will pass the `n` selected arguments
+as *one* single tuple to the predicate.
+
+### Using Matchers as Predicates
+
+Have you noticed that a matcher is just a fancy predicate that also knows how to
+describe itself? Many existing algorithms take predicates as arguments (e.g.
+those defined in STL's `` header), and it would be a shame if gMock
+matchers were not allowed to participate.
+
+Luckily, you can use a matcher where a unary predicate functor is expected by
+wrapping it inside the `Matches()` function. For example,
+
+```cpp
+#include 
+#include 
+
+using ::testing::Matches;
+using ::testing::Ge;
+
+vector v;
+...
+// How many elements in v are >= 10?
+const int count = count_if(v.begin(), v.end(), Matches(Ge(10)));
+```
+
+Since you can build complex matchers from simpler ones easily using gMock, this
+gives you a way to conveniently construct composite predicates (doing the same
+using STL's `` header is just painful). For example, here's a
+predicate that's satisfied by any number that is >= 0, <= 100, and != 50:
+
+```cpp
+using testing::AllOf;
+using testing::Ge;
+using testing::Le;
+using testing::Matches;
+using testing::Ne;
+...
+Matches(AllOf(Ge(0), Le(100), Ne(50)))
+```
+
+### Using Matchers in googletest Assertions
+
+Since matchers are basically predicates that also know how to describe
+themselves, there is a way to take advantage of them in googletest assertions.
+It's called `ASSERT_THAT` and `EXPECT_THAT`:
+
+```cpp
+  ASSERT_THAT(value, matcher);  // Asserts that value matches matcher.
+  EXPECT_THAT(value, matcher);  // The non-fatal version.
+```
+
+For example, in a googletest test you can write:
+
+```cpp
+#include "gmock/gmock.h"
+
+using ::testing::AllOf;
+using ::testing::Ge;
+using ::testing::Le;
+using ::testing::MatchesRegex;
+using ::testing::StartsWith;
+
+...
+  EXPECT_THAT(Foo(), StartsWith("Hello"));
+  EXPECT_THAT(Bar(), MatchesRegex("Line \\d+"));
+  ASSERT_THAT(Baz(), AllOf(Ge(5), Le(10)));
+```
+
+which (as you can probably guess) executes `Foo()`, `Bar()`, and `Baz()`, and
+verifies that:
+
+*   `Foo()` returns a string that starts with `"Hello"`.
+*   `Bar()` returns a string that matches regular expression `"Line \\d+"`.
+*   `Baz()` returns a number in the range [5, 10].
+
+The nice thing about these macros is that *they read like English*. They
+generate informative messages too. For example, if the first `EXPECT_THAT()`
+above fails, the message will be something like:
+
+```cpp
+Value of: Foo()
+  Actual: "Hi, world!"
+Expected: starts with "Hello"
+```
+
+**Credit:** The idea of `(ASSERT|EXPECT)_THAT` was borrowed from Joe Walnes'
+Hamcrest project, which adds `assertThat()` to JUnit.
+
+### Using Predicates as Matchers
+
+gMock provides a [built-in set](cheat_sheet.md#MatcherList) of matchers. In case
+you find them lacking, you can use an arbitrary unary predicate function or
+functor as a matcher - as long as the predicate accepts a value of the type you
+want. You do this by wrapping the predicate inside the `Truly()` function, for
+example:
+
+```cpp
+using ::testing::Truly;
+
+int IsEven(int n) { return (n % 2) == 0 ? 1 : 0; }
+...
+  // Bar() must be called with an even number.
+  EXPECT_CALL(foo, Bar(Truly(IsEven)));
+```
+
+Note that the predicate function / functor doesn't have to return `bool`. It
+works as long as the return value can be used as the condition in in statement
+`if (condition) ...`.
+
+
+
+### Matching Arguments that Are Not Copyable
+
+When you do an `EXPECT_CALL(mock_obj, Foo(bar))`, gMock saves away a copy of
+`bar`. When `Foo()` is called later, gMock compares the argument to `Foo()` with
+the saved copy of `bar`. This way, you don't need to worry about `bar` being
+modified or destroyed after the `EXPECT_CALL()` is executed. The same is true
+when you use matchers like `Eq(bar)`, `Le(bar)`, and so on.
+
+But what if `bar` cannot be copied (i.e. has no copy constructor)? You could
+define your own matcher function or callback and use it with `Truly()`, as the
+previous couple of recipes have shown. Or, you may be able to get away from it
+if you can guarantee that `bar` won't be changed after the `EXPECT_CALL()` is
+executed. Just tell gMock that it should save a reference to `bar`, instead of a
+copy of it. Here's how:
+
+```cpp
+using ::testing::Eq;
+using ::testing::Lt;
+...
+  // Expects that Foo()'s argument == bar.
+  EXPECT_CALL(mock_obj, Foo(Eq(std::ref(bar))));
+
+  // Expects that Foo()'s argument < bar.
+  EXPECT_CALL(mock_obj, Foo(Lt(std::ref(bar))));
+```
+
+Remember: if you do this, don't change `bar` after the `EXPECT_CALL()`, or the
+result is undefined.
+
+### Validating a Member of an Object
+
+Often a mock function takes a reference to object as an argument. When matching
+the argument, you may not want to compare the entire object against a fixed
+object, as that may be over-specification. Instead, you may need to validate a
+certain member variable or the result of a certain getter method of the object.
+You can do this with `Field()` and `Property()`. More specifically,
+
+```cpp
+Field(&Foo::bar, m)
+```
+
+is a matcher that matches a `Foo` object whose `bar` member variable satisfies
+matcher `m`.
+
+```cpp
+Property(&Foo::baz, m)
+```
+
+is a matcher that matches a `Foo` object whose `baz()` method returns a value
+that satisfies matcher `m`.
+
+For example:
+
+
+| Expression                   | Description                              |
+| :--------------------------- | :--------------------------------------- |
+| `Field(&Foo::number, Ge(3))` | Matches `x` where `x.number >= 3`.       |
+| `Property(&Foo::name,  StartsWith("John "))` | Matches `x` where `x.name()` starts with  `"John "`. |
+
+
+Note that in `Property(&Foo::baz, ...)`, method `baz()` must take no argument
+and be declared as `const`.
+
+BTW, `Field()` and `Property()` can also match plain pointers to objects. For
+instance,
+
+```cpp
+using ::testing::Field;
+using ::testing::Ge;
+...
+Field(&Foo::number, Ge(3))
+```
+
+matches a plain pointer `p` where `p->number >= 3`. If `p` is `NULL`, the match
+will always fail regardless of the inner matcher.
+
+What if you want to validate more than one members at the same time? Remember
+that there are [`AllOf()` and `AllOfArray()`](#CombiningMatchers).
+
+Finally `Field()` and `Property()` provide overloads that take the field or
+property names as the first argument to include it in the error message. This
+can be useful when creating combined matchers.
+
+```cpp
+using ::testing::AllOf;
+using ::testing::Field;
+using ::testing::Matcher;
+using ::testing::SafeMatcherCast;
+
+Matcher IsFoo(const Foo& foo) {
+  return AllOf(Field("some_field", &Foo::some_field, foo.some_field),
+               Field("other_field", &Foo::other_field, foo.other_field),
+               Field("last_field", &Foo::last_field, foo.last_field));
+}
+```
+
+### Validating the Value Pointed to by a Pointer Argument
+
+C++ functions often take pointers as arguments. You can use matchers like
+`IsNull()`, `NotNull()`, and other comparison matchers to match a pointer, but
+what if you want to make sure the value *pointed to* by the pointer, instead of
+the pointer itself, has a certain property? Well, you can use the `Pointee(m)`
+matcher.
+
+`Pointee(m)` matches a pointer if and only if `m` matches the value the pointer
+points to. For example:
+
+```cpp
+using ::testing::Ge;
+using ::testing::Pointee;
+...
+  EXPECT_CALL(foo, Bar(Pointee(Ge(3))));
+```
+
+expects `foo.Bar()` to be called with a pointer that points to a value greater
+than or equal to 3.
+
+One nice thing about `Pointee()` is that it treats a `NULL` pointer as a match
+failure, so you can write `Pointee(m)` instead of
+
+```cpp
+using ::testing::AllOf;
+using ::testing::NotNull;
+using ::testing::Pointee;
+...
+  AllOf(NotNull(), Pointee(m))
+```
+
+without worrying that a `NULL` pointer will crash your test.
+
+Also, did we tell you that `Pointee()` works with both raw pointers **and**
+smart pointers (`std::unique_ptr`, `std::shared_ptr`, etc)?
+
+What if you have a pointer to pointer? You guessed it - you can use nested
+`Pointee()` to probe deeper inside the value. For example,
+`Pointee(Pointee(Lt(3)))` matches a pointer that points to a pointer that points
+to a number less than 3 (what a mouthful...).
+
+### Testing a Certain Property of an Object
+
+Sometimes you want to specify that an object argument has a certain property,
+but there is no existing matcher that does this. If you want good error
+messages, you should [define a matcher](#NewMatchers). If you want to do it
+quick and dirty, you could get away with writing an ordinary function.
+
+Let's say you have a mock function that takes an object of type `Foo`, which has
+an `int bar()` method and an `int baz()` method, and you want to constrain that
+the argument's `bar()` value plus its `baz()` value is a given number. Here's
+how you can define a matcher to do it:
+
+```cpp
+using ::testing::Matcher;
+using ::testing::MatcherInterface;
+using ::testing::MatchResultListener;
+
+class BarPlusBazEqMatcher : public MatcherInterface {
+ public:
+  explicit BarPlusBazEqMatcher(int expected_sum)
+      : expected_sum_(expected_sum) {}
+
+  bool MatchAndExplain(const Foo& foo,
+                       MatchResultListener* /* listener */) const override {
+    return (foo.bar() + foo.baz()) == expected_sum_;
+  }
+
+  void DescribeTo(std::ostream* os) const override {
+    *os << "bar() + baz() equals " << expected_sum_;
+  }
+
+  void DescribeNegationTo(std::ostream* os) const override {
+    *os << "bar() + baz() does not equal " << expected_sum_;
+  }
+ private:
+  const int expected_sum_;
+};
+
+Matcher BarPlusBazEq(int expected_sum) {
+  return MakeMatcher(new BarPlusBazEqMatcher(expected_sum));
+}
+
+...
+  EXPECT_CALL(..., DoThis(BarPlusBazEq(5)))...;
+```
+
+### Matching Containers
+
+Sometimes an STL container (e.g. list, vector, map, ...) is passed to a mock
+function and you may want to validate it. Since most STL containers support the
+`==` operator, you can write `Eq(expected_container)` or simply
+`expected_container` to match a container exactly.
+
+Sometimes, though, you may want to be more flexible (for example, the first
+element must be an exact match, but the second element can be any positive
+number, and so on). Also, containers used in tests often have a small number of
+elements, and having to define the expected container out-of-line is a bit of a
+hassle.
+
+You can use the `ElementsAre()` or `UnorderedElementsAre()` matcher in such
+cases:
+
+```cpp
+using ::testing::_;
+using ::testing::ElementsAre;
+using ::testing::Gt;
+...
+  MOCK_METHOD(void, Foo, (const vector& numbers), (override));
+...
+  EXPECT_CALL(mock, Foo(ElementsAre(1, Gt(0), _, 5)));
+```
+
+The above matcher says that the container must have 4 elements, which must be 1,
+greater than 0, anything, and 5 respectively.
+
+If you instead write:
+
+```cpp
+using ::testing::_;
+using ::testing::Gt;
+using ::testing::UnorderedElementsAre;
+...
+  MOCK_METHOD(void, Foo, (const vector& numbers), (override));
+...
+  EXPECT_CALL(mock, Foo(UnorderedElementsAre(1, Gt(0), _, 5)));
+```
+
+It means that the container must have 4 elements, which (under some permutation)
+must be 1, greater than 0, anything, and 5 respectively.
+
+As an alternative you can place the arguments in a C-style array and use
+`ElementsAreArray()` or `UnorderedElementsAreArray()` instead:
+
+```cpp
+using ::testing::ElementsAreArray;
+...
+  // ElementsAreArray accepts an array of element values.
+  const int expected_vector1[] = {1, 5, 2, 4, ...};
+  EXPECT_CALL(mock, Foo(ElementsAreArray(expected_vector1)));
+
+  // Or, an array of element matchers.
+  Matcher expected_vector2[] = {1, Gt(2), _, 3, ...};
+  EXPECT_CALL(mock, Foo(ElementsAreArray(expected_vector2)));
+```
+
+In case the array needs to be dynamically created (and therefore the array size
+cannot be inferred by the compiler), you can give `ElementsAreArray()` an
+additional argument to specify the array size:
+
+```cpp
+using ::testing::ElementsAreArray;
+...
+  int* const expected_vector3 = new int[count];
+  ... fill expected_vector3 with values ...
+  EXPECT_CALL(mock, Foo(ElementsAreArray(expected_vector3, count)));
+```
+
+Use `Pair` when comparing maps or other associative containers.
+
+```cpp
+using testing::ElementsAre;
+using testing::Pair;
+...
+  std::map m = {{"a", 1}, {"b", 2}, {"c", 3}};
+  EXPECT_THAT(m, ElementsAre(Pair("a", 1), Pair("b", 2), Pair("c", 3)));
+```
+
+**Tips:**
+
+*   `ElementsAre*()` can be used to match *any* container that implements the
+    STL iterator pattern (i.e. it has a `const_iterator` type and supports
+    `begin()/end()`), not just the ones defined in STL. It will even work with
+    container types yet to be written - as long as they follows the above
+    pattern.
+*   You can use nested `ElementsAre*()` to match nested (multi-dimensional)
+    containers.
+*   If the container is passed by pointer instead of by reference, just write
+    `Pointee(ElementsAre*(...))`.
+*   The order of elements *matters* for `ElementsAre*()`. If you are using it
+    with containers whose element order are undefined (e.g. `hash_map`) you
+    should use `WhenSorted` around `ElementsAre`.
+
+### Sharing Matchers
+
+Under the hood, a gMock matcher object consists of a pointer to a ref-counted
+implementation object. Copying matchers is allowed and very efficient, as only
+the pointer is copied. When the last matcher that references the implementation
+object dies, the implementation object will be deleted.
+
+Therefore, if you have some complex matcher that you want to use again and
+again, there is no need to build it everytime. Just assign it to a matcher
+variable and use that variable repeatedly! For example,
+
+```cpp
+using ::testing::AllOf;
+using ::testing::Gt;
+using ::testing::Le;
+using ::testing::Matcher;
+...
+  Matcher in_range = AllOf(Gt(5), Le(10));
+  ... use in_range as a matcher in multiple EXPECT_CALLs ...
+```
+
+### Matchers must have no side-effects {#PureMatchers}
+
+WARNING: gMock does not guarantee when or how many times a matcher will be
+invoked. Therefore, all matchers must be *purely functional*: they cannot have
+any side effects, and the match result must not depend on anything other than
+the matcher's parameters and the value being matched.
+
+This requirement must be satisfied no matter how a matcher is defined (e.g., if
+it is one of the standard matchers, or a custom matcher). In particular, a
+matcher can never call a mock function, as that will affect the state of the
+mock object and gMock.
+
+## Setting Expectations
+
+### Knowing When to Expect {#UseOnCall}
+
+
+
+**`ON_CALL`** is likely the *single most under-utilized construct* in gMock.
+
+There are basically two constructs for defining the behavior of a mock object:
+`ON_CALL` and `EXPECT_CALL`. The difference? `ON_CALL` defines what happens when
+a mock method is called, but doesn't imply any expectation on the method
+being called. `EXPECT_CALL` not only defines the behavior, but also sets an
+expectation that the method will be called with the given arguments, for the
+given number of times (and *in the given order* when you specify the order
+too).
+
+Since `EXPECT_CALL` does more, isn't it better than `ON_CALL`? Not really. Every
+`EXPECT_CALL` adds a constraint on the behavior of the code under test. Having
+more constraints than necessary is *baaad* - even worse than not having enough
+constraints.
+
+This may be counter-intuitive. How could tests that verify more be worse than
+tests that verify less? Isn't verification the whole point of tests?
+
+The answer lies in *what* a test should verify. **A good test verifies the
+contract of the code.** If a test over-specifies, it doesn't leave enough
+freedom to the implementation. As a result, changing the implementation without
+breaking the contract (e.g. refactoring and optimization), which should be
+perfectly fine to do, can break such tests. Then you have to spend time fixing
+them, only to see them broken again the next time the implementation is changed.
+
+Keep in mind that one doesn't have to verify more than one property in one test.
+In fact, **it's a good style to verify only one thing in one test.** If you do
+that, a bug will likely break only one or two tests instead of dozens (which
+case would you rather debug?). If you are also in the habit of giving tests
+descriptive names that tell what they verify, you can often easily guess what's
+wrong just from the test log itself.
+
+So use `ON_CALL` by default, and only use `EXPECT_CALL` when you actually intend
+to verify that the call is made. For example, you may have a bunch of `ON_CALL`s
+in your test fixture to set the common mock behavior shared by all tests in the
+same group, and write (scarcely) different `EXPECT_CALL`s in different `TEST_F`s
+to verify different aspects of the code's behavior. Compared with the style
+where each `TEST` has many `EXPECT_CALL`s, this leads to tests that are more
+resilient to implementational changes (and thus less likely to require
+maintenance) and makes the intent of the tests more obvious (so they are easier
+to maintain when you do need to maintain them).
+
+If you are bothered by the "Uninteresting mock function call" message printed
+when a mock method without an `EXPECT_CALL` is called, you may use a `NiceMock`
+instead to suppress all such messages for the mock object, or suppress the
+message for specific methods by adding `EXPECT_CALL(...).Times(AnyNumber())`. DO
+NOT suppress it by blindly adding an `EXPECT_CALL(...)`, or you'll have a test
+that's a pain to maintain.
+
+### Ignoring Uninteresting Calls
+
+If you are not interested in how a mock method is called, just don't say
+anything about it. In this case, if the method is ever called, gMock will
+perform its default action to allow the test program to continue. If you are not
+happy with the default action taken by gMock, you can override it using
+`DefaultValue::Set()` (described [here](#DefaultValue)) or `ON_CALL()`.
+
+Please note that once you expressed interest in a particular mock method (via
+`EXPECT_CALL()`), all invocations to it must match some expectation. If this
+function is called but the arguments don't match any `EXPECT_CALL()` statement,
+it will be an error.
+
+### Disallowing Unexpected Calls
+
+If a mock method shouldn't be called at all, explicitly say so:
+
+```cpp
+using ::testing::_;
+...
+  EXPECT_CALL(foo, Bar(_))
+      .Times(0);
+```
+
+If some calls to the method are allowed, but the rest are not, just list all the
+expected calls:
+
+```cpp
+using ::testing::AnyNumber;
+using ::testing::Gt;
+...
+  EXPECT_CALL(foo, Bar(5));
+  EXPECT_CALL(foo, Bar(Gt(10)))
+      .Times(AnyNumber());
+```
+
+A call to `foo.Bar()` that doesn't match any of the `EXPECT_CALL()` statements
+will be an error.
+
+### Understanding Uninteresting vs Unexpected Calls {#uninteresting-vs-unexpected}
+
+*Uninteresting* calls and *unexpected* calls are different concepts in gMock.
+*Very* different.
+
+A call `x.Y(...)` is **uninteresting** if there's *not even a single*
+`EXPECT_CALL(x, Y(...))` set. In other words, the test isn't interested in the
+`x.Y()` method at all, as evident in that the test doesn't care to say anything
+about it.
+
+A call `x.Y(...)` is **unexpected** if there are *some* `EXPECT_CALL(x,
+Y(...))`s set, but none of them matches the call. Put another way, the test is
+interested in the `x.Y()` method (therefore it explicitly sets some
+`EXPECT_CALL` to verify how it's called); however, the verification fails as the
+test doesn't expect this particular call to happen.
+
+**An unexpected call is always an error,** as the code under test doesn't behave
+the way the test expects it to behave.
+
+**By default, an uninteresting call is not an error,** as it violates no
+constraint specified by the test. (gMock's philosophy is that saying nothing
+means there is no constraint.) However, it leads to a warning, as it *might*
+indicate a problem (e.g. the test author might have forgotten to specify a
+constraint).
+
+In gMock, `NiceMock` and `StrictMock` can be used to make a mock class "nice" or
+"strict". How does this affect uninteresting calls and unexpected calls?
+
+A **nice mock** suppresses uninteresting call *warnings*. It is less chatty than
+the default mock, but otherwise is the same. If a test fails with a default
+mock, it will also fail using a nice mock instead. And vice versa. Don't expect
+making a mock nice to change the test's result.
+
+A **strict mock** turns uninteresting call warnings into errors. So making a
+mock strict may change the test's result.
+
+Let's look at an example:
+
+```cpp
+TEST(...) {
+  NiceMock mock_registry;
+  EXPECT_CALL(mock_registry, GetDomainOwner("google.com"))
+          .WillRepeatedly(Return("Larry Page"));
+
+  // Use mock_registry in code under test.
+  ... &mock_registry ...
+}
+```
+
+The sole `EXPECT_CALL` here says that all calls to `GetDomainOwner()` must have
+`"google.com"` as the argument. If `GetDomainOwner("yahoo.com")` is called, it
+will be an unexpected call, and thus an error. *Having a nice mock doesn't
+change the severity of an unexpected call.*
+
+So how do we tell gMock that `GetDomainOwner()` can be called with some other
+arguments as well? The standard technique is to add a "catch all" `EXPECT_CALL`:
+
+```cpp
+  EXPECT_CALL(mock_registry, GetDomainOwner(_))
+        .Times(AnyNumber());  // catches all other calls to this method.
+  EXPECT_CALL(mock_registry, GetDomainOwner("google.com"))
+        .WillRepeatedly(Return("Larry Page"));
+```
+
+Remember that `_` is the wildcard matcher that matches anything. With this, if
+`GetDomainOwner("google.com")` is called, it will do what the second
+`EXPECT_CALL` says; if it is called with a different argument, it will do what
+the first `EXPECT_CALL` says.
+
+Note that the order of the two `EXPECT_CALL`s is important, as a newer
+`EXPECT_CALL` takes precedence over an older one.
+
+For more on uninteresting calls, nice mocks, and strict mocks, read
+["The Nice, the Strict, and the Naggy"](#NiceStrictNaggy).
+
+### Ignoring Uninteresting Arguments {#ParameterlessExpectations}
+
+If your test doesn't care about the parameters (it only cares about the number
+or order of calls), you can often simply omit the parameter list:
+
+```cpp
+  // Expect foo.Bar( ... ) twice with any arguments.
+  EXPECT_CALL(foo, Bar).Times(2);
+
+  // Delegate to the given method whenever the factory is invoked.
+  ON_CALL(foo_factory, MakeFoo)
+      .WillByDefault(&BuildFooForTest);
+```
+
+This functionality is only available when a method is not overloaded; to prevent
+unexpected behavior it is a compilation error to try to set an expectation on a
+method where the specific overload is ambiguous. You can work around this by
+supplying a [simpler mock interface](#SimplerInterfaces) than the mocked class
+provides.
+
+This pattern is also useful when the arguments are interesting, but match logic
+is substantially complex. You can leave the argument list unspecified and use
+SaveArg actions to [save the values for later verification](#SaveArgVerify). If
+you do that, you can easily differentiate calling the method the wrong number of
+times from calling it with the wrong arguments.
+
+### Expecting Ordered Calls {#OrderedCalls}
+
+Although an `EXPECT_CALL()` statement defined later takes precedence when gMock
+tries to match a function call with an expectation, by default calls don't have
+to happen in the order `EXPECT_CALL()` statements are written. For example, if
+the arguments match the matchers in the second `EXPECT_CALL()`, but not those in
+the first and third, then the second expectation will be used.
+
+If you would rather have all calls occur in the order of the expectations, put
+the `EXPECT_CALL()` statements in a block where you define a variable of type
+`InSequence`:
+
+```cpp
+using ::testing::_;
+using ::testing::InSequence;
+
+  {
+    InSequence s;
+
+    EXPECT_CALL(foo, DoThis(5));
+    EXPECT_CALL(bar, DoThat(_))
+        .Times(2);
+    EXPECT_CALL(foo, DoThis(6));
+  }
+```
+
+In this example, we expect a call to `foo.DoThis(5)`, followed by two calls to
+`bar.DoThat()` where the argument can be anything, which are in turn followed by
+a call to `foo.DoThis(6)`. If a call occurred out-of-order, gMock will report an
+error.
+
+### Expecting Partially Ordered Calls {#PartialOrder}
+
+Sometimes requiring everything to occur in a predetermined order can lead to
+brittle tests. For example, we may care about `A` occurring before both `B` and
+`C`, but aren't interested in the relative order of `B` and `C`. In this case,
+the test should reflect our real intent, instead of being overly constraining.
+
+gMock allows you to impose an arbitrary DAG (directed acyclic graph) on the
+calls. One way to express the DAG is to use the
+[After](cheat_sheet.md#AfterClause) clause of `EXPECT_CALL`.
+
+Another way is via the `InSequence()` clause (not the same as the `InSequence`
+class), which we borrowed from jMock 2. It's less flexible than `After()`, but
+more convenient when you have long chains of sequential calls, as it doesn't
+require you to come up with different names for the expectations in the chains.
+Here's how it works:
+
+If we view `EXPECT_CALL()` statements as nodes in a graph, and add an edge from
+node A to node B wherever A must occur before B, we can get a DAG. We use the
+term "sequence" to mean a directed path in this DAG. Now, if we decompose the
+DAG into sequences, we just need to know which sequences each `EXPECT_CALL()`
+belongs to in order to be able to reconstruct the original DAG.
+
+So, to specify the partial order on the expectations we need to do two things:
+first to define some `Sequence` objects, and then for each `EXPECT_CALL()` say
+which `Sequence` objects it is part of.
+
+Expectations in the same sequence must occur in the order they are written. For
+example,
+
+```cpp
+using ::testing::Sequence;
+...
+  Sequence s1, s2;
+
+  EXPECT_CALL(foo, A())
+      .InSequence(s1, s2);
+  EXPECT_CALL(bar, B())
+      .InSequence(s1);
+  EXPECT_CALL(bar, C())
+      .InSequence(s2);
+  EXPECT_CALL(foo, D())
+      .InSequence(s2);
+```
+
+specifies the following DAG (where `s1` is `A -> B`, and `s2` is `A -> C -> D`):
+
+```text
+       +---> B
+       |
+  A ---|
+       |
+        +---> C ---> D
+```
+
+This means that A must occur before B and C, and C must occur before D. There's
+no restriction about the order other than these.
+
+### Controlling When an Expectation Retires
+
+When a mock method is called, gMock only considers expectations that are still
+active. An expectation is active when created, and becomes inactive (aka
+*retires*) when a call that has to occur later has occurred. For example, in
+
+```cpp
+using ::testing::_;
+using ::testing::Sequence;
+...
+  Sequence s1, s2;
+
+  EXPECT_CALL(log, Log(WARNING, _, "File too large."))      // #1
+      .Times(AnyNumber())
+      .InSequence(s1, s2);
+  EXPECT_CALL(log, Log(WARNING, _, "Data set is empty."))   // #2
+      .InSequence(s1);
+  EXPECT_CALL(log, Log(WARNING, _, "User not found."))      // #3
+      .InSequence(s2);
+```
+
+as soon as either #2 or #3 is matched, #1 will retire. If a warning `"File too
+large."` is logged after this, it will be an error.
+
+Note that an expectation doesn't retire automatically when it's saturated. For
+example,
+
+```cpp
+using ::testing::_;
+...
+  EXPECT_CALL(log, Log(WARNING, _, _));                     // #1
+  EXPECT_CALL(log, Log(WARNING, _, "File too large."));     // #2
+```
+
+says that there will be exactly one warning with the message `"File too
+large."`. If the second warning contains this message too, #2 will match again
+and result in an upper-bound-violated error.
+
+If this is not what you want, you can ask an expectation to retire as soon as it
+becomes saturated:
+
+```cpp
+using ::testing::_;
+...
+  EXPECT_CALL(log, Log(WARNING, _, _));                     // #1
+  EXPECT_CALL(log, Log(WARNING, _, "File too large."))      // #2
+      .RetiresOnSaturation();
+```
+
+Here #2 can be used only once, so if you have two warnings with the message
+`"File too large."`, the first will match #2 and the second will match #1 -
+there will be no error.
+
+## Using Actions
+
+### Returning References from Mock Methods
+
+If a mock function's return type is a reference, you need to use `ReturnRef()`
+instead of `Return()` to return a result:
+
+```cpp
+using ::testing::ReturnRef;
+
+class MockFoo : public Foo {
+ public:
+  MOCK_METHOD(Bar&, GetBar, (), (override));
+};
+...
+  MockFoo foo;
+  Bar bar;
+  EXPECT_CALL(foo, GetBar())
+      .WillOnce(ReturnRef(bar));
+...
+```
+
+### Returning Live Values from Mock Methods
+
+The `Return(x)` action saves a copy of `x` when the action is created, and
+always returns the same value whenever it's executed. Sometimes you may want to
+instead return the *live* value of `x` (i.e. its value at the time when the
+action is *executed*.). Use either `ReturnRef()` or `ReturnPointee()` for this
+purpose.
+
+If the mock function's return type is a reference, you can do it using
+`ReturnRef(x)`, as shown in the previous recipe ("Returning References from Mock
+Methods"). However, gMock doesn't let you use `ReturnRef()` in a mock function
+whose return type is not a reference, as doing that usually indicates a user
+error. So, what shall you do?
+
+Though you may be tempted, DO NOT use `std::ref()`:
+
+```cpp
+using testing::Return;
+
+class MockFoo : public Foo {
+ public:
+  MOCK_METHOD(int, GetValue, (), (override));
+};
+...
+  int x = 0;
+  MockFoo foo;
+  EXPECT_CALL(foo, GetValue())
+      .WillRepeatedly(Return(std::ref(x)));  // Wrong!
+  x = 42;
+  EXPECT_EQ(42, foo.GetValue());
+```
+
+Unfortunately, it doesn't work here. The above code will fail with error:
+
+```text
+Value of: foo.GetValue()
+  Actual: 0
+Expected: 42
+```
+
+The reason is that `Return(*value*)` converts `value` to the actual return type
+of the mock function at the time when the action is *created*, not when it is
+*executed*. (This behavior was chosen for the action to be safe when `value` is
+a proxy object that references some temporary objects.) As a result,
+`std::ref(x)` is converted to an `int` value (instead of a `const int&`) when
+the expectation is set, and `Return(std::ref(x))` will always return 0.
+
+`ReturnPointee(pointer)` was provided to solve this problem specifically. It
+returns the value pointed to by `pointer` at the time the action is *executed*:
+
+```cpp
+using testing::ReturnPointee;
+...
+  int x = 0;
+  MockFoo foo;
+  EXPECT_CALL(foo, GetValue())
+      .WillRepeatedly(ReturnPointee(&x));  // Note the & here.
+  x = 42;
+  EXPECT_EQ(42, foo.GetValue());  // This will succeed now.
+```
+
+### Combining Actions
+
+Want to do more than one thing when a function is called? That's fine. `DoAll()`
+allow you to do sequence of actions every time. Only the return value of the
+last action in the sequence will be used.
+
+```cpp
+using ::testing::_;
+using ::testing::DoAll;
+
+class MockFoo : public Foo {
+ public:
+  MOCK_METHOD(bool, Bar, (int n), (override));
+};
+...
+  EXPECT_CALL(foo, Bar(_))
+      .WillOnce(DoAll(action_1,
+                      action_2,
+                      ...
+                      action_n));
+```
+
+### Verifying Complex Arguments {#SaveArgVerify}
+
+If you want to verify that a method is called with a particular argument but the
+match criteria is complex, it can be difficult to distinguish between
+cardinality failures (calling the method the wrong number of times) and argument
+match failures. Similarly, if you are matching multiple parameters, it may not
+be easy to distinguishing which argument failed to match. For example:
+
+```cpp
+  // Not ideal: this could fail because of a problem with arg1 or arg2, or maybe
+  // just the method wasn't called.
+  EXPECT_CALL(foo, SendValues(_, ElementsAre(1, 4, 4, 7), EqualsProto( ... )));
+```
+
+You can instead save the arguments and test them individually:
+
+```cpp
+  EXPECT_CALL(foo, SendValues)
+      .WillOnce(DoAll(SaveArg<1>(&actual_array), SaveArg<2>(&actual_proto)));
+  ... run the test
+  EXPECT_THAT(actual_array, ElementsAre(1, 4, 4, 7));
+  EXPECT_THAT(actual_proto, EqualsProto( ... ));
+```
+
+### Mocking Side Effects {#MockingSideEffects}
+
+Sometimes a method exhibits its effect not via returning a value but via side
+effects. For example, it may change some global state or modify an output
+argument. To mock side effects, in general you can define your own action by
+implementing `::testing::ActionInterface`.
+
+If all you need to do is to change an output argument, the built-in
+`SetArgPointee()` action is convenient:
+
+```cpp
+using ::testing::_;
+using ::testing::SetArgPointee;
+
+class MockMutator : public Mutator {
+ public:
+  MOCK_METHOD(void, Mutate, (bool mutate, int* value), (override));
+  ...
+}
+...
+  MockMutator mutator;
+  EXPECT_CALL(mutator, Mutate(true, _))
+      .WillOnce(SetArgPointee<1>(5));
+```
+
+In this example, when `mutator.Mutate()` is called, we will assign 5 to the
+`int` variable pointed to by argument #1 (0-based).
+
+`SetArgPointee()` conveniently makes an internal copy of the value you pass to
+it, removing the need to keep the value in scope and alive. The implication
+however is that the value must have a copy constructor and assignment operator.
+
+If the mock method also needs to return a value as well, you can chain
+`SetArgPointee()` with `Return()` using `DoAll()`, remembering to put the
+`Return()` statement last:
+
+```cpp
+using ::testing::_;
+using ::testing::Return;
+using ::testing::SetArgPointee;
+
+class MockMutator : public Mutator {
+ public:
+  ...
+  MOCK_METHOD(bool, MutateInt, (int* value), (override));
+}
+...
+  MockMutator mutator;
+  EXPECT_CALL(mutator, MutateInt(_))
+      .WillOnce(DoAll(SetArgPointee<0>(5),
+                      Return(true)));
+```
+
+Note, however, that if you use the `ReturnOKWith()` method, it will override the
+values provided by `SetArgPointee()` in the response parameters of your function
+call.
+
+If the output argument is an array, use the `SetArrayArgument(first, last)`
+action instead. It copies the elements in source range `[first, last)` to the
+array pointed to by the `N`-th (0-based) argument:
+
+```cpp
+using ::testing::NotNull;
+using ::testing::SetArrayArgument;
+
+class MockArrayMutator : public ArrayMutator {
+ public:
+  MOCK_METHOD(void, Mutate, (int* values, int num_values), (override));
+  ...
+}
+...
+  MockArrayMutator mutator;
+  int values[5] = {1, 2, 3, 4, 5};
+  EXPECT_CALL(mutator, Mutate(NotNull(), 5))
+      .WillOnce(SetArrayArgument<0>(values, values + 5));
+```
+
+This also works when the argument is an output iterator:
+
+```cpp
+using ::testing::_;
+using ::testing::SetArrayArgument;
+
+class MockRolodex : public Rolodex {
+ public:
+  MOCK_METHOD(void, GetNames, (std::back_insert_iterator>),
+              (override));
+  ...
+}
+...
+  MockRolodex rolodex;
+  vector names;
+  names.push_back("George");
+  names.push_back("John");
+  names.push_back("Thomas");
+  EXPECT_CALL(rolodex, GetNames(_))
+      .WillOnce(SetArrayArgument<0>(names.begin(), names.end()));
+```
+
+### Changing a Mock Object's Behavior Based on the State
+
+If you expect a call to change the behavior of a mock object, you can use
+`::testing::InSequence` to specify different behaviors before and after the
+call:
+
+```cpp
+using ::testing::InSequence;
+using ::testing::Return;
+
+...
+  {
+     InSequence seq;
+     EXPECT_CALL(my_mock, IsDirty())
+         .WillRepeatedly(Return(true));
+     EXPECT_CALL(my_mock, Flush());
+     EXPECT_CALL(my_mock, IsDirty())
+         .WillRepeatedly(Return(false));
+  }
+  my_mock.FlushIfDirty();
+```
+
+This makes `my_mock.IsDirty()` return `true` before `my_mock.Flush()` is called
+and return `false` afterwards.
+
+If the behavior change is more complex, you can store the effects in a variable
+and make a mock method get its return value from that variable:
+
+```cpp
+using ::testing::_;
+using ::testing::SaveArg;
+using ::testing::Return;
+
+ACTION_P(ReturnPointee, p) { return *p; }
+...
+  int previous_value = 0;
+  EXPECT_CALL(my_mock, GetPrevValue)
+      .WillRepeatedly(ReturnPointee(&previous_value));
+  EXPECT_CALL(my_mock, UpdateValue)
+      .WillRepeatedly(SaveArg<0>(&previous_value));
+  my_mock.DoSomethingToUpdateValue();
+```
+
+Here `my_mock.GetPrevValue()` will always return the argument of the last
+`UpdateValue()` call.
+
+### Setting the Default Value for a Return Type {#DefaultValue}
+
+If a mock method's return type is a built-in C++ type or pointer, by default it
+will return 0 when invoked. Also, in C++ 11 and above, a mock method whose
+return type has a default constructor will return a default-constructed value by
+default. You only need to specify an action if this default value doesn't work
+for you.
+
+Sometimes, you may want to change this default value, or you may want to specify
+a default value for types gMock doesn't know about. You can do this using the
+`::testing::DefaultValue` class template:
+
+```cpp
+using ::testing::DefaultValue;
+
+class MockFoo : public Foo {
+ public:
+  MOCK_METHOD(Bar, CalculateBar, (), (override));
+};
+
+
+...
+  Bar default_bar;
+  // Sets the default return value for type Bar.
+  DefaultValue::Set(default_bar);
+
+  MockFoo foo;
+
+  // We don't need to specify an action here, as the default
+  // return value works for us.
+  EXPECT_CALL(foo, CalculateBar());
+
+  foo.CalculateBar();  // This should return default_bar.
+
+  // Unsets the default return value.
+  DefaultValue::Clear();
+```
+
+Please note that changing the default value for a type can make your tests hard
+to understand. We recommend you to use this feature judiciously. For example,
+you may want to make sure the `Set()` and `Clear()` calls are right next to the
+code that uses your mock.
+
+### Setting the Default Actions for a Mock Method
+
+You've learned how to change the default value of a given type. However, this
+may be too coarse for your purpose: perhaps you have two mock methods with the
+same return type and you want them to have different behaviors. The `ON_CALL()`
+macro allows you to customize your mock's behavior at the method level:
+
+```cpp
+using ::testing::_;
+using ::testing::AnyNumber;
+using ::testing::Gt;
+using ::testing::Return;
+...
+  ON_CALL(foo, Sign(_))
+      .WillByDefault(Return(-1));
+  ON_CALL(foo, Sign(0))
+      .WillByDefault(Return(0));
+  ON_CALL(foo, Sign(Gt(0)))
+      .WillByDefault(Return(1));
+
+  EXPECT_CALL(foo, Sign(_))
+      .Times(AnyNumber());
+
+  foo.Sign(5);   // This should return 1.
+  foo.Sign(-9);  // This should return -1.
+  foo.Sign(0);   // This should return 0.
+```
+
+As you may have guessed, when there are more than one `ON_CALL()` statements,
+the newer ones in the order take precedence over the older ones. In other words,
+the **last** one that matches the function arguments will be used. This matching
+order allows you to set up the common behavior in a mock object's constructor or
+the test fixture's set-up phase and specialize the mock's behavior later.
+
+Note that both `ON_CALL` and `EXPECT_CALL` have the same "later statements take
+precedence" rule, but they don't interact. That is, `EXPECT_CALL`s have their
+own precedence order distinct from the `ON_CALL` precedence order.
+
+### Using Functions/Methods/Functors/Lambdas as Actions {#FunctionsAsActions}
+
+If the built-in actions don't suit you, you can use an existing callable
+(function, `std::function`, method, functor, lambda) as an action.
+
+
+
+```cpp
+using ::testing::_; using ::testing::Invoke;
+
+class MockFoo : public Foo {
+ public:
+  MOCK_METHOD(int, Sum, (int x, int y), (override));
+  MOCK_METHOD(bool, ComplexJob, (int x), (override));
+};
+
+int CalculateSum(int x, int y) { return x + y; }
+int Sum3(int x, int y, int z) { return x + y + z; }
+
+class Helper {
+ public:
+  bool ComplexJob(int x);
+};
+
+...
+  MockFoo foo;
+  Helper helper;
+  EXPECT_CALL(foo, Sum(_, _))
+      .WillOnce(&CalculateSum)
+      .WillRepeatedly(Invoke(NewPermanentCallback(Sum3, 1)));
+  EXPECT_CALL(foo, ComplexJob(_))
+      .WillOnce(Invoke(&helper, &Helper::ComplexJob))
+      .WillOnce([] { return true; })
+      .WillRepeatedly([](int x) { return x > 0; });
+
+  foo.Sum(5, 6);         // Invokes CalculateSum(5, 6).
+  foo.Sum(2, 3);         // Invokes Sum3(1, 2, 3).
+  foo.ComplexJob(10);    // Invokes helper.ComplexJob(10).
+  foo.ComplexJob(-1);    // Invokes the inline lambda.
+```
+
+The only requirement is that the type of the function, etc must be *compatible*
+with the signature of the mock function, meaning that the latter's arguments (if
+it takes any) can be implicitly converted to the corresponding arguments of the
+former, and the former's return type can be implicitly converted to that of the
+latter. So, you can invoke something whose type is *not* exactly the same as the
+mock function, as long as it's safe to do so - nice, huh?
+
+**`Note:`{.escaped}**
+
+*   The action takes ownership of the callback and will delete it when the
+    action itself is destructed.
+*   If the type of a callback is derived from a base callback type `C`, you need
+    to implicitly cast it to `C` to resolve the overloading, e.g.
+
+    ```cpp
+    using ::testing::Invoke;
+    ...
+      ResultCallback* is_ok = ...;
+      ... Invoke(is_ok) ...;  // This works.
+
+      BlockingClosure* done = new BlockingClosure;
+      ... Invoke(implicit_cast(done)) ...;  // The cast is necessary.
+    ```
+
+### Using Functions with Extra Info as Actions
+
+The function or functor you call using `Invoke()` must have the same number of
+arguments as the mock function you use it for. Sometimes you may have a function
+that takes more arguments, and you are willing to pass in the extra arguments
+yourself to fill the gap. You can do this in gMock using callbacks with
+pre-bound arguments. Here's an example:
+
+```cpp
+using ::testing::Invoke;
+
+class MockFoo : public Foo {
+ public:
+  MOCK_METHOD(char, DoThis, (int n), (override));
+};
+
+char SignOfSum(int x, int y) {
+  const int sum = x + y;
+  return (sum > 0) ? '+' : (sum < 0) ? '-' : '0';
+}
+
+TEST_F(FooTest, Test) {
+  MockFoo foo;
+
+  EXPECT_CALL(foo, DoThis(2))
+      .WillOnce(Invoke(NewPermanentCallback(SignOfSum, 5)));
+  EXPECT_EQ('+', foo.DoThis(2));  // Invokes SignOfSum(5, 2).
+}
+```
+
+### Invoking a Function/Method/Functor/Lambda/Callback Without Arguments
+
+`Invoke()` passes the mock function's arguments to the function, etc being
+invoked such that the callee has the full context of the call to work with. If
+the invoked function is not interested in some or all of the arguments, it can
+simply ignore them.
+
+Yet, a common pattern is that a test author wants to invoke a function without
+the arguments of the mock function. She could do that using a wrapper function
+that throws away the arguments before invoking an underlining nullary function.
+Needless to say, this can be tedious and obscures the intent of the test.
+
+There are two solutions to this problem. First, you can pass any callable of
+zero args as an action. Alternatively, use `InvokeWithoutArgs()`, which is like
+`Invoke()` except that it doesn't pass the mock function's arguments to the
+callee. Here's an example of each:
+
+```cpp
+using ::testing::_;
+using ::testing::InvokeWithoutArgs;
+
+class MockFoo : public Foo {
+ public:
+  MOCK_METHOD(bool, ComplexJob, (int n), (override));
+};
+
+bool Job1() { ... }
+bool Job2(int n, char c) { ... }
+
+...
+  MockFoo foo;
+  EXPECT_CALL(foo, ComplexJob(_))
+      .WillOnce([] { Job1(); });
+      .WillOnce(InvokeWithoutArgs(NewPermanentCallback(Job2, 5, 'a')));
+
+  foo.ComplexJob(10);  // Invokes Job1().
+  foo.ComplexJob(20);  // Invokes Job2(5, 'a').
+```
+
+**`Note:`{.escaped}**
+
+*   The action takes ownership of the callback and will delete it when the
+    action itself is destructed.
+*   If the type of a callback is derived from a base callback type `C`, you need
+    to implicitly cast it to `C` to resolve the overloading, e.g.
+
+    ```cpp
+    using ::testing::InvokeWithoutArgs;
+    ...
+      ResultCallback* is_ok = ...;
+      ... InvokeWithoutArgs(is_ok) ...;  // This works.
+
+      BlockingClosure* done = ...;
+      ... InvokeWithoutArgs(implicit_cast(done)) ...;
+      // The cast is necessary.
+    ```
+
+### Invoking an Argument of the Mock Function
+
+Sometimes a mock function will receive a function pointer, a functor (in other
+words, a "callable") as an argument, e.g.
+
+```cpp
+class MockFoo : public Foo {
+ public:
+  MOCK_METHOD(bool, DoThis, (int n, (ResultCallback1* callback)),
+              (override));
+};
+```
+
+and you may want to invoke this callable argument:
+
+```cpp
+using ::testing::_;
+...
+  MockFoo foo;
+  EXPECT_CALL(foo, DoThis(_, _))
+      .WillOnce(...);
+      // Will execute callback->Run(5), where callback is the
+      // second argument DoThis() receives.
+```
+
+NOTE: The section below is legacy documentation from before C++ had lambdas:
+
+Arghh, you need to refer to a mock function argument but C++ has no lambda
+(yet), so you have to define your own action. :-( Or do you really?
+
+Well, gMock has an action to solve *exactly* this problem:
+
+```cpp
+InvokeArgument(arg_1, arg_2, ..., arg_m)
+```
+
+will invoke the `N`-th (0-based) argument the mock function receives, with
+`arg_1`, `arg_2`, ..., and `arg_m`. No matter if the argument is a function
+pointer, a functor, or a callback. gMock handles them all.
+
+With that, you could write:
+
+```cpp
+using ::testing::_;
+using ::testing::InvokeArgument;
+...
+  EXPECT_CALL(foo, DoThis(_, _))
+      .WillOnce(InvokeArgument<1>(5));
+      // Will execute callback->Run(5), where callback is the
+      // second argument DoThis() receives.
+```
+
+What if the callable takes an argument by reference? No problem - just wrap it
+inside `std::ref()`:
+
+```cpp
+  ...
+  MOCK_METHOD(bool, Bar,
+              ((ResultCallback2* callback)),
+              (override));
+  ...
+  using ::testing::_;
+  using ::testing::InvokeArgument;
+  ...
+  MockFoo foo;
+  Helper helper;
+  ...
+  EXPECT_CALL(foo, Bar(_))
+      .WillOnce(InvokeArgument<0>(5, std::ref(helper)));
+      // std::ref(helper) guarantees that a reference to helper, not a copy of
+      // it, will be passed to the callback.
+```
+
+What if the callable takes an argument by reference and we do **not** wrap the
+argument in `std::ref()`? Then `InvokeArgument()` will *make a copy* of the
+argument, and pass a *reference to the copy*, instead of a reference to the
+original value, to the callable. This is especially handy when the argument is a
+temporary value:
+
+```cpp
+  ...
+  MOCK_METHOD(bool, DoThat, (bool (*f)(const double& x, const string& s)),
+              (override));
+  ...
+  using ::testing::_;
+  using ::testing::InvokeArgument;
+  ...
+  MockFoo foo;
+  ...
+  EXPECT_CALL(foo, DoThat(_))
+      .WillOnce(InvokeArgument<0>(5.0, string("Hi")));
+      // Will execute (*f)(5.0, string("Hi")), where f is the function pointer
+      // DoThat() receives.  Note that the values 5.0 and string("Hi") are
+      // temporary and dead once the EXPECT_CALL() statement finishes.  Yet
+      // it's fine to perform this action later, since a copy of the values
+      // are kept inside the InvokeArgument action.
+```
+
+### Ignoring an Action's Result
+
+Sometimes you have an action that returns *something*, but you need an action
+that returns `void` (perhaps you want to use it in a mock function that returns
+`void`, or perhaps it needs to be used in `DoAll()` and it's not the last in the
+list). `IgnoreResult()` lets you do that. For example:
+
+```cpp
+using ::testing::_;
+using ::testing::DoAll;
+using ::testing::IgnoreResult;
+using ::testing::Return;
+
+int Process(const MyData& data);
+string DoSomething();
+
+class MockFoo : public Foo {
+ public:
+  MOCK_METHOD(void, Abc, (const MyData& data), (override));
+  MOCK_METHOD(bool, Xyz, (), (override));
+};
+
+  ...
+  MockFoo foo;
+  EXPECT_CALL(foo, Abc(_))
+      // .WillOnce(Invoke(Process));
+      // The above line won't compile as Process() returns int but Abc() needs
+      // to return void.
+      .WillOnce(IgnoreResult(Process));
+  EXPECT_CALL(foo, Xyz())
+      .WillOnce(DoAll(IgnoreResult(DoSomething),
+                      // Ignores the string DoSomething() returns.
+                      Return(true)));
+```
+
+Note that you **cannot** use `IgnoreResult()` on an action that already returns
+`void`. Doing so will lead to ugly compiler errors.
+
+### Selecting an Action's Arguments {#SelectingArgs}
+
+Say you have a mock function `Foo()` that takes seven arguments, and you have a
+custom action that you want to invoke when `Foo()` is called. Trouble is, the
+custom action only wants three arguments:
+
+```cpp
+using ::testing::_;
+using ::testing::Invoke;
+...
+  MOCK_METHOD(bool, Foo,
+              (bool visible, const string& name, int x, int y,
+               (const map>), double& weight, double min_weight,
+               double max_wight));
+...
+bool IsVisibleInQuadrant1(bool visible, int x, int y) {
+  return visible && x >= 0 && y >= 0;
+}
+...
+  EXPECT_CALL(mock, Foo)
+      .WillOnce(Invoke(IsVisibleInQuadrant1));  // Uh, won't compile. :-(
+```
+
+To please the compiler God, you need to define an "adaptor" that has the same
+signature as `Foo()` and calls the custom action with the right arguments:
+
+```cpp
+using ::testing::_;
+using ::testing::Invoke;
+...
+bool MyIsVisibleInQuadrant1(bool visible, const string& name, int x, int y,
+                            const map, double>& weight,
+                            double min_weight, double max_wight) {
+  return IsVisibleInQuadrant1(visible, x, y);
+}
+...
+  EXPECT_CALL(mock, Foo)
+      .WillOnce(Invoke(MyIsVisibleInQuadrant1));  // Now it works.
+```
+
+But isn't this awkward?
+
+gMock provides a generic *action adaptor*, so you can spend your time minding
+more important business than writing your own adaptors. Here's the syntax:
+
+```cpp
+WithArgs(action)
+```
+
+creates an action that passes the arguments of the mock function at the given
+indices (0-based) to the inner `action` and performs it. Using `WithArgs`, our
+original example can be written as:
+
+```cpp
+using ::testing::_;
+using ::testing::Invoke;
+using ::testing::WithArgs;
+...
+  EXPECT_CALL(mock, Foo)
+      .WillOnce(WithArgs<0, 2, 3>(Invoke(IsVisibleInQuadrant1)));  // No need to define your own adaptor.
+```
+
+For better readability, gMock also gives you:
+
+*   `WithoutArgs(action)` when the inner `action` takes *no* argument, and
+*   `WithArg(action)` (no `s` after `Arg`) when the inner `action` takes
+    *one* argument.
+
+As you may have realized, `InvokeWithoutArgs(...)` is just syntactic sugar for
+`WithoutArgs(Invoke(...))`.
+
+Here are more tips:
+
+*   The inner action used in `WithArgs` and friends does not have to be
+    `Invoke()` -- it can be anything.
+*   You can repeat an argument in the argument list if necessary, e.g.
+    `WithArgs<2, 3, 3, 5>(...)`.
+*   You can change the order of the arguments, e.g. `WithArgs<3, 2, 1>(...)`.
+*   The types of the selected arguments do *not* have to match the signature of
+    the inner action exactly. It works as long as they can be implicitly
+    converted to the corresponding arguments of the inner action. For example,
+    if the 4-th argument of the mock function is an `int` and `my_action` takes
+    a `double`, `WithArg<4>(my_action)` will work.
+
+### Ignoring Arguments in Action Functions
+
+The [selecting-an-action's-arguments](#SelectingArgs) recipe showed us one way
+to make a mock function and an action with incompatible argument lists fit
+together. The downside is that wrapping the action in `WithArgs<...>()` can get
+tedious for people writing the tests.
+
+If you are defining a function (or method, functor, lambda, callback) to be used
+with `Invoke*()`, and you are not interested in some of its arguments, an
+alternative to `WithArgs` is to declare the uninteresting arguments as `Unused`.
+This makes the definition less cluttered and less fragile in case the types of
+the uninteresting arguments change. It could also increase the chance the action
+function can be reused. For example, given
+
+```cpp
+ public:
+  MOCK_METHOD(double, Foo, double(const string& label, double x, double y),
+              (override));
+  MOCK_METHOD(double, Bar, (int index, double x, double y), (override));
+```
+
+instead of
+
+```cpp
+using ::testing::_;
+using ::testing::Invoke;
+
+double DistanceToOriginWithLabel(const string& label, double x, double y) {
+  return sqrt(x*x + y*y);
+}
+double DistanceToOriginWithIndex(int index, double x, double y) {
+  return sqrt(x*x + y*y);
+}
+...
+  EXPECT_CALL(mock, Foo("abc", _, _))
+      .WillOnce(Invoke(DistanceToOriginWithLabel));
+  EXPECT_CALL(mock, Bar(5, _, _))
+      .WillOnce(Invoke(DistanceToOriginWithIndex));
+```
+
+you could write
+
+```cpp
+using ::testing::_;
+using ::testing::Invoke;
+using ::testing::Unused;
+
+double DistanceToOrigin(Unused, double x, double y) {
+  return sqrt(x*x + y*y);
+}
+...
+  EXPECT_CALL(mock, Foo("abc", _, _))
+      .WillOnce(Invoke(DistanceToOrigin));
+  EXPECT_CALL(mock, Bar(5, _, _))
+      .WillOnce(Invoke(DistanceToOrigin));
+```
+
+### Sharing Actions
+
+Just like matchers, a gMock action object consists of a pointer to a ref-counted
+implementation object. Therefore copying actions is also allowed and very
+efficient. When the last action that references the implementation object dies,
+the implementation object will be deleted.
+
+If you have some complex action that you want to use again and again, you may
+not have to build it from scratch everytime. If the action doesn't have an
+internal state (i.e. if it always does the same thing no matter how many times
+it has been called), you can assign it to an action variable and use that
+variable repeatedly. For example:
+
+```cpp
+using ::testing::Action;
+using ::testing::DoAll;
+using ::testing::Return;
+using ::testing::SetArgPointee;
+...
+  Action set_flag = DoAll(SetArgPointee<0>(5),
+                                      Return(true));
+  ... use set_flag in .WillOnce() and .WillRepeatedly() ...
+```
+
+However, if the action has its own state, you may be surprised if you share the
+action object. Suppose you have an action factory `IncrementCounter(init)` which
+creates an action that increments and returns a counter whose initial value is
+`init`, using two actions created from the same expression and using a shared
+action will exhibit different behaviors. Example:
+
+```cpp
+  EXPECT_CALL(foo, DoThis())
+      .WillRepeatedly(IncrementCounter(0));
+  EXPECT_CALL(foo, DoThat())
+      .WillRepeatedly(IncrementCounter(0));
+  foo.DoThis();  // Returns 1.
+  foo.DoThis();  // Returns 2.
+  foo.DoThat();  // Returns 1 - Blah() uses a different
+                 // counter than Bar()'s.
+```
+
+versus
+
+```cpp
+using ::testing::Action;
+...
+  Action increment = IncrementCounter(0);
+  EXPECT_CALL(foo, DoThis())
+      .WillRepeatedly(increment);
+  EXPECT_CALL(foo, DoThat())
+      .WillRepeatedly(increment);
+  foo.DoThis();  // Returns 1.
+  foo.DoThis();  // Returns 2.
+  foo.DoThat();  // Returns 3 - the counter is shared.
+```
+
+### Testing Asynchronous Behavior
+
+One oft-encountered problem with gMock is that it can be hard to test
+asynchronous behavior. Suppose you had a `EventQueue` class that you wanted to
+test, and you created a separate `EventDispatcher` interface so that you could
+easily mock it out. However, the implementation of the class fired all the
+events on a background thread, which made test timings difficult. You could just
+insert `sleep()` statements and hope for the best, but that makes your test
+behavior nondeterministic. A better way is to use gMock actions and
+`Notification` objects to force your asynchronous test to behave synchronously.
+
+```cpp
+using ::testing::DoAll;
+using ::testing::InvokeWithoutArgs;
+using ::testing::Return;
+
+class MockEventDispatcher : public EventDispatcher {
+  MOCK_METHOD(bool, DispatchEvent, (int32), (override));
+};
+
+ACTION_P(Notify, notification) {
+  notification->Notify();
+}
+
+TEST(EventQueueTest, EnqueueEventTest) {
+  MockEventDispatcher mock_event_dispatcher;
+  EventQueue event_queue(&mock_event_dispatcher);
+
+  const int32 kEventId = 321;
+  absl::Notification done;
+  EXPECT_CALL(mock_event_dispatcher, DispatchEvent(kEventId))
+      .WillOnce(Notify(&done));
+
+  event_queue.EnqueueEvent(kEventId);
+  done.WaitForNotification();
+}
+```
+
+In the example above, we set our normal gMock expectations, but then add an
+additional action to notify the `Notification` object. Now we can just call
+`Notification::WaitForNotification()` in the main thread to wait for the
+asynchronous call to finish. After that, our test suite is complete and we can
+safely exit.
+
+Note: this example has a downside: namely, if the expectation is not satisfied,
+our test will run forever. It will eventually time-out and fail, but it will
+take longer and be slightly harder to debug. To alleviate this problem, you can
+use `WaitForNotificationWithTimeout(ms)` instead of `WaitForNotification()`.
+
+## Misc Recipes on Using gMock
+
+### Mocking Methods That Use Move-Only Types
+
+C++11 introduced *move-only types*. A move-only-typed value can be moved from
+one object to another, but cannot be copied. `std::unique_ptr` is probably
+the most commonly used move-only type.
+
+Mocking a method that takes and/or returns move-only types presents some
+challenges, but nothing insurmountable. This recipe shows you how you can do it.
+Note that the support for move-only method arguments was only introduced to
+gMock in April 2017; in older code, you may find more complex
+[workarounds](#LegacyMoveOnly) for lack of this feature.
+
+Let’s say we are working on a fictional project that lets one post and share
+snippets called “buzzes”. Your code uses these types:
+
+```cpp
+enum class AccessLevel { kInternal, kPublic };
+
+class Buzz {
+ public:
+  explicit Buzz(AccessLevel access) { ... }
+  ...
+};
+
+class Buzzer {
+ public:
+  virtual ~Buzzer() {}
+  virtual std::unique_ptr MakeBuzz(StringPiece text) = 0;
+  virtual bool ShareBuzz(std::unique_ptr buzz, int64_t timestamp) = 0;
+  ...
+};
+```
+
+A `Buzz` object represents a snippet being posted. A class that implements the
+`Buzzer` interface is capable of creating and sharing `Buzz`es. Methods in
+`Buzzer` may return a `unique_ptr` or take a `unique_ptr`. Now we
+need to mock `Buzzer` in our tests.
+
+To mock a method that accepts or returns move-only types, you just use the
+familiar `MOCK_METHOD` syntax as usual:
+
+```cpp
+class MockBuzzer : public Buzzer {
+ public:
+  MOCK_METHOD(std::unique_ptr, MakeBuzz, (StringPiece text), (override));
+  MOCK_METHOD(bool, ShareBuzz, (std::unique_ptr buzz, int64_t timestamp),
+              (override));
+};
+```
+
+Now that we have the mock class defined, we can use it in tests. In the
+following code examples, we assume that we have defined a `MockBuzzer` object
+named `mock_buzzer_`:
+
+```cpp
+  MockBuzzer mock_buzzer_;
+```
+
+First let’s see how we can set expectations on the `MakeBuzz()` method, which
+returns a `unique_ptr`.
+
+As usual, if you set an expectation without an action (i.e. the `.WillOnce()` or
+`.WillRepeatedly()` clause), when that expectation fires, the default action for
+that method will be taken. Since `unique_ptr<>` has a default constructor that
+returns a null `unique_ptr`, that’s what you’ll get if you don’t specify an
+action:
+
+```cpp
+  // Use the default action.
+  EXPECT_CALL(mock_buzzer_, MakeBuzz("hello"));
+
+  // Triggers the previous EXPECT_CALL.
+  EXPECT_EQ(nullptr, mock_buzzer_.MakeBuzz("hello"));
+```
+
+If you are not happy with the default action, you can tweak it as usual; see
+[Setting Default Actions](#OnCall).
+
+If you just need to return a pre-defined move-only value, you can use the
+`Return(ByMove(...))` action:
+
+```cpp
+  // When this fires, the unique_ptr<> specified by ByMove(...) will
+  // be returned.
+  EXPECT_CALL(mock_buzzer_, MakeBuzz("world"))
+      .WillOnce(Return(ByMove(MakeUnique(AccessLevel::kInternal))));
+
+  EXPECT_NE(nullptr, mock_buzzer_.MakeBuzz("world"));
+```
+
+Note that `ByMove()` is essential here - if you drop it, the code won’t compile.
+
+Quiz time! What do you think will happen if a `Return(ByMove(...))` action is
+performed more than once (e.g. you write `...
+.WillRepeatedly(Return(ByMove(...)));`)? Come think of it, after the first time
+the action runs, the source value will be consumed (since it’s a move-only
+value), so the next time around, there’s no value to move from -- you’ll get a
+run-time error that `Return(ByMove(...))` can only be run once.
+
+If you need your mock method to do more than just moving a pre-defined value,
+remember that you can always use a lambda or a callable object, which can do
+pretty much anything you want:
+
+```cpp
+  EXPECT_CALL(mock_buzzer_, MakeBuzz("x"))
+      .WillRepeatedly([](StringPiece text) {
+        return MakeUnique(AccessLevel::kInternal);
+      });
+
+  EXPECT_NE(nullptr, mock_buzzer_.MakeBuzz("x"));
+  EXPECT_NE(nullptr, mock_buzzer_.MakeBuzz("x"));
+```
+
+Every time this `EXPECT_CALL` fires, a new `unique_ptr` will be created
+and returned. You cannot do this with `Return(ByMove(...))`.
+
+That covers returning move-only values; but how do we work with methods
+accepting move-only arguments? The answer is that they work normally, although
+some actions will not compile when any of method's arguments are move-only. You
+can always use `Return`, or a [lambda or functor](#FunctionsAsActions):
+
+```cpp
+  using ::testing::Unused;
+
+  EXPECT_CALL(mock_buzzer_, ShareBuzz(NotNull(), _)).WillOnce(Return(true));
+  EXPECT_TRUE(mock_buzzer_.ShareBuzz(MakeUnique(AccessLevel::kInternal)),
+              0);
+
+  EXPECT_CALL(mock_buzzer_, ShareBuzz(_, _)).WillOnce(
+      [](std::unique_ptr buzz, Unused) { return buzz != nullptr; });
+  EXPECT_FALSE(mock_buzzer_.ShareBuzz(nullptr, 0));
+```
+
+Many built-in actions (`WithArgs`, `WithoutArgs`,`DeleteArg`, `SaveArg`, ...)
+could in principle support move-only arguments, but the support for this is not
+implemented yet. If this is blocking you, please file a bug.
+
+A few actions (e.g. `DoAll`) copy their arguments internally, so they can never
+work with non-copyable objects; you'll have to use functors instead.
+
+#### Legacy workarounds for move-only types {#LegacyMoveOnly}
+
+Support for move-only function arguments was only introduced to gMock in April
+2017. In older code, you may encounter the following workaround for the lack of
+this feature (it is no longer necessary - we're including it just for
+reference):
+
+```cpp
+class MockBuzzer : public Buzzer {
+ public:
+  MOCK_METHOD(bool, DoShareBuzz, (Buzz* buzz, Time timestamp));
+  bool ShareBuzz(std::unique_ptr buzz, Time timestamp) override {
+    return DoShareBuzz(buzz.get(), timestamp);
+  }
+};
+```
+
+The trick is to delegate the `ShareBuzz()` method to a mock method (let’s call
+it `DoShareBuzz()`) that does not take move-only parameters. Then, instead of
+setting expectations on `ShareBuzz()`, you set them on the `DoShareBuzz()` mock
+method:
+
+```cpp
+  MockBuzzer mock_buzzer_;
+  EXPECT_CALL(mock_buzzer_, DoShareBuzz(NotNull(), _));
+
+  // When one calls ShareBuzz() on the MockBuzzer like this, the call is
+  // forwarded to DoShareBuzz(), which is mocked.  Therefore this statement
+  // will trigger the above EXPECT_CALL.
+  mock_buzzer_.ShareBuzz(MakeUnique(AccessLevel::kInternal), 0);
+```
+
+### Making the Compilation Faster
+
+Believe it or not, the *vast majority* of the time spent on compiling a mock
+class is in generating its constructor and destructor, as they perform
+non-trivial tasks (e.g. verification of the expectations). What's more, mock
+methods with different signatures have different types and thus their
+constructors/destructors need to be generated by the compiler separately. As a
+result, if you mock many different types of methods, compiling your mock class
+can get really slow.
+
+If you are experiencing slow compilation, you can move the definition of your
+mock class' constructor and destructor out of the class body and into a `.cc`
+file. This way, even if you `#include` your mock class in N files, the compiler
+only needs to generate its constructor and destructor once, resulting in a much
+faster compilation.
+
+Let's illustrate the idea using an example. Here's the definition of a mock
+class before applying this recipe:
+
+```cpp
+// File mock_foo.h.
+...
+class MockFoo : public Foo {
+ public:
+  // Since we don't declare the constructor or the destructor,
+  // the compiler will generate them in every translation unit
+  // where this mock class is used.
+
+  MOCK_METHOD(int, DoThis, (), (override));
+  MOCK_METHOD(bool, DoThat, (const char* str), (override));
+  ... more mock methods ...
+};
+```
+
+After the change, it would look like:
+
+```cpp
+// File mock_foo.h.
+...
+class MockFoo : public Foo {
+ public:
+  // The constructor and destructor are declared, but not defined, here.
+  MockFoo();
+  virtual ~MockFoo();
+
+  MOCK_METHOD(int, DoThis, (), (override));
+  MOCK_METHOD(bool, DoThat, (const char* str), (override));
+  ... more mock methods ...
+};
+```
+
+and
+
+```cpp
+// File mock_foo.cc.
+#include "path/to/mock_foo.h"
+
+// The definitions may appear trivial, but the functions actually do a
+// lot of things through the constructors/destructors of the member
+// variables used to implement the mock methods.
+MockFoo::MockFoo() {}
+MockFoo::~MockFoo() {}
+```
+
+### Forcing a Verification
+
+When it's being destroyed, your friendly mock object will automatically verify
+that all expectations on it have been satisfied, and will generate googletest
+failures if not. This is convenient as it leaves you with one less thing to
+worry about. That is, unless you are not sure if your mock object will be
+destroyed.
+
+How could it be that your mock object won't eventually be destroyed? Well, it
+might be created on the heap and owned by the code you are testing. Suppose
+there's a bug in that code and it doesn't delete the mock object properly - you
+could end up with a passing test when there's actually a bug.
+
+Using a heap checker is a good idea and can alleviate the concern, but its
+implementation is not 100% reliable. So, sometimes you do want to *force* gMock
+to verify a mock object before it is (hopefully) destructed. You can do this
+with `Mock::VerifyAndClearExpectations(&mock_object)`:
+
+```cpp
+TEST(MyServerTest, ProcessesRequest) {
+  using ::testing::Mock;
+
+  MockFoo* const foo = new MockFoo;
+  EXPECT_CALL(*foo, ...)...;
+  // ... other expectations ...
+
+  // server now owns foo.
+  MyServer server(foo);
+  server.ProcessRequest(...);
+
+  // In case that server's destructor will forget to delete foo,
+  // this will verify the expectations anyway.
+  Mock::VerifyAndClearExpectations(foo);
+}  // server is destroyed when it goes out of scope here.
+```
+
+**Tip:** The `Mock::VerifyAndClearExpectations()` function returns a `bool` to
+indicate whether the verification was successful (`true` for yes), so you can
+wrap that function call inside a `ASSERT_TRUE()` if there is no point going
+further when the verification has failed.
+
+### Using Check Points {#UsingCheckPoints}
+
+Sometimes you may want to "reset" a mock object at various check points in your
+test: at each check point, you verify that all existing expectations on the mock
+object have been satisfied, and then you set some new expectations on it as if
+it's newly created. This allows you to work with a mock object in "phases" whose
+sizes are each manageable.
+
+One such scenario is that in your test's `SetUp()` function, you may want to put
+the object you are testing into a certain state, with the help from a mock
+object. Once in the desired state, you want to clear all expectations on the
+mock, such that in the `TEST_F` body you can set fresh expectations on it.
+
+As you may have figured out, the `Mock::VerifyAndClearExpectations()` function
+we saw in the previous recipe can help you here. Or, if you are using
+`ON_CALL()` to set default actions on the mock object and want to clear the
+default actions as well, use `Mock::VerifyAndClear(&mock_object)` instead. This
+function does what `Mock::VerifyAndClearExpectations(&mock_object)` does and
+returns the same `bool`, **plus** it clears the `ON_CALL()` statements on
+`mock_object` too.
+
+Another trick you can use to achieve the same effect is to put the expectations
+in sequences and insert calls to a dummy "check-point" function at specific
+places. Then you can verify that the mock function calls do happen at the right
+time. For example, if you are exercising code:
+
+```cpp
+  Foo(1);
+  Foo(2);
+  Foo(3);
+```
+
+and want to verify that `Foo(1)` and `Foo(3)` both invoke `mock.Bar("a")`, but
+`Foo(2)` doesn't invoke anything. You can write:
+
+```cpp
+using ::testing::MockFunction;
+
+TEST(FooTest, InvokesBarCorrectly) {
+  MyMock mock;
+  // Class MockFunction has exactly one mock method.  It is named
+  // Call() and has type F.
+  MockFunction check;
+  {
+    InSequence s;
+
+    EXPECT_CALL(mock, Bar("a"));
+    EXPECT_CALL(check, Call("1"));
+    EXPECT_CALL(check, Call("2"));
+    EXPECT_CALL(mock, Bar("a"));
+  }
+  Foo(1);
+  check.Call("1");
+  Foo(2);
+  check.Call("2");
+  Foo(3);
+}
+```
+
+The expectation spec says that the first `Bar("a")` must happen before check
+point "1", the second `Bar("a")` must happen after check point "2", and nothing
+should happen between the two check points. The explicit check points make it
+easy to tell which `Bar("a")` is called by which call to `Foo()`.
+
+### Mocking Destructors
+
+Sometimes you want to make sure a mock object is destructed at the right time,
+e.g. after `bar->A()` is called but before `bar->B()` is called. We already know
+that you can specify constraints on the [order](#OrderedCalls) of mock function
+calls, so all we need to do is to mock the destructor of the mock function.
+
+This sounds simple, except for one problem: a destructor is a special function
+with special syntax and special semantics, and the `MOCK_METHOD` macro doesn't
+work for it:
+
+```cpp
+MOCK_METHOD(void, ~MockFoo, ());  // Won't compile!
+```
+
+The good news is that you can use a simple pattern to achieve the same effect.
+First, add a mock function `Die()` to your mock class and call it in the
+destructor, like this:
+
+```cpp
+class MockFoo : public Foo {
+  ...
+  // Add the following two lines to the mock class.
+  MOCK_METHOD(void, Die, ());
+  virtual ~MockFoo() { Die(); }
+};
+```
+
+(If the name `Die()` clashes with an existing symbol, choose another name.) Now,
+we have translated the problem of testing when a `MockFoo` object dies to
+testing when its `Die()` method is called:
+
+```cpp
+  MockFoo* foo = new MockFoo;
+  MockBar* bar = new MockBar;
+  ...
+  {
+    InSequence s;
+
+    // Expects *foo to die after bar->A() and before bar->B().
+    EXPECT_CALL(*bar, A());
+    EXPECT_CALL(*foo, Die());
+    EXPECT_CALL(*bar, B());
+  }
+```
+
+And that's that.
+
+### Using gMock and Threads {#UsingThreads}
+
+In a **unit** test, it's best if you could isolate and test a piece of code in a
+single-threaded context. That avoids race conditions and dead locks, and makes
+debugging your test much easier.
+
+Yet most programs are multi-threaded, and sometimes to test something we need to
+pound on it from more than one thread. gMock works for this purpose too.
+
+Remember the steps for using a mock:
+
+1.  Create a mock object `foo`.
+2.  Set its default actions and expectations using `ON_CALL()` and
+    `EXPECT_CALL()`.
+3.  The code under test calls methods of `foo`.
+4.  Optionally, verify and reset the mock.
+5.  Destroy the mock yourself, or let the code under test destroy it. The
+    destructor will automatically verify it.
+
+If you follow the following simple rules, your mocks and threads can live
+happily together:
+
+*   Execute your *test code* (as opposed to the code being tested) in *one*
+    thread. This makes your test easy to follow.
+*   Obviously, you can do step #1 without locking.
+*   When doing step #2 and #5, make sure no other thread is accessing `foo`.
+    Obvious too, huh?
+*   #3 and #4 can be done either in one thread or in multiple threads - anyway
+    you want. gMock takes care of the locking, so you don't have to do any -
+    unless required by your test logic.
+
+If you violate the rules (for example, if you set expectations on a mock while
+another thread is calling its methods), you get undefined behavior. That's not
+fun, so don't do it.
+
+gMock guarantees that the action for a mock function is done in the same thread
+that called the mock function. For example, in
+
+```cpp
+  EXPECT_CALL(mock, Foo(1))
+      .WillOnce(action1);
+  EXPECT_CALL(mock, Foo(2))
+      .WillOnce(action2);
+```
+
+if `Foo(1)` is called in thread 1 and `Foo(2)` is called in thread 2, gMock will
+execute `action1` in thread 1 and `action2` in thread 2.
+
+gMock does *not* impose a sequence on actions performed in different threads
+(doing so may create deadlocks as the actions may need to cooperate). This means
+that the execution of `action1` and `action2` in the above example *may*
+interleave. If this is a problem, you should add proper synchronization logic to
+`action1` and `action2` to make the test thread-safe.
+
+Also, remember that `DefaultValue` is a global resource that potentially
+affects *all* living mock objects in your program. Naturally, you won't want to
+mess with it from multiple threads or when there still are mocks in action.
+
+### Controlling How Much Information gMock Prints
+
+When gMock sees something that has the potential of being an error (e.g. a mock
+function with no expectation is called, a.k.a. an uninteresting call, which is
+allowed but perhaps you forgot to explicitly ban the call), it prints some
+warning messages, including the arguments of the function, the return value, and
+the stack trace. Hopefully this will remind you to take a look and see if there
+is indeed a problem.
+
+Sometimes you are confident that your tests are correct and may not appreciate
+such friendly messages. Some other times, you are debugging your tests or
+learning about the behavior of the code you are testing, and wish you could
+observe every mock call that happens (including argument values, the return
+value, and the stack trace). Clearly, one size doesn't fit all.
+
+You can control how much gMock tells you using the `--gmock_verbose=LEVEL`
+command-line flag, where `LEVEL` is a string with three possible values:
+
+*   `info`: gMock will print all informational messages, warnings, and errors
+    (most verbose). At this setting, gMock will also log any calls to the
+    `ON_CALL/EXPECT_CALL` macros. It will include a stack trace in
+    "uninteresting call" warnings.
+*   `warning`: gMock will print both warnings and errors (less verbose); it will
+    omit the stack traces in "uninteresting call" warnings. This is the default.
+*   `error`: gMock will print errors only (least verbose).
+
+Alternatively, you can adjust the value of that flag from within your tests like
+so:
+
+```cpp
+  ::testing::FLAGS_gmock_verbose = "error";
+```
+
+If you find gMock printing too many stack frames with its informational or
+warning messages, remember that you can control their amount with the
+`--gtest_stack_trace_depth=max_depth` flag.
+
+Now, judiciously use the right flag to enable gMock serve you better!
+
+### Gaining Super Vision into Mock Calls
+
+You have a test using gMock. It fails: gMock tells you some expectations aren't
+satisfied. However, you aren't sure why: Is there a typo somewhere in the
+matchers? Did you mess up the order of the `EXPECT_CALL`s? Or is the code under
+test doing something wrong? How can you find out the cause?
+
+Won't it be nice if you have X-ray vision and can actually see the trace of all
+`EXPECT_CALL`s and mock method calls as they are made? For each call, would you
+like to see its actual argument values and which `EXPECT_CALL` gMock thinks it
+matches? If you still need some help to figure out who made these calls, how
+about being able to see the complete stack trace at each mock call?
+
+You can unlock this power by running your test with the `--gmock_verbose=info`
+flag. For example, given the test program:
+
+```cpp
+#include "gmock/gmock.h"
+
+using testing::_;
+using testing::HasSubstr;
+using testing::Return;
+
+class MockFoo {
+ public:
+  MOCK_METHOD(void, F, (const string& x, const string& y));
+};
+
+TEST(Foo, Bar) {
+  MockFoo mock;
+  EXPECT_CALL(mock, F(_, _)).WillRepeatedly(Return());
+  EXPECT_CALL(mock, F("a", "b"));
+  EXPECT_CALL(mock, F("c", HasSubstr("d")));
+
+  mock.F("a", "good");
+  mock.F("a", "b");
+}
+```
+
+if you run it with `--gmock_verbose=info`, you will see this output:
+
+```shell
+[ RUN       ] Foo.Bar
+
+foo_test.cc:14: EXPECT_CALL(mock, F(_, _)) invoked
+Stack trace: ...
+
+foo_test.cc:15: EXPECT_CALL(mock, F("a", "b")) invoked
+Stack trace: ...
+
+foo_test.cc:16: EXPECT_CALL(mock, F("c", HasSubstr("d"))) invoked
+Stack trace: ...
+
+foo_test.cc:14: Mock function call matches EXPECT_CALL(mock, F(_, _))...
+    Function call: F(@0x7fff7c8dad40"a",@0x7fff7c8dad10"good")
+Stack trace: ...
+
+foo_test.cc:15: Mock function call matches EXPECT_CALL(mock, F("a", "b"))...
+    Function call: F(@0x7fff7c8dada0"a",@0x7fff7c8dad70"b")
+Stack trace: ...
+
+foo_test.cc:16: Failure
+Actual function call count doesn't match EXPECT_CALL(mock, F("c", HasSubstr("d")))...
+         Expected: to be called once
+           Actual: never called - unsatisfied and active
+[  FAILED  ] Foo.Bar
+```
+
+Suppose the bug is that the `"c"` in the third `EXPECT_CALL` is a typo and
+should actually be `"a"`. With the above message, you should see that the actual
+`F("a", "good")` call is matched by the first `EXPECT_CALL`, not the third as
+you thought. From that it should be obvious that the third `EXPECT_CALL` is
+written wrong. Case solved.
+
+If you are interested in the mock call trace but not the stack traces, you can
+combine `--gmock_verbose=info` with `--gtest_stack_trace_depth=0` on the test
+command line.
+
+
+
+### Running Tests in Emacs
+
+If you build and run your tests in Emacs using the `M-x google-compile` command
+(as many googletest users do), the source file locations of gMock and googletest
+errors will be highlighted. Just press `` on one of them and you'll be
+taken to the offending line. Or, you can just type `C-x`` to jump to the next
+error.
+
+To make it even easier, you can add the following lines to your `~/.emacs` file:
+
+```text
+(global-set-key "\M-m"  'google-compile)  ; m is for make
+(global-set-key [M-down] 'next-error)
+(global-set-key [M-up]  '(lambda () (interactive) (next-error -1)))
+```
+
+Then you can type `M-m` to start a build (if you want to run the test as well,
+just make sure `foo_test.run` or `runtests` is in the build command you supply
+after typing `M-m`), or `M-up`/`M-down` to move back and forth between errors.
+
+## Extending gMock
+
+### Writing New Matchers Quickly {#NewMatchers}
+
+WARNING: gMock does not guarantee when or how many times a matcher will be
+invoked. Therefore, all matchers must be functionally pure. See
+[this section](#PureMatchers) for more details.
+
+The `MATCHER*` family of macros can be used to define custom matchers easily.
+The syntax:
+
+```cpp
+MATCHER(name, description_string_expression) { statements; }
+```
+
+will define a matcher with the given name that executes the statements, which
+must return a `bool` to indicate if the match succeeds. Inside the statements,
+you can refer to the value being matched by `arg`, and refer to its type by
+`arg_type`.
+
+The *description string* is a `string`-typed expression that documents what the
+matcher does, and is used to generate the failure message when the match fails.
+It can (and should) reference the special `bool` variable `negation`, and should
+evaluate to the description of the matcher when `negation` is `false`, or that
+of the matcher's negation when `negation` is `true`.
+
+For convenience, we allow the description string to be empty (`""`), in which
+case gMock will use the sequence of words in the matcher name as the
+description.
+
+For example:
+
+```cpp
+MATCHER(IsDivisibleBy7, "") { return (arg % 7) == 0; }
+```
+
+allows you to write
+
+```cpp
+  // Expects mock_foo.Bar(n) to be called where n is divisible by 7.
+  EXPECT_CALL(mock_foo, Bar(IsDivisibleBy7()));
+```
+
+or,
+
+```cpp
+  using ::testing::Not;
+  ...
+  // Verifies that two values are divisible by 7.
+  EXPECT_THAT(some_expression, IsDivisibleBy7());
+  EXPECT_THAT(some_other_expression, Not(IsDivisibleBy7()));
+```
+
+If the above assertions fail, they will print something like:
+
+```shell
+  Value of: some_expression
+  Expected: is divisible by 7
+    Actual: 27
+  ...
+  Value of: some_other_expression
+  Expected: not (is divisible by 7)
+    Actual: 21
+```
+
+where the descriptions `"is divisible by 7"` and `"not (is divisible by 7)"` are
+automatically calculated from the matcher name `IsDivisibleBy7`.
+
+As you may have noticed, the auto-generated descriptions (especially those for
+the negation) may not be so great. You can always override them with a `string`
+expression of your own:
+
+```cpp
+MATCHER(IsDivisibleBy7,
+        absl::StrCat(negation ? "isn't" : "is", " divisible by 7")) {
+  return (arg % 7) == 0;
+}
+```
+
+Optionally, you can stream additional information to a hidden argument named
+`result_listener` to explain the match result. For example, a better definition
+of `IsDivisibleBy7` is:
+
+```cpp
+MATCHER(IsDivisibleBy7, "") {
+  if ((arg % 7) == 0)
+    return true;
+
+  *result_listener << "the remainder is " << (arg % 7);
+  return false;
+}
+```
+
+With this definition, the above assertion will give a better message:
+
+```shell
+  Value of: some_expression
+  Expected: is divisible by 7
+    Actual: 27 (the remainder is 6)
+```
+
+You should let `MatchAndExplain()` print *any additional information* that can
+help a user understand the match result. Note that it should explain why the
+match succeeds in case of a success (unless it's obvious) - this is useful when
+the matcher is used inside `Not()`. There is no need to print the argument value
+itself, as gMock already prints it for you.
+
+NOTE: The type of the value being matched (`arg_type`) is determined by the
+context in which you use the matcher and is supplied to you by the compiler, so
+you don't need to worry about declaring it (nor can you). This allows the
+matcher to be polymorphic. For example, `IsDivisibleBy7()` can be used to match
+any type where the value of `(arg % 7) == 0` can be implicitly converted to a
+`bool`. In the `Bar(IsDivisibleBy7())` example above, if method `Bar()` takes an
+`int`, `arg_type` will be `int`; if it takes an `unsigned long`, `arg_type` will
+be `unsigned long`; and so on.
+
+### Writing New Parameterized Matchers Quickly
+
+Sometimes you'll want to define a matcher that has parameters. For that you can
+use the macro:
+
+```cpp
+MATCHER_P(name, param_name, description_string) { statements; }
+```
+
+where the description string can be either `""` or a `string` expression that
+references `negation` and `param_name`.
+
+For example:
+
+```cpp
+MATCHER_P(HasAbsoluteValue, value, "") { return abs(arg) == value; }
+```
+
+will allow you to write:
+
+```cpp
+  EXPECT_THAT(Blah("a"), HasAbsoluteValue(n));
+```
+
+which may lead to this message (assuming `n` is 10):
+
+```shell
+  Value of: Blah("a")
+  Expected: has absolute value 10
+    Actual: -9
+```
+
+Note that both the matcher description and its parameter are printed, making the
+message human-friendly.
+
+In the matcher definition body, you can write `foo_type` to reference the type
+of a parameter named `foo`. For example, in the body of
+`MATCHER_P(HasAbsoluteValue, value)` above, you can write `value_type` to refer
+to the type of `value`.
+
+gMock also provides `MATCHER_P2`, `MATCHER_P3`, ..., up to `MATCHER_P10` to
+support multi-parameter matchers:
+
+```cpp
+MATCHER_Pk(name, param_1, ..., param_k, description_string) { statements; }
+```
+
+Please note that the custom description string is for a particular *instance* of
+the matcher, where the parameters have been bound to actual values. Therefore
+usually you'll want the parameter values to be part of the description. gMock
+lets you do that by referencing the matcher parameters in the description string
+expression.
+
+For example,
+
+```cpp
+using ::testing::PrintToString;
+MATCHER_P2(InClosedRange, low, hi,
+           absl::StrFormat("%s in range [%s, %s]", negation ? "isn't" : "is",
+                           PrintToString(low), PrintToString(hi))) {
+  return low <= arg && arg <= hi;
+}
+...
+EXPECT_THAT(3, InClosedRange(4, 6));
+```
+
+would generate a failure that contains the message:
+
+```shell
+  Expected: is in range [4, 6]
+```
+
+If you specify `""` as the description, the failure message will contain the
+sequence of words in the matcher name followed by the parameter values printed
+as a tuple. For example,
+
+```cpp
+  MATCHER_P2(InClosedRange, low, hi, "") { ... }
+  ...
+  EXPECT_THAT(3, InClosedRange(4, 6));
+```
+
+would generate a failure that contains the text:
+
+```shell
+  Expected: in closed range (4, 6)
+```
+
+For the purpose of typing, you can view
+
+```cpp
+MATCHER_Pk(Foo, p1, ..., pk, description_string) { ... }
+```
+
+as shorthand for
+
+```cpp
+template 
+FooMatcherPk
+Foo(p1_type p1, ..., pk_type pk) { ... }
+```
+
+When you write `Foo(v1, ..., vk)`, the compiler infers the types of the
+parameters `v1`, ..., and `vk` for you. If you are not happy with the result of
+the type inference, you can specify the types by explicitly instantiating the
+template, as in `Foo(5, false)`. As said earlier, you don't get to
+(or need to) specify `arg_type` as that's determined by the context in which the
+matcher is used.
+
+You can assign the result of expression `Foo(p1, ..., pk)` to a variable of type
+`FooMatcherPk`. This can be useful when composing
+matchers. Matchers that don't have a parameter or have only one parameter have
+special types: you can assign `Foo()` to a `FooMatcher`-typed variable, and
+assign `Foo(p)` to a `FooMatcherP`-typed variable.
+
+While you can instantiate a matcher template with reference types, passing the
+parameters by pointer usually makes your code more readable. If, however, you
+still want to pass a parameter by reference, be aware that in the failure
+message generated by the matcher you will see the value of the referenced object
+but not its address.
+
+You can overload matchers with different numbers of parameters:
+
+```cpp
+MATCHER_P(Blah, a, description_string_1) { ... }
+MATCHER_P2(Blah, a, b, description_string_2) { ... }
+```
+
+While it's tempting to always use the `MATCHER*` macros when defining a new
+matcher, you should also consider implementing `MatcherInterface` or using
+`MakePolymorphicMatcher()` instead (see the recipes that follow), especially if
+you need to use the matcher a lot. While these approaches require more work,
+they give you more control on the types of the value being matched and the
+matcher parameters, which in general leads to better compiler error messages
+that pay off in the long run. They also allow overloading matchers based on
+parameter types (as opposed to just based on the number of parameters).
+
+### Writing New Monomorphic Matchers
+
+A matcher of argument type `T` implements `::testing::MatcherInterface` and
+does two things: it tests whether a value of type `T` matches the matcher, and
+can describe what kind of values it matches. The latter ability is used for
+generating readable error messages when expectations are violated.
+
+The interface looks like this:
+
+```cpp
+class MatchResultListener {
+ public:
+  ...
+  // Streams x to the underlying ostream; does nothing if the ostream
+  // is NULL.
+  template 
+  MatchResultListener& operator<<(const T& x);
+
+  // Returns the underlying ostream.
+  std::ostream* stream();
+};
+
+template 
+class MatcherInterface {
+ public:
+  virtual ~MatcherInterface();
+
+  // Returns true if and only if the matcher matches x; also explains the match
+  // result to 'listener'.
+  virtual bool MatchAndExplain(T x, MatchResultListener* listener) const = 0;
+
+  // Describes this matcher to an ostream.
+  virtual void DescribeTo(std::ostream* os) const = 0;
+
+  // Describes the negation of this matcher to an ostream.
+  virtual void DescribeNegationTo(std::ostream* os) const;
+};
+```
+
+If you need a custom matcher but `Truly()` is not a good option (for example,
+you may not be happy with the way `Truly(predicate)` describes itself, or you
+may want your matcher to be polymorphic as `Eq(value)` is), you can define a
+matcher to do whatever you want in two steps: first implement the matcher
+interface, and then define a factory function to create a matcher instance. The
+second step is not strictly needed but it makes the syntax of using the matcher
+nicer.
+
+For example, you can define a matcher to test whether an `int` is divisible by 7
+and then use it like this:
+
+```cpp
+using ::testing::MakeMatcher;
+using ::testing::Matcher;
+using ::testing::MatcherInterface;
+using ::testing::MatchResultListener;
+
+class DivisibleBy7Matcher : public MatcherInterface {
+ public:
+  bool MatchAndExplain(int n,
+                       MatchResultListener* /* listener */) const override {
+    return (n % 7) == 0;
+  }
+
+  void DescribeTo(std::ostream* os) const override {
+    *os << "is divisible by 7";
+  }
+
+  void DescribeNegationTo(std::ostream* os) const override {
+    *os << "is not divisible by 7";
+  }
+};
+
+Matcher DivisibleBy7() {
+  return MakeMatcher(new DivisibleBy7Matcher);
+}
+
+...
+  EXPECT_CALL(foo, Bar(DivisibleBy7()));
+```
+
+You may improve the matcher message by streaming additional information to the
+`listener` argument in `MatchAndExplain()`:
+
+```cpp
+class DivisibleBy7Matcher : public MatcherInterface {
+ public:
+  bool MatchAndExplain(int n,
+                       MatchResultListener* listener) const override {
+    const int remainder = n % 7;
+    if (remainder != 0) {
+      *listener << "the remainder is " << remainder;
+    }
+    return remainder == 0;
+  }
+  ...
+};
+```
+
+Then, `EXPECT_THAT(x, DivisibleBy7());` may generate a message like this:
+
+```shell
+Value of: x
+Expected: is divisible by 7
+  Actual: 23 (the remainder is 2)
+```
+
+### Writing New Polymorphic Matchers
+
+You've learned how to write your own matchers in the previous recipe. Just one
+problem: a matcher created using `MakeMatcher()` only works for one particular
+type of arguments. If you want a *polymorphic* matcher that works with arguments
+of several types (for instance, `Eq(x)` can be used to match a *`value`* as long
+as `value == x` compiles -- *`value`* and `x` don't have to share the same
+type), you can learn the trick from `testing/base/public/gmock-matchers.h` but
+it's a bit involved.
+
+Fortunately, most of the time you can define a polymorphic matcher easily with
+the help of `MakePolymorphicMatcher()`. Here's how you can define `NotNull()` as
+an example:
+
+```cpp
+using ::testing::MakePolymorphicMatcher;
+using ::testing::MatchResultListener;
+using ::testing::PolymorphicMatcher;
+
+class NotNullMatcher {
+ public:
+  // To implement a polymorphic matcher, first define a COPYABLE class
+  // that has three members MatchAndExplain(), DescribeTo(), and
+  // DescribeNegationTo(), like the following.
+
+  // In this example, we want to use NotNull() with any pointer, so
+  // MatchAndExplain() accepts a pointer of any type as its first argument.
+  // In general, you can define MatchAndExplain() as an ordinary method or
+  // a method template, or even overload it.
+  template 
+  bool MatchAndExplain(T* p,
+                       MatchResultListener* /* listener */) const {
+    return p != NULL;
+  }
+
+  // Describes the property of a value matching this matcher.
+  void DescribeTo(std::ostream* os) const { *os << "is not NULL"; }
+
+  // Describes the property of a value NOT matching this matcher.
+  void DescribeNegationTo(std::ostream* os) const { *os << "is NULL"; }
+};
+
+// To construct a polymorphic matcher, pass an instance of the class
+// to MakePolymorphicMatcher().  Note the return type.
+PolymorphicMatcher NotNull() {
+  return MakePolymorphicMatcher(NotNullMatcher());
+}
+
+...
+
+  EXPECT_CALL(foo, Bar(NotNull()));  // The argument must be a non-NULL pointer.
+```
+
+**Note:** Your polymorphic matcher class does **not** need to inherit from
+`MatcherInterface` or any other class, and its methods do **not** need to be
+virtual.
+
+Like in a monomorphic matcher, you may explain the match result by streaming
+additional information to the `listener` argument in `MatchAndExplain()`.
+
+### Writing New Cardinalities
+
+A cardinality is used in `Times()` to tell gMock how many times you expect a
+call to occur. It doesn't have to be exact. For example, you can say
+`AtLeast(5)` or `Between(2, 4)`.
+
+If the [built-in set](cheat_sheet.md#CardinalityList) of cardinalities doesn't
+suit you, you are free to define your own by implementing the following
+interface (in namespace `testing`):
+
+```cpp
+class CardinalityInterface {
+ public:
+  virtual ~CardinalityInterface();
+
+  // Returns true if and only if call_count calls will satisfy this cardinality.
+  virtual bool IsSatisfiedByCallCount(int call_count) const = 0;
+
+  // Returns true if and only if call_count calls will saturate this
+  // cardinality.
+  virtual bool IsSaturatedByCallCount(int call_count) const = 0;
+
+  // Describes self to an ostream.
+  virtual void DescribeTo(std::ostream* os) const = 0;
+};
+```
+
+For example, to specify that a call must occur even number of times, you can
+write
+
+```cpp
+using ::testing::Cardinality;
+using ::testing::CardinalityInterface;
+using ::testing::MakeCardinality;
+
+class EvenNumberCardinality : public CardinalityInterface {
+ public:
+  bool IsSatisfiedByCallCount(int call_count) const override {
+    return (call_count % 2) == 0;
+  }
+
+  bool IsSaturatedByCallCount(int call_count) const override {
+    return false;
+  }
+
+  void DescribeTo(std::ostream* os) const {
+    *os << "called even number of times";
+  }
+};
+
+Cardinality EvenNumber() {
+  return MakeCardinality(new EvenNumberCardinality);
+}
+
+...
+  EXPECT_CALL(foo, Bar(3))
+      .Times(EvenNumber());
+```
+
+### Writing New Actions Quickly {#QuickNewActions}
+
+If the built-in actions don't work for you, you can easily define your own one.
+Just define a functor class with a (possibly templated) call operator, matching
+the signature of your action.
+
+```cpp
+struct Increment {
+  template 
+  T operator()(T* arg) {
+    return ++(*arg);
+  }
+}
+```
+
+The same approach works with stateful functors (or any callable, really):
+
+```
+struct MultiplyBy {
+  template 
+  T operator()(T arg) { return arg * multiplier; }
+
+  int multiplier;
+}
+
+// Then use:
+// EXPECT_CALL(...).WillOnce(MultiplyBy{7});
+```
+
+#### Legacy macro-based Actions
+
+Before C++11, the functor-based actions were not supported; the old way of
+writing actions was through a set of `ACTION*` macros. We suggest to avoid them
+in new code; they hide a lot of logic behind the macro, potentially leading to
+harder-to-understand compiler errors. Nevertheless, we cover them here for
+completeness.
+
+By writing
+
+```cpp
+ACTION(name) { statements; }
+```
+
+in a namespace scope (i.e. not inside a class or function), you will define an
+action with the given name that executes the statements. The value returned by
+`statements` will be used as the return value of the action. Inside the
+statements, you can refer to the K-th (0-based) argument of the mock function as
+`argK`. For example:
+
+```cpp
+ACTION(IncrementArg1) { return ++(*arg1); }
+```
+
+allows you to write
+
+```cpp
+... WillOnce(IncrementArg1());
+```
+
+Note that you don't need to specify the types of the mock function arguments.
+Rest assured that your code is type-safe though: you'll get a compiler error if
+`*arg1` doesn't support the `++` operator, or if the type of `++(*arg1)` isn't
+compatible with the mock function's return type.
+
+Another example:
+
+```cpp
+ACTION(Foo) {
+  (*arg2)(5);
+  Blah();
+  *arg1 = 0;
+  return arg0;
+}
+```
+
+defines an action `Foo()` that invokes argument #2 (a function pointer) with 5,
+calls function `Blah()`, sets the value pointed to by argument #1 to 0, and
+returns argument #0.
+
+For more convenience and flexibility, you can also use the following pre-defined
+symbols in the body of `ACTION`:
+
+`argK_type`     | The type of the K-th (0-based) argument of the mock function
+:-------------- | :-----------------------------------------------------------
+`args`          | All arguments of the mock function as a tuple
+`args_type`     | The type of all arguments of the mock function as a tuple
+`return_type`   | The return type of the mock function
+`function_type` | The type of the mock function
+
+For example, when using an `ACTION` as a stub action for mock function:
+
+```cpp
+int DoSomething(bool flag, int* ptr);
+```
+
+we have:
+
+Pre-defined Symbol | Is Bound To
+------------------ | ---------------------------------
+`arg0`             | the value of `flag`
+`arg0_type`        | the type `bool`
+`arg1`             | the value of `ptr`
+`arg1_type`        | the type `int*`
+`args`             | the tuple `(flag, ptr)`
+`args_type`        | the type `std::tuple`
+`return_type`      | the type `int`
+`function_type`    | the type `int(bool, int*)`
+
+#### Legacy macro-based parameterized Actions
+
+Sometimes you'll want to parameterize an action you define. For that we have
+another macro
+
+```cpp
+ACTION_P(name, param) { statements; }
+```
+
+For example,
+
+```cpp
+ACTION_P(Add, n) { return arg0 + n; }
+```
+
+will allow you to write
+
+```cpp
+// Returns argument #0 + 5.
+... WillOnce(Add(5));
+```
+
+For convenience, we use the term *arguments* for the values used to invoke the
+mock function, and the term *parameters* for the values used to instantiate an
+action.
+
+Note that you don't need to provide the type of the parameter either. Suppose
+the parameter is named `param`, you can also use the gMock-defined symbol
+`param_type` to refer to the type of the parameter as inferred by the compiler.
+For example, in the body of `ACTION_P(Add, n)` above, you can write `n_type` for
+the type of `n`.
+
+gMock also provides `ACTION_P2`, `ACTION_P3`, and etc to support multi-parameter
+actions. For example,
+
+```cpp
+ACTION_P2(ReturnDistanceTo, x, y) {
+  double dx = arg0 - x;
+  double dy = arg1 - y;
+  return sqrt(dx*dx + dy*dy);
+}
+```
+
+lets you write
+
+```cpp
+... WillOnce(ReturnDistanceTo(5.0, 26.5));
+```
+
+You can view `ACTION` as a degenerated parameterized action where the number of
+parameters is 0.
+
+You can also easily define actions overloaded on the number of parameters:
+
+```cpp
+ACTION_P(Plus, a) { ... }
+ACTION_P2(Plus, a, b) { ... }
+```
+
+### Restricting the Type of an Argument or Parameter in an ACTION
+
+For maximum brevity and reusability, the `ACTION*` macros don't ask you to
+provide the types of the mock function arguments and the action parameters.
+Instead, we let the compiler infer the types for us.
+
+Sometimes, however, we may want to be more explicit about the types. There are
+several tricks to do that. For example:
+
+```cpp
+ACTION(Foo) {
+  // Makes sure arg0 can be converted to int.
+  int n = arg0;
+  ... use n instead of arg0 here ...
+}
+
+ACTION_P(Bar, param) {
+  // Makes sure the type of arg1 is const char*.
+  ::testing::StaticAssertTypeEq();
+
+  // Makes sure param can be converted to bool.
+  bool flag = param;
+}
+```
+
+where `StaticAssertTypeEq` is a compile-time assertion in googletest that
+verifies two types are the same.
+
+### Writing New Action Templates Quickly
+
+Sometimes you want to give an action explicit template parameters that cannot be
+inferred from its value parameters. `ACTION_TEMPLATE()` supports that and can be
+viewed as an extension to `ACTION()` and `ACTION_P*()`.
+
+The syntax:
+
+```cpp
+ACTION_TEMPLATE(ActionName,
+                HAS_m_TEMPLATE_PARAMS(kind1, name1, ..., kind_m, name_m),
+                AND_n_VALUE_PARAMS(p1, ..., p_n)) { statements; }
+```
+
+defines an action template that takes *m* explicit template parameters and *n*
+value parameters, where *m* is in [1, 10] and *n* is in [0, 10]. `name_i` is the
+name of the *i*-th template parameter, and `kind_i` specifies whether it's a
+`typename`, an integral constant, or a template. `p_i` is the name of the *i*-th
+value parameter.
+
+Example:
+
+```cpp
+// DuplicateArg(output) converts the k-th argument of the mock
+// function to type T and copies it to *output.
+ACTION_TEMPLATE(DuplicateArg,
+                // Note the comma between int and k:
+                HAS_2_TEMPLATE_PARAMS(int, k, typename, T),
+                AND_1_VALUE_PARAMS(output)) {
+  *output = T(std::get(args));
+}
+```
+
+To create an instance of an action template, write:
+
+```cpp
+ActionName(v1, ..., v_n)
+```
+
+where the `t`s are the template arguments and the `v`s are the value arguments.
+The value argument types are inferred by the compiler. For example:
+
+```cpp
+using ::testing::_;
+...
+  int n;
+  EXPECT_CALL(mock, Foo).WillOnce(DuplicateArg<1, unsigned char>(&n));
+```
+
+If you want to explicitly specify the value argument types, you can provide
+additional template arguments:
+
+```cpp
+ActionName(v1, ..., v_n)
+```
+
+where `u_i` is the desired type of `v_i`.
+
+`ACTION_TEMPLATE` and `ACTION`/`ACTION_P*` can be overloaded on the number of
+value parameters, but not on the number of template parameters. Without the
+restriction, the meaning of the following is unclear:
+
+```cpp
+  OverloadedAction(x);
+```
+
+Are we using a single-template-parameter action where `bool` refers to the type
+of `x`, or a two-template-parameter action where the compiler is asked to infer
+the type of `x`?
+
+### Using the ACTION Object's Type
+
+If you are writing a function that returns an `ACTION` object, you'll need to
+know its type. The type depends on the macro used to define the action and the
+parameter types. The rule is relatively simple:
+
+| Given Definition              | Expression          | Has Type              |
+| ----------------------------- | ------------------- | --------------------- |
+| `ACTION(Foo)`                 | `Foo()`             | `FooAction`           |
+| `ACTION_TEMPLATE(Foo,`        | `Foo()`             : t_m>`                 :
+: `AND_0_VALUE_PARAMS())`       :                     :                       :
+| `ACTION_P(Bar, param)`        | `Bar(int_value)`    | `BarActionP`     |
+| `ACTION_TEMPLATE(Bar,`        | `Bar` | `FooActionP`            :
+: `AND_1_VALUE_PARAMS(p1))`     :                     :                       :
+| `ACTION_P2(Baz, p1, p2)`      | `Baz(bool_value,`   | `BazActionP2`                 :
+| `ACTION_TEMPLATE(Baz,`        | `Baz` | `FooActionP2`    :
+: `AND_2_VALUE_PARAMS(p1, p2))` : `int_value)`        :                       :
+| ...                           | ...                 | ...                   |
+
+Note that we have to pick different suffixes (`Action`, `ActionP`, `ActionP2`,
+and etc) for actions with different numbers of value parameters, or the action
+definitions cannot be overloaded on the number of them.
+
+### Writing New Monomorphic Actions {#NewMonoActions}
+
+While the `ACTION*` macros are very convenient, sometimes they are
+inappropriate. For example, despite the tricks shown in the previous recipes,
+they don't let you directly specify the types of the mock function arguments and
+the action parameters, which in general leads to unoptimized compiler error
+messages that can baffle unfamiliar users. They also don't allow overloading
+actions based on parameter types without jumping through some hoops.
+
+An alternative to the `ACTION*` macros is to implement
+`::testing::ActionInterface`, where `F` is the type of the mock function in
+which the action will be used. For example:
+
+```cpp
+template 
+class ActionInterface {
+ public:
+  virtual ~ActionInterface();
+
+  // Performs the action.  Result is the return type of function type
+  // F, and ArgumentTuple is the tuple of arguments of F.
+  //
+
+  // For example, if F is int(bool, const string&), then Result would
+  // be int, and ArgumentTuple would be std::tuple.
+  virtual Result Perform(const ArgumentTuple& args) = 0;
+};
+```
+
+```cpp
+using ::testing::_;
+using ::testing::Action;
+using ::testing::ActionInterface;
+using ::testing::MakeAction;
+
+typedef int IncrementMethod(int*);
+
+class IncrementArgumentAction : public ActionInterface {
+ public:
+  int Perform(const std::tuple& args) override {
+    int* p = std::get<0>(args);  // Grabs the first argument.
+    return *p++;
+  }
+};
+
+Action IncrementArgument() {
+  return MakeAction(new IncrementArgumentAction);
+}
+
+...
+  EXPECT_CALL(foo, Baz(_))
+      .WillOnce(IncrementArgument());
+
+  int n = 5;
+  foo.Baz(&n);  // Should return 5 and change n to 6.
+```
+
+### Writing New Polymorphic Actions {#NewPolyActions}
+
+The previous recipe showed you how to define your own action. This is all good,
+except that you need to know the type of the function in which the action will
+be used. Sometimes that can be a problem. For example, if you want to use the
+action in functions with *different* types (e.g. like `Return()` and
+`SetArgPointee()`).
+
+If an action can be used in several types of mock functions, we say it's
+*polymorphic*. The `MakePolymorphicAction()` function template makes it easy to
+define such an action:
+
+```cpp
+namespace testing {
+template 
+PolymorphicAction MakePolymorphicAction(const Impl& impl);
+}  // namespace testing
+```
+
+As an example, let's define an action that returns the second argument in the
+mock function's argument list. The first step is to define an implementation
+class:
+
+```cpp
+class ReturnSecondArgumentAction {
+ public:
+  template 
+  Result Perform(const ArgumentTuple& args) const {
+    // To get the i-th (0-based) argument, use std::get(args).
+    return std::get<1>(args);
+  }
+};
+```
+
+This implementation class does *not* need to inherit from any particular class.
+What matters is that it must have a `Perform()` method template. This method
+template takes the mock function's arguments as a tuple in a **single**
+argument, and returns the result of the action. It can be either `const` or not,
+but must be invokable with exactly one template argument, which is the result
+type. In other words, you must be able to call `Perform(args)` where `R` is
+the mock function's return type and `args` is its arguments in a tuple.
+
+Next, we use `MakePolymorphicAction()` to turn an instance of the implementation
+class into the polymorphic action we need. It will be convenient to have a
+wrapper for this:
+
+```cpp
+using ::testing::MakePolymorphicAction;
+using ::testing::PolymorphicAction;
+
+PolymorphicAction ReturnSecondArgument() {
+  return MakePolymorphicAction(ReturnSecondArgumentAction());
+}
+```
+
+Now, you can use this polymorphic action the same way you use the built-in ones:
+
+```cpp
+using ::testing::_;
+
+class MockFoo : public Foo {
+ public:
+  MOCK_METHOD(int, DoThis, (bool flag, int n), (override));
+  MOCK_METHOD(string, DoThat, (int x, const char* str1, const char* str2),
+              (override));
+};
+
+  ...
+  MockFoo foo;
+  EXPECT_CALL(foo, DoThis).WillOnce(ReturnSecondArgument());
+  EXPECT_CALL(foo, DoThat).WillOnce(ReturnSecondArgument());
+  ...
+  foo.DoThis(true, 5);  // Will return 5.
+  foo.DoThat(1, "Hi", "Bye");  // Will return "Hi".
+```
+
+### Teaching gMock How to Print Your Values
+
+When an uninteresting or unexpected call occurs, gMock prints the argument
+values and the stack trace to help you debug. Assertion macros like
+`EXPECT_THAT` and `EXPECT_EQ` also print the values in question when the
+assertion fails. gMock and googletest do this using googletest's user-extensible
+value printer.
+
+This printer knows how to print built-in C++ types, native arrays, STL
+containers, and any type that supports the `<<` operator. For other types, it
+prints the raw bytes in the value and hopes that you the user can figure it out.
+[googletest's advanced guide](../../googletest/docs/advanced.md#teaching-googletest-how-to-print-your-values)
+explains how to extend the printer to do a better job at printing your
+particular type than to dump the bytes.
+
+## Useful Mocks Created Using gMock
+
+
+
+
+### Mock std::function {#MockFunction}
+
+`std::function` is a general function type introduced in C++11. It is a
+preferred way of passing callbacks to new interfaces. Functions are copiable,
+and are not usually passed around by pointer, which makes them tricky to mock.
+But fear not - `MockFunction` can help you with that.
+
+`MockFunction` has a mock method `Call()` with the signature:
+
+```cpp
+  R Call(T1, ..., Tn);
+```
+
+It also has a `AsStdFunction()` method, which creates a `std::function` proxy
+forwarding to Call:
+
+```cpp
+  std::function AsStdFunction();
+```
+
+To use `MockFunction`, first create `MockFunction` object and set up
+expectations on its `Call` method. Then pass proxy obtained from
+`AsStdFunction()` to the code you are testing. For example:
+
+```cpp
+TEST(FooTest, RunsCallbackWithBarArgument) {
+  // 1. Create a mock object.
+  MockFunction mock_function;
+
+  // 2. Set expectations on Call() method.
+  EXPECT_CALL(mock_function, Call("bar")).WillOnce(Return(1));
+
+  // 3. Exercise code that uses std::function.
+  Foo(mock_function.AsStdFunction());
+  // Foo's signature can be either of:
+  // void Foo(const std::function& fun);
+  // void Foo(std::function fun);
+
+  // 4. All expectations will be verified when mock_function
+  //     goes out of scope and is destroyed.
+}
+```
+
+Remember that function objects created with `AsStdFunction()` are just
+forwarders. If you create multiple of them, they will share the same set of
+expectations.
+
+Although `std::function` supports unlimited number of arguments, `MockFunction`
+implementation is limited to ten. If you ever hit that limit... well, your
+callback has bigger problems than being mockable. :-)
+
+
diff --git a/thirdparty/GTest/googletest/googlemock/docs/for_dummies.md b/thirdparty/GTest/googletest/googlemock/docs/for_dummies.md
new file mode 100644
index 0000000000..8ba164f9a1
--- /dev/null
+++ b/thirdparty/GTest/googletest/googlemock/docs/for_dummies.md
@@ -0,0 +1,702 @@
+# gMock for Dummies {#GMockForDummies}
+
+
+
+
+
+## What Is gMock?
+
+When you write a prototype or test, often it's not feasible or wise to rely on
+real objects entirely. A **mock object** implements the same interface as a real
+object (so it can be used as one), but lets you specify at run time how it will
+be used and what it should do (which methods will be called? in which order? how
+many times? with what arguments? what will they return? etc).
+
+**Note:** It is easy to confuse the term *fake objects* with mock objects. Fakes
+and mocks actually mean very different things in the Test-Driven Development
+(TDD) community:
+
+*   **Fake** objects have working implementations, but usually take some
+    shortcut (perhaps to make the operations less expensive), which makes them
+    not suitable for production. An in-memory file system would be an example of
+    a fake.
+*   **Mocks** are objects pre-programmed with *expectations*, which form a
+    specification of the calls they are expected to receive.
+
+If all this seems too abstract for you, don't worry - the most important thing
+to remember is that a mock allows you to check the *interaction* between itself
+and code that uses it. The difference between fakes and mocks shall become much
+clearer once you start to use mocks.
+
+**gMock** is a library (sometimes we also call it a "framework" to make it sound
+cool) for creating mock classes and using them. It does to C++ what
+jMock/EasyMock does to Java (well, more or less).
+
+When using gMock,
+
+1.  first, you use some simple macros to describe the interface you want to
+    mock, and they will expand to the implementation of your mock class;
+2.  next, you create some mock objects and specify its expectations and behavior
+    using an intuitive syntax;
+3.  then you exercise code that uses the mock objects. gMock will catch any
+    violation to the expectations as soon as it arises.
+
+## Why gMock?
+
+While mock objects help you remove unnecessary dependencies in tests and make
+them fast and reliable, using mocks manually in C++ is *hard*:
+
+*   Someone has to implement the mocks. The job is usually tedious and
+    error-prone. No wonder people go great distance to avoid it.
+*   The quality of those manually written mocks is a bit, uh, unpredictable. You
+    may see some really polished ones, but you may also see some that were
+    hacked up in a hurry and have all sorts of ad hoc restrictions.
+*   The knowledge you gained from using one mock doesn't transfer to the next
+    one.
+
+In contrast, Java and Python programmers have some fine mock frameworks (jMock,
+EasyMock, [Mox](http://wtf/mox), etc), which automate the creation of mocks. As
+a result, mocking is a proven effective technique and widely adopted practice in
+those communities. Having the right tool absolutely makes the difference.
+
+gMock was built to help C++ programmers. It was inspired by jMock and EasyMock,
+but designed with C++'s specifics in mind. It is your friend if any of the
+following problems is bothering you:
+
+*   You are stuck with a sub-optimal design and wish you had done more
+    prototyping before it was too late, but prototyping in C++ is by no means
+    "rapid".
+*   Your tests are slow as they depend on too many libraries or use expensive
+    resources (e.g. a database).
+*   Your tests are brittle as some resources they use are unreliable (e.g. the
+    network).
+*   You want to test how your code handles a failure (e.g. a file checksum
+    error), but it's not easy to cause one.
+*   You need to make sure that your module interacts with other modules in the
+    right way, but it's hard to observe the interaction; therefore you resort to
+    observing the side effects at the end of the action, but it's awkward at
+    best.
+*   You want to "mock out" your dependencies, except that they don't have mock
+    implementations yet; and, frankly, you aren't thrilled by some of those
+    hand-written mocks.
+
+We encourage you to use gMock as
+
+*   a *design* tool, for it lets you experiment with your interface design early
+    and often. More iterations lead to better designs!
+*   a *testing* tool to cut your tests' outbound dependencies and probe the
+    interaction between your module and its collaborators.
+
+## Getting Started
+
+gMock is bundled with googletest.
+
+## A Case for Mock Turtles
+
+Let's look at an example. Suppose you are developing a graphics program that
+relies on a [LOGO](http://en.wikipedia.org/wiki/Logo_programming_language)-like
+API for drawing. How would you test that it does the right thing? Well, you can
+run it and compare the screen with a golden screen snapshot, but let's admit it:
+tests like this are expensive to run and fragile (What if you just upgraded to a
+shiny new graphics card that has better anti-aliasing? Suddenly you have to
+update all your golden images.). It would be too painful if all your tests are
+like this. Fortunately, you learned about
+[Dependency Injection](http://en.wikipedia.org/wiki/Dependency_injection) and know the right thing
+to do: instead of having your application talk to the system API directly, wrap
+the API in an interface (say, `Turtle`) and code to that interface:
+
+```cpp
+class Turtle {
+  ...
+  virtual ~Turtle() {}
+  virtual void PenUp() = 0;
+  virtual void PenDown() = 0;
+  virtual void Forward(int distance) = 0;
+  virtual void Turn(int degrees) = 0;
+  virtual void GoTo(int x, int y) = 0;
+  virtual int GetX() const = 0;
+  virtual int GetY() const = 0;
+};
+```
+
+(Note that the destructor of `Turtle` **must** be virtual, as is the case for
+**all** classes you intend to inherit from - otherwise the destructor of the
+derived class will not be called when you delete an object through a base
+pointer, and you'll get corrupted program states like memory leaks.)
+
+You can control whether the turtle's movement will leave a trace using `PenUp()`
+and `PenDown()`, and control its movement using `Forward()`, `Turn()`, and
+`GoTo()`. Finally, `GetX()` and `GetY()` tell you the current position of the
+turtle.
+
+Your program will normally use a real implementation of this interface. In
+tests, you can use a mock implementation instead. This allows you to easily
+check what drawing primitives your program is calling, with what arguments, and
+in which order. Tests written this way are much more robust (they won't break
+because your new machine does anti-aliasing differently), easier to read and
+maintain (the intent of a test is expressed in the code, not in some binary
+images), and run *much, much faster*.
+
+## Writing the Mock Class
+
+If you are lucky, the mocks you need to use have already been implemented by
+some nice people. If, however, you find yourself in the position to write a mock
+class, relax - gMock turns this task into a fun game! (Well, almost.)
+
+### How to Define It
+
+Using the `Turtle` interface as example, here are the simple steps you need to
+follow:
+
+*   Derive a class `MockTurtle` from `Turtle`.
+*   Take a *virtual* function of `Turtle` (while it's possible to
+    [mock non-virtual methods using templates](cook_book.md#MockingNonVirtualMethods),
+    it's much more involved).
+*   In the `public:` section of the child class, write `MOCK_METHOD();`
+*   Now comes the fun part: you take the function signature, cut-and-paste it
+    into the macro, and add two commas - one between the return type and the
+    name, another between the name and the argument list.
+*   If you're mocking a const method, add a 4th parameter containing `(const)`
+    (the parentheses are required).
+*   Since you're overriding a virtual method, we suggest adding the `override`
+    keyword. For const methods the 4th parameter becomes `(const, override)`,
+    for non-const methods just `(override)`. This isn't mandatory.
+*   Repeat until all virtual functions you want to mock are done. (It goes
+    without saying that *all* pure virtual methods in your abstract class must
+    be either mocked or overridden.)
+
+After the process, you should have something like:
+
+```cpp
+#include "gmock/gmock.h"  // Brings in gMock.
+
+class MockTurtle : public Turtle {
+ public:
+  ...
+  MOCK_METHOD(void, PenUp, (), (override));
+  MOCK_METHOD(void, PenDown, (), (override));
+  MOCK_METHOD(void, Forward, (int distance), (override));
+  MOCK_METHOD(void, Turn, (int degrees), (override));
+  MOCK_METHOD(void, GoTo, (int x, int y), (override));
+  MOCK_METHOD(int, GetX, (), (const, override));
+  MOCK_METHOD(int, GetY, (), (const, override));
+};
+```
+
+You don't need to define these mock methods somewhere else - the `MOCK_METHOD`
+macro will generate the definitions for you. It's that simple!
+
+### Where to Put It
+
+When you define a mock class, you need to decide where to put its definition.
+Some people put it in a `_test.cc`. This is fine when the interface being mocked
+(say, `Foo`) is owned by the same person or team. Otherwise, when the owner of
+`Foo` changes it, your test could break. (You can't really expect `Foo`'s
+maintainer to fix every test that uses `Foo`, can you?)
+
+So, the rule of thumb is: if you need to mock `Foo` and it's owned by others,
+define the mock class in `Foo`'s package (better, in a `testing` sub-package
+such that you can clearly separate production code and testing utilities), put
+it in a `.h` and a `cc_library`. Then everyone can reference them from their
+tests. If `Foo` ever changes, there is only one copy of `MockFoo` to change, and
+only tests that depend on the changed methods need to be fixed.
+
+Another way to do it: you can introduce a thin layer `FooAdaptor` on top of
+`Foo` and code to this new interface. Since you own `FooAdaptor`, you can absorb
+changes in `Foo` much more easily. While this is more work initially, carefully
+choosing the adaptor interface can make your code easier to write and more
+readable (a net win in the long run), as you can choose `FooAdaptor` to fit your
+specific domain much better than `Foo` does.
+
+
+
+## Using Mocks in Tests
+
+Once you have a mock class, using it is easy. The typical work flow is:
+
+1.  Import the gMock names from the `testing` namespace such that you can use
+    them unqualified (You only have to do it once per file). Remember that
+    namespaces are a good idea.
+2.  Create some mock objects.
+3.  Specify your expectations on them (How many times will a method be called?
+    With what arguments? What should it do? etc.).
+4.  Exercise some code that uses the mocks; optionally, check the result using
+    googletest assertions. If a mock method is called more than expected or with
+    wrong arguments, you'll get an error immediately.
+5.  When a mock is destructed, gMock will automatically check whether all
+    expectations on it have been satisfied.
+
+Here's an example:
+
+```cpp
+#include "path/to/mock-turtle.h"
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+
+using ::testing::AtLeast;                         // #1
+
+TEST(PainterTest, CanDrawSomething) {
+  MockTurtle turtle;                              // #2
+  EXPECT_CALL(turtle, PenDown())                  // #3
+      .Times(AtLeast(1));
+
+  Painter painter(&turtle);                       // #4
+
+  EXPECT_TRUE(painter.DrawCircle(0, 0, 10));      // #5
+}
+```
+
+As you might have guessed, this test checks that `PenDown()` is called at least
+once. If the `painter` object didn't call this method, your test will fail with
+a message like this:
+
+```text
+path/to/my_test.cc:119: Failure
+Actual function call count doesn't match this expectation:
+Actually: never called;
+Expected: called at least once.
+Stack trace:
+...
+```
+
+**Tip 1:** If you run the test from an Emacs buffer, you can hit `` on
+the line number to jump right to the failed expectation.
+
+**Tip 2:** If your mock objects are never deleted, the final verification won't
+happen. Therefore it's a good idea to turn on the heap checker in your tests
+when you allocate mocks on the heap. You get that automatically if you use the
+`gtest_main` library already.
+
+**Important note:** gMock requires expectations to be set **before** the mock
+functions are called, otherwise the behavior is **undefined**. In particular,
+you mustn't interleave `EXPECT_CALL()s` and calls to the mock functions.
+
+This means `EXPECT_CALL()` should be read as expecting that a call will occur
+*in the future*, not that a call has occurred. Why does gMock work like that?
+Well, specifying the expectation beforehand allows gMock to report a violation
+as soon as it rises, when the context (stack trace, etc) is still available.
+This makes debugging much easier.
+
+Admittedly, this test is contrived and doesn't do much. You can easily achieve
+the same effect without using gMock. However, as we shall reveal soon, gMock
+allows you to do *so much more* with the mocks.
+
+## Setting Expectations
+
+The key to using a mock object successfully is to set the *right expectations*
+on it. If you set the expectations too strict, your test will fail as the result
+of unrelated changes. If you set them too loose, bugs can slip through. You want
+to do it just right such that your test can catch exactly the kind of bugs you
+intend it to catch. gMock provides the necessary means for you to do it "just
+right."
+
+### General Syntax
+
+In gMock we use the `EXPECT_CALL()` macro to set an expectation on a mock
+method. The general syntax is:
+
+```cpp
+EXPECT_CALL(mock_object, method(matchers))
+    .Times(cardinality)
+    .WillOnce(action)
+    .WillRepeatedly(action);
+```
+
+The macro has two arguments: first the mock object, and then the method and its
+arguments. Note that the two are separated by a comma (`,`), not a period (`.`).
+(Why using a comma? The answer is that it was necessary for technical reasons.)
+If the method is not overloaded, the macro can also be called without matchers:
+
+```cpp
+EXPECT_CALL(mock_object, non-overloaded-method)
+    .Times(cardinality)
+    .WillOnce(action)
+    .WillRepeatedly(action);
+```
+
+This syntax allows the test writer to specify "called with any arguments"
+without explicitly specifying the number or types of arguments. To avoid
+unintended ambiguity, this syntax may only be used for methods which are not
+overloaded
+
+Either form of the macro can be followed by some optional *clauses* that provide
+more information about the expectation. We'll discuss how each clause works in
+the coming sections.
+
+This syntax is designed to make an expectation read like English. For example,
+you can probably guess that
+
+```cpp
+using ::testing::Return;
+...
+EXPECT_CALL(turtle, GetX())
+    .Times(5)
+    .WillOnce(Return(100))
+    .WillOnce(Return(150))
+    .WillRepeatedly(Return(200));
+```
+
+says that the `turtle` object's `GetX()` method will be called five times, it
+will return 100 the first time, 150 the second time, and then 200 every time.
+Some people like to call this style of syntax a Domain-Specific Language (DSL).
+
+**Note:** Why do we use a macro to do this? Well it serves two purposes: first
+it makes expectations easily identifiable (either by `gsearch` or by a human
+reader), and second it allows gMock to include the source file location of a
+failed expectation in messages, making debugging easier.
+
+### Matchers: What Arguments Do We Expect?
+
+When a mock function takes arguments, we may specify what arguments we are
+expecting, for example:
+
+```cpp
+// Expects the turtle to move forward by 100 units.
+EXPECT_CALL(turtle, Forward(100));
+```
+
+Oftentimes you do not want to be too specific. Remember that talk about tests
+being too rigid? Over specification leads to brittle tests and obscures the
+intent of tests. Therefore we encourage you to specify only what's necessary—no
+more, no less. If you aren't interested in the value of an argument, write `_`
+as the argument, which means "anything goes":
+
+```cpp
+using ::testing::_;
+...
+// Expects that the turtle jumps to somewhere on the x=50 line.
+EXPECT_CALL(turtle, GoTo(50, _));
+```
+
+`_` is an instance of what we call **matchers**. A matcher is like a predicate
+and can test whether an argument is what we'd expect. You can use a matcher
+inside `EXPECT_CALL()` wherever a function argument is expected. `_` is a
+convenient way of saying "any value".
+
+In the above examples, `100` and `50` are also matchers; implicitly, they are
+the same as `Eq(100)` and `Eq(50)`, which specify that the argument must be
+equal (using `operator==`) to the matcher argument. There are many
+[built-in matchers](cheat_sheet.md#MatcherList) for common types (as well as
+[custom matchers](cook_book.md#NewMatchers)); for example:
+
+```cpp
+using ::testing::Ge;
+...
+// Expects the turtle moves forward by at least 100.
+EXPECT_CALL(turtle, Forward(Ge(100)));
+```
+
+If you don't care about *any* arguments, rather than specify `_` for each of
+them you may instead omit the parameter list:
+
+```cpp
+// Expects the turtle to move forward.
+EXPECT_CALL(turtle, Forward);
+// Expects the turtle to jump somewhere.
+EXPECT_CALL(turtle, GoTo);
+```
+
+This works for all non-overloaded methods; if a method is overloaded, you need
+to help gMock resolve which overload is expected by specifying the number of
+arguments and possibly also the
+[types of the arguments](cook_book.md#SelectOverload).
+
+### Cardinalities: How Many Times Will It Be Called?
+
+The first clause we can specify following an `EXPECT_CALL()` is `Times()`. We
+call its argument a **cardinality** as it tells *how many times* the call should
+occur. It allows us to repeat an expectation many times without actually writing
+it as many times. More importantly, a cardinality can be "fuzzy", just like a
+matcher can be. This allows a user to express the intent of a test exactly.
+
+An interesting special case is when we say `Times(0)`. You may have guessed - it
+means that the function shouldn't be called with the given arguments at all, and
+gMock will report a googletest failure whenever the function is (wrongfully)
+called.
+
+We've seen `AtLeast(n)` as an example of fuzzy cardinalities earlier. For the
+list of built-in cardinalities you can use, see
+[here](cheat_sheet.md#CardinalityList).
+
+The `Times()` clause can be omitted. **If you omit `Times()`, gMock will infer
+the cardinality for you.** The rules are easy to remember:
+
+*   If **neither** `WillOnce()` **nor** `WillRepeatedly()` is in the
+    `EXPECT_CALL()`, the inferred cardinality is `Times(1)`.
+*   If there are *n* `WillOnce()`'s but **no** `WillRepeatedly()`, where *n* >=
+    1, the cardinality is `Times(n)`.
+*   If there are *n* `WillOnce()`'s and **one** `WillRepeatedly()`, where *n* >=
+    0, the cardinality is `Times(AtLeast(n))`.
+
+**Quick quiz:** what do you think will happen if a function is expected to be
+called twice but actually called four times?
+
+### Actions: What Should It Do?
+
+Remember that a mock object doesn't really have a working implementation? We as
+users have to tell it what to do when a method is invoked. This is easy in
+gMock.
+
+First, if the return type of a mock function is a built-in type or a pointer,
+the function has a **default action** (a `void` function will just return, a
+`bool` function will return `false`, and other functions will return 0). In
+addition, in C++ 11 and above, a mock function whose return type is
+default-constructible (i.e. has a default constructor) has a default action of
+returning a default-constructed value. If you don't say anything, this behavior
+will be used.
+
+Second, if a mock function doesn't have a default action, or the default action
+doesn't suit you, you can specify the action to be taken each time the
+expectation matches using a series of `WillOnce()` clauses followed by an
+optional `WillRepeatedly()`. For example,
+
+```cpp
+using ::testing::Return;
+...
+EXPECT_CALL(turtle, GetX())
+     .WillOnce(Return(100))
+     .WillOnce(Return(200))
+     .WillOnce(Return(300));
+```
+
+says that `turtle.GetX()` will be called *exactly three times* (gMock inferred
+this from how many `WillOnce()` clauses we've written, since we didn't
+explicitly write `Times()`), and will return 100, 200, and 300 respectively.
+
+```cpp
+using ::testing::Return;
+...
+EXPECT_CALL(turtle, GetY())
+     .WillOnce(Return(100))
+     .WillOnce(Return(200))
+     .WillRepeatedly(Return(300));
+```
+
+says that `turtle.GetY()` will be called *at least twice* (gMock knows this as
+we've written two `WillOnce()` clauses and a `WillRepeatedly()` while having no
+explicit `Times()`), will return 100 and 200 respectively the first two times,
+and 300 from the third time on.
+
+Of course, if you explicitly write a `Times()`, gMock will not try to infer the
+cardinality itself. What if the number you specified is larger than there are
+`WillOnce()` clauses? Well, after all `WillOnce()`s are used up, gMock will do
+the *default* action for the function every time (unless, of course, you have a
+`WillRepeatedly()`.).
+
+What can we do inside `WillOnce()` besides `Return()`? You can return a
+reference using `ReturnRef(*variable*)`, or invoke a pre-defined function, among
+[others](cook_book.md#using-actions).
+
+**Important note:** The `EXPECT_CALL()` statement evaluates the action clause
+only once, even though the action may be performed many times. Therefore you
+must be careful about side effects. The following may not do what you want:
+
+```cpp
+using ::testing::Return;
+...
+int n = 100;
+EXPECT_CALL(turtle, GetX())
+    .Times(4)
+    .WillRepeatedly(Return(n++));
+```
+
+Instead of returning 100, 101, 102, ..., consecutively, this mock function will
+always return 100 as `n++` is only evaluated once. Similarly, `Return(new Foo)`
+will create a new `Foo` object when the `EXPECT_CALL()` is executed, and will
+return the same pointer every time. If you want the side effect to happen every
+time, you need to define a custom action, which we'll teach in the
+[cook book](http://).
+
+Time for another quiz! What do you think the following means?
+
+```cpp
+using ::testing::Return;
+...
+EXPECT_CALL(turtle, GetY())
+    .Times(4)
+    .WillOnce(Return(100));
+```
+
+Obviously `turtle.GetY()` is expected to be called four times. But if you think
+it will return 100 every time, think twice! Remember that one `WillOnce()`
+clause will be consumed each time the function is invoked and the default action
+will be taken afterwards. So the right answer is that `turtle.GetY()` will
+return 100 the first time, but **return 0 from the second time on**, as
+returning 0 is the default action for `int` functions.
+
+### Using Multiple Expectations {#MultiExpectations}
+
+So far we've only shown examples where you have a single expectation. More
+realistically, you'll specify expectations on multiple mock methods which may be
+from multiple mock objects.
+
+By default, when a mock method is invoked, gMock will search the expectations in
+the **reverse order** they are defined, and stop when an active expectation that
+matches the arguments is found (you can think of it as "newer rules override
+older ones."). If the matching expectation cannot take any more calls, you will
+get an upper-bound-violated failure. Here's an example:
+
+```cpp
+using ::testing::_;
+...
+EXPECT_CALL(turtle, Forward(_));  // #1
+EXPECT_CALL(turtle, Forward(10))  // #2
+    .Times(2);
+```
+
+If `Forward(10)` is called three times in a row, the third time it will be an
+error, as the last matching expectation (#2) has been saturated. If, however,
+the third `Forward(10)` call is replaced by `Forward(20)`, then it would be OK,
+as now #1 will be the matching expectation.
+
+**Note:** Why does gMock search for a match in the *reverse* order of the
+expectations? The reason is that this allows a user to set up the default
+expectations in a mock object's constructor or the test fixture's set-up phase
+and then customize the mock by writing more specific expectations in the test
+body. So, if you have two expectations on the same method, you want to put the
+one with more specific matchers **after** the other, or the more specific rule
+would be shadowed by the more general one that comes after it.
+
+**Tip:** It is very common to start with a catch-all expectation for a method
+and `Times(AnyNumber())` (omitting arguments, or with `_` for all arguments, if
+overloaded). This makes any calls to the method expected. This is not necessary
+for methods that are not mentioned at all (these are "uninteresting"), but is
+useful for methods that have some expectations, but for which other calls are
+ok. See
+[Understanding Uninteresting vs Unexpected Calls](cook_book.md#uninteresting-vs-unexpected).
+
+### Ordered vs Unordered Calls {#OrderedCalls}
+
+By default, an expectation can match a call even though an earlier expectation
+hasn't been satisfied. In other words, the calls don't have to occur in the
+order the expectations are specified.
+
+Sometimes, you may want all the expected calls to occur in a strict order. To
+say this in gMock is easy:
+
+```cpp
+using ::testing::InSequence;
+...
+TEST(FooTest, DrawsLineSegment) {
+  ...
+  {
+    InSequence seq;
+
+    EXPECT_CALL(turtle, PenDown());
+    EXPECT_CALL(turtle, Forward(100));
+    EXPECT_CALL(turtle, PenUp());
+  }
+  Foo();
+}
+```
+
+By creating an object of type `InSequence`, all expectations in its scope are
+put into a *sequence* and have to occur *sequentially*. Since we are just
+relying on the constructor and destructor of this object to do the actual work,
+its name is really irrelevant.
+
+In this example, we test that `Foo()` calls the three expected functions in the
+order as written. If a call is made out-of-order, it will be an error.
+
+(What if you care about the relative order of some of the calls, but not all of
+them? Can you specify an arbitrary partial order? The answer is ... yes! The
+details can be found [here](cook_book.md#OrderedCalls).)
+
+### All Expectations Are Sticky (Unless Said Otherwise) {#StickyExpectations}
+
+Now let's do a quick quiz to see how well you can use this mock stuff already.
+How would you test that the turtle is asked to go to the origin *exactly twice*
+(you want to ignore any other instructions it receives)?
+
+After you've come up with your answer, take a look at ours and compare notes
+(solve it yourself first - don't cheat!):
+
+```cpp
+using ::testing::_;
+using ::testing::AnyNumber;
+...
+EXPECT_CALL(turtle, GoTo(_, _))  // #1
+     .Times(AnyNumber());
+EXPECT_CALL(turtle, GoTo(0, 0))  // #2
+     .Times(2);
+```
+
+Suppose `turtle.GoTo(0, 0)` is called three times. In the third time, gMock will
+see that the arguments match expectation #2 (remember that we always pick the
+last matching expectation). Now, since we said that there should be only two
+such calls, gMock will report an error immediately. This is basically what we've
+told you in the [Using Multiple Expectations](#MultiExpectations) section above.
+
+This example shows that **expectations in gMock are "sticky" by default**, in
+the sense that they remain active even after we have reached their invocation
+upper bounds. This is an important rule to remember, as it affects the meaning
+of the spec, and is **different** to how it's done in many other mocking
+frameworks (Why'd we do that? Because we think our rule makes the common cases
+easier to express and understand.).
+
+Simple? Let's see if you've really understood it: what does the following code
+say?
+
+```cpp
+using ::testing::Return;
+...
+for (int i = n; i > 0; i--) {
+  EXPECT_CALL(turtle, GetX())
+      .WillOnce(Return(10*i));
+}
+```
+
+If you think it says that `turtle.GetX()` will be called `n` times and will
+return 10, 20, 30, ..., consecutively, think twice! The problem is that, as we
+said, expectations are sticky. So, the second time `turtle.GetX()` is called,
+the last (latest) `EXPECT_CALL()` statement will match, and will immediately
+lead to an "upper bound violated" error - this piece of code is not very useful!
+
+One correct way of saying that `turtle.GetX()` will return 10, 20, 30, ..., is
+to explicitly say that the expectations are *not* sticky. In other words, they
+should *retire* as soon as they are saturated:
+
+```cpp
+using ::testing::Return;
+...
+for (int i = n; i > 0; i--) {
+  EXPECT_CALL(turtle, GetX())
+      .WillOnce(Return(10*i))
+      .RetiresOnSaturation();
+}
+```
+
+And, there's a better way to do it: in this case, we expect the calls to occur
+in a specific order, and we line up the actions to match the order. Since the
+order is important here, we should make it explicit using a sequence:
+
+```cpp
+using ::testing::InSequence;
+using ::testing::Return;
+...
+{
+  InSequence s;
+
+  for (int i = 1; i <= n; i++) {
+    EXPECT_CALL(turtle, GetX())
+        .WillOnce(Return(10*i))
+        .RetiresOnSaturation();
+  }
+}
+```
+
+By the way, the other situation where an expectation may *not* be sticky is when
+it's in a sequence - as soon as another expectation that comes after it in the
+sequence has been used, it automatically retires (and will never be used to
+match any call).
+
+### Uninteresting Calls
+
+A mock object may have many methods, and not all of them are that interesting.
+For example, in some tests we may not care about how many times `GetX()` and
+`GetY()` get called.
+
+In gMock, if you are not interested in a method, just don't say anything about
+it. If a call to this method occurs, you'll see a warning in the test output,
+but it won't be a failure. This is called "naggy" behavior; to change, see
+[The Nice, the Strict, and the Naggy](cook_book.md#NiceStrictNaggy).
diff --git a/thirdparty/GTest/googletest/googlemock/docs/gmock_faq.md b/thirdparty/GTest/googletest/googlemock/docs/gmock_faq.md
new file mode 100644
index 0000000000..14acae5302
--- /dev/null
+++ b/thirdparty/GTest/googletest/googlemock/docs/gmock_faq.md
@@ -0,0 +1,398 @@
+## Legacy gMock FAQ {#GMockFaq}
+
+
+
+
+
+### When I call a method on my mock object, the method for the real object is invoked instead. What's the problem?
+
+In order for a method to be mocked, it must be *virtual*, unless you use the
+[high-perf dependency injection technique](cook_book.md#MockingNonVirtualMethods).
+
+### Can I mock a variadic function?
+
+You cannot mock a variadic function (i.e. a function taking ellipsis (`...`)
+arguments) directly in gMock.
+
+The problem is that in general, there is *no way* for a mock object to know how
+many arguments are passed to the variadic method, and what the arguments' types
+are. Only the *author of the base class* knows the protocol, and we cannot look
+into his or her head.
+
+Therefore, to mock such a function, the *user* must teach the mock object how to
+figure out the number of arguments and their types. One way to do it is to
+provide overloaded versions of the function.
+
+Ellipsis arguments are inherited from C and not really a C++ feature. They are
+unsafe to use and don't work with arguments that have constructors or
+destructors. Therefore we recommend to avoid them in C++ as much as possible.
+
+### MSVC gives me warning C4301 or C4373 when I define a mock method with a const parameter. Why?
+
+If you compile this using Microsoft Visual C++ 2005 SP1:
+
+```cpp
+class Foo {
+  ...
+  virtual void Bar(const int i) = 0;
+};
+
+class MockFoo : public Foo {
+  ...
+  MOCK_METHOD(void, Bar, (const int i), (override));
+};
+```
+
+You may get the following warning:
+
+```shell
+warning C4301: 'MockFoo::Bar': overriding virtual function only differs from 'Foo::Bar' by const/volatile qualifier
+```
+
+This is a MSVC bug. The same code compiles fine with gcc, for example. If you
+use Visual C++ 2008 SP1, you would get the warning:
+
+```shell
+warning C4373: 'MockFoo::Bar': virtual function overrides 'Foo::Bar', previous versions of the compiler did not override when parameters only differed by const/volatile qualifiers
+```
+
+In C++, if you *declare* a function with a `const` parameter, the `const`
+modifier is ignored. Therefore, the `Foo` base class above is equivalent to:
+
+```cpp
+class Foo {
+  ...
+  virtual void Bar(int i) = 0;  // int or const int?  Makes no difference.
+};
+```
+
+In fact, you can *declare* `Bar()` with an `int` parameter, and define it with a
+`const int` parameter. The compiler will still match them up.
+
+Since making a parameter `const` is meaningless in the method declaration, we
+recommend to remove it in both `Foo` and `MockFoo`. That should workaround the
+VC bug.
+
+Note that we are talking about the *top-level* `const` modifier here. If the
+function parameter is passed by pointer or reference, declaring the pointee or
+referee as `const` is still meaningful. For example, the following two
+declarations are *not* equivalent:
+
+```cpp
+void Bar(int* p);         // Neither p nor *p is const.
+void Bar(const int* p);  // p is not const, but *p is.
+```
+
+
+
+### I can't figure out why gMock thinks my expectations are not satisfied. What should I do?
+
+You might want to run your test with `--gmock_verbose=info`. This flag lets
+gMock print a trace of every mock function call it receives. By studying the
+trace, you'll gain insights on why the expectations you set are not met.
+
+If you see the message "The mock function has no default action set, and its
+return type has no default value set.", then try
+[adding a default action](for_dummies.md#DefaultValue). Due to a known issue,
+unexpected calls on mocks without default actions don't print out a detailed
+comparison between the actual arguments and the expected arguments.
+
+### My program crashed and `ScopedMockLog` spit out tons of messages. Is it a gMock bug?
+
+gMock and `ScopedMockLog` are likely doing the right thing here.
+
+When a test crashes, the failure signal handler will try to log a lot of
+information (the stack trace, and the address map, for example). The messages
+are compounded if you have many threads with depth stacks. When `ScopedMockLog`
+intercepts these messages and finds that they don't match any expectations, it
+prints an error for each of them.
+
+You can learn to ignore the errors, or you can rewrite your expectations to make
+your test more robust, for example, by adding something like:
+
+```cpp
+using ::testing::AnyNumber;
+using ::testing::Not;
+...
+  // Ignores any log not done by us.
+  EXPECT_CALL(log, Log(_, Not(EndsWith("/my_file.cc")), _))
+      .Times(AnyNumber());
+```
+
+### How can I assert that a function is NEVER called?
+
+```cpp
+using ::testing::_;
+...
+  EXPECT_CALL(foo, Bar(_))
+      .Times(0);
+```
+
+
+
+### I have a failed test where gMock tells me TWICE that a particular expectation is not satisfied. Isn't this redundant?
+
+When gMock detects a failure, it prints relevant information (the mock function
+arguments, the state of relevant expectations, and etc) to help the user debug.
+If another failure is detected, gMock will do the same, including printing the
+state of relevant expectations.
+
+Sometimes an expectation's state didn't change between two failures, and you'll
+see the same description of the state twice. They are however *not* redundant,
+as they refer to *different points in time*. The fact they are the same *is*
+interesting information.
+
+### I get a heapcheck failure when using a mock object, but using a real object is fine. What can be wrong?
+
+Does the class (hopefully a pure interface) you are mocking have a virtual
+destructor?
+
+Whenever you derive from a base class, make sure its destructor is virtual.
+Otherwise Bad Things will happen. Consider the following code:
+
+```cpp
+class Base {
+ public:
+  // Not virtual, but should be.
+  ~Base() { ... }
+  ...
+};
+
+class Derived : public Base {
+ public:
+  ...
+ private:
+  std::string value_;
+};
+
+...
+  Base* p = new Derived;
+  ...
+  delete p;  // Surprise! ~Base() will be called, but ~Derived() will not
+                 // - value_ is leaked.
+```
+
+By changing `~Base()` to virtual, `~Derived()` will be correctly called when
+`delete p` is executed, and the heap checker will be happy.
+
+### The "newer expectations override older ones" rule makes writing expectations awkward. Why does gMock do that?
+
+When people complain about this, often they are referring to code like:
+
+```cpp
+using ::testing::Return;
+...
+  // foo.Bar() should be called twice, return 1 the first time, and return
+  // 2 the second time.  However, I have to write the expectations in the
+  // reverse order.  This sucks big time!!!
+  EXPECT_CALL(foo, Bar())
+      .WillOnce(Return(2))
+      .RetiresOnSaturation();
+  EXPECT_CALL(foo, Bar())
+      .WillOnce(Return(1))
+      .RetiresOnSaturation();
+```
+
+The problem, is that they didn't pick the **best** way to express the test's
+intent.
+
+By default, expectations don't have to be matched in *any* particular order. If
+you want them to match in a certain order, you need to be explicit. This is
+gMock's (and jMock's) fundamental philosophy: it's easy to accidentally
+over-specify your tests, and we want to make it harder to do so.
+
+There are two better ways to write the test spec. You could either put the
+expectations in sequence:
+
+```cpp
+using ::testing::Return;
+...
+  // foo.Bar() should be called twice, return 1 the first time, and return
+  // 2 the second time.  Using a sequence, we can write the expectations
+  // in their natural order.
+  {
+    InSequence s;
+    EXPECT_CALL(foo, Bar())
+        .WillOnce(Return(1))
+        .RetiresOnSaturation();
+    EXPECT_CALL(foo, Bar())
+        .WillOnce(Return(2))
+        .RetiresOnSaturation();
+  }
+```
+
+or you can put the sequence of actions in the same expectation:
+
+```cpp
+using ::testing::Return;
+...
+  // foo.Bar() should be called twice, return 1 the first time, and return
+  // 2 the second time.
+  EXPECT_CALL(foo, Bar())
+      .WillOnce(Return(1))
+      .WillOnce(Return(2))
+      .RetiresOnSaturation();
+```
+
+Back to the original questions: why does gMock search the expectations (and
+`ON_CALL`s) from back to front? Because this allows a user to set up a mock's
+behavior for the common case early (e.g. in the mock's constructor or the test
+fixture's set-up phase) and customize it with more specific rules later. If
+gMock searches from front to back, this very useful pattern won't be possible.
+
+### gMock prints a warning when a function without EXPECT_CALL is called, even if I have set its behavior using ON_CALL. Would it be reasonable not to show the warning in this case?
+
+When choosing between being neat and being safe, we lean toward the latter. So
+the answer is that we think it's better to show the warning.
+
+Often people write `ON_CALL`s in the mock object's constructor or `SetUp()`, as
+the default behavior rarely changes from test to test. Then in the test body
+they set the expectations, which are often different for each test. Having an
+`ON_CALL` in the set-up part of a test doesn't mean that the calls are expected.
+If there's no `EXPECT_CALL` and the method is called, it's possibly an error. If
+we quietly let the call go through without notifying the user, bugs may creep in
+unnoticed.
+
+If, however, you are sure that the calls are OK, you can write
+
+```cpp
+using ::testing::_;
+...
+  EXPECT_CALL(foo, Bar(_))
+      .WillRepeatedly(...);
+```
+
+instead of
+
+```cpp
+using ::testing::_;
+...
+  ON_CALL(foo, Bar(_))
+      .WillByDefault(...);
+```
+
+This tells gMock that you do expect the calls and no warning should be printed.
+
+Also, you can control the verbosity by specifying `--gmock_verbose=error`. Other
+values are `info` and `warning`. If you find the output too noisy when
+debugging, just choose a less verbose level.
+
+### How can I delete the mock function's argument in an action?
+
+If your mock function takes a pointer argument and you want to delete that
+argument, you can use testing::DeleteArg() to delete the N'th (zero-indexed)
+argument:
+
+```cpp
+using ::testing::_;
+  ...
+  MOCK_METHOD(void, Bar, (X* x, const Y& y));
+  ...
+  EXPECT_CALL(mock_foo_, Bar(_, _))
+      .WillOnce(testing::DeleteArg<0>()));
+```
+
+### How can I perform an arbitrary action on a mock function's argument?
+
+If you find yourself needing to perform some action that's not supported by
+gMock directly, remember that you can define your own actions using
+[`MakeAction()`](#NewMonoActions) or
+[`MakePolymorphicAction()`](#NewPolyActions), or you can write a stub function
+and invoke it using [`Invoke()`](#FunctionsAsActions).
+
+```cpp
+using ::testing::_;
+using ::testing::Invoke;
+  ...
+  MOCK_METHOD(void, Bar, (X* p));
+  ...
+  EXPECT_CALL(mock_foo_, Bar(_))
+      .WillOnce(Invoke(MyAction(...)));
+```
+
+### My code calls a static/global function. Can I mock it?
+
+You can, but you need to make some changes.
+
+In general, if you find yourself needing to mock a static function, it's a sign
+that your modules are too tightly coupled (and less flexible, less reusable,
+less testable, etc). You are probably better off defining a small interface and
+call the function through that interface, which then can be easily mocked. It's
+a bit of work initially, but usually pays for itself quickly.
+
+This Google Testing Blog
+[post](https://testing.googleblog.com/2008/06/defeat-static-cling.html) says it
+excellently. Check it out.
+
+### My mock object needs to do complex stuff. It's a lot of pain to specify the actions. gMock sucks!
+
+I know it's not a question, but you get an answer for free any way. :-)
+
+With gMock, you can create mocks in C++ easily. And people might be tempted to
+use them everywhere. Sometimes they work great, and sometimes you may find them,
+well, a pain to use. So, what's wrong in the latter case?
+
+When you write a test without using mocks, you exercise the code and assert that
+it returns the correct value or that the system is in an expected state. This is
+sometimes called "state-based testing".
+
+Mocks are great for what some call "interaction-based" testing: instead of
+checking the system state at the very end, mock objects verify that they are
+invoked the right way and report an error as soon as it arises, giving you a
+handle on the precise context in which the error was triggered. This is often
+more effective and economical to do than state-based testing.
+
+If you are doing state-based testing and using a test double just to simulate
+the real object, you are probably better off using a fake. Using a mock in this
+case causes pain, as it's not a strong point for mocks to perform complex
+actions. If you experience this and think that mocks suck, you are just not
+using the right tool for your problem. Or, you might be trying to solve the
+wrong problem. :-)
+
+### I got a warning "Uninteresting function call encountered - default action taken.." Should I panic?
+
+By all means, NO! It's just an FYI. :-)
+
+What it means is that you have a mock function, you haven't set any expectations
+on it (by gMock's rule this means that you are not interested in calls to this
+function and therefore it can be called any number of times), and it is called.
+That's OK - you didn't say it's not OK to call the function!
+
+What if you actually meant to disallow this function to be called, but forgot to
+write `EXPECT_CALL(foo, Bar()).Times(0)`? While one can argue that it's the
+user's fault, gMock tries to be nice and prints you a note.
+
+So, when you see the message and believe that there shouldn't be any
+uninteresting calls, you should investigate what's going on. To make your life
+easier, gMock dumps the stack trace when an uninteresting call is encountered.
+From that you can figure out which mock function it is, and how it is called.
+
+### I want to define a custom action. Should I use Invoke() or implement the ActionInterface interface?
+
+Either way is fine - you want to choose the one that's more convenient for your
+circumstance.
+
+Usually, if your action is for a particular function type, defining it using
+`Invoke()` should be easier; if your action can be used in functions of
+different types (e.g. if you are defining `Return(*value*)`),
+`MakePolymorphicAction()` is easiest. Sometimes you want precise control on what
+types of functions the action can be used in, and implementing `ActionInterface`
+is the way to go here. See the implementation of `Return()` in
+`testing/base/public/gmock-actions.h` for an example.
+
+### I use SetArgPointee() in WillOnce(), but gcc complains about "conflicting return type specified". What does it mean?
+
+You got this error as gMock has no idea what value it should return when the
+mock method is called. `SetArgPointee()` says what the side effect is, but
+doesn't say what the return value should be. You need `DoAll()` to chain a
+`SetArgPointee()` with a `Return()` that provides a value appropriate to the API
+being mocked.
+
+See this [recipe](cook_book.md#mocking-side-effects) for more details and an
+example.
+
+### I have a huge mock class, and Microsoft Visual C++ runs out of memory when compiling it. What can I do?
+
+We've noticed that when the `/clr` compiler flag is used, Visual C++ uses 5~6
+times as much memory when compiling a mock class. We suggest to avoid `/clr`
+when compiling native C++ mocks.
diff --git a/thirdparty/GTest/googletest/googlemock/docs/pump_manual.md b/thirdparty/GTest/googletest/googlemock/docs/pump_manual.md
new file mode 100644
index 0000000000..17fb370dee
--- /dev/null
+++ b/thirdparty/GTest/googletest/googlemock/docs/pump_manual.md
@@ -0,0 +1,189 @@
+Pump is Useful for Meta Programming.
+
+
+
+# The Problem
+
+Template and macro libraries often need to define many classes, functions, or
+macros that vary only (or almost only) in the number of arguments they take.
+It's a lot of repetitive, mechanical, and error-prone work.
+
+Our experience is that it's tedious to write custom scripts, which tend to
+reflect the structure of the generated code poorly and are often hard to read
+and edit. For example, a small change needed in the generated code may require
+some non-intuitive, non-trivial changes in the script. This is especially
+painful when experimenting with the code.
+
+This script may be useful for generating meta code, for example a series of
+macros of FOO1, FOO2, etc. Nevertheless, please make it your last resort
+technique by favouring C++ template metaprogramming or variadic macros.
+
+# Our Solution
+
+Pump (for Pump is Useful for Meta Programming, Pretty Useful for Meta
+Programming, or Practical Utility for Meta Programming, whichever you prefer) is
+a simple meta-programming tool for C++. The idea is that a programmer writes a
+`foo.pump` file which contains C++ code plus meta code that manipulates the C++
+code. The meta code can handle iterations over a range, nested iterations, local
+meta variable definitions, simple arithmetic, and conditional expressions. You
+can view it as a small Domain-Specific Language. The meta language is designed
+to be non-intrusive (s.t. it won't confuse Emacs' C++ mode, for example) and
+concise, making Pump code intuitive and easy to maintain.
+
+## Highlights
+
+*   The implementation is in a single Python script and thus ultra portable: no
+    build or installation is needed and it works cross platforms.
+*   Pump tries to be smart with respect to
+    [Google's style guide](https://github.com/google/styleguide): it breaks long
+    lines (easy to have when they are generated) at acceptable places to fit
+    within 80 columns and indent the continuation lines correctly.
+*   The format is human-readable and more concise than XML.
+*   The format works relatively well with Emacs' C++ mode.
+
+## Examples
+
+The following Pump code (where meta keywords start with `$`, `[[` and `]]` are
+meta brackets, and `$$` starts a meta comment that ends with the line):
+
+```
+$var n = 3     $$ Defines a meta variable n.
+$range i 0..n  $$ Declares the range of meta iterator i (inclusive).
+$for i [[
+               $$ Meta loop.
+// Foo$i does blah for $i-ary predicates.
+$range j 1..i
+template 
+class Foo$i {
+$if i == 0 [[
+  blah a;
+]] $elif i <= 2 [[
+  blah b;
+]] $else [[
+  blah c;
+]]
+};
+
+]]
+```
+
+will be translated by the Pump compiler to:
+
+```cpp
+// Foo0 does blah for 0-ary predicates.
+template 
+class Foo0 {
+  blah a;
+};
+
+// Foo1 does blah for 1-ary predicates.
+template 
+class Foo1 {
+  blah b;
+};
+
+// Foo2 does blah for 2-ary predicates.
+template 
+class Foo2 {
+  blah b;
+};
+
+// Foo3 does blah for 3-ary predicates.
+template 
+class Foo3 {
+  blah c;
+};
+```
+
+In another example,
+
+```
+$range i 1..n
+Func($for i + [[a$i]]);
+$$ The text between i and [[ is the separator between iterations.
+```
+
+will generate one of the following lines (without the comments), depending on
+the value of `n`:
+
+```cpp
+Func();              // If n is 0.
+Func(a1);            // If n is 1.
+Func(a1 + a2);       // If n is 2.
+Func(a1 + a2 + a3);  // If n is 3.
+// And so on...
+```
+
+## Constructs
+
+We support the following meta programming constructs:
+
+| `$var id = exp`                  | Defines a named constant value. `$id` is |
+:                                  : valid until the end of the current meta  :
+:                                  : lexical block.                           :
+| :------------------------------- | :--------------------------------------- |
+| `$range id exp..exp`             | Sets the range of an iteration variable, |
+:                                  : which can be reused in multiple loops    :
+:                                  : later.                                   :
+| `$for id sep [[ code ]]`         | Iteration. The range of `id` must have   |
+:                                  : been defined earlier. `$id` is valid in  :
+:                                  : `code`.                                  :
+| `$($)`                           | Generates a single `$` character.        |
+| `$id`                            | Value of the named constant or iteration |
+:                                  : variable.                                :
+| `$(exp)`                         | Value of the expression.                 |
+| `$if exp [[ code ]] else_branch` | Conditional.                             |
+| `[[ code ]]`                     | Meta lexical block.                      |
+| `cpp_code`                       | Raw C++ code.                            |
+| `$$ comment`                     | Meta comment.                            |
+
+**Note:** To give the user some freedom in formatting the Pump source code, Pump
+ignores a new-line character if it's right after `$for foo` or next to `[[` or
+`]]`. Without this rule you'll often be forced to write very long lines to get
+the desired output. Therefore sometimes you may need to insert an extra new-line
+in such places for a new-line to show up in your output.
+
+## Grammar
+
+```ebnf
+code ::= atomic_code*
+atomic_code ::= $var id = exp
+    | $var id = [[ code ]]
+    | $range id exp..exp
+    | $for id sep [[ code ]]
+    | $($)
+    | $id
+    | $(exp)
+    | $if exp [[ code ]] else_branch
+    | [[ code ]]
+    | cpp_code
+sep ::= cpp_code | empty_string
+else_branch ::= $else [[ code ]]
+    | $elif exp [[ code ]] else_branch
+    | empty_string
+exp ::= simple_expression_in_Python_syntax
+```
+
+## Code
+
+You can find the source code of Pump in [scripts/pump.py](../scripts/pump.py).
+It is still very unpolished and lacks automated tests, although it has been
+successfully used many times. If you find a chance to use it in your project,
+please let us know what you think! We also welcome help on improving Pump.
+
+## Real Examples
+
+You can find real-world applications of Pump in
+[Google Test](https://github.com/google/googletest/tree/master/googletest) and
+[Google Mock](https://github.com/google/googletest/tree/master/googlemock). The
+source file `foo.h.pump` generates `foo.h`.
+
+## Tips
+
+*   If a meta variable is followed by a letter or digit, you can separate them
+    using `[[]]`, which inserts an empty string. For example `Foo$j[[]]Helper`
+    generate `Foo1Helper` when `j` is 1.
+*   To avoid extra-long Pump source lines, you can break a line anywhere you
+    want by inserting `[[]]` followed by a new line. Since any new-line
+    character next to `[[` or `]]` is ignored, the generated code won't contain
+    this new line.
diff --git a/thirdparty/GTest/googletest/googlemock/include/gmock/gmock-actions.h b/thirdparty/GTest/googletest/googlemock/include/gmock/gmock-actions.h
index 90fd2ea69d..02b17c7d9d 100644
--- a/thirdparty/GTest/googletest/googlemock/include/gmock/gmock-actions.h
+++ b/thirdparty/GTest/googletest/googlemock/include/gmock/gmock-actions.h
@@ -26,12 +26,106 @@
 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-//
-// Author: wan@google.com (Zhanyong Wan)
+
 
 // Google Mock - a framework for writing C++ mock classes.
 //
-// This file implements some commonly used actions.
+// The ACTION* family of macros can be used in a namespace scope to
+// define custom actions easily.  The syntax:
+//
+//   ACTION(name) { statements; }
+//
+// will define an action with the given name that executes the
+// statements.  The value returned by the statements will be used as
+// the return value of the action.  Inside the statements, you can
+// refer to the K-th (0-based) argument of the mock function by
+// 'argK', and refer to its type by 'argK_type'.  For example:
+//
+//   ACTION(IncrementArg1) {
+//     arg1_type temp = arg1;
+//     return ++(*temp);
+//   }
+//
+// allows you to write
+//
+//   ...WillOnce(IncrementArg1());
+//
+// You can also refer to the entire argument tuple and its type by
+// 'args' and 'args_type', and refer to the mock function type and its
+// return type by 'function_type' and 'return_type'.
+//
+// Note that you don't need to specify the types of the mock function
+// arguments.  However rest assured that your code is still type-safe:
+// you'll get a compiler error if *arg1 doesn't support the ++
+// operator, or if the type of ++(*arg1) isn't compatible with the
+// mock function's return type, for example.
+//
+// Sometimes you'll want to parameterize the action.   For that you can use
+// another macro:
+//
+//   ACTION_P(name, param_name) { statements; }
+//
+// For example:
+//
+//   ACTION_P(Add, n) { return arg0 + n; }
+//
+// will allow you to write:
+//
+//   ...WillOnce(Add(5));
+//
+// Note that you don't need to provide the type of the parameter
+// either.  If you need to reference the type of a parameter named
+// 'foo', you can write 'foo_type'.  For example, in the body of
+// ACTION_P(Add, n) above, you can write 'n_type' to refer to the type
+// of 'n'.
+//
+// We also provide ACTION_P2, ACTION_P3, ..., up to ACTION_P10 to support
+// multi-parameter actions.
+//
+// For the purpose of typing, you can view
+//
+//   ACTION_Pk(Foo, p1, ..., pk) { ... }
+//
+// as shorthand for
+//
+//   template 
+//   FooActionPk Foo(p1_type p1, ..., pk_type pk) { ... }
+//
+// In particular, you can provide the template type arguments
+// explicitly when invoking Foo(), as in Foo(5, false);
+// although usually you can rely on the compiler to infer the types
+// for you automatically.  You can assign the result of expression
+// Foo(p1, ..., pk) to a variable of type FooActionPk.  This can be useful when composing actions.
+//
+// You can also overload actions with different numbers of parameters:
+//
+//   ACTION_P(Plus, a) { ... }
+//   ACTION_P2(Plus, a, b) { ... }
+//
+// While it's tempting to always use the ACTION* macros when defining
+// a new action, you should also consider implementing ActionInterface
+// or using MakePolymorphicAction() instead, especially if you need to
+// use the action a lot.  While these approaches require more work,
+// they give you more control on the types of the mock function
+// arguments and the action parameters, which in general leads to
+// better compiler error messages that pay off in the long run.  They
+// also allow overloading actions based on parameter types (as opposed
+// to just based on the number of parameters).
+//
+// CAVEAT:
+//
+// ACTION*() can only be used in a namespace scope as templates cannot be
+// declared inside of a local class.
+// Users can, however, define any local functors (e.g. a lambda) that
+// can be used as actions.
+//
+// MORE INFORMATION:
+//
+// To learn more about using these macros, please search for 'ACTION' on
+// https://github.com/google/googletest/blob/master/googlemock/docs/cook_book.md
+
+// GOOGLETEST_CM0002 DO NOT DELETE
 
 #ifndef GMOCK_INCLUDE_GMOCK_GMOCK_ACTIONS_H_
 #define GMOCK_INCLUDE_GMOCK_GMOCK_ACTIONS_H_
@@ -41,15 +135,21 @@
 #endif
 
 #include 
+#include 
+#include 
 #include 
+#include 
+#include 
+#include 
 
 #include "gmock/internal/gmock-internal-utils.h"
 #include "gmock/internal/gmock-port.h"
+#include "gmock/internal/gmock-pp.h"
 
-#if GTEST_LANG_CXX11  // Defined by gtest-port.h via gmock-port.h.
-#include 
-#include 
-#endif  // GTEST_LANG_CXX11
+#ifdef _MSC_VER
+# pragma warning(push)
+# pragma warning(disable:4100)
+#endif
 
 namespace testing {
 
@@ -64,9 +164,6 @@ namespace testing {
 
 namespace internal {
 
-template 
-class ActionAdaptor;
-
 // BuiltInDefaultValueGetter::Get() returns a
 // default-constructed T value.  BuiltInDefaultValueGetter::Get() crashes with an error.
@@ -97,8 +194,8 @@ struct BuiltInDefaultValueGetter {
 template 
 class BuiltInDefaultValue {
  public:
-#if GTEST_LANG_CXX11
-  // This function returns true iff type T has a built-in default value.
+  // This function returns true if and only if type T has a built-in default
+  // value.
   static bool Exists() {
     return ::std::is_default_constructible::value;
   }
@@ -107,18 +204,6 @@ class BuiltInDefaultValue {
     return BuiltInDefaultValueGetter<
         T, ::std::is_default_constructible::value>::Get();
   }
-
-#else  // GTEST_LANG_CXX11
-  // This function returns true iff type T has a built-in default value.
-  static bool Exists() {
-    return false;
-  }
-
-  static T Get() {
-    return BuiltInDefaultValueGetter::Get();
-  }
-
-#endif  // GTEST_LANG_CXX11
 };
 
 // This partial specialization says that we use the same built-in
@@ -136,7 +221,7 @@ template 
 class BuiltInDefaultValue {
  public:
   static bool Exists() { return true; }
-  static T* Get() { return NULL; }
+  static T* Get() { return nullptr; }
 };
 
 // The following specializations define the default values for
@@ -150,9 +235,6 @@ class BuiltInDefaultValue {
   }
 
 GMOCK_DEFINE_DEFAULT_ACTION_FOR_RETURN_TYPE_(void, );  // NOLINT
-#if GTEST_HAS_GLOBAL_STRING
-GMOCK_DEFINE_DEFAULT_ACTION_FOR_RETURN_TYPE_(::string, "");
-#endif  // GTEST_HAS_GLOBAL_STRING
 GMOCK_DEFINE_DEFAULT_ACTION_FOR_RETURN_TYPE_(::std::string, "");
 GMOCK_DEFINE_DEFAULT_ACTION_FOR_RETURN_TYPE_(bool, false);
 GMOCK_DEFINE_DEFAULT_ACTION_FOR_RETURN_TYPE_(unsigned char, '\0');
@@ -175,13 +257,17 @@ GMOCK_DEFINE_DEFAULT_ACTION_FOR_RETURN_TYPE_(unsigned int, 0U);
 GMOCK_DEFINE_DEFAULT_ACTION_FOR_RETURN_TYPE_(signed int, 0);
 GMOCK_DEFINE_DEFAULT_ACTION_FOR_RETURN_TYPE_(unsigned long, 0UL);  // NOLINT
 GMOCK_DEFINE_DEFAULT_ACTION_FOR_RETURN_TYPE_(signed long, 0L);     // NOLINT
-GMOCK_DEFINE_DEFAULT_ACTION_FOR_RETURN_TYPE_(UInt64, 0);
-GMOCK_DEFINE_DEFAULT_ACTION_FOR_RETURN_TYPE_(Int64, 0);
+GMOCK_DEFINE_DEFAULT_ACTION_FOR_RETURN_TYPE_(unsigned long long, 0);  // NOLINT
+GMOCK_DEFINE_DEFAULT_ACTION_FOR_RETURN_TYPE_(signed long long, 0);  // NOLINT
 GMOCK_DEFINE_DEFAULT_ACTION_FOR_RETURN_TYPE_(float, 0);
 GMOCK_DEFINE_DEFAULT_ACTION_FOR_RETURN_TYPE_(double, 0);
 
 #undef GMOCK_DEFINE_DEFAULT_ACTION_FOR_RETURN_TYPE_
 
+// Simple two-arg form of std::disjunction.
+template 
+using disjunction = typename ::std::conditional::type;
+
 }  // namespace internal
 
 // When an unexpected function call is encountered, Google Mock will
@@ -219,11 +305,11 @@ class DefaultValue {
   // Unsets the default value for type T.
   static void Clear() {
     delete producer_;
-    producer_ = NULL;
+    producer_ = nullptr;
   }
 
-  // Returns true iff the user has set the default value for type T.
-  static bool IsSet() { return producer_ != NULL; }
+  // Returns true if and only if the user has set the default value for type T.
+  static bool IsSet() { return producer_ != nullptr; }
 
   // Returns true if T has a default return value set by the user or there
   // exists a built-in default value.
@@ -235,8 +321,8 @@ class DefaultValue {
   // otherwise returns the built-in default value. Requires that Exists()
   // is true, which ensures that the return value is well-defined.
   static T Get() {
-    return producer_ == NULL ?
-        internal::BuiltInDefaultValue::Get() : producer_->Produce();
+    return producer_ == nullptr ? internal::BuiltInDefaultValue::Get()
+                                : producer_->Produce();
   }
 
  private:
@@ -249,7 +335,7 @@ class DefaultValue {
   class FixedValueProducer : public ValueProducer {
    public:
     explicit FixedValueProducer(T value) : value_(value) {}
-    virtual T Produce() { return value_; }
+    T Produce() override { return value_; }
 
    private:
     const T value_;
@@ -260,7 +346,7 @@ class DefaultValue {
    public:
     explicit FactoryValueProducer(FactoryFunction factory)
         : factory_(factory) {}
-    virtual T Produce() { return factory_(); }
+    T Produce() override { return factory_(); }
 
    private:
     const FactoryFunction factory_;
@@ -281,12 +367,10 @@ class DefaultValue {
   }
 
   // Unsets the default value for type T&.
-  static void Clear() {
-    address_ = NULL;
-  }
+  static void Clear() { address_ = nullptr; }
 
-  // Returns true iff the user has set the default value for type T&.
-  static bool IsSet() { return address_ != NULL; }
+  // Returns true if and only if the user has set the default value for type T&.
+  static bool IsSet() { return address_ != nullptr; }
 
   // Returns true if T has a default return value set by the user or there
   // exists a built-in default value.
@@ -298,8 +382,8 @@ class DefaultValue {
   // otherwise returns the built-in default value if there is one;
   // otherwise aborts the process.
   static T& Get() {
-    return address_ == NULL ?
-        internal::BuiltInDefaultValue::Get() : *address_;
+    return address_ == nullptr ? internal::BuiltInDefaultValue::Get()
+                               : *address_;
   }
 
  private:
@@ -317,11 +401,11 @@ class DefaultValue {
 
 // Points to the user-set default value for type T.
 template 
-typename DefaultValue::ValueProducer* DefaultValue::producer_ = NULL;
+typename DefaultValue::ValueProducer* DefaultValue::producer_ = nullptr;
 
 // Points to the user-set default value for type T&.
 template 
-T* DefaultValue::address_ = NULL;
+T* DefaultValue::address_ = nullptr;
 
 // Implement this interface to define an action for function type F.
 template 
@@ -346,38 +430,60 @@ class ActionInterface {
 // An Action is a copyable and IMMUTABLE (except by assignment)
 // object that represents an action to be taken when a mock function
 // of type F is called.  The implementation of Action is just a
-// linked_ptr to const ActionInterface, so copying is fairly cheap.
-// Don't inherit from Action!
-//
+// std::shared_ptr to const ActionInterface. Don't inherit from Action!
 // You can view an object implementing ActionInterface as a
 // concrete action (including its current state), and an Action
 // object as a handle to it.
 template 
 class Action {
+  // Adapter class to allow constructing Action from a legacy ActionInterface.
+  // New code should create Actions from functors instead.
+  struct ActionAdapter {
+    // Adapter must be copyable to satisfy std::function requirements.
+    ::std::shared_ptr> impl_;
+
+    template 
+    typename internal::Function::Result operator()(Args&&... args) {
+      return impl_->Perform(
+          ::std::forward_as_tuple(::std::forward(args)...));
+    }
+  };
+
+  template 
+  using IsCompatibleFunctor = std::is_constructible, G>;
+
  public:
   typedef typename internal::Function::Result Result;
   typedef typename internal::Function::ArgumentTuple ArgumentTuple;
 
   // Constructs a null Action.  Needed for storing Action objects in
   // STL containers.
-  Action() : impl_(NULL) {}
-
-  // Constructs an Action from its implementation.  A NULL impl is
-  // used to represent the "do-default" action.
-  explicit Action(ActionInterface* impl) : impl_(impl) {}
+  Action() {}
+
+  // Construct an Action from a specified callable.
+  // This cannot take std::function directly, because then Action would not be
+  // directly constructible from lambda (it would require two conversions).
+  template <
+      typename G,
+      typename = typename std::enable_if, std::is_constructible,
+                                                        G>>::value>::type>
+  Action(G&& fun) {  // NOLINT
+    Init(::std::forward(fun), IsCompatibleFunctor());
+  }
 
-  // Copy constructor.
-  Action(const Action& action) : impl_(action.impl_) {}
+  // Constructs an Action from its implementation.
+  explicit Action(ActionInterface* impl)
+      : fun_(ActionAdapter{::std::shared_ptr>(impl)}) {}
 
   // This constructor allows us to turn an Action object into an
   // Action, as long as F's arguments can be implicitly converted
-  // to Func's and Func's return type can be implicitly converted to
-  // F's.
+  // to Func's and Func's return type can be implicitly converted to F's.
   template 
-  explicit Action(const Action& action);
+  explicit Action(const Action& action) : fun_(action.fun_) {}
 
-  // Returns true iff this is the DoDefault() action.
-  bool IsDoDefault() const { return impl_.get() == NULL; }
+  // Returns true if and only if this is the DoDefault() action.
+  bool IsDoDefault() const { return fun_ == nullptr; }
 
   // Performs the action.  Note that this method is const even though
   // the corresponding method in ActionInterface is not.  The reason
@@ -385,22 +491,39 @@ class Action {
   // another concrete action, not that the concrete action it binds to
   // cannot change state.  (Think of the difference between a const
   // pointer and a pointer to const.)
-  Result Perform(const ArgumentTuple& args) const {
-    internal::Assert(
-        !IsDoDefault(), __FILE__, __LINE__,
-        "You are using DoDefault() inside a composite action like "
-        "DoAll() or WithArgs().  This is not supported for technical "
-        "reasons.  Please instead spell out the default action, or "
-        "assign the default action to an Action variable and use "
-        "the variable in various places.");
-    return impl_->Perform(args);
+  Result Perform(ArgumentTuple args) const {
+    if (IsDoDefault()) {
+      internal::IllegalDoDefault(__FILE__, __LINE__);
+    }
+    return internal::Apply(fun_, ::std::move(args));
   }
 
  private:
-  template 
-  friend class internal::ActionAdaptor;
+  template 
+  friend class Action;
+
+  template 
+  void Init(G&& g, ::std::true_type) {
+    fun_ = ::std::forward(g);
+  }
+
+  template 
+  void Init(G&& g, ::std::false_type) {
+    fun_ = IgnoreArgs::type>{::std::forward(g)};
+  }
+
+  template 
+  struct IgnoreArgs {
+    template 
+    Result operator()(const Args&...) const {
+      return function_impl();
+    }
+
+    FunctionImpl function_impl;
+  };
 
-  internal::linked_ptr > impl_;
+  // fun_ is an empty function if and only if this is the DoDefault() action.
+  ::std::function fun_;
 };
 
 // The PolymorphicAction class template makes it easy to implement a
@@ -415,7 +538,7 @@ class Action {
 //     template 
 //     Result Perform(const ArgumentTuple& args) const {
 //       // Processes the arguments and returns a result, using
-//       // tr1::get(args) to get the N-th (0-based) argument in the tuple.
+//       // std::get(args) to get the N-th (0-based) argument in the tuple.
 //     }
 //     ...
 //   };
@@ -443,19 +566,15 @@ class PolymorphicAction {
 
     explicit MonomorphicImpl(const Impl& impl) : impl_(impl) {}
 
-    virtual Result Perform(const ArgumentTuple& args) {
+    Result Perform(const ArgumentTuple& args) override {
       return impl_.template Perform(args);
     }
 
    private:
     Impl impl_;
-
-    GTEST_DISALLOW_ASSIGN_(MonomorphicImpl);
   };
 
   Impl impl_;
-
-  GTEST_DISALLOW_ASSIGN_(PolymorphicAction);
 };
 
 // Creates an Action from its implementation and returns it.  The
@@ -479,31 +598,11 @@ inline PolymorphicAction MakePolymorphicAction(const Impl& impl) {
 
 namespace internal {
 
-// Allows an Action object to pose as an Action, as long as F2
-// and F1 are compatible.
-template 
-class ActionAdaptor : public ActionInterface {
- public:
-  typedef typename internal::Function::Result Result;
-  typedef typename internal::Function::ArgumentTuple ArgumentTuple;
-
-  explicit ActionAdaptor(const Action& from) : impl_(from.impl_) {}
-
-  virtual Result Perform(const ArgumentTuple& args) {
-    return impl_->Perform(args);
-  }
-
- private:
-  const internal::linked_ptr > impl_;
-
-  GTEST_DISALLOW_ASSIGN_(ActionAdaptor);
-};
-
 // Helper struct to specialize ReturnAction to execute a move instead of a copy
 // on return. Useful for move-only types, but could be used on any type.
 template 
 struct ByMoveWrapper {
-  explicit ByMoveWrapper(T value) : payload(internal::move(value)) {}
+  explicit ByMoveWrapper(T value) : payload(std::move(value)) {}
   T payload;
 };
 
@@ -531,18 +630,21 @@ struct ByMoveWrapper {
 // statement, and conversion of the result of Return to Action is a
 // good place for that.
 //
+// The real life example of the above scenario happens when an invocation
+// of gtl::Container() is passed into Return.
+//
 template 
 class ReturnAction {
  public:
   // Constructs a ReturnAction object from the value to be returned.
   // 'value' is passed by value instead of by const reference in order
   // to allow Return("string literal") to compile.
-  explicit ReturnAction(R value) : value_(new R(internal::move(value))) {}
+  explicit ReturnAction(R value) : value_(new R(std::move(value))) {}
 
   // This template type conversion operator allows Return(x) to be
   // used in ANY function that returns x's type.
   template 
-  operator Action() const {
+  operator Action() const {  // NOLINT
     // Assert statement belongs here because this is the best place to verify
     // conditions on F. It produces the clearest error messages
     // in most compilers.
@@ -553,8 +655,10 @@ class ReturnAction {
     // in the Impl class. But both definitions must be the same.
     typedef typename Function::Result Result;
     GTEST_COMPILE_ASSERT_(
-        !is_reference::value,
+        !std::is_reference::value,
         use_ReturnRef_instead_of_Return_to_return_a_reference);
+    static_assert(!std::is_void::value,
+                  "Can't use Return() on an action expected to return `void`.");
     return Action(new Impl(value_));
   }
 
@@ -573,14 +677,14 @@ class ReturnAction {
     // Result to call.  ImplicitCast_ forces the compiler to convert R to
     // Result without considering explicit constructors, thus resolving the
     // ambiguity. value_ is then initialized using its copy constructor.
-    explicit Impl(const linked_ptr& value)
+    explicit Impl(const std::shared_ptr& value)
         : value_before_cast_(*value),
           value_(ImplicitCast_(value_before_cast_)) {}
 
-    virtual Result Perform(const ArgumentTuple&) { return value_; }
+    Result Perform(const ArgumentTuple&) override { return value_; }
 
    private:
-    GTEST_COMPILE_ASSERT_(!is_reference::value,
+    GTEST_COMPILE_ASSERT_(!std::is_reference::value,
                           Result_cannot_be_a_reference_type);
     // We save the value before casting just in case it is being cast to a
     // wrapper type.
@@ -598,26 +702,22 @@ class ReturnAction {
     typedef typename Function::Result Result;
     typedef typename Function::ArgumentTuple ArgumentTuple;
 
-    explicit Impl(const linked_ptr& wrapper)
+    explicit Impl(const std::shared_ptr& wrapper)
         : performed_(false), wrapper_(wrapper) {}
 
-    virtual Result Perform(const ArgumentTuple&) {
+    Result Perform(const ArgumentTuple&) override {
       GTEST_CHECK_(!performed_)
           << "A ByMove() action should only be performed once.";
       performed_ = true;
-      return internal::move(wrapper_->payload);
+      return std::move(wrapper_->payload);
     }
 
    private:
     bool performed_;
-    const linked_ptr wrapper_;
-
-    GTEST_DISALLOW_ASSIGN_(Impl);
+    const std::shared_ptr wrapper_;
   };
 
-  const linked_ptr value_;
-
-  GTEST_DISALLOW_ASSIGN_(ReturnAction);
+  const std::shared_ptr value_;
 };
 
 // Implements the ReturnNull() action.
@@ -628,13 +728,7 @@ class ReturnNullAction {
   // pointer type on compile time.
   template 
   static Result Perform(const ArgumentTuple&) {
-#if GTEST_LANG_CXX11
     return nullptr;
-#else
-    GTEST_COMPILE_ASSERT_(internal::is_pointer::value,
-                          ReturnNull_can_be_used_to_return_a_pointer_only);
-    return NULL;
-#endif  // GTEST_LANG_CXX11
   }
 };
 
@@ -644,7 +738,7 @@ class ReturnVoidAction {
   // Allows Return() to be used in any void-returning function.
   template 
   static void Perform(const ArgumentTuple&) {
-    CompileAssertTypesEqual();
+    static_assert(std::is_void::value, "Result should be void.");
   }
 };
 
@@ -665,7 +759,7 @@ class ReturnRefAction {
     // Asserts that the function return type is a reference.  This
     // catches the user error of using ReturnRef(x) when Return(x)
     // should be used, and generates some helpful error message.
-    GTEST_COMPILE_ASSERT_(internal::is_reference::value,
+    GTEST_COMPILE_ASSERT_(std::is_reference::value,
                           use_Return_instead_of_ReturnRef_to_return_a_value);
     return Action(new Impl(ref_));
   }
@@ -680,19 +774,13 @@ class ReturnRefAction {
 
     explicit Impl(T& ref) : ref_(ref) {}  // NOLINT
 
-    virtual Result Perform(const ArgumentTuple&) {
-      return ref_;
-    }
+    Result Perform(const ArgumentTuple&) override { return ref_; }
 
    private:
     T& ref_;
-
-    GTEST_DISALLOW_ASSIGN_(Impl);
   };
 
   T& ref_;
-
-  GTEST_DISALLOW_ASSIGN_(ReturnRefAction);
 };
 
 // Implements the polymorphic ReturnRefOfCopy(x) action, which can be
@@ -714,7 +802,7 @@ class ReturnRefOfCopyAction {
     // catches the user error of using ReturnRefOfCopy(x) when Return(x)
     // should be used, and generates some helpful error message.
     GTEST_COMPILE_ASSERT_(
-        internal::is_reference::value,
+        std::is_reference::value,
         use_Return_instead_of_ReturnRefOfCopy_to_return_a_value);
     return Action(new Impl(value_));
   }
@@ -729,19 +817,43 @@ class ReturnRefOfCopyAction {
 
     explicit Impl(const T& value) : value_(value) {}  // NOLINT
 
-    virtual Result Perform(const ArgumentTuple&) {
-      return value_;
-    }
+    Result Perform(const ArgumentTuple&) override { return value_; }
 
    private:
     T value_;
-
-    GTEST_DISALLOW_ASSIGN_(Impl);
   };
 
   const T value_;
+};
 
-  GTEST_DISALLOW_ASSIGN_(ReturnRefOfCopyAction);
+// Implements the polymorphic ReturnRoundRobin(v) action, which can be
+// used in any function that returns the element_type of v.
+template 
+class ReturnRoundRobinAction {
+ public:
+  explicit ReturnRoundRobinAction(std::vector values) {
+    GTEST_CHECK_(!values.empty())
+        << "ReturnRoundRobin requires at least one element.";
+    state_->values = std::move(values);
+  }
+
+  template 
+  T operator()(Args&&...) const {
+     return state_->Next();
+  }
+
+ private:
+  struct State {
+    T Next() {
+      T ret_val = values[i++];
+      if (i == values.size()) i = 0;
+      return ret_val;
+    }
+
+    std::vector values;
+    size_t i = 0;
+  };
+  std::shared_ptr state_ = std::make_shared();
 };
 
 // Implements the polymorphic DoDefault() action.
@@ -750,7 +862,7 @@ class DoDefaultAction {
   // This template type conversion operator allows DoDefault() to be
   // used in any function.
   template 
-  operator Action() const { return Action(NULL); }
+  operator Action() const { return Action(); }  // NOLINT
 };
 
 // Implements the Assign action to set a given pointer referent to a
@@ -768,8 +880,6 @@ class AssignAction {
  private:
   T1* const ptr_;
   const T2 value_;
-
-  GTEST_DISALLOW_ASSIGN_(AssignAction);
 };
 
 #if !GTEST_OS_WINDOWS_MOBILE
@@ -791,99 +901,64 @@ class SetErrnoAndReturnAction {
  private:
   const int errno_;
   const T result_;
-
-  GTEST_DISALLOW_ASSIGN_(SetErrnoAndReturnAction);
 };
 
 #endif  // !GTEST_OS_WINDOWS_MOBILE
 
 // Implements the SetArgumentPointee(x) action for any function
-// whose N-th argument (0-based) is a pointer to x's type.  The
-// template parameter kIsProto is true iff type A is ProtocolMessage,
-// proto2::Message, or a sub-class of those.
-template 
-class SetArgumentPointeeAction {
- public:
-  // Constructs an action that sets the variable pointed to by the
-  // N-th function argument to 'value'.
-  explicit SetArgumentPointeeAction(const A& value) : value_(value) {}
-
-  template 
-  void Perform(const ArgumentTuple& args) const {
-    CompileAssertTypesEqual();
-    *::testing::get(args) = value_;
+// whose N-th argument (0-based) is a pointer to x's type.
+template 
+struct SetArgumentPointeeAction {
+  A value;
+
+  template 
+  void operator()(const Args&... args) const {
+    *::std::get(std::tie(args...)) = value;
   }
-
- private:
-  const A value_;
-
-  GTEST_DISALLOW_ASSIGN_(SetArgumentPointeeAction);
 };
 
-template 
-class SetArgumentPointeeAction {
- public:
-  // Constructs an action that sets the variable pointed to by the
-  // N-th function argument to 'proto'.  Both ProtocolMessage and
-  // proto2::Message have the CopyFrom() method, so the same
-  // implementation works for both.
-  explicit SetArgumentPointeeAction(const Proto& proto) : proto_(new Proto) {
-    proto_->CopyFrom(proto);
-  }
-
-  template 
-  void Perform(const ArgumentTuple& args) const {
-    CompileAssertTypesEqual();
-    ::testing::get(args)->CopyFrom(*proto_);
+// Implements the Invoke(object_ptr, &Class::Method) action.
+template 
+struct InvokeMethodAction {
+  Class* const obj_ptr;
+  const MethodPtr method_ptr;
+
+  template 
+  auto operator()(Args&&... args) const
+      -> decltype((obj_ptr->*method_ptr)(std::forward(args)...)) {
+    return (obj_ptr->*method_ptr)(std::forward(args)...);
   }
-
- private:
-  const internal::linked_ptr proto_;
-
-  GTEST_DISALLOW_ASSIGN_(SetArgumentPointeeAction);
 };
 
 // Implements the InvokeWithoutArgs(f) action.  The template argument
 // FunctionImpl is the implementation type of f, which can be either a
 // function pointer or a functor.  InvokeWithoutArgs(f) can be used as an
-// Action as long as f's type is compatible with F (i.e. f can be
-// assigned to a tr1::function).
+// Action as long as f's type is compatible with F.
 template 
-class InvokeWithoutArgsAction {
- public:
-  // The c'tor makes a copy of function_impl (either a function
-  // pointer or a functor).
-  explicit InvokeWithoutArgsAction(FunctionImpl function_impl)
-      : function_impl_(function_impl) {}
+struct InvokeWithoutArgsAction {
+  FunctionImpl function_impl;
 
   // Allows InvokeWithoutArgs(f) to be used as any action whose type is
   // compatible with f.
-  template 
-  Result Perform(const ArgumentTuple&) { return function_impl_(); }
-
- private:
-  FunctionImpl function_impl_;
-
-  GTEST_DISALLOW_ASSIGN_(InvokeWithoutArgsAction);
+  template 
+  auto operator()(const Args&...) -> decltype(function_impl()) {
+    return function_impl();
+  }
 };
 
 // Implements the InvokeWithoutArgs(object_ptr, &Class::Method) action.
 template 
-class InvokeMethodWithoutArgsAction {
- public:
-  InvokeMethodWithoutArgsAction(Class* obj_ptr, MethodPtr method_ptr)
-      : obj_ptr_(obj_ptr), method_ptr_(method_ptr) {}
+struct InvokeMethodWithoutArgsAction {
+  Class* const obj_ptr;
+  const MethodPtr method_ptr;
 
-  template 
-  Result Perform(const ArgumentTuple&) const {
-    return (obj_ptr_->*method_ptr_)();
-  }
-
- private:
-  Class* const obj_ptr_;
-  const MethodPtr method_ptr_;
+  using ReturnType =
+      decltype((std::declval()->*std::declval())());
 
-  GTEST_DISALLOW_ASSIGN_(InvokeMethodWithoutArgsAction);
+  template 
+  ReturnType operator()(const Args&...) const {
+    return (obj_ptr->*method_ptr)();
+  }
 };
 
 // Implements the IgnoreResult(action) action.
@@ -905,7 +980,7 @@ class IgnoreResultAction {
     typedef typename internal::Function::Result Result;
 
     // Asserts at compile time that F returns void.
-    CompileAssertTypesEqual();
+    static_assert(std::is_void::value, "Result type should be void.");
 
     return Action(new Impl(action_));
   }
@@ -919,7 +994,7 @@ class IgnoreResultAction {
 
     explicit Impl(const A& action) : action_(action) {}
 
-    virtual void Perform(const ArgumentTuple& args) {
+    void Perform(const ArgumentTuple& args) override {
       // Performs the action and ignores its result.
       action_.Perform(args);
     }
@@ -931,86 +1006,162 @@ class IgnoreResultAction {
         OriginalFunction;
 
     const Action action_;
-
-    GTEST_DISALLOW_ASSIGN_(Impl);
   };
 
   const A action_;
+};
 
-  GTEST_DISALLOW_ASSIGN_(IgnoreResultAction);
+template 
+struct WithArgsAction {
+  InnerAction action;
+
+  // The inner action could be anything convertible to Action.
+  // We use the conversion operator to detect the signature of the inner Action.
+  template 
+  operator Action() const {  // NOLINT
+    using TupleType = std::tuple;
+    Action::type...)>
+        converted(action);
+
+    return [converted](Args... args) -> R {
+      return converted.Perform(std::forward_as_tuple(
+        std::get(std::forward_as_tuple(std::forward(args)...))...));
+    };
+  }
 };
 
-// A ReferenceWrapper object represents a reference to type T,
-// which can be either const or not.  It can be explicitly converted
-// from, and implicitly converted to, a T&.  Unlike a reference,
-// ReferenceWrapper can be copied and can survive template type
-// inference.  This is used to support by-reference arguments in the
-// InvokeArgument(...) action.  The idea was from "reference
-// wrappers" in tr1, which we don't have in our source tree yet.
-template 
-class ReferenceWrapper {
+template 
+struct DoAllAction {
+ private:
+  template 
+  using NonFinalType =
+      typename std::conditional::value, T, const T&>::type;
+
+  template 
+  std::vector Convert(IndexSequence) const {
+    return {ActionT(std::get(actions))...};
+  }
+
  public:
-  // Constructs a ReferenceWrapper object from a T&.
-  explicit ReferenceWrapper(T& l_value) : pointer_(&l_value) {}  // NOLINT
+  std::tuple actions;
+
+  template 
+  operator Action() const {  // NOLINT
+    struct Op {
+      std::vector...)>> converted;
+      Action last;
+      R operator()(Args... args) const {
+        auto tuple_args = std::forward_as_tuple(std::forward(args)...);
+        for (auto& a : converted) {
+          a.Perform(tuple_args);
+        }
+        return last.Perform(std::move(tuple_args));
+      }
+    };
+    return Op{Convert...)>>(
+                  MakeIndexSequence()),
+              std::get(actions)};
+  }
+};
 
-  // Allows a ReferenceWrapper object to be implicitly converted to
-  // a T&.
-  operator T&() const { return *pointer_; }
- private:
-  T* pointer_;
+template 
+struct ReturnNewAction {
+  T* operator()() const {
+    return internal::Apply(
+        [](const Params&... unpacked_params) {
+          return new T(unpacked_params...);
+        },
+        params);
+  }
+  std::tuple params;
 };
 
-// Allows the expression ByRef(x) to be printed as a reference to x.
-template 
-void PrintTo(const ReferenceWrapper& ref, ::std::ostream* os) {
-  T& value = ref;
-  UniversalPrinter::Print(value, os);
-}
+template 
+struct ReturnArgAction {
+  template 
+  auto operator()(const Args&... args) const ->
+      typename std::tuple_element>::type {
+    return std::get(std::tie(args...));
+  }
+};
 
-// Does two actions sequentially.  Used for implementing the DoAll(a1,
-// a2, ...) action.
-template 
-class DoBothAction {
- public:
-  DoBothAction(Action1 action1, Action2 action2)
-      : action1_(action1), action2_(action2) {}
+template 
+struct SaveArgAction {
+  Ptr pointer;
 
-  // This template type conversion operator allows DoAll(a1, ..., a_n)
-  // to be used in ANY function of compatible type.
-  template 
-  operator Action() const {
-    return Action(new Impl(action1_, action2_));
+  template 
+  void operator()(const Args&... args) const {
+    *pointer = std::get(std::tie(args...));
   }
+};
 
- private:
-  // Implements the DoAll(...) action for a particular function type F.
-  template 
-  class Impl : public ActionInterface {
-   public:
-    typedef typename Function::Result Result;
-    typedef typename Function::ArgumentTuple ArgumentTuple;
-    typedef typename Function::MakeResultVoid VoidResult;
+template 
+struct SaveArgPointeeAction {
+  Ptr pointer;
 
-    Impl(const Action& action1, const Action& action2)
-        : action1_(action1), action2_(action2) {}
+  template 
+  void operator()(const Args&... args) const {
+    *pointer = *std::get(std::tie(args...));
+  }
+};
 
-    virtual Result Perform(const ArgumentTuple& args) {
-      action1_.Perform(args);
-      return action2_.Perform(args);
-    }
+template 
+struct SetArgRefereeAction {
+  T value;
+
+  template 
+  void operator()(Args&&... args) const {
+    using argk_type =
+        typename ::std::tuple_element>::type;
+    static_assert(std::is_lvalue_reference::value,
+                  "Argument must be a reference type.");
+    std::get(std::tie(args...)) = value;
+  }
+};
 
-   private:
-    const Action action1_;
-    const Action action2_;
+template 
+struct SetArrayArgumentAction {
+  I1 first;
+  I2 last;
 
-    GTEST_DISALLOW_ASSIGN_(Impl);
-  };
+  template 
+  void operator()(const Args&... args) const {
+    auto value = std::get(std::tie(args...));
+    for (auto it = first; it != last; ++it, (void)++value) {
+      *value = *it;
+    }
+  }
+};
+
+template 
+struct DeleteArgAction {
+  template 
+  void operator()(const Args&... args) const {
+    delete std::get(std::tie(args...));
+  }
+};
 
-  Action1 action1_;
-  Action2 action2_;
+template 
+struct ReturnPointeeAction {
+  Ptr pointer;
+  template 
+  auto operator()(const Args&...) const -> decltype(*pointer) {
+    return *pointer;
+  }
+};
 
-  GTEST_DISALLOW_ASSIGN_(DoBothAction);
+#if GTEST_HAS_EXCEPTIONS
+template 
+struct ThrowAction {
+  T exception;
+  // We use a conversion operator to adapt to any return type.
+  template 
+  operator Action() const {  // NOLINT
+    T copy = exception;
+    return [copy](Args...) -> R { throw copy; };
+  }
 };
+#endif  // GTEST_HAS_EXCEPTIONS
 
 }  // namespace internal
 
@@ -1046,21 +1197,52 @@ class DoBothAction {
 //   EXPECT_CALL(mock, Bar(5, _, _)).WillOnce(Invoke(DistanceToOrigin));
 typedef internal::IgnoredValue Unused;
 
-// This constructor allows us to turn an Action object into an
-// Action, as long as To's arguments can be implicitly converted
-// to From's and From's return type cann be implicitly converted to
-// To's.
-template 
-template 
-Action::Action(const Action& from)
-    : impl_(new internal::ActionAdaptor(from)) {}
+// Creates an action that does actions a1, a2, ..., sequentially in
+// each invocation. All but the last action will have a readonly view of the
+// arguments.
+template 
+internal::DoAllAction::type...> DoAll(
+    Action&&... action) {
+  return {std::forward_as_tuple(std::forward(action)...)};
+}
+
+// WithArg(an_action) creates an action that passes the k-th
+// (0-based) argument of the mock function to an_action and performs
+// it.  It adapts an action accepting one argument to one that accepts
+// multiple arguments.  For convenience, we also provide
+// WithArgs(an_action) (defined below) as a synonym.
+template 
+internal::WithArgsAction::type, k>
+WithArg(InnerAction&& action) {
+  return {std::forward(action)};
+}
+
+// WithArgs(an_action) creates an action that passes
+// the selected arguments of the mock function to an_action and
+// performs it.  It serves as an adaptor between actions with
+// different argument lists.
+template 
+internal::WithArgsAction::type, k, ks...>
+WithArgs(InnerAction&& action) {
+  return {std::forward(action)};
+}
+
+// WithoutArgs(inner_action) can be used in a mock function with a
+// non-empty argument list to perform inner_action, which takes no
+// argument.  In other words, it adapts an action accepting no
+// argument to one that accepts (and ignores) arguments.
+template 
+internal::WithArgsAction::type>
+WithoutArgs(InnerAction&& action) {
+  return {std::forward(action)};
+}
 
 // Creates an action that returns 'value'.  'value' is passed by value
 // instead of const reference - otherwise Return("string literal")
 // will trigger a compiler error about using array as initializer.
 template 
 internal::ReturnAction Return(R value) {
-  return internal::ReturnAction(internal::move(value));
+  return internal::ReturnAction(std::move(value));
 }
 
 // Creates an action that returns NULL.
@@ -1079,6 +1261,10 @@ inline internal::ReturnRefAction ReturnRef(R& x) {  // NOLINT
   return internal::ReturnRefAction(x);
 }
 
+// Prevent using ReturnRef on reference to temporary.
+template 
+internal::ReturnRefAction ReturnRef(R&&) = delete;
+
 // Creates an action that returns the reference to a copy of the
 // argument.  The copy is created when the action is constructed and
 // lives as long as the action.
@@ -1093,7 +1279,24 @@ inline internal::ReturnRefOfCopyAction ReturnRefOfCopy(const R& x) {
 // invariant.
 template 
 internal::ByMoveWrapper ByMove(R x) {
-  return internal::ByMoveWrapper(internal::move(x));
+  return internal::ByMoveWrapper(std::move(x));
+}
+
+// Creates an action that returns an element of `vals`. Calling this action will
+// repeatedly return the next value from `vals` until it reaches the end and
+// will restart from the beginning.
+template 
+internal::ReturnRoundRobinAction ReturnRoundRobin(std::vector vals) {
+  return internal::ReturnRoundRobinAction(std::move(vals));
+}
+
+// Creates an action that returns an element of `vals`. Calling this action will
+// repeatedly return the next value from `vals` until it reaches the end and
+// will restart from the beginning.
+template 
+internal::ReturnRoundRobinAction ReturnRoundRobin(
+    std::initializer_list vals) {
+  return internal::ReturnRoundRobinAction(std::vector(vals));
 }
 
 // Creates an action that does the default action for the give mock function.
@@ -1104,43 +1307,14 @@ inline internal::DoDefaultAction DoDefault() {
 // Creates an action that sets the variable pointed by the N-th
 // (0-based) function argument to 'value'.
 template 
-PolymorphicAction<
-  internal::SetArgumentPointeeAction<
-    N, T, internal::IsAProtocolMessage::value> >
-SetArgPointee(const T& x) {
-  return MakePolymorphicAction(internal::SetArgumentPointeeAction<
-      N, T, internal::IsAProtocolMessage::value>(x));
+internal::SetArgumentPointeeAction SetArgPointee(T value) {
+  return {std::move(value)};
 }
 
-#if !((GTEST_GCC_VER_ && GTEST_GCC_VER_ < 40000) || GTEST_OS_SYMBIAN)
-// This overload allows SetArgPointee() to accept a string literal.
-// GCC prior to the version 4.0 and Symbian C++ compiler cannot distinguish
-// this overload from the templated version and emit a compile error.
-template 
-PolymorphicAction<
-  internal::SetArgumentPointeeAction >
-SetArgPointee(const char* p) {
-  return MakePolymorphicAction(internal::SetArgumentPointeeAction<
-      N, const char*, false>(p));
-}
-
-template 
-PolymorphicAction<
-  internal::SetArgumentPointeeAction >
-SetArgPointee(const wchar_t* p) {
-  return MakePolymorphicAction(internal::SetArgumentPointeeAction<
-      N, const wchar_t*, false>(p));
-}
-#endif
-
 // The following version is DEPRECATED.
 template 
-PolymorphicAction<
-  internal::SetArgumentPointeeAction<
-    N, T, internal::IsAProtocolMessage::value> >
-SetArgumentPointee(const T& x) {
-  return MakePolymorphicAction(internal::SetArgumentPointeeAction<
-      N, T, internal::IsAProtocolMessage::value>(x));
+internal::SetArgumentPointeeAction SetArgumentPointee(T value) {
+  return {std::move(value)};
 }
 
 // Creates an action that sets a pointer referent to a given value.
@@ -1161,24 +1335,38 @@ SetErrnoAndReturn(int errval, T result) {
 
 #endif  // !GTEST_OS_WINDOWS_MOBILE
 
-// Various overloads for InvokeWithoutArgs().
+// Various overloads for Invoke().
+
+// Legacy function.
+// Actions can now be implicitly constructed from callables. No need to create
+// wrapper objects.
+// This function exists for backwards compatibility.
+template 
+typename std::decay::type Invoke(FunctionImpl&& function_impl) {
+  return std::forward(function_impl);
+}
+
+// Creates an action that invokes the given method on the given object
+// with the mock function's arguments.
+template 
+internal::InvokeMethodAction Invoke(Class* obj_ptr,
+                                                      MethodPtr method_ptr) {
+  return {obj_ptr, method_ptr};
+}
 
 // Creates an action that invokes 'function_impl' with no argument.
 template 
-PolymorphicAction >
+internal::InvokeWithoutArgsAction::type>
 InvokeWithoutArgs(FunctionImpl function_impl) {
-  return MakePolymorphicAction(
-      internal::InvokeWithoutArgsAction(function_impl));
+  return {std::move(function_impl)};
 }
 
 // Creates an action that invokes the given method on the given object
 // with no argument.
 template 
-PolymorphicAction >
-InvokeWithoutArgs(Class* obj_ptr, MethodPtr method_ptr) {
-  return MakePolymorphicAction(
-      internal::InvokeMethodWithoutArgsAction(
-          obj_ptr, method_ptr));
+internal::InvokeMethodWithoutArgsAction InvokeWithoutArgs(
+    Class* obj_ptr, MethodPtr method_ptr) {
+  return {obj_ptr, method_ptr};
 }
 
 // Creates an action that performs an_action and throws away its
@@ -1196,11 +1384,323 @@ inline internal::IgnoreResultAction IgnoreResult(const A& an_action) {
 // where Base is a base class of Derived, just write:
 //
 //   ByRef(derived)
+//
+// N.B. ByRef is redundant with std::ref, std::cref and std::reference_wrapper.
+// However, it may still be used for consistency with ByMove().
 template 
-inline internal::ReferenceWrapper ByRef(T& l_value) {  // NOLINT
-  return internal::ReferenceWrapper(l_value);
+inline ::std::reference_wrapper ByRef(T& l_value) {  // NOLINT
+  return ::std::reference_wrapper(l_value);
+}
+
+// The ReturnNew(a1, a2, ..., a_k) action returns a pointer to a new
+// instance of type T, constructed on the heap with constructor arguments
+// a1, a2, ..., and a_k. The caller assumes ownership of the returned value.
+template 
+internal::ReturnNewAction::type...> ReturnNew(
+    Params&&... params) {
+  return {std::forward_as_tuple(std::forward(params)...)};
+}
+
+// Action ReturnArg() returns the k-th argument of the mock function.
+template 
+internal::ReturnArgAction ReturnArg() {
+  return {};
+}
+
+// Action SaveArg(pointer) saves the k-th (0-based) argument of the
+// mock function to *pointer.
+template 
+internal::SaveArgAction SaveArg(Ptr pointer) {
+  return {pointer};
+}
+
+// Action SaveArgPointee(pointer) saves the value pointed to
+// by the k-th (0-based) argument of the mock function to *pointer.
+template 
+internal::SaveArgPointeeAction SaveArgPointee(Ptr pointer) {
+  return {pointer};
+}
+
+// Action SetArgReferee(value) assigns 'value' to the variable
+// referenced by the k-th (0-based) argument of the mock function.
+template 
+internal::SetArgRefereeAction::type> SetArgReferee(
+    T&& value) {
+  return {std::forward(value)};
+}
+
+// Action SetArrayArgument(first, last) copies the elements in
+// source range [first, last) to the array pointed to by the k-th
+// (0-based) argument, which can be either a pointer or an
+// iterator. The action does not take ownership of the elements in the
+// source range.
+template 
+internal::SetArrayArgumentAction SetArrayArgument(I1 first,
+                                                             I2 last) {
+  return {first, last};
+}
+
+// Action DeleteArg() deletes the k-th (0-based) argument of the mock
+// function.
+template 
+internal::DeleteArgAction DeleteArg() {
+  return {};
+}
+
+// This action returns the value pointed to by 'pointer'.
+template 
+internal::ReturnPointeeAction ReturnPointee(Ptr pointer) {
+  return {pointer};
+}
+
+// Action Throw(exception) can be used in a mock function of any type
+// to throw the given exception.  Any copyable value can be thrown.
+#if GTEST_HAS_EXCEPTIONS
+template 
+internal::ThrowAction::type> Throw(T&& exception) {
+  return {std::forward(exception)};
+}
+#endif  // GTEST_HAS_EXCEPTIONS
+
+namespace internal {
+
+// A macro from the ACTION* family (defined later in gmock-generated-actions.h)
+// defines an action that can be used in a mock function.  Typically,
+// these actions only care about a subset of the arguments of the mock
+// function.  For example, if such an action only uses the second
+// argument, it can be used in any mock function that takes >= 2
+// arguments where the type of the second argument is compatible.
+//
+// Therefore, the action implementation must be prepared to take more
+// arguments than it needs.  The ExcessiveArg type is used to
+// represent those excessive arguments.  In order to keep the compiler
+// error messages tractable, we define it in the testing namespace
+// instead of testing::internal.  However, this is an INTERNAL TYPE
+// and subject to change without notice, so a user MUST NOT USE THIS
+// TYPE DIRECTLY.
+struct ExcessiveArg {};
+
+// A helper class needed for implementing the ACTION* macros.
+template 
+class ActionHelper {
+ public:
+  template 
+  static Result Perform(Impl* impl, const std::tuple& args) {
+    static constexpr size_t kMaxArgs = sizeof...(Ts) <= 10 ? sizeof...(Ts) : 10;
+    return Apply(impl, args, MakeIndexSequence{},
+                 MakeIndexSequence<10 - kMaxArgs>{});
+  }
+
+ private:
+  template 
+  static Result Apply(Impl* impl, const std::tuple& args,
+                      IndexSequence, IndexSequence) {
+    return impl->template gmock_PerformImpl<
+        typename std::tuple_element>::type...>(
+        args, std::get(args)...,
+        ((void)rest_ids, ExcessiveArg())...);
+  }
+};
+
+// A helper base class needed for implementing the ACTION* macros.
+// Implements constructor and conversion operator for Action.
+//
+// Template specialization for parameterless Action.
+template 
+class ActionImpl {
+ public:
+  ActionImpl() = default;
+
+  template 
+  operator ::testing::Action() const {  // NOLINT(runtime/explicit)
+    return ::testing::Action(new typename Derived::template gmock_Impl());
+  }
+};
+
+// Template specialization for parameterized Action.
+template