More common tests

source/adios2/toolkit/sst/cp/ffs_marshal.c

@@ -894,6 +894,7 @@ static void DecodeAndPrepareData(SstStream Stream, int Writer)
     FMFieldList FieldList;
     FMStructDescList FormatList;
     void *BaseData;
+    int DumpData = -1;
 
     FFSformat = FFSTypeHandle_from_encode(Stream->ReaderFFSContext,
                                           WriterInfo->RawBuffer);
@@ -925,8 +926,16 @@ static void DecodeAndPrepareData(SstStream Stream, int Writer)
         FFSdecode_to_buffer(Stream->ReaderFFSContext, WriterInfo->RawBuffer,
                             decode_buf);
     }
-    //    printf("\nIncomingDatablock is %p :\n", BaseData);
-    //    FMdump_data(FMFormat_of_original(FFSformat), BaseData, 1024000);
+    if (DumpData == -1)
+    {
+        DumpData = (getenv("SstDumpData") != NULL);
+    }
+    if (DumpData)
+    {
+        printf("\nOn Rank %d, IncomingDatablock from writer %d is %p :\n",
+               Stream->Rank, Writer, BaseData);
+        FMdump_data(FMFormat_of_original(FFSformat), BaseData, 1024000);
+    }
     Info->DataBaseAddrs[Writer] = BaseData;
     FormatList = format_list_of_FMFormat(FMFormat_of_original(FFSformat));
     FieldList = FormatList[0].field_list;
@@ -1220,10 +1229,110 @@ void ExtractSelectionFromPartialCM(int ElementSize, size_t Dims,
     free(FirstIndex);
 }
 
+typedef struct _range_list
+{
+    size_t start;
+    size_t end;
+    struct _range_list *next;
+} * range_list;
+
+range_list static OneDCoverage(size_t start, size_t end,
+                               range_list uncovered_list)
+{
+    if (uncovered_list == NULL)
+        return NULL;
+
+    if ((start <= uncovered_list->start) && (end >= uncovered_list->end))
+    {
+        /* this uncovered element is covered now, recurse on next */
+        range_list next = uncovered_list->next;
+        free(uncovered_list);
+        return OneDCoverage(start, end, next);
+    }
+    else if ((end < uncovered_list->end) && (start > uncovered_list->start))
+    {
+        /* covering a bit in the middle */
+        range_list new = malloc(sizeof(*new));
+        new->next = uncovered_list->next;
+        new->end = uncovered_list->end;
+        new->start = end + 1;
+        uncovered_list->end = start - 1;
+        uncovered_list->next = new;
+        return (uncovered_list);
+    }
+    else if ((end < uncovered_list->start) || (start > uncovered_list->end))
+    {
+        uncovered_list->next = OneDCoverage(start, end, uncovered_list->next);
+        return uncovered_list;
+    }
+    else if (start <= uncovered_list->start)
+    {
+        /* we don't cover completely nor a middle portion, so this means we span
+         * the beginning */
+        uncovered_list->start = end + 1;
+        uncovered_list->next = OneDCoverage(start, end, uncovered_list->next);
+        return uncovered_list;
+    }
+    else if (end >= uncovered_list->end)
+    {
+        /* we don't cover completely nor a middle portion, so this means we span
+         * the end */
+        uncovered_list->end = start - 1;
+        uncovered_list->next = OneDCoverage(start, end, uncovered_list->next);
+        return uncovered_list;
+    }
+    return NULL;
+}
+
+static void DumpCoverageList(range_list list)
+{
+    if (!list)
+        return;
+    printf("%ld - %ld", list->start, list->end);
+    if (list->next != NULL)
+    {
+        printf(", ");
+        DumpCoverageList(list->next);
+    }
+}
+
+static void ImplementGapWarning(SstStream Stream, FFSArrayRequest Req)
+{
+    if (Req->RequestType == Local)
+    {
+        /* no analysis here */
+        return;
+    }
+    if (Req->VarRec->DimCount != 1)
+    {
+        /* at this point, multidimensional fill analysis is too much */
+        return;
+    }
+    struct _range_list *Required = malloc(sizeof(*Required));
+    Required->next = NULL;
+    Required->start = Req->Start[0];
+    Required->end = Req->Start[0] + Req->Count[0] - 1;
+    for (int i = 0; i < Stream->WriterCohortSize; i++)
+    {
+        size_t start = Req->VarRec->PerWriterStart[i][0];
+        size_t end = start + Req->VarRec->PerWriterCounts[i][0] - 1;
+        Required = OneDCoverage(start, end, Required);
+    }
+    if (Required != NULL)
+    {
+        printf("WARNING:   Reader Rank %d requested elements %lu - %lu,\n\tbut "
+               "these elements were not written by any writer rank: \n",
+               Stream->Rank, (unsigned long)Req->Start[0],
+               (unsigned long)Req->Start[0] + Req->Count[0] - 1);
+        DumpCoverageList(Required);
+    }
+}
+
 static void FillReadRequests(SstStream Stream, FFSArrayRequest Reqs)
 {
     while (Reqs)
     {
+        ImplementGapWarning(Stream, Reqs);
         for (int i = 0; i < Stream->WriterCohortSize; i++)
         {
             if (NeedWriter(Reqs, i))

testing/adios2/engine/staging-common/CMakeLists.txt

@@ -29,9 +29,11 @@ add_executable(TestCommonWrite TestCommonWrite.cpp)
 add_executable(TestCommonWriteAttrs TestCommonWriteAttrs.cpp)
 add_executable(TestCommonWriteModes TestCommonWriteModes.cpp)
 add_executable(TestCommonWriteLocal TestCommonWriteLocal.cpp)
+add_executable(TestCommonWriteShared TestCommonWriteShared.cpp)
 add_executable(TestCommonRead TestCommonRead.cpp)
 add_executable(TestCommonReadAttrs TestCommonReadAttrs.cpp)
 add_executable(TestCommonReadLocal TestCommonReadLocal.cpp)
+add_executable(TestCommonReadShared TestCommonReadShared.cpp)
 add_executable(TestCommonServer TestCommonServer.cpp)
 add_executable(TestCommonClient TestCommonClient.cpp)
 if(ADIOS2_HAVE_Fortran)
@@ -49,17 +51,21 @@ if(SST_INCLUDE_DIRS)
   target_include_directories(TestCommonWriteModes PRIVATE ${SST_INCLUDE_DIRS})
   target_include_directories(TestCommonWriteAttrs PRIVATE ${SST_INCLUDE_DIRS})
   target_include_directories(TestCommonWriteLocal PRIVATE ${SST_INCLUDE_DIRS})
+  target_include_directories(TestCommonWriteShared PRIVATE ${SST_INCLUDE_DIRS})
   target_include_directories(TestCommonRead PRIVATE ${SST_INCLUDE_DIRS})
   target_include_directories(TestCommonReadLocal PRIVATE ${SST_INCLUDE_DIRS})
+  target_include_directories(TestCommonReadShared PRIVATE ${SST_INCLUDE_DIRS})
   target_include_directories(TestCommonServer PRIVATE ${SST_INCLUDE_DIRS})
   target_include_directories(TestCommonClient PRIVATE ${SST_INCLUDE_DIRS})
 endif()
 target_link_libraries(TestCommonWrite adios2 gtest_interface ${Sst_LIBRARY})
 target_link_libraries(TestCommonWriteModes adios2 gtest_interface ${Sst_LIBRARY})
 target_link_libraries(TestCommonWriteAttrs adios2 gtest_interface ${Sst_LIBRARY})
 target_link_libraries(TestCommonWriteLocal adios2 gtest_interface ${Sst_LIBRARY})
+target_link_libraries(TestCommonWriteShared adios2 gtest_interface ${Sst_LIBRARY})
 target_link_libraries(TestCommonRead adios2 gtest_interface ${Sst_LIBRARY})
 target_link_libraries(TestCommonReadLocal adios2 gtest_interface ${Sst_LIBRARY})
+target_link_libraries(TestCommonReadShared adios2 gtest_interface ${Sst_LIBRARY})
 target_link_libraries(TestCommonReadAttrs adios2 gtest_interface ${Sst_LIBRARY})
 target_link_libraries(TestCommonServer adios2 gtest_interface ${Sst_LIBRARY})
 target_link_libraries(TestCommonClient adios2 gtest_interface ${Sst_LIBRARY})
@@ -69,8 +75,10 @@ if(ADIOS2_HAVE_MPI)
   target_link_libraries(TestCommonWriteModes MPI::MPI_C)
   target_link_libraries(TestCommonWriteAttrs MPI::MPI_C)
   target_link_libraries(TestCommonWriteLocal MPI::MPI_C)
+  target_link_libraries(TestCommonWriteShared MPI::MPI_C)
   target_link_libraries(TestCommonRead MPI::MPI_C)
   target_link_libraries(TestCommonReadLocal MPI::MPI_C)
+  target_link_libraries(TestCommonReadShared MPI::MPI_C)
   target_link_libraries(TestCommonReadAttrs MPI::MPI_C)
   target_link_libraries(TestCommonServer MPI::MPI_C)
   target_link_libraries(TestCommonClient MPI::MPI_C)
@@ -109,7 +117,7 @@ if (ADIOS2_HAVE_MPI AND NOT ("${MPIEXEC_EXECUTABLE}" STREQUAL "MPIEXEC_EXECUTABL
    endforeach()
 endif()
 
-set (SIMPLE_TESTS "1x1;NoReaderNoWait;TimeoutOnOpen;Modes;1x1.Attrs;1x1.Local")
+set (SIMPLE_TESTS "1x1;NoReaderNoWait;TimeoutOnOpen;Modes;1x1.Attrs;1x1.Local;1x1.SharedNothing;1x1.SharedIO;1x1.SharedVar;1x1.SharedNothingSync;1x1.SharedIOSync;1x1.SharedVarSync;2x1.SharedNothing;2x1.SharedIO;2x1.SharedVar;2x1.SharedNothingSync;2x1.SharedIOSync;2x1.SharedVarSync")
 set (SIMPLE_FORTRAN_TESTS "")
 if(ADIOS2_HAVE_Fortran)
   set (SIMPLE_FORTRAN_TESTS "FtoC.1x1;CtoF.1x1;FtoF.1x1")
@@ -175,6 +183,10 @@ if(ADIOS2_HAVE_MPI)
     list (REMOVE_ITEM INSITU_TESTS "TimeoutOnOpen" "Modes" "1x1.Attrs")
     # Local Vars don't work for InSitu
     list (FILTER INSITU_TESTS EXCLUDE REGEX ".*Local$")
+    # Multiple streams don't work for InSitu
+    list (FILTER INSITU_TESTS EXCLUDE REGEX "x1.Shared")
+    # Fortran Destination doesn't work for InSitu
+    list (FILTER INSITU_TESTS EXCLUDE REGEX "toF")
     # DelayedReader doesn't make sense for InSitu
     list (FILTER INSITU_TESTS EXCLUDE REGEX "DelayedReader.*")
     # Fortran scalar reads don't work for InSitu
@@ -188,9 +200,12 @@ endif()
 #   Setup tests for BP engine
 #
 if (NOT MSVC)    # not on windows
+    # Local Vars don't work for InSitu
     set (BP_TESTS ${ALL_SIMPLE_TESTS})
+    list (FILTER BP_TESTS EXCLUDE REGEX ".*SharedVar$")
     foreach(test ${BP_TESTS})
-        add_common_test(${test} BP)
+        add_common_test(${test} BP3)
+        add_common_test(${test} BP4)
     endforeach()
 endif(NOT MSVC)
 

testing/adios2/engine/staging-common/ParseArgs.h

@@ -0,0 +1,180 @@
+#ifndef _WIN32
+#include "strings.h"
+#else
+#define strcasecmp _stricmp
+#endif
+std::string fname = "ADIOS2Common";
+std::string engine = "sst";
+adios2::Params engineParams = {}; // parsed from command line
+
+bool SharedIO = false;
+bool SharedVar = false;
+int DurationSeconds = 60 * 60 * 24 * 365; // one year default
+int DelayMS = 1000;                       // one step per sec default
+int CompressSz = 0;
+int CompressZfp = 0;
+int TimeGapExpected = 0;
+int IgnoreTimeGap = 1;
+int ExpectOpenTimeout = 0;
+std::string shutdown_name = "DieTest";
+adios2::Mode GlobalWriteMode = adios2::Mode::Deferred;
+
+static std::string Trim(std::string &str)
+{
+    size_t first = str.find_first_not_of(' ');
+    size_t last = str.find_last_not_of(' ');
+    return str.substr(first, (last - first + 1));
+}
+
+/*
+ * Engine parameters spec is a poor-man's JSON.  name:value pairs are separated
+ * by commas.  White space is trimmed off front and back.  No quotes or anything
+ * fancy allowed.
+ */
+static adios2::Params ParseEngineParams(std::string Input)
+{
+    std::istringstream ss(Input);
+    std::string Param;
+    adios2::Params Ret = {};
+
+    while (std::getline(ss, Param, ','))
+    {
+        std::istringstream ss2(Param);
+        std::string ParamName;
+        std::string ParamValue;
+        std::getline(ss2, ParamName, ':');
+        if (!std::getline(ss2, ParamValue, ':'))
+        {
+            throw std::invalid_argument("Engine parameter \"" + Param +
+                                        "\" missing value");
+        }
+        Ret[Trim(ParamName)] = Trim(ParamValue);
+    }
+    return Ret;
+}
+
+static void ParseArgs(int argc, char **argv)
+{
+    int bare_arg = 0;
+    while (argc > 1)
+    {
+        if (std::string(argv[1]) == "--expect_time_gap")
+        {
+
+            TimeGapExpected++;
+            IgnoreTimeGap = 0;
+        }
+        else if (std::string(argv[1]) == "--expect_contiguous_time")
+        {
+            TimeGapExpected = 0;
+            IgnoreTimeGap = 0;
+        }
+        else if (std::string(argv[1]) == "--compress_sz")
+        {
+            CompressSz++;
+        }
+        else if (std::string(argv[1]) == "--shared_io")
+        {
+            SharedIO = true;
+        }
+        else if (std::string(argv[1]) == "--shared_var")
+        {
+            SharedVar = true;
+            SharedIO = true;
+        }
+        else if (std::string(argv[1]) == "--compress_zfp")
+        {
+            CompressZfp++;
+        }
+        else if (std::string(argv[1]) == "--filename")
+        {
+            fname = std::string(argv[2]);
+            argv++;
+            argc--;
+        }
+        else if (std::string(argv[1]) == "--write_mode")
+        {
+            if (strcasecmp(argv[2], "sync") == 0)
+            {
+                GlobalWriteMode = adios2::Mode::Sync;
+            }
+            else if (strcasecmp(argv[2], "deferred") == 0)
+            {
+                GlobalWriteMode = adios2::Mode::Deferred;
+            }
+            else
+            {
+                std::cerr << "Invalid mode for --write_mode " << argv[2]
+                          << std::endl;
+            }
+            argv++;
+            argc--;
+        }
+        else if (std::string(argv[1]) == "--engine_params")
+        {
+            engineParams = ParseEngineParams(argv[2]);
+            argv++;
+            argc--;
+        }
+        else if (std::string(argv[1]) == "--engine")
+        {
+            engine = std::string(argv[2]);
+            argv++;
+            argc--;
+        }
+        else if (std::string(argv[1]) == "--expect_timeout")
+        {
+            ExpectOpenTimeout++;
+        }
+        else if (std::string(argv[1]) == "--duration")
+        {
+            std::istringstream ss(argv[2]);
+            if (!(ss >> DurationSeconds))
+                std::cerr << "Invalid number for duration " << argv[1] << '\n';
+            argv++;
+            argc--;
+        }
+        else if (std::string(argv[1]) == "--shutdown_filename")
+        {
+            shutdown_name = std::string(argv[2]);
+            argv++;
+            argc--;
+        }
+        else if (std::string(argv[1]) == "--ms_delay")
+        {
+            std::istringstream ss(argv[2]);
+            if (!(ss >> DelayMS))
+                std::cerr << "Invalid number for ms_delay " << argv[1] << '\n';
+            argv++;
+            argc--;
+        }
+        else
+        {
+            if (bare_arg == 0)
+            {
+                /* first arg without -- is engine */
+                engine = std::string(argv[1]);
+                bare_arg++;
+            }
+            else if (bare_arg == 1)
+            {
+                /* second arg without -- is filename */
+                fname = std::string(argv[1]);
+                bare_arg++;
+            }
+            else if (bare_arg == 2)
+            {
+                engineParams = ParseEngineParams(argv[1]);
+                bare_arg++;
+            }
+            else
+            {
+
+                throw std::invalid_argument("Unknown argument \"" +
+                                            std::string(argv[1]) + "\"");
+            }
+        }
+        argv++;
+        argc--;
+    }
+}