Merge pull request #52 from eseiler/misc/upgrade_index

[MISC] Add raptor upgrade
seqan · Aug 13, 2021 · dac69f3 · dac69f3
2 parents 22d14b2 + c4740ae
commit dac69f3
Show file tree

Hide file tree

Showing 19 changed files with 323 additions and 5 deletions.
diff --git a/include/raptor/argument_parsing/upgrade.hpp b/include/raptor/argument_parsing/upgrade.hpp
@@ -0,0 +1,11 @@
+#pragma once
+
+#include <raptor/argument_parsing/shared.hpp>
+
+namespace raptor
+{
+
+void init_upgrade_parser(seqan3::argument_parser & parser, upgrade_arguments & arguments);
+void run_upgrade(seqan3::argument_parser & parser);
+
+} // namespace raptor
diff --git a/include/raptor/build/store_index.hpp b/include/raptor/build/store_index.hpp
@@ -7,10 +7,10 @@
 namespace raptor
 {
 
-template <auto layout>
+template <auto layout, typename arguments_t>
 static inline void store_index(std::filesystem::path const & path,
                                seqan3::interleaved_bloom_filter<layout> const & ibf,
-                               build_arguments const & arguments)
+                               arguments_t const & arguments)
 {
     std::ofstream os{path, std::ios::binary};
     cereal::BinaryOutputArchive oarchive{os};

diff --git a/include/raptor/shared.hpp b/include/raptor/shared.hpp
@@ -72,4 +72,18 @@ struct search_arguments
     bool is_socks{false};
 };
 
+struct upgrade_arguments
+{
+    uint32_t window_size{};
+    uint8_t kmer_size{};
+    uint8_t parts{1u};
+    bool compressed{false};
+
+    std::filesystem::path bin_file{};
+    std::filesystem::path in_file{};
+    std::filesystem::path out_file{};
+
+    std::vector<std::vector<std::string>> bin_path{};
+};
+
 } // namespace raptor
diff --git a/include/raptor/upgrade/upgrade.hpp b/include/raptor/upgrade/upgrade.hpp
@@ -0,0 +1,10 @@
+#pragma once
+
+#include <raptor/shared.hpp>
+
+namespace raptor
+{
+
+void raptor_upgrade(upgrade_arguments const & arguments);
+
+} // namespace raptor
diff --git a/include/raptor/upgrade/upgrade_index.hpp b/include/raptor/upgrade/upgrade_index.hpp
@@ -0,0 +1,40 @@
+#pragma once
+
+#include <raptor/argument_parsing/shared.hpp>
+#include <raptor/build/store_index.hpp>
+
+namespace raptor
+{
+
+template <bool compressed>
+void upgrade_index(upgrade_arguments const & arguments)
+{
+    constexpr seqan3::data_layout layout = compressed ? seqan3::data_layout::compressed : seqan3::data_layout::uncompressed;
+    seqan3::interleaved_bloom_filter<layout> original_ibf{};
+
+    if (arguments.parts == 1u)
+    {
+        std::ifstream is{arguments.in_file, std::ios::binary};
+        cereal::BinaryInputArchive iarchive{is};
+        iarchive(original_ibf);
+        store_index(arguments.out_file, original_ibf, arguments);
+    }
+    else
+    {
+        for (size_t part : std::views::iota(0u, arguments.parts))
+        {
+            std::filesystem::path in_file{arguments.in_file};
+            in_file += "_" + std::to_string(part);
+
+            std::ifstream is{in_file, std::ios::binary};
+            cereal::BinaryInputArchive iarchive{is};
+            iarchive(original_ibf);
+
+            std::filesystem::path out_file{arguments.out_file};
+            out_file += "_" + std::to_string(part);
+            store_index(out_file, original_ibf, arguments);
+        }
+    }
+}
+
+} // namespace raptor
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
@@ -26,6 +26,10 @@ target_link_libraries ("${PROJECT_NAME}_simple_model_lib" PUBLIC "${PROJECT_NAME
 add_library ("${PROJECT_NAME}_search_lib" STATIC raptor_search.cpp)
 target_link_libraries ("${PROJECT_NAME}_search_lib" PUBLIC "${PROJECT_NAME}_simple_model_lib")
 
+# Raptor upgrade
+add_library ("${PROJECT_NAME}_upgrade_lib" STATIC raptor_upgrade.cpp)
+target_link_libraries ("${PROJECT_NAME}_upgrade_lib" PUBLIC "${PROJECT_NAME}_interface")
+
 # Raptor argument parsing
 add_library ("${PROJECT_NAME}_argument_parsing_shared_lib" STATIC argument_parsing/shared.cpp)
 target_link_libraries ("${PROJECT_NAME}_argument_parsing_shared_lib" PUBLIC "${PROJECT_NAME}_interface")
@@ -39,14 +43,19 @@ target_link_libraries ("${PROJECT_NAME}_argument_parsing_search_lib" PUBLIC "${P
 add_library ("${PROJECT_NAME}_argument_parsing_top_level_lib" STATIC argument_parsing/top_level.cpp)
 target_link_libraries ("${PROJECT_NAME}_argument_parsing_top_level_lib" PUBLIC "${PROJECT_NAME}_argument_parsing_shared_lib")
 
+add_library ("${PROJECT_NAME}_argument_parsing_upgrade_lib" STATIC argument_parsing/upgrade.cpp)
+target_link_libraries ("${PROJECT_NAME}_argument_parsing_upgrade_lib" PUBLIC "${PROJECT_NAME}_argument_parsing_shared_lib")
+
 # Raptor library
 add_library ("${PROJECT_NAME}_lib" INTERFACE)
 target_link_libraries ("${PROJECT_NAME}_lib" INTERFACE "${PROJECT_NAME}_argument_parsing_build_lib")
 target_link_libraries ("${PROJECT_NAME}_lib" INTERFACE "${PROJECT_NAME}_argument_parsing_search_lib")
 target_link_libraries ("${PROJECT_NAME}_lib" INTERFACE "${PROJECT_NAME}_argument_parsing_shared_lib")
 target_link_libraries ("${PROJECT_NAME}_lib" INTERFACE "${PROJECT_NAME}_argument_parsing_top_level_lib")
+target_link_libraries ("${PROJECT_NAME}_lib" INTERFACE "${PROJECT_NAME}_argument_parsing_upgrade_lib")
 target_link_libraries ("${PROJECT_NAME}_lib" INTERFACE "${PROJECT_NAME}_build_lib")
 target_link_libraries ("${PROJECT_NAME}_lib" INTERFACE "${PROJECT_NAME}_search_lib")
+target_link_libraries ("${PROJECT_NAME}_lib" INTERFACE "${PROJECT_NAME}_upgrade_lib")
 
 # Raptor executable
 add_executable ("${PROJECT_NAME}" raptor_main.cpp)

diff --git a/src/argument_parsing/upgrade.cpp b/src/argument_parsing/upgrade.cpp
@@ -0,0 +1,96 @@
+#include <raptor/argument_parsing/upgrade.hpp>
+#include <raptor/upgrade/upgrade.hpp>
+
+namespace raptor
+{
+
+void init_upgrade_parser(seqan3::argument_parser & parser, upgrade_arguments & arguments)
+{
+    init_shared_meta(parser);
+    parser.add_option(arguments.bin_file,
+                      '\0',
+                      "bins",
+                      "File containing one file per line per bin.",
+                      seqan3::option_spec::required,
+                      seqan3::input_file_validator{});
+    parser.add_option(arguments.in_file,
+                      '\0',
+                      "input",
+                      "The index to upgrade. Parts: Without suffix _0",
+                      seqan3::option_spec::required);
+    parser.add_option(arguments.out_file,
+                      '\0',
+                      "output",
+                      "Path to new index.",
+                      seqan3::option_spec::required,
+                      seqan3::output_file_validator{});
+    parser.add_option(arguments.window_size,
+                      '\0',
+                      "window",
+                      "The original window size.",
+                      seqan3::option_spec::required,
+                      positive_integer_validator{});
+    parser.add_option(arguments.kmer_size,
+                      '\0',
+                      "kmer",
+                      "The original kmer size.",
+                      seqan3::option_spec::required,
+                      seqan3::arithmetic_range_validator{1, 32});
+    parser.add_option(arguments.parts,
+                      '\0',
+                      "parts",
+                      "Original index consisted of this many parts.",
+                      seqan3::option_spec::standard,
+                      power_of_two_validator{});
+    parser.add_flag(arguments.compressed,
+                    '\0',
+                    "compressed",
+                    "Original IBF was compressed.");
+}
+
+void run_upgrade(seqan3::argument_parser & parser)
+{
+    upgrade_arguments arguments{};
+    init_upgrade_parser(parser, arguments);
+    try_parsing(parser);
+
+    // ==========================================
+    // Various checks.
+    // ==========================================
+
+    if (arguments.parts == 1)
+    {
+        seqan3::input_file_validator{}(arguments.in_file);
+    }
+    else
+    {
+        seqan3::input_file_validator validator{};
+        for (size_t part{0}; part < arguments.parts; ++part)
+        {
+            validator(arguments.in_file.string() + std::string{"_"} + std::to_string(part));
+        }
+    }
+
+    // ==========================================
+    // Process bin_path
+    // ==========================================
+    std::ifstream istrm{arguments.bin_file};
+    std::string line;
+    auto sequence_file_validator{bin_validator{}.sequence_file_validator};
+
+    while (std::getline(istrm, line))
+    {
+        if (!line.empty())
+        {
+            sequence_file_validator(line);
+            arguments.bin_path.emplace_back(std::vector<std::string>{line});
+        }
+    }
+
+    // ==========================================
+    // Dispatch
+    // ==========================================
+    raptor_upgrade(arguments);
+};
+
+} // namespace raptor
diff --git a/src/raptor_main.cpp b/src/raptor_main.cpp
@@ -3,14 +3,15 @@
 #include <raptor/argument_parsing/build.hpp>
 #include <raptor/argument_parsing/search.hpp>
 #include <raptor/argument_parsing/shared.hpp>
+#include <raptor/argument_parsing/upgrade.hpp>
 #include <raptor/argument_parsing/top_level.hpp>
 #include <raptor/raptor.hpp>
 
 int main(int argc, char ** argv)
 {
     try
     {
-        seqan3::argument_parser top_level_parser{"raptor", argc, argv, seqan3::update_notifications::on, {"build", "search", "socks"}};
+        seqan3::argument_parser top_level_parser{"raptor", argc, argv, seqan3::update_notifications::on, {"build", "search", "socks", "upgrade"}};
         raptor::init_top_level_parser(top_level_parser);
 
         raptor::try_parsing(top_level_parser);
@@ -30,6 +31,8 @@ int main(int argc, char ** argv)
             if (socks_sub_parser.info.app_name == std::string_view{"socks-lookup-kmer"})
                 raptor::run_search(socks_sub_parser, true);
         }
+        if (sub_parser.info.app_name == std::string_view{"raptor-upgrade"})
+            raptor::run_upgrade(sub_parser);
     }
     catch (seqan3::argument_parser_error const & ext)
     {

diff --git a/src/raptor_upgrade.cpp b/src/raptor_upgrade.cpp
@@ -0,0 +1,16 @@
+#include <raptor/upgrade/upgrade.hpp>
+#include <raptor/upgrade/upgrade_index.hpp>
+
+namespace raptor
+{
+
+void raptor_upgrade(upgrade_arguments const & arguments)
+{
+    if (arguments.compressed)
+        upgrade_index<true>(arguments);
+    else
+        upgrade_index<false>(arguments);
+    return;
+}
+
+} // namespace raptor
diff --git a/test/cli/CMakeLists.txt b/test/cli/CMakeLists.txt
@@ -17,6 +17,11 @@ target_use_datasources (raptor_test FILES 64bins19window.ibf)
 target_use_datasources (raptor_test FILES 64bins23window.ibf)
 target_use_datasources (raptor_test FILES 128bins19window.ibf)
 target_use_datasources (raptor_test FILES 128bins23window.ibf)
+target_use_datasources (raptor_test FILES 1_1.ibf)
+target_use_datasources (raptor_test FILES 1_1.ibf_0)
+target_use_datasources (raptor_test FILES 1_1.ibf_1)
+target_use_datasources (raptor_test FILES 1_1.ibf_2)
+target_use_datasources (raptor_test FILES 1_1.ibf_3)
 target_use_datasources (raptor_test FILES 1bins19window0error.out)
 target_use_datasources (raptor_test FILES 1bins19window1error.out)
 target_use_datasources (raptor_test FILES 1bins23window1error.out)

diff --git a/test/cli/cli_test.hpp b/test/cli/cli_test.hpp
@@ -157,3 +157,4 @@ struct raptor : public cli_test
 struct raptor_build : public raptor, public testing::WithParamInterface<std::tuple<size_t, size_t, bool>> {};
 struct raptor_search : public raptor, public testing::WithParamInterface<std::tuple<size_t, size_t, size_t>> {};
 struct raptor_parts : public raptor, public testing::WithParamInterface<std::tuple<size_t, size_t, bool, size_t>> {};
+struct raptor_upgrade : public raptor {};
diff --git a/test/cli/raptor_options_test.cpp b/test/cli/raptor_options_test.cpp
@@ -58,7 +58,7 @@ TEST_F(raptor, no_subparser)
     std::string const expected
     {
         "[Error] You either forgot or misspelled the subcommand! Please specify which sub-program you want to use: one "
-        "of [build,search,socks]. Use -h/--help for more information.\n"
+        "of [build,search,socks,upgrade]. Use -h/--help for more information.\n"
     };
     EXPECT_NE(result.exit_code, 0);
     EXPECT_EQ(result.out, std::string{});
@@ -71,7 +71,7 @@ TEST_F(raptor, unknown_option)
     std::string const expected
     {
         "[Error] You either forgot or misspelled the subcommand! Please specify which sub-program you want to use: one "
-        "of [build,search,socks]. Use -h/--help for more information.\n"
+        "of [build,search,socks,upgrade]. Use -h/--help for more information.\n"
     };
     EXPECT_NE(result.exit_code, 0);
     EXPECT_EQ(result.out, std::string{});