optimize manifest reading

moonshadow565 · Oct 26, 2022 · 53d54f3 · 53d54f3
1 parent 70e3e73
commit 53d54f3
Show file tree

Hide file tree

Showing 6 changed files with 79 additions and 52 deletions.
diff --git a/lib/rlib/common.hpp b/lib/rlib/common.hpp
@@ -163,6 +163,12 @@ namespace rlib {
         return {str, {}};
     }
 
+    inline auto str_strip(std::string_view str) noexcept -> std::string_view {
+        while (str.empty() && ::isspace(str.front())) str.remove_prefix(1);
+        while (str.empty() && ::isspace(str.back())) str.remove_suffix(1);
+        return str;
+    }
+
     template <typename Signature>
     struct function_ref;
 

diff --git a/lib/rlib/rmanifest.cpp b/lib/rlib/rmanifest.cpp
@@ -17,6 +17,16 @@
 using namespace rlib;
 using json = nlohmann::json;
 
+auto RMAN::Filter::operator()(File const& file) const noexcept -> bool {
+    if (langs && !std::regex_search(file.langs, *langs)) {
+        return true;
+    }
+    if (path && !std::regex_search(file.path, *path)) {
+        return true;
+    }
+    return false;
+}
+
 struct RMAN::Raw {
     struct Offset {
         char const* beg = {};
@@ -179,7 +189,7 @@ struct RMAN::Raw {
     std::vector<RBUN> bundles;
     std::vector<RMAN::File> files;
 
-    auto parse(std::span<char const> src) -> void {
+    auto parse(std::span<char const> src) && -> RMAN {
         this->parse_header(src);
         auto body = zstd_decompress(src.subspan(header.offset, header.length), header.body_length);
         auto offset = Offset{body.data(), 0, (std::int32_t)body.size()};
@@ -190,6 +200,11 @@ struct RMAN::Raw {
         // this->parse_keys(body_table[4].as<std::vector<Table>>());
         this->parse_bundles(body_table[0].as<std::vector<Table>>());
         this->parse_files(body_table[2].as<std::vector<Table>>());
+        return RMAN{
+            .manifestId = std::move(this->header.manifestId),
+            .files = std::move(this->files),
+            .bundles = std::move(this->bundles),
+        };
     }
 
 private:
@@ -351,36 +366,40 @@ struct RMAN::Raw {
     }
 };
 
-RMAN RMAN::read(std::span<char const> data) {
-    rlib_assert(data.size() >= 5);
-    if (memcmp(data.data(), "JRMAN", 5) == 0) {
-        auto files = std::vector<RMAN::File>{};
-        auto const eof = data.data() + data.size();
-        // spliting lines in C++ is hard
-        for (auto i = std::find(data.data(), eof, '\n'); i != eof;) {
-            while (i != eof && (*i == '\r' || *i == '\n' || *i == ' ' || *i == '\t' || *i == '\v' || *i == '\f')) ++i;
-            auto const end = std::find(i, eof, '\n');
-            if (i != eof) {
-                auto line = std::string_view{i, end};
-                files.push_back(File::undump(line));
-            }
-            i = end;
+auto RMAN::read_jrman(std::span<char const> data, Filter const& filter) -> RMAN {
+    auto files = std::vector<RMAN::File>{};
+    auto [line, iter] = str_split({data.data(), data.size()}, '\n');
+    while (!iter.empty()) {
+        std::tie(line, iter) = str_split(iter, '\n');
+        line = str_strip(line);
+        if (line.empty()) {
+            continue;
+        }
+        auto file = File::undump(line);
+        if (!filter(file)) {
+            files.push_back(std::move(file));
         }
-        return RMAN{.files = std::move(files)};
     }
-    auto raw = Raw{};
-    raw.parse(data);
-    return RMAN{
-        .manifestId = std::move(raw.header.manifestId),
-        .files = std::move(raw.files),
-        .bundles = std::move(raw.bundles),
-    };
+    return RMAN{.files = std::move(files)};
+}
+
+auto RMAN::read_zrman(std::span<char const> data, Filter const& filter) -> RMAN {
+    rlib_assert(!"not implemented");
+    return {};
 }
 
-RMAN RMAN::read_file(fs::path const& path) {
+auto RMAN::read(std::span<char const> data, Filter const& filter) -> RMAN {
+    rlib_assert(data.size() >= 5);
+    if (std::memcmp(data.data(), "ZRMAN", 5) == 0) {
+        return read_jrman(data, filter);
+    }
+    return Raw{}.parse(data);
+}
+
+auto RMAN::read_file(fs::path const& path, Filter const& filter) -> RMAN {
     auto infile = IO::MMap(path, IO::READ);
     auto data = infile.copy(0, infile.size());
-    return RMAN::read(data);
+    return RMAN::read(data, filter);
 }
 
 auto RMAN::lookup() const -> std::unordered_map<std::string, File const*> {
@@ -437,16 +456,6 @@ auto RMAN::File::undump(std::string_view data) -> File {
     return file;
 }
 
-auto RMAN::File::matches(Filter const& filter) const noexcept -> bool {
-    if (filter.langs && !std::regex_search(langs, *filter.langs)) {
-        return false;
-    }
-    if (filter.path && !std::regex_search(path, *filter.path)) {
-        return false;
-    }
-    return true;
-}
-
 auto RMAN::File::verify(fs::path const& path, RChunk::Dst::data_cb on_data) const -> std::vector<RChunk::Dst> {
     if (!fs::exists(path)) {
         return chunks;

diff --git a/lib/rlib/rmanifest.hpp b/lib/rlib/rmanifest.hpp
@@ -19,11 +19,6 @@ namespace rlib {
     enum class ManifestID : std::uint64_t { None };
 
     struct RMAN {
-        struct Filter {
-            std::optional<std::regex> path;
-            std::optional<std::regex> langs;
-        };
-
         struct File {
             FileID fileId;
             std::uint8_t permissions;
@@ -37,21 +32,29 @@ namespace rlib {
 
             static auto undump(std::string_view data) -> File;
 
-            auto matches(Filter const& filter) const noexcept -> bool;
-
             auto verify(fs::path const& path, RChunk::Dst::data_cb on_good) const -> std::vector<RChunk::Dst>;
         };
 
+        struct Filter {
+            std::optional<std::regex> path;
+            std::optional<std::regex> langs;
+
+            auto operator()(File const& file) const noexcept -> bool;
+        };
+
         ManifestID manifestId;
         std::vector<File> files;
         std::vector<RBUN> bundles;
 
-        static RMAN read(std::span<char const> data);
-        static RMAN read_file(fs::path const& path);
+        static auto read(std::span<char const> data, Filter const& filter = {}) -> RMAN;
+        static auto read_file(fs::path const& path, Filter const& filter = {}) -> RMAN;
 
         auto lookup() const -> std::unordered_map<std::string, File const*>;
 
     private:
+        static auto read_jrman(std::span<char const> data, Filter const& filter) -> RMAN;
+        static auto read_zrman(std::span<char const> data, Filter const& filter) -> RMAN;
+
         struct Raw;
     };
 }

diff --git a/src/rman_dl.cpp b/src/rman_dl.cpp
@@ -142,7 +142,7 @@ struct Main {
 
     auto run() -> void {
         rlib_trace("Manifest file: %s", cli.manifest.c_str());
-        auto manifest = RMAN::read_file(cli.manifest);
+        auto manifest = RMAN::read_file(cli.manifest, cli.filter);
 
         if (!cli.no_write) {
             fs::create_directories(cli.output);
@@ -159,8 +159,6 @@ struct Main {
             cdn = std::make_unique<RCDN>(cli.cdn, cache.get());
         }
 
-        remove_if(manifest.files, [&](RMAN::File const& rfile) { return !rfile.matches(cli.filter); });
-
         for (std::uint32_t index = manifest.files.size(); auto const& rfile : manifest.files) {
             download_file(rfile, index--);
         }

diff --git a/src/rman_ls.cpp b/src/rman_ls.cpp
@@ -55,12 +55,9 @@ struct Main {
 
     auto run() -> void {
         rlib_trace("Manifest file: %s", cli.manifest.c_str());
-        auto manifest = RMAN::read_file(cli.manifest);
+        auto manifest = RMAN::read_file(cli.manifest, cli.filter);
 
         for (auto const& rfile : manifest.files) {
-            if (!rfile.matches(cli.filter)) {
-                continue;
-            }
             fmt::dynamic_format_arg_store<fmt::format_context> store{};
             store.push_back(fmt::arg("path", rfile.path));
             store.push_back(fmt::arg("size", rfile.size));

diff --git a/src/rman_rads.cpp b/src/rman_rads.cpp
@@ -42,7 +42,7 @@ struct Main {
 
         rlib_trace("Manifest file: %s", cli.inmanifest.c_str());
         std::cerr << "Reading input manifest ... " << std::endl;
-        auto manifest = RMAN::read_file(cli.inmanifest);
+        auto manifest = RMAN::read_file(cli.inmanifest, make_realm_filter());
 
         std::cerr << "Indexing input manifest ... " << std::endl;
         auto lookup = manifest.lookup();
@@ -59,6 +59,20 @@ struct Main {
         });
     }
 
+    auto make_realm_filter() const noexcept -> RMAN::Filter {
+        if (auto [realm, rest] = str_split(cli.inrelease, "projects/"); !realm.empty()) {
+            return RMAN::Filter{
+                .path = std::regex{std::string(realm) + ".*", std::regex::optimize | std::regex::icase},
+            };
+        }
+        if (auto [realm, rest] = str_split(cli.inrelease, "solutions/"); !realm.empty()) {
+            return RMAN::Filter{
+                .path = std::regex{std::string(realm) + ".*", std::regex::optimize | std::regex::icase},
+            };
+        }
+        return {};
+    }
+
     auto find_file(std::string path, auto const& lookup) const noexcept -> RMAN::File const* {
         std::transform(path.begin(), path.end(), path.begin(), ::tolower);
         if (auto i = lookup.find(path); i != lookup.end()) {