Skip to content

Commit

Permalink
rewrite splitting
Browse files Browse the repository at this point in the history
  • Loading branch information
moonshadow565 committed Oct 13, 2022
1 parent 7a4a2e5 commit 6398838
Show file tree
Hide file tree
Showing 14 changed files with 431 additions and 461 deletions.
11 changes: 4 additions & 7 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,9 @@ add_subdirectory(dep)
add_library(rlib STATIC
lib/rlib/ar.hpp
lib/rlib/ar.cpp
lib/rlib/ar/bnk.hpp
lib/rlib/ar/bnk.cpp
lib/rlib/ar/wad.hpp
lib/rlib/ar/wad.cpp
lib/rlib/ar/wpk.hpp
lib/rlib/ar/wpk.cpp
lib/rlib/ar_wad.cpp
lib/rlib/ar_wpk.cpp
lib/rlib/ar_zip.cpp
lib/rlib/common.hpp
lib/rlib/common.cpp
lib/rlib/iofile.cpp
Expand All @@ -32,7 +29,7 @@ add_library(rlib STATIC
lib/rlib/rmanifest.hpp
)
target_include_directories(rlib PUBLIC lib/)
target_link_libraries(rlib PUBLIC argparse libcurl zstd digestpp fmt json)
target_link_libraries(rlib PUBLIC argparse libcurl zstd digestpp fmt json miniz)

add_executable(rman-dl src/rman_dl.cpp)
target_link_libraries(rman-dl PRIVATE rlib)
Expand Down
20 changes: 19 additions & 1 deletion dep/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -109,4 +109,22 @@ if(NOT json_POPULATED)
FetchContent_Populate(json)
add_library(json INTERFACE)
target_include_directories(json INTERFACE ${json_SOURCE_DIR}/include)
endif()
endif()

FetchContent_Declare(
miniz
GIT_REPOSITORY https://github.com/richgel999/miniz.git
GIT_TAG 76b3a872855388c735c564905da030f26334f3b3
)
FetchContent_GetProperties(miniz)
if(NOT miniz_POPULATED)
FetchContent_Populate(miniz)
add_subdirectory(${miniz_SOURCE_DIR} ${miniz_BINARY_DIR})
target_compile_definitions(miniz PUBLIC
-DMINIZ_NO_DEFLATE_APIS=1
-DMINIZ_NO_ZLIB_APIS=1
-DMINIZ_NO_TIME=1
-DMINIZ_NO_STDIO=1
-DMINIZ_DISABLE_ZIP_READER_CRC32_CHECKS=1
)
endif()
93 changes: 32 additions & 61 deletions lib/rlib/ar.cpp
Original file line number Diff line number Diff line change
@@ -1,81 +1,52 @@
#include "ar.hpp"

#include "ar/bnk.hpp"
#include "ar/wad.hpp"
#include "ar/wpk.hpp"

using namespace rlib;
using namespace rlib::ar;

auto ArSplit::operator()(IO const& io, offset_cb cb) const -> void {
process(io, cb, 0, {.offset = 0, .size = io.size()});
auto Ar::operator()(IO const& io, offset_cb cb) const -> void {
process(io, cb, {.offset = 0, .size = io.size(), .nest = true});
}

template <typename T>
auto ArSplit::process_ar(IO const& io, offset_cb cb, Entry top_entry) const -> void {
auto archive = T{};
if (auto error = archive.read(io, top_entry.offset, top_entry.size)) rlib_error(error);

// ensure offsets are processed in order
std::sort(archive.entries.begin(), archive.entries.end(), [](auto const& lhs, auto const& rhs) {
if (lhs.offset < rhs.offset) return true;
if (lhs.offset == rhs.offset && lhs.size > rhs.size) return true;
return false;
});

auto cur = top_entry.offset;
for (auto entry : archive.entries) {
// skip empty entries
if (!entry.size) continue;

// skip duplicate or overlapping entries
if (entry.offset < cur) {
continue;
}

// process any skipped data
if (auto leftover = entry.offset - cur) {
process(io, cb, -1, {.offset = cur, .size = leftover, .compressed = top_entry.compressed});
}

// process current entry
process(io,
cb,
T::can_nest && !no_nest && !entry.compressed ? 1 : -1,
{
.offset = entry.offset,
.size = entry.size,
.compressed = entry.compressed,
});

// go to next entry
cur = entry.offset + entry.size;
auto Ar::process_iter_next(IO const& io,
offset_cb cb,
Entry const& top_entry,
std::size_t& cur,
Entry const& entry) const -> void {
// skip empty entries
if (!entry.size) return;
// skip duplicate or overlapping entries
if (entry.offset < cur) return;
// process any skipped data
if (auto leftover = entry.offset - cur) {
process(io, cb, {.offset = cur, .size = leftover, .compressed = top_entry.compressed});
cur += leftover;
}
// process current entry
process(io, cb, entry);
// go to next entry
cur += entry.size;
}

// process any remaining data
auto Ar::process_iter_end(IO const& io, offset_cb cb, Entry const& top_entry, std::size_t cur) const -> void {
if (auto remain = (top_entry.offset + top_entry.size) - cur) {
process(io, cb, -1, {.offset = cur, .size = remain, .compressed = top_entry.compressed});
process(io, cb, {.offset = cur, .size = remain, .compressed = top_entry.compressed});
}
}

auto ArSplit::process(IO const& io, offset_cb cb, int depth, Entry top_entry) const -> void {
if (depth >= 0 && top_entry.size >= 64) {
char buffer[8] = {};
rlib_assert(io.read(top_entry.offset, buffer));
if (!no_bnk && BNK::check_magic(buffer)) {
return process_ar<BNK>(io, cb, top_entry);
}
if (!no_wad && depth < 1 && WAD::check_magic(buffer)) {
return process_ar<WAD>(io, cb, top_entry);
}
if (!no_wpk && WPK::check_magic(buffer)) {
return process_ar<WPK>(io, cb, top_entry);
}
auto Ar::process(IO const& io, offset_cb cb, Entry const& top_entry) const -> void {
if (top_entry.nest && !no_wad && process_try_wad(io, cb, top_entry)) {
return;
}
if (top_entry.nest && !no_wpk && process_try_wpk(io, cb, top_entry)) {
return;
}
if (top_entry.nest && !no_zip && process_try_zip(io, cb, top_entry)) {
return;
}
for (auto i = top_entry.offset, remain = top_entry.size; remain;) {
auto size = std::min(chunk_size, remain);
cb({.offset = i, .size = size, .compressed = top_entry.compressed});
i += size;
remain -= size;
}
return;
}
27 changes: 21 additions & 6 deletions lib/rlib/ar.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -3,26 +3,41 @@
#include <rlib/iofile.hpp>

namespace rlib {
struct ArSplit {
struct Ar {
struct Entry {
std::size_t offset;
std::size_t size;
bool compressed;
bool nest;
};
using offset_cb = function_ref<void(Entry)>;

std::size_t chunk_size;
bool no_bnk;
bool no_wad;
bool no_wpk;
bool no_nest;
bool no_zip;

auto operator()(IO const& io, offset_cb cb) const -> void;

private:
auto process(IO const& io, offset_cb cb, int depth, Entry top_entry) const -> void;
struct WAD;
struct WPK;
struct ZIP;

template <typename T>
auto process_ar(IO const& io, offset_cb cb, Entry top_entry) const -> void;
auto process(IO const& io, offset_cb cb, Entry const& top_entry) const -> void;

auto process_try_wad(IO const& io, offset_cb cb, Entry const& top_entry) const -> bool;

auto process_try_wpk(IO const& io, offset_cb cb, Entry const& top_entry) const -> bool;

auto process_try_zip(IO const& io, offset_cb cb, Entry const& top_entry) const -> bool;

auto process_iter_next(IO const& io,
offset_cb cb,
Entry const& top_entry,
std::size_t& cur,
Entry const& entry) const -> void;

auto process_iter_end(IO const& io, offset_cb cb, Entry const& top_entry, std::size_t cur) const -> void;
};
}
89 changes: 0 additions & 89 deletions lib/rlib/ar/bnk.cpp

This file was deleted.

21 changes: 0 additions & 21 deletions lib/rlib/ar/bnk.hpp

This file was deleted.

Loading

0 comments on commit 6398838

Please sign in to comment.