Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(clp-s): Add end to end search tests for clp-s. #668

Open
wants to merge 10 commits into
base: main
Choose a base branch
from
56 changes: 54 additions & 2 deletions components/core/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -299,18 +299,30 @@ set(SOURCE_FILES_clp_s_unitTest
src/clp_s/SchemaTree.hpp
src/clp_s/SchemaWriter.cpp
src/clp_s/SchemaWriter.hpp
src/clp_s/search/AddTimestampConditions.cpp
src/clp_s/search/AddTimestampConditions.hpp
src/clp_s/search/AndExpr.cpp
src/clp_s/search/AndExpr.hpp
src/clp_s/search/BooleanLiteral.cpp
src/clp_s/search/BooleanLiteral.hpp
src/clp_s/search/clp_search/EncodedVariableInterpreter.cpp
src/clp_s/search/clp_search/EncodedVariableInterpreter.hpp
src/clp_s/search/clp_search/Grep.cpp
src/clp_s/search/clp_search/Grep.hpp
src/clp_s/search/clp_search/Query.cpp
src/clp_s/search/clp_search/Query.hpp
src/clp_s/search/ColumnDescriptor.cpp
src/clp_s/search/ColumnDescriptor.hpp
src/clp_s/search/ConstantProp.cpp
src/clp_s/search/ConstantProp.hpp
src/clp_s/search/ConvertToExists.cpp
src/clp_s/search/ConvertToExists.hpp
src/clp_s/search/DateLiteral.cpp
src/clp_s/search/DateLiteral.hpp
src/clp_s/search/EmptyExpr.cpp
src/clp_s/search/EmptyExpr.hpp
src/clp_s/search/EvaluateTimestampIndex.cpp
src/clp_s/search/EvaluateTimestampIndex.hpp
src/clp_s/search/Expression.cpp
src/clp_s/search/Expression.hpp
src/clp_s/search/FilterExpr.cpp
Expand All @@ -319,12 +331,22 @@ set(SOURCE_FILES_clp_s_unitTest
src/clp_s/search/Integral.cpp
src/clp_s/search/Integral.hpp
src/clp_s/search/Literal.hpp
src/clp_s/search/NarrowTypes.cpp
src/clp_s/search/NarrowTypes.hpp
src/clp_s/search/NullLiteral.cpp
src/clp_s/search/NullLiteral.hpp
src/clp_s/search/OrExpr.cpp
src/clp_s/search/OrExpr.hpp
src/clp_s/search/OrOfAndForm.cpp
src/clp_s/search/OrOfAndForm.hpp
src/clp_s/search/Output.cpp
src/clp_s/search/Output.hpp
src/clp_s/search/OutputHandler.cpp
src/clp_s/search/OutputHandler.hpp
src/clp_s/search/Projection.cpp
src/clp_s/search/Projection.hpp
src/clp_s/search/SchemaMatch.cpp
src/clp_s/search/SchemaMatch.hpp
src/clp_s/search/SearchUtils.cpp
src/clp_s/search/SearchUtils.hpp
src/clp_s/search/StringLiteral.cpp
Expand All @@ -349,7 +371,29 @@ set(SOURCE_FILES_clp_s_unitTest
src/clp_s/ZstdCompressor.hpp
src/clp_s/ZstdDecompressor.cpp
src/clp_s/ZstdDecompressor.hpp
)
)

set(SOURCE_FILES_reducer_unitTest
src/reducer/BufferedSocketWriter.cpp
src/reducer/BufferedSocketWriter.hpp
src/reducer/ConstRecordIterator.hpp
src/reducer/CountOperator.cpp
src/reducer/CountOperator.hpp
src/reducer/DeserializedRecordGroup.cpp
src/reducer/DeserializedRecordGroup.hpp
src/reducer/GroupTags.hpp
src/reducer/network_utils.cpp
src/reducer/network_utils.hpp
src/reducer/Operator.cpp
src/reducer/Operator.hpp
src/reducer/Pipeline.cpp
src/reducer/Pipeline.hpp
src/reducer/Record.hpp
src/reducer/RecordGroup.hpp
src/reducer/RecordGroupIterator.hpp
src/reducer/RecordTypedKeyIterator.hpp
src/reducer/types.hpp
)

set(SOURCE_FILES_unitTest
src/clp/Array.hpp
Expand Down Expand Up @@ -491,6 +535,8 @@ set(SOURCE_FILES_unitTest
src/clp/MySQLParamBindings.hpp
src/clp/MySQLPreparedStatement.cpp
src/clp/MySQLPreparedStatement.hpp
src/clp/networking/socket_utils.cpp
src/clp/networking/socket_utils.hpp
src/clp/NetworkReader.cpp
src/clp/NetworkReader.hpp
src/clp/PageAllocatedVector.hpp
Expand Down Expand Up @@ -576,10 +622,12 @@ set(SOURCE_FILES_unitTest
submodules/sqlite3/sqlite3.h
submodules/sqlite3/sqlite3ext.h
tests/LogSuppressor.hpp
tests/TestOutputCleaner.hpp
tests/test-Array.cpp
tests/test-BoundedReader.cpp
tests/test-BufferedFileReader.cpp
tests/test-clp_s-end_to_end.cpp
tests/test-clp_s-search.cpp
tests/test-EncodedVariableInterpreter.cpp
tests/test-encoding_methods.cpp
tests/test-error_handling.cpp
Expand Down Expand Up @@ -609,7 +657,11 @@ set(SOURCE_FILES_unitTest
tests/test-utf8_utils.cpp
tests/test-Utils.cpp
)
add_executable(unitTest ${SOURCE_FILES_unitTest} ${SOURCE_FILES_clp_s_unitTest})
add_executable(unitTest
${SOURCE_FILES_unitTest}
${SOURCE_FILES_clp_s_unitTest}
${SOURCE_FILES_reducer_unitTest}
)
target_include_directories(unitTest
PRIVATE
${CMAKE_SOURCE_DIR}/submodules
Expand Down
49 changes: 49 additions & 0 deletions components/core/src/clp_s/search/OutputHandler.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
#include <queue>
#include <string>
#include <string_view>
#include <vector>

#include <mongocxx/client.hpp>
#include <mongocxx/collection.hpp>
Expand Down Expand Up @@ -294,6 +295,54 @@ class CountByTimeOutputHandler : public OutputHandler {
std::map<int64_t, int64_t> m_bucket_counts;
int64_t m_count_by_time_bucket_size;
};

/**
* Output handler that records all results in a provided vector.
*/
class VectorOutputHandler : public OutputHandler {
public:
// Types
struct QueryResult {
// Constructors
QueryResult(
std::string_view message,
epochtime_t timestamp,
std::string_view archive_id,
int64_t log_event_idx
)
: message{message},
timestamp{timestamp},
archive_id{archive_id},
log_event_idx{log_event_idx} {}

std::string message;
epochtime_t timestamp;
std::string archive_id;
int64_t log_event_idx;
};

// Constructors
VectorOutputHandler(std::vector<QueryResult>& output)
: OutputHandler{true, true},
m_output(output) {}

// Methods inherited from OutputHandler
void write(
std::string_view message,
epochtime_t timestamp,
std::string_view archive_id,
int64_t log_event_idx
) override {
m_output.emplace_back(message, timestamp, archive_id, log_event_idx);
}

void write(std::string_view message) override {
m_output.emplace_back(message, epochtime_t{}, std::string_view{}, int64_t{});
}

private:
std::vector<QueryResult>& m_output;
};
} // namespace clp_s::search

#endif // CLP_S_SEARCH_OUTPUTHANDLER_HPP
19 changes: 13 additions & 6 deletions components/core/tests/LogSuppressor.hpp
Original file line number Diff line number Diff line change
@@ -1,3 +1,8 @@
#ifndef TEST_LOGSUPPRESSOR_HPP
#define TEST_LOGSUPPRESSOR_HPP

#include <spdlog/spdlog.h>

/**
* A class that suppresses logs so long as it's instantiated.
*/
Expand All @@ -8,14 +13,16 @@ class LogSuppressor {
spdlog::default_logger()->set_level(spdlog::level::off);
}

LogSuppressor(LogSuppressor const& other) = default;
LogSuppressor(LogSuppressor&& other) noexcept = default;

LogSuppressor& operator=(LogSuppressor const& other) = default;
LogSuppressor& operator=(LogSuppressor&& other) noexcept = default;

~LogSuppressor() { spdlog::default_logger()->set_level(m_previous_logging_level); }

// Delete copy & move constructors and assignment operators
LogSuppressor(LogSuppressor const& other) = delete;
LogSuppressor(LogSuppressor&& other) = delete;
auto operator=(LogSuppressor const& other) -> LogSuppressor& = delete;
auto operator=(LogSuppressor&& other) -> LogSuppressor& = delete;

private:
spdlog::level::level_enum m_previous_logging_level;
};

#endif // TEST_LOGSUPPRESSOR_HPP
36 changes: 36 additions & 0 deletions components/core/tests/TestOutputCleaner.hpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
#ifndef TEST_TESTOUTPUTCLEANER_HPP
#define TEST_TESTOUTPUTCLEANER_HPP

#include <filesystem>
#include <string>
#include <vector>

/**
* A class that deletes the directories and files created by test cases, both before and after each
* test case where the class is instantiated.
*/
class TestOutputCleaner {
public:
explicit TestOutputCleaner(std::vector<std::string> const& paths) : m_paths(paths) {
delete_paths();
}

~TestOutputCleaner() { delete_paths(); }

// Delete copy & move constructors and assignment operators
TestOutputCleaner(TestOutputCleaner const&) = delete;
TestOutputCleaner(TestOutputCleaner&&) = delete;
auto operator=(TestOutputCleaner const&) -> TestOutputCleaner& = delete;
auto operator=(TestOutputCleaner&&) -> TestOutputCleaner& = delete;

private:
void delete_paths() const {
for (auto const& path : m_paths) {
std::filesystem::remove_all(path);
}
}
gibber9809 marked this conversation as resolved.
Show resolved Hide resolved

std::vector<std::string> m_paths;
};

#endif // TEST_TESTOUTPUTCLEANER_HPP
45 changes: 11 additions & 34 deletions components/core/tests/test-clp_s-end_to_end.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,14 @@
#include <filesystem>
#include <string>
#include <string_view>
#include <vector>

#include <Catch2/single_include/catch2/catch.hpp>
#include <fmt/format.h>

#include "../src/clp_s/InputConfig.hpp"
#include "../src/clp_s/JsonConstructor.hpp"
#include "../src/clp_s/JsonParser.hpp"
#include "TestOutputCleaner.hpp"

constexpr std::string_view cTestEndToEndArchiveDirectory{"test-end-to-end-archive"};
constexpr std::string_view cTestEndToEndOutputDirectory{"test-end-to-end-out"};
Expand All @@ -20,33 +20,9 @@ constexpr std::string_view cTestEndToEndInputFileDirectory{"test_log_files"};
constexpr std::string_view cTestEndToEndInputFile{"test_no_floats_sorted.jsonl"};

namespace {
/**
* A class that deletes the directories and files created by test cases, both before and after each
* test case where the class is instantiated.
*/
class TestOutputCleaner {
public:
TestOutputCleaner() { delete_files(); }

~TestOutputCleaner() { delete_files(); }

// Delete copy & move constructors and assignment operators
TestOutputCleaner(TestOutputCleaner const&) = delete;
TestOutputCleaner(TestOutputCleaner&&) = delete;
auto operator=(TestOutputCleaner const&) -> TestOutputCleaner& = delete;
auto operator=(TestOutputCleaner&&) -> TestOutputCleaner& = delete;

private:
static void delete_files() {
std::filesystem::remove_all(cTestEndToEndArchiveDirectory);
std::filesystem::remove_all(cTestEndToEndOutputDirectory);
std::filesystem::remove(cTestEndToEndOutputSortedJson);
}
};

auto get_test_input_path_relative_to_tests_dir() -> std::filesystem::path;
auto get_test_input_local_path() -> std::string;
void compress(bool structurize_arrays);
void compress(bool structurize_arrays, bool single_file_archive);
auto extract() -> std::filesystem::path;
void compare(std::filesystem::path const& extracted_json_path);

Expand All @@ -60,7 +36,7 @@ auto get_test_input_local_path() -> std::string {
return (tests_dir / get_test_input_path_relative_to_tests_dir()).string();
}

void compress(bool structurize_arrays) {
void compress(bool structurize_arrays, bool single_file_archive) {
constexpr auto cDefaultTargetEncodedSize = 8ULL * 1024 * 1024 * 1024; // 8 GiB
constexpr auto cDefaultMaxDocumentSize = 512ULL * 1024 * 1024; // 512 MiB
constexpr auto cDefaultMinTableSize = 1ULL * 1024 * 1024; // 1 MiB
Expand All @@ -82,6 +58,7 @@ void compress(bool structurize_arrays) {
parser_option.compression_level = cDefaultCompressionLevel;
parser_option.print_archive_stats = cDefaultPrintArchiveStats;
parser_option.structurize_arrays = structurize_arrays;
parser_option.single_file_archive = single_file_archive;

clp_s::JsonParser parser{parser_option};
REQUIRE(parser.parse());
Expand All @@ -102,11 +79,6 @@ auto extract() -> std::filesystem::path {
constructor_option.ordered = cDefaultOrdered;
constructor_option.target_ordered_chunk_size = cDefaultTargetOrderedChunkSize;
for (auto const& entry : std::filesystem::directory_iterator(cTestEndToEndArchiveDirectory)) {
if (false == entry.is_directory()) {
// Skip non-directories
continue;
}

constructor_option.archive_path = clp_s::Path{
.source{clp_s::InputSource::Filesystem},
.path{entry.path().string()}
Expand Down Expand Up @@ -153,10 +125,15 @@ void compare(std::filesystem::path const& extracted_json_path) {

TEST_CASE("clp-s-compress-extract-no-floats", "[clp-s][end-to-end]") {
auto structurize_arrays = GENERATE(true, false);
auto single_file_archive = GENERATE(true, false);

TestOutputCleaner const test_cleanup;
TestOutputCleaner const test_cleanup{
{std::string{cTestEndToEndArchiveDirectory},
std::string{cTestEndToEndOutputDirectory},
std::string{cTestEndToEndOutputSortedJson}}
};

compress(structurize_arrays);
compress(structurize_arrays, single_file_archive);

auto extracted_json_path = extract();

Expand Down
Loading
Loading