From 5382e8099de924e6de0779c905e590f72c97cc9e Mon Sep 17 00:00:00 2001 From: Abigail Matthews Date: Mon, 18 Nov 2024 15:42:20 -0500 Subject: [PATCH 01/18] Adding end to end test case for clp-s --- components/core/CMakeLists.txt | 54 ++++++++- components/core/tests/test-end_to_end.cpp | 112 ++++++++++++++++++ .../test_log_files/test_no_floats_sorted.json | 4 + .../tests/test_log_files/test_sorted.json | 4 + 4 files changed, 173 insertions(+), 1 deletion(-) create mode 100644 components/core/tests/test-end_to_end.cpp create mode 100644 components/core/tests/test_log_files/test_no_floats_sorted.json create mode 100644 components/core/tests/test_log_files/test_sorted.json diff --git a/components/core/CMakeLists.txt b/components/core/CMakeLists.txt index e5c9b06c8..d4f9ff918 100644 --- a/components/core/CMakeLists.txt +++ b/components/core/CMakeLists.txt @@ -241,6 +241,42 @@ add_subdirectory(src/clp_s) add_subdirectory(src/reducer) set(SOURCE_FILES_clp_s_unitTest + src/clp_s/ArchiveReader.cpp + src/clp_s/ArchiveReader.hpp + src/clp_s/ArchiveWriter.cpp + src/clp_s/ArchiveWriter.hpp + src/clp_s/ColumnReader.cpp + src/clp_s/ColumnReader.hpp + src/clp_s/ColumnWriter.cpp + src/clp_s/ColumnWriter.hpp + src/clp_s/DictionaryEntry.cpp + src/clp_s/DictionaryEntry.hpp + src/clp_s/DictionaryWriter.cpp + src/clp_s/DictionaryWriter.hpp + src/clp_s/FileReader.cpp + src/clp_s/FileReader.hpp + src/clp_s/FileWriter.cpp + src/clp_s/FileWriter.hpp + src/clp_s/JsonConstructor.cpp + src/clp_s/JsonConstructor.hpp + src/clp_s/JsonFileIterator.cpp + src/clp_s/JsonFileIterator.hpp + src/clp_s/JsonParser.cpp + src/clp_s/JsonParser.hpp + src/clp_s/PackedStreamReader.cpp + src/clp_s/PackedStreamReader.hpp + src/clp_s/ReaderUtils.cpp + src/clp_s/ReaderUtils.hpp + src/clp_s/Schema.cpp + src/clp_s/Schema.hpp + src/clp_s/SchemaMap.cpp + src/clp_s/SchemaMap.hpp + src/clp_s/SchemaReader.cpp + src/clp_s/SchemaReader.hpp + src/clp_s/SchemaTree.cpp + src/clp_s/SchemaTree.hpp + src/clp_s/SchemaWriter.cpp + src/clp_s/SchemaWriter.hpp src/clp_s/search/AndExpr.cpp src/clp_s/search/AndExpr.hpp src/clp_s/search/BooleanLiteral.cpp @@ -273,11 +309,24 @@ set(SOURCE_FILES_clp_s_unitTest src/clp_s/search/StringLiteral.hpp src/clp_s/search/Transformation.hpp src/clp_s/search/Value.hpp - src/clp_s/SchemaTree.hpp + src/clp_s/TimestampDictionaryReader.cpp + src/clp_s/TimestampDictionaryReader.hpp + src/clp_s/TimestampDictionaryWriter.cpp + src/clp_s/TimestampDictionaryWriter.hpp + src/clp_s/TimestampEntry.cpp + src/clp_s/TimestampEntry.hpp src/clp_s/TimestampPattern.cpp src/clp_s/TimestampPattern.hpp src/clp_s/Utils.cpp src/clp_s/Utils.hpp + src/clp_s/VariableDecoder.cpp + src/clp_s/VariableDecoder.hpp + src/clp_s/VariableEncoder.cpp + src/clp_s/VariableEncoder.hpp + src/clp_s/ZstdCompressor.cpp + src/clp_s/ZstdCompressor.hpp + src/clp_s/ZstdDecompressor.cpp + src/clp_s/ZstdDecompressor.hpp ) set(SOURCE_FILES_unitTest @@ -501,6 +550,7 @@ set(SOURCE_FILES_unitTest tests/test-BufferedFileReader.cpp tests/test-EncodedVariableInterpreter.cpp tests/test-encoding_methods.cpp + tests/test-end_to_end.cpp tests/test-ffi_IrUnitHandlerInterface.cpp tests/test-ffi_KeyValuePairLogEvent.cpp tests/test-ffi_SchemaTree.cpp @@ -542,6 +592,8 @@ target_link_libraries(unitTest log_surgeon::log_surgeon LibArchive::LibArchive MariaDBClient::MariaDBClient + ${MONGOCXX_TARGET} + simdjson spdlog::spdlog OpenSSL::Crypto ${sqlite_LIBRARY_DEPENDENCIES} diff --git a/components/core/tests/test-end_to_end.cpp b/components/core/tests/test-end_to_end.cpp new file mode 100644 index 000000000..c2192f9ce --- /dev/null +++ b/components/core/tests/test-end_to_end.cpp @@ -0,0 +1,112 @@ +#include +#include +#include +#include +#include + +#include +#include + +#include "../src/clp/BufferReader.hpp" +#include "../src/clp/ffi/ir_stream/decoding_methods.hpp" +#include "../src/clp/ffi/ir_stream/Deserializer.hpp" +#include "../src/clp/ffi/ir_stream/Serializer.hpp" +#include "../src/clp/ffi/KeyValuePairLogEvent.hpp" +#include "../src/clp/ir/types.hpp" +#include "../src/clp/time_types.hpp" +#include "../src/clp_s/JsonConstructor.hpp" +#include "../src/clp_s/JsonParser.hpp" + +using clp::BufferReader; +using clp::ffi::ir_stream::Deserializer; +using clp::ffi::ir_stream::IRErrorCode; +using clp::ffi::ir_stream::Serializer; +using clp::ffi::KeyValuePairLogEvent; +using clp::ir::eight_byte_encoded_variable_t; +using clp::ir::four_byte_encoded_variable_t; +using clp::size_checked_pointer_cast; +using clp::UtcOffset; +using std::string; +using std::string_view; +using std::vector; + +auto const cDefaultTargetEncodedSize = 8ULL * 1024 * 1024 * 1024; +auto const cDefaultMaxDocumentSize = 512ULL * 1024 * 1024; +auto const cDefaultMinTableSize = 1ULL * 1024 * 1024; +auto const cDeaultCompressionLevel = 3; +auto const cDefaultPrintArchiveStats = false; +auto const cDefaultStructurizeArrays = false; + +namespace { +auto get_test_input_path_relative_to_tests_dir() -> std::filesystem::path { + return std::filesystem::path{"test_log_files"} / "test_no_floats_sorted.json"; +} + +auto get_test_input_local_path() -> std::string { + std::filesystem::path const current_file_path{__FILE__}; + auto const tests_dir{current_file_path.parent_path()}; + return (tests_dir / get_test_input_path_relative_to_tests_dir()).string(); +} +} // namespace + +// NOLINTNEXTLINE(readability-function-cognitive-complexity) +TEMPLATE_TEST_CASE( + "clp-s_compression_and_extraction_no_floats", + "[clp-s][end-to-end]", + four_byte_encoded_variable_t, + eight_byte_encoded_variable_t +) { + std::filesystem::remove_all("test-end-to-end-archive"); + std::filesystem::remove_all("test-end-to-end-out"); + std::filesystem::remove("test-end-to-end_sorted.json"); + std::filesystem::remove("diff_out.txt"); + + std::filesystem::create_directory("test-end-to-end-archive"); + REQUIRE(std::filesystem::is_directory("test-end-to-end-archive")); + + clp_s::JsonParserOption parser_option{}; + parser_option.file_paths.push_back(get_test_input_local_path()); + parser_option.archives_dir = "test-end-to-end-archive"; + parser_option.target_encoded_size = cDefaultTargetEncodedSize; + parser_option.max_document_size = cDefaultMaxDocumentSize; + parser_option.min_table_size = cDefaultMinTableSize; + parser_option.compression_level = cDeaultCompressionLevel; + parser_option.print_archive_stats = cDefaultPrintArchiveStats; + parser_option.structurize_arrays = cDefaultStructurizeArrays; + + clp_s::JsonParser parser(parser_option); + REQUIRE(parser.parse()); + parser.store(); + + REQUIRE(false == std::filesystem::is_empty("test-end-to-end-archive")); + + std::filesystem::create_directory("test-end-to-end-out"); + REQUIRE(std::filesystem::is_directory("test-end-to-end-out")); + + clp_s::JsonConstructorOption constructor_option{}; + constructor_option.output_dir = "test-end-to-end-out"; + constructor_option.ordered = false; + constructor_option.archives_dir = parser_option.archives_dir; + constructor_option.ordered_chunk_size = 0; + for (auto const& entry : std::filesystem::directory_iterator(constructor_option.archives_dir)) { + if (false == entry.is_directory()) { + // Skip non-directories + continue; + } + + constructor_option.archive_id = entry.path().filename(); + clp_s::JsonConstructor constructor(constructor_option); + constructor.store(); + } + + REQUIRE(std::filesystem::exists("test-end-to-end-out/original")); + + std::system("jq -S -c '.' test-end-to-end-out/original | sort > test-end-to-end_sorted.json"); + + REQUIRE(false == std::filesystem::is_empty("test-end-to-end_sorted.json")); + + std::string const command = "diff -u test-end-to-end_sorted.json " + get_test_input_local_path() + + " > diff_out.txt"; + std::system(command.c_str()); + REQUIRE(std::filesystem::is_empty("diff_out.txt")); +} diff --git a/components/core/tests/test_log_files/test_no_floats_sorted.json b/components/core/tests/test_log_files/test_no_floats_sorted.json new file mode 100644 index 000000000..db4a54fea --- /dev/null +++ b/components/core/tests/test_log_files/test_no_floats_sorted.json @@ -0,0 +1,4 @@ +{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"empty_object":{},"false":false,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max":0,"int64_min":1,"int8_max":127,"int8_min":-128,"null":null,"string":"short_string","true":true} +{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"false":false,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max":0,"int64_min":1,"int8_max":127,"int8_min":-128,"nonempty_object":{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"empty_object":{},"false":false,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max":0,"int64_min":1,"int8_max":127,"int8_min":-128,"null":null,"string":"short_string","true":true},"null":null,"string":"short_string","true":true} +{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"false":false,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max":0,"int64_min":1,"int8_max":127,"int8_min":-128,"nonempty_object":{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"false":false,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max":0,"int64_min":1,"int8_max":127,"int8_min":-128,"non_empty_object2":{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"empty_object":{},"false":false,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max":0,"int64_min":1,"int8_max":127,"int8_min":-128,"null":null,"string":"short_string","true":true},"null":null,"string":"short_string","true":true},"null":null,"string":"short_string","true":true} +{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_object":{},"false":false,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max":0,"int64_min":1,"int8_max":127,"int8_min":-128,"nonempty_array":[1,2,3,4,5],"null":null,"string":"short_string","true":true} diff --git a/components/core/tests/test_log_files/test_sorted.json b/components/core/tests/test_log_files/test_sorted.json new file mode 100644 index 000000000..0ac463d7b --- /dev/null +++ b/components/core/tests/test_log_files/test_sorted.json @@ -0,0 +1,4 @@ +{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"empty_object":{},"false":false,"float_neg":-1.01,"float_pos":1.01,"float_zero":0.0,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max":0,"int64_min":1,"int8_max":127,"int8_min":-128,"null":null,"string":"short_string","true":true} +{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"false":false,"float_neg":-1.01,"float_pos":1.01,"float_zero":0.0,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max":0,"int64_min":1,"int8_max":127,"int8_min":-128,"nonempty_object":{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"empty_object":{},"false":false,"float_neg":-1.01,"float_pos":1.01,"float_zero":0.0,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max":0,"int64_min":1,"int8_max":127,"int8_min":-128,"null":null,"string":"short_string","true":true},"null":null,"string":"short_string","true":true} +{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"false":false,"float_neg":-1.01,"float_pos":1.01,"float_zero":0.0,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max":0,"int64_min":1,"int8_max":127,"int8_min":-128,"nonempty_object":{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"false":false,"float_neg":-1.01,"float_pos":1.01,"float_zero":0.0,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max":0,"int64_min":1,"int8_max":127,"int8_min":-128,"non_empty_object2":{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"empty_object":{},"false":false,"float_neg":-1.01,"float_pos":1.01,"float_zero":0.0,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max":0,"int64_min":1,"int8_max":127,"int8_min":-128,"null":null,"string":"short_string","true":true},"null":null,"string":"short_string","true":true},"null":null,"string":"short_string","true":true} +{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_object":{},"false":false,"float_neg":-1.01,"float_pos":1.01,"float_zero":0.0,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max":0,"int64_min":1,"int8_max":127,"int8_min":-128,"nonempty_array":[1,2,3,4,5],"null":null,"string":"short_string","true":true} From acce22c2b00d857f027af7fd61f9cbae9c0a697e Mon Sep 17 00:00:00 2001 From: Abigail Matthews Date: Mon, 18 Nov 2024 16:55:24 -0500 Subject: [PATCH 02/18] Add required jq install --- .../lib_install/centos-stream-9/install-prebuilt-packages.sh | 1 + .../lib_install/ubuntu-focal/install-prebuilt-packages.sh | 1 + .../lib_install/ubuntu-jammy/install-prebuilt-packages.sh | 1 + 3 files changed, 3 insertions(+) diff --git a/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh index e90f54733..9d5b58f34 100755 --- a/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh @@ -11,6 +11,7 @@ dnf install -y \ gcc-c++ \ git \ java-11-openjdk \ + jq \ libarchive-devel \ libcurl-devel \ libzstd-devel \ diff --git a/components/core/tools/scripts/lib_install/ubuntu-focal/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/ubuntu-focal/install-prebuilt-packages.sh index 706674764..8997ffe01 100755 --- a/components/core/tools/scripts/lib_install/ubuntu-focal/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/ubuntu-focal/install-prebuilt-packages.sh @@ -17,6 +17,7 @@ DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ gcc \ gcc-10 \ git \ + jq \ libcurl4 \ libcurl4-openssl-dev \ libmariadb-dev \ diff --git a/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh index 92d965b9b..9ed6b9b10 100755 --- a/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh @@ -14,6 +14,7 @@ DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ curl \ build-essential \ git \ + jq \ libboost-filesystem-dev \ libboost-iostreams-dev \ libboost-program-options-dev \ From ad187f767b60e628aacf21ae52e0ad2fc91e570e Mon Sep 17 00:00:00 2001 From: Abigail Matthews Date: Tue, 19 Nov 2024 12:37:52 -0500 Subject: [PATCH 03/18] Additional error checking and small modification to test input files --- components/core/tests/test-end_to_end.cpp | 11 +++++++++-- .../tests/test_log_files/test_no_floats_sorted.json | 8 ++++---- components/core/tests/test_log_files/test_sorted.json | 8 ++++---- 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/components/core/tests/test-end_to_end.cpp b/components/core/tests/test-end_to_end.cpp index c2192f9ce..5ced3b7f4 100644 --- a/components/core/tests/test-end_to_end.cpp +++ b/components/core/tests/test-end_to_end.cpp @@ -101,12 +101,19 @@ TEMPLATE_TEST_CASE( REQUIRE(std::filesystem::exists("test-end-to-end-out/original")); - std::system("jq -S -c '.' test-end-to-end-out/original | sort > test-end-to-end_sorted.json"); + int result = std::system("command -v jq >/dev/null 2>&1"); + REQUIRE(0 == result); + result = std::system( + "jq -S -c '.' test-end-to-end-out/original | sort > test-end-to-end_sorted.json" + ); + REQUIRE(0 == result); REQUIRE(false == std::filesystem::is_empty("test-end-to-end_sorted.json")); std::string const command = "diff -u test-end-to-end_sorted.json " + get_test_input_local_path() + " > diff_out.txt"; - std::system(command.c_str()); + result = std::system(command.c_str()); + REQUIRE((0 == result || 1 == result)); + REQUIRE(std::filesystem::is_empty("diff_out.txt")); } diff --git a/components/core/tests/test_log_files/test_no_floats_sorted.json b/components/core/tests/test_log_files/test_no_floats_sorted.json index db4a54fea..cd1075fb1 100644 --- a/components/core/tests/test_log_files/test_no_floats_sorted.json +++ b/components/core/tests/test_log_files/test_no_floats_sorted.json @@ -1,4 +1,4 @@ -{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"empty_object":{},"false":false,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max":0,"int64_min":1,"int8_max":127,"int8_min":-128,"null":null,"string":"short_string","true":true} -{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"false":false,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max":0,"int64_min":1,"int8_max":127,"int8_min":-128,"nonempty_object":{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"empty_object":{},"false":false,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max":0,"int64_min":1,"int8_max":127,"int8_min":-128,"null":null,"string":"short_string","true":true},"null":null,"string":"short_string","true":true} -{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"false":false,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max":0,"int64_min":1,"int8_max":127,"int8_min":-128,"nonempty_object":{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"false":false,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max":0,"int64_min":1,"int8_max":127,"int8_min":-128,"non_empty_object2":{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"empty_object":{},"false":false,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max":0,"int64_min":1,"int8_max":127,"int8_min":-128,"null":null,"string":"short_string","true":true},"null":null,"string":"short_string","true":true},"null":null,"string":"short_string","true":true} -{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_object":{},"false":false,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max":0,"int64_min":1,"int8_max":127,"int8_min":-128,"nonempty_array":[1,2,3,4,5],"null":null,"string":"short_string","true":true} +{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"empty_object":{},"false":false,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max":9223372036854775807,"int64_min":-9223372036854775808,"int8_max":127,"int8_min":-128,"null":null,"string":"short_string","true":true} +{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"false":false,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max":9223372036854775807,"int64_min":-9223372036854775808,"int8_max":127,"int8_min":-128,"nonempty_object":{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"empty_object":{},"false":false,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max":9223372036854775807,"int64_min":-9223372036854775808,"int8_max":127,"int8_min":-128,"null":null,"string":"short_string","true":true},"null":null,"string":"short_string","true":true} +{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"false":false,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max":9223372036854775807,"int64_min":-9223372036854775808,"int8_max":127,"int8_min":-128,"nonempty_object":{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"false":false,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max":9223372036854775807,"int64_min":-9223372036854775808,"int8_max":127,"int8_min":-128,"non_empty_object2":{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"empty_object":{},"false":false,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max":9223372036854775807,"int64_min":-9223372036854775808,"int8_max":127,"int8_min":-128,"null":null,"string":"short_string","true":true},"null":null,"string":"short_string","true":true},"null":null,"string":"short_string","true":true} +{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_object":{},"false":false,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max":9223372036854775807,"int64_min":-9223372036854775808,"int8_max":127,"int8_min":-128,"nonempty_array":[1,2,3,4,5],"null":null,"string":"short_string","true":true} diff --git a/components/core/tests/test_log_files/test_sorted.json b/components/core/tests/test_log_files/test_sorted.json index 0ac463d7b..ec08731dd 100644 --- a/components/core/tests/test_log_files/test_sorted.json +++ b/components/core/tests/test_log_files/test_sorted.json @@ -1,4 +1,4 @@ -{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"empty_object":{},"false":false,"float_neg":-1.01,"float_pos":1.01,"float_zero":0.0,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max":0,"int64_min":1,"int8_max":127,"int8_min":-128,"null":null,"string":"short_string","true":true} -{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"false":false,"float_neg":-1.01,"float_pos":1.01,"float_zero":0.0,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max":0,"int64_min":1,"int8_max":127,"int8_min":-128,"nonempty_object":{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"empty_object":{},"false":false,"float_neg":-1.01,"float_pos":1.01,"float_zero":0.0,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max":0,"int64_min":1,"int8_max":127,"int8_min":-128,"null":null,"string":"short_string","true":true},"null":null,"string":"short_string","true":true} -{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"false":false,"float_neg":-1.01,"float_pos":1.01,"float_zero":0.0,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max":0,"int64_min":1,"int8_max":127,"int8_min":-128,"nonempty_object":{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"false":false,"float_neg":-1.01,"float_pos":1.01,"float_zero":0.0,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max":0,"int64_min":1,"int8_max":127,"int8_min":-128,"non_empty_object2":{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"empty_object":{},"false":false,"float_neg":-1.01,"float_pos":1.01,"float_zero":0.0,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max":0,"int64_min":1,"int8_max":127,"int8_min":-128,"null":null,"string":"short_string","true":true},"null":null,"string":"short_string","true":true},"null":null,"string":"short_string","true":true} -{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_object":{},"false":false,"float_neg":-1.01,"float_pos":1.01,"float_zero":0.0,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max":0,"int64_min":1,"int8_max":127,"int8_min":-128,"nonempty_array":[1,2,3,4,5],"null":null,"string":"short_string","true":true} +{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"empty_object":{},"false":false,"float_neg":-1.01,"float_pos":1.01,"float_zero":0.0,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max":9223372036854775807,"int64_min":-9223372036854775808,"int8_max":127,"int8_min":-128,"null":null,"string":"short_string","true":true} +{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"false":false,"float_neg":-1.01,"float_pos":1.01,"float_zero":0.0,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max":9223372036854775807,"int64_min":-9223372036854775808,"int8_max":127,"int8_min":-128,"nonempty_object":{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"empty_object":{},"false":false,"float_neg":-1.01,"float_pos":1.01,"float_zero":0.0,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max":9223372036854775807,"int64_min":-9223372036854775808,"int8_max":127,"int8_min":-128,"null":null,"string":"short_string","true":true},"null":null,"string":"short_string","true":true} +{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"false":false,"float_neg":-1.01,"float_pos":1.01,"float_zero":0.0,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max":9223372036854775807,"int64_min":-9223372036854775808,"int8_max":127,"int8_min":-128,"nonempty_object":{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"false":false,"float_neg":-1.01,"float_pos":1.01,"float_zero":0.0,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max":9223372036854775807,"int64_min":-9223372036854775808,"int8_max":127,"int8_min":-128,"non_empty_object2":{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"empty_object":{},"false":false,"float_neg":-1.01,"float_pos":1.01,"float_zero":0.0,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max":9223372036854775807,"int64_min":-9223372036854775808,"int8_max":127,"int8_min":-128,"null":null,"string":"short_string","true":true},"null":null,"string":"short_string","true":true},"null":null,"string":"short_string","true":true} +{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_object":{},"false":false,"float_neg":-1.01,"float_pos":1.01,"float_zero":0.0,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max":9223372036854775807,"int64_min":-9223372036854775808,"int8_max":127,"int8_min":-128,"nonempty_array":[1,2,3,4,5],"null":null,"string":"short_string","true":true} From 692d8d89131a572c2be3950ff722533fc1e3d29c Mon Sep 17 00:00:00 2001 From: Abigail Matthews Date: Tue, 19 Nov 2024 13:22:39 -0500 Subject: [PATCH 04/18] Add install and check for diff command --- components/core/tests/test-end_to_end.cpp | 4 ++++ .../lib_install/centos-stream-9/install-prebuilt-packages.sh | 1 + .../lib_install/ubuntu-focal/install-prebuilt-packages.sh | 1 + .../lib_install/ubuntu-jammy/install-prebuilt-packages.sh | 1 + 4 files changed, 7 insertions(+) diff --git a/components/core/tests/test-end_to_end.cpp b/components/core/tests/test-end_to_end.cpp index 5ced3b7f4..a5dcf299b 100644 --- a/components/core/tests/test-end_to_end.cpp +++ b/components/core/tests/test-end_to_end.cpp @@ -110,9 +110,13 @@ TEMPLATE_TEST_CASE( REQUIRE(false == std::filesystem::is_empty("test-end-to-end_sorted.json")); + result = std::system("command -v diff >/dev/null 2>&1"); + REQUIRE(0 == result); std::string const command = "diff -u test-end-to-end_sorted.json " + get_test_input_local_path() + " > diff_out.txt"; + std::cout << command << "\n"; result = std::system(command.c_str()); + std::cout << result << "\n"; REQUIRE((0 == result || 1 == result)); REQUIRE(std::filesystem::is_empty("diff_out.txt")); diff --git a/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh index 9d5b58f34..66ea4ac4f 100755 --- a/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh @@ -8,6 +8,7 @@ set -u dnf install -y \ cmake \ + diffutils \ gcc-c++ \ git \ java-11-openjdk \ diff --git a/components/core/tools/scripts/lib_install/ubuntu-focal/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/ubuntu-focal/install-prebuilt-packages.sh index 8997ffe01..9e278468d 100755 --- a/components/core/tools/scripts/lib_install/ubuntu-focal/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/ubuntu-focal/install-prebuilt-packages.sh @@ -12,6 +12,7 @@ DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ checkinstall \ cmake \ curl \ + diffutils \ g++ \ g++-10 \ gcc \ diff --git a/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh index 9ed6b9b10..ffde62210 100755 --- a/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh @@ -13,6 +13,7 @@ DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ cmake \ curl \ build-essential \ + diffutils \ git \ jq \ libboost-filesystem-dev \ From 8f76f0a7281c436864a744c0708b059e3335d822 Mon Sep 17 00:00:00 2001 From: Abigail Matthews Date: Tue, 19 Nov 2024 14:26:57 -0500 Subject: [PATCH 05/18] Adding extra debug prints to help solve ubuntu build problems --- components/core/tests/test-end_to_end.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/components/core/tests/test-end_to_end.cpp b/components/core/tests/test-end_to_end.cpp index a5dcf299b..d3f1f732a 100644 --- a/components/core/tests/test-end_to_end.cpp +++ b/components/core/tests/test-end_to_end.cpp @@ -117,6 +117,8 @@ TEMPLATE_TEST_CASE( std::cout << command << "\n"; result = std::system(command.c_str()); std::cout << result << "\n"; + std::cout << WEXITSTATUS(result) << "\n"; + std::system("cat diff_out.txt"); REQUIRE((0 == result || 1 == result)); REQUIRE(std::filesystem::is_empty("diff_out.txt")); From 5486700a9e3c66a2332b7f842106774eafe5a682 Mon Sep 17 00:00:00 2001 From: Abigail Matthews Date: Tue, 19 Nov 2024 15:32:56 -0500 Subject: [PATCH 06/18] More debug prints --- components/core/tests/test-end_to_end.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/components/core/tests/test-end_to_end.cpp b/components/core/tests/test-end_to_end.cpp index d3f1f732a..d8769a33c 100644 --- a/components/core/tests/test-end_to_end.cpp +++ b/components/core/tests/test-end_to_end.cpp @@ -118,6 +118,7 @@ TEMPLATE_TEST_CASE( result = std::system(command.c_str()); std::cout << result << "\n"; std::cout << WEXITSTATUS(result) << "\n"; + std::system("cat test-end-to-end-out/original"); std::system("cat diff_out.txt"); REQUIRE((0 == result || 1 == result)); From df6122c136b6b344be95617a30cee1115a809f45 Mon Sep 17 00:00:00 2001 From: Abigail Matthews Date: Tue, 19 Nov 2024 16:07:56 -0500 Subject: [PATCH 07/18] change input files only include max int supported by jq --- components/core/tests/test-end_to_end.cpp | 6 +++--- .../core/tests/test_log_files/test_no_floats_sorted.json | 8 ++++---- components/core/tests/test_log_files/test_sorted.json | 8 ++++---- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/components/core/tests/test-end_to_end.cpp b/components/core/tests/test-end_to_end.cpp index d8769a33c..4ea958d43 100644 --- a/components/core/tests/test-end_to_end.cpp +++ b/components/core/tests/test-end_to_end.cpp @@ -114,13 +114,13 @@ TEMPLATE_TEST_CASE( REQUIRE(0 == result); std::string const command = "diff -u test-end-to-end_sorted.json " + get_test_input_local_path() + " > diff_out.txt"; - std::cout << command << "\n"; + //std::cout << command << "\n"; result = std::system(command.c_str()); - std::cout << result << "\n"; + //std::cout << result << "\n"; std::cout << WEXITSTATUS(result) << "\n"; std::system("cat test-end-to-end-out/original"); std::system("cat diff_out.txt"); - REQUIRE((0 == result || 1 == result)); + REQUIRE((0 == WEXITSTATUS(result) || 1 == WEXITSTATUS(result))); REQUIRE(std::filesystem::is_empty("diff_out.txt")); } diff --git a/components/core/tests/test_log_files/test_no_floats_sorted.json b/components/core/tests/test_log_files/test_no_floats_sorted.json index cd1075fb1..8d82aea33 100644 --- a/components/core/tests/test_log_files/test_no_floats_sorted.json +++ b/components/core/tests/test_log_files/test_no_floats_sorted.json @@ -1,4 +1,4 @@ -{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"empty_object":{},"false":false,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max":9223372036854775807,"int64_min":-9223372036854775808,"int8_max":127,"int8_min":-128,"null":null,"string":"short_string","true":true} -{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"false":false,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max":9223372036854775807,"int64_min":-9223372036854775808,"int8_max":127,"int8_min":-128,"nonempty_object":{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"empty_object":{},"false":false,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max":9223372036854775807,"int64_min":-9223372036854775808,"int8_max":127,"int8_min":-128,"null":null,"string":"short_string","true":true},"null":null,"string":"short_string","true":true} -{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"false":false,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max":9223372036854775807,"int64_min":-9223372036854775808,"int8_max":127,"int8_min":-128,"nonempty_object":{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"false":false,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max":9223372036854775807,"int64_min":-9223372036854775808,"int8_max":127,"int8_min":-128,"non_empty_object2":{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"empty_object":{},"false":false,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max":9223372036854775807,"int64_min":-9223372036854775808,"int8_max":127,"int8_min":-128,"null":null,"string":"short_string","true":true},"null":null,"string":"short_string","true":true},"null":null,"string":"short_string","true":true} -{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_object":{},"false":false,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max":9223372036854775807,"int64_min":-9223372036854775808,"int8_max":127,"int8_min":-128,"nonempty_array":[1,2,3,4,5],"null":null,"string":"short_string","true":true} +{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"empty_object":{},"false":false,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max_supported":9223372036854776,"int64_min_supported":-9223372036854776,"int8_max":127,"int8_min":-128,"null":null,"string":"short_string","true":true} +{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"false":false,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max_supported":9223372036854776,"int64_min_supported":-9223372036854776,"int8_max":127,"int8_min":-128,"nonempty_object":{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"empty_object":{},"false":false,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max_supported":9223372036854776,"int64_min_supported":-9223372036854776,"int8_max":127,"int8_min":-128,"null":null,"string":"short_string","true":true},"null":null,"string":"short_string","true":true} +{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"false":false,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max_supported":9223372036854776,"int64_min_supported":-9223372036854776,"int8_max":127,"int8_min":-128,"nonempty_object":{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"false":false,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max_supported":9223372036854776,"int64_min_supported":-9223372036854776,"int8_max":127,"int8_min":-128,"non_empty_object2":{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"empty_object":{},"false":false,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max_supported":9223372036854776,"int64_min_supported":-9223372036854776,"int8_max":127,"int8_min":-128,"null":null,"string":"short_string","true":true},"null":null,"string":"short_string","true":true},"null":null,"string":"short_string","true":true} +{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_object":{},"false":false,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max_supported":9223372036854776,"int64_min_supported":-9223372036854776,"int8_max":127,"int8_min":-128,"nonempty_array":[1,2,3,4,5],"null":null,"string":"short_string","true":true} diff --git a/components/core/tests/test_log_files/test_sorted.json b/components/core/tests/test_log_files/test_sorted.json index ec08731dd..50804bb98 100644 --- a/components/core/tests/test_log_files/test_sorted.json +++ b/components/core/tests/test_log_files/test_sorted.json @@ -1,4 +1,4 @@ -{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"empty_object":{},"false":false,"float_neg":-1.01,"float_pos":1.01,"float_zero":0.0,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max":9223372036854775807,"int64_min":-9223372036854775808,"int8_max":127,"int8_min":-128,"null":null,"string":"short_string","true":true} -{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"false":false,"float_neg":-1.01,"float_pos":1.01,"float_zero":0.0,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max":9223372036854775807,"int64_min":-9223372036854775808,"int8_max":127,"int8_min":-128,"nonempty_object":{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"empty_object":{},"false":false,"float_neg":-1.01,"float_pos":1.01,"float_zero":0.0,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max":9223372036854775807,"int64_min":-9223372036854775808,"int8_max":127,"int8_min":-128,"null":null,"string":"short_string","true":true},"null":null,"string":"short_string","true":true} -{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"false":false,"float_neg":-1.01,"float_pos":1.01,"float_zero":0.0,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max":9223372036854775807,"int64_min":-9223372036854775808,"int8_max":127,"int8_min":-128,"nonempty_object":{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"false":false,"float_neg":-1.01,"float_pos":1.01,"float_zero":0.0,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max":9223372036854775807,"int64_min":-9223372036854775808,"int8_max":127,"int8_min":-128,"non_empty_object2":{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"empty_object":{},"false":false,"float_neg":-1.01,"float_pos":1.01,"float_zero":0.0,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max":9223372036854775807,"int64_min":-9223372036854775808,"int8_max":127,"int8_min":-128,"null":null,"string":"short_string","true":true},"null":null,"string":"short_string","true":true},"null":null,"string":"short_string","true":true} -{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_object":{},"false":false,"float_neg":-1.01,"float_pos":1.01,"float_zero":0.0,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max":9223372036854775807,"int64_min":-9223372036854775808,"int8_max":127,"int8_min":-128,"nonempty_array":[1,2,3,4,5],"null":null,"string":"short_string","true":true} +{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"empty_object":{},"false":false,"float_neg":-1.01,"float_pos":1.01,"float_zero":0.0,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max_supported":9223372036854776,"int64_min_supported":-9223372036854776,"int8_max":127,"int8_min":-128,"null":null,"string":"short_string","true":true} +{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"false":false,"float_neg":-1.01,"float_pos":1.01,"float_zero":0.0,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max_supported":9223372036854776,"int64_min_supported":-9223372036854776,"int8_max":127,"int8_min":-128,"nonempty_object":{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"empty_object":{},"false":false,"float_neg":-1.01,"float_pos":1.01,"float_zero":0.0,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max_supported":9223372036854776,"int64_min_supported":-9223372036854776,"int8_max":127,"int8_min":-128,"null":null,"string":"short_string","true":true},"null":null,"string":"short_string","true":true} +{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"false":false,"float_neg":-1.01,"float_pos":1.01,"float_zero":0.0,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max_supported":9223372036854776,"int64_min_supported":-9223372036854776,"int8_max":127,"int8_min":-128,"nonempty_object":{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"false":false,"float_neg":-1.01,"float_pos":1.01,"float_zero":0.0,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max_supported":9223372036854776,"int64_min_supported":-9223372036854776,"int8_max":127,"int8_min":-128,"non_empty_object2":{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"empty_object":{},"false":false,"float_neg":-1.01,"float_pos":1.01,"float_zero":0.0,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max_supported":9223372036854776,"int64_min_supported":-9223372036854776,"int8_max":127,"int8_min":-128,"null":null,"string":"short_string","true":true},"null":null,"string":"short_string","true":true},"null":null,"string":"short_string","true":true} +{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_object":{},"false":false,"float_neg":-1.01,"float_pos":1.01,"float_zero":0.0,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max_supported":9223372036854776,"int64_min_supported":-9223372036854776,"int8_max":127,"int8_min":-128,"nonempty_array":[1,2,3,4,5],"null":null,"string":"short_string","true":true} From e1b636da54c94e6bd4c4553e80912d82b75cafd9 Mon Sep 17 00:00:00 2001 From: Abigail Matthews Date: Tue, 19 Nov 2024 16:42:11 -0500 Subject: [PATCH 08/18] Remove debug prints --- components/core/tests/test-end_to_end.cpp | 5 ----- 1 file changed, 5 deletions(-) diff --git a/components/core/tests/test-end_to_end.cpp b/components/core/tests/test-end_to_end.cpp index 4ea958d43..7346cf8c4 100644 --- a/components/core/tests/test-end_to_end.cpp +++ b/components/core/tests/test-end_to_end.cpp @@ -114,12 +114,7 @@ TEMPLATE_TEST_CASE( REQUIRE(0 == result); std::string const command = "diff -u test-end-to-end_sorted.json " + get_test_input_local_path() + " > diff_out.txt"; - //std::cout << command << "\n"; result = std::system(command.c_str()); - //std::cout << result << "\n"; - std::cout << WEXITSTATUS(result) << "\n"; - std::system("cat test-end-to-end-out/original"); - std::system("cat diff_out.txt"); REQUIRE((0 == WEXITSTATUS(result) || 1 == WEXITSTATUS(result))); REQUIRE(std::filesystem::is_empty("diff_out.txt")); From 7a21d858d2d3655b9c3ca4a3e3cd0270de5a3f66 Mon Sep 17 00:00:00 2001 From: Abigail Matthews Date: Wed, 20 Nov 2024 16:54:19 -0500 Subject: [PATCH 09/18] remove magic values, add addtional file cleanup, test on structurized arrays --- components/core/tests/test-end_to_end.cpp | 116 +++++++++++----------- 1 file changed, 57 insertions(+), 59 deletions(-) diff --git a/components/core/tests/test-end_to_end.cpp b/components/core/tests/test-end_to_end.cpp index 7346cf8c4..571842b9d 100644 --- a/components/core/tests/test-end_to_end.cpp +++ b/components/core/tests/test-end_to_end.cpp @@ -2,44 +2,46 @@ #include #include #include +#include #include #include -#include - -#include "../src/clp/BufferReader.hpp" -#include "../src/clp/ffi/ir_stream/decoding_methods.hpp" -#include "../src/clp/ffi/ir_stream/Deserializer.hpp" -#include "../src/clp/ffi/ir_stream/Serializer.hpp" -#include "../src/clp/ffi/KeyValuePairLogEvent.hpp" -#include "../src/clp/ir/types.hpp" -#include "../src/clp/time_types.hpp" + #include "../src/clp_s/JsonConstructor.hpp" #include "../src/clp_s/JsonParser.hpp" -using clp::BufferReader; -using clp::ffi::ir_stream::Deserializer; -using clp::ffi::ir_stream::IRErrorCode; -using clp::ffi::ir_stream::Serializer; -using clp::ffi::KeyValuePairLogEvent; -using clp::ir::eight_byte_encoded_variable_t; -using clp::ir::four_byte_encoded_variable_t; -using clp::size_checked_pointer_cast; -using clp::UtcOffset; -using std::string; -using std::string_view; -using std::vector; - -auto const cDefaultTargetEncodedSize = 8ULL * 1024 * 1024 * 1024; -auto const cDefaultMaxDocumentSize = 512ULL * 1024 * 1024; -auto const cDefaultMinTableSize = 1ULL * 1024 * 1024; -auto const cDeaultCompressionLevel = 3; +auto const cDefaultTargetEncodedSize = 8ULL * 1024 * 1024 * 1024; // 8 GB +auto const cDefaultMaxDocumentSize = 512ULL * 1024 * 1024; // 512 MB +auto const cDefaultMinTableSize = 1ULL * 1024 * 1024; // 1 MB +auto const cDefaultCompressionLevel = 3; auto const cDefaultPrintArchiveStats = false; -auto const cDefaultStructurizeArrays = false; + +constexpr char cTestEndToEndArchiveDirectory[] = "test-end-to-end-archive"; +constexpr char cTestEndToEndOutputDirectory[] = "test-end-to-end-out"; +constexpr char cTestEndToEndOutputSortedJson[] = "test-end-to-end_sorted.json"; +constexpr char cTestEndToEndInputFileDirectory[] = "test_log_files"; +constexpr char cTestEndToEndInputFile[] = "test_no_floats_sorted.json"; + namespace { + +class Cleanup { + public: + Cleanup() { + std::filesystem::remove_all(cTestEndToEndArchiveDirectory); + std::filesystem::remove_all(cTestEndToEndOutputDirectory); + std::filesystem::remove(cTestEndToEndOutputSortedJson); + } + ~Cleanup() { + std::filesystem::remove_all(cTestEndToEndArchiveDirectory); + std::filesystem::remove_all(cTestEndToEndOutputDirectory); + std::filesystem::remove(cTestEndToEndOutputSortedJson); + } + +}; + auto get_test_input_path_relative_to_tests_dir() -> std::filesystem::path { - return std::filesystem::path{"test_log_files"} / "test_no_floats_sorted.json"; + return std::filesystem::path{cTestEndToEndInputFileDirectory} / cTestEndToEndInputFile; } auto get_test_input_local_path() -> std::string { @@ -47,44 +49,40 @@ auto get_test_input_local_path() -> std::string { auto const tests_dir{current_file_path.parent_path()}; return (tests_dir / get_test_input_path_relative_to_tests_dir()).string(); } -} // namespace +} // NOLINTNEXTLINE(readability-function-cognitive-complexity) -TEMPLATE_TEST_CASE( - "clp-s_compression_and_extraction_no_floats", - "[clp-s][end-to-end]", - four_byte_encoded_variable_t, - eight_byte_encoded_variable_t -) { - std::filesystem::remove_all("test-end-to-end-archive"); - std::filesystem::remove_all("test-end-to-end-out"); - std::filesystem::remove("test-end-to-end_sorted.json"); - std::filesystem::remove("diff_out.txt"); - - std::filesystem::create_directory("test-end-to-end-archive"); - REQUIRE(std::filesystem::is_directory("test-end-to-end-archive")); +TEST_CASE("clp-s_compression_and_extraction_no_floats", + "[clp-s][end-to-end]") { + + auto structurize_arrays = GENERATE(true, false); + + Cleanup test_cleanup; + + std::filesystem::create_directory(cTestEndToEndArchiveDirectory); + REQUIRE(std::filesystem::is_directory(cTestEndToEndArchiveDirectory)); clp_s::JsonParserOption parser_option{}; parser_option.file_paths.push_back(get_test_input_local_path()); - parser_option.archives_dir = "test-end-to-end-archive"; + parser_option.archives_dir = cTestEndToEndArchiveDirectory; parser_option.target_encoded_size = cDefaultTargetEncodedSize; parser_option.max_document_size = cDefaultMaxDocumentSize; parser_option.min_table_size = cDefaultMinTableSize; - parser_option.compression_level = cDeaultCompressionLevel; + parser_option.compression_level = cDefaultCompressionLevel; parser_option.print_archive_stats = cDefaultPrintArchiveStats; - parser_option.structurize_arrays = cDefaultStructurizeArrays; + parser_option.structurize_arrays = structurize_arrays; clp_s::JsonParser parser(parser_option); REQUIRE(parser.parse()); parser.store(); - REQUIRE(false == std::filesystem::is_empty("test-end-to-end-archive")); + REQUIRE(false == std::filesystem::is_empty(cTestEndToEndArchiveDirectory)); - std::filesystem::create_directory("test-end-to-end-out"); - REQUIRE(std::filesystem::is_directory("test-end-to-end-out")); + std::filesystem::create_directory(cTestEndToEndOutputDirectory); + REQUIRE(std::filesystem::is_directory(cTestEndToEndOutputDirectory)); clp_s::JsonConstructorOption constructor_option{}; - constructor_option.output_dir = "test-end-to-end-out"; + constructor_option.output_dir = cTestEndToEndOutputDirectory; constructor_option.ordered = false; constructor_option.archives_dir = parser_option.archives_dir; constructor_option.ordered_chunk_size = 0; @@ -99,23 +97,23 @@ TEMPLATE_TEST_CASE( constructor.store(); } - REQUIRE(std::filesystem::exists("test-end-to-end-out/original")); + std::string command = cTestEndToEndOutputDirectory; + command += "/original"; + REQUIRE(std::filesystem::exists(command.c_str())); int result = std::system("command -v jq >/dev/null 2>&1"); REQUIRE(0 == result); - result = std::system( - "jq -S -c '.' test-end-to-end-out/original | sort > test-end-to-end_sorted.json" - ); + command = ""; + command = (((command += "jq -S -c '.' ") += cTestEndToEndOutputDirectory) += "/original | sort > ") += cTestEndToEndOutputSortedJson; + result = std::system(command.c_str()); REQUIRE(0 == result); - REQUIRE(false == std::filesystem::is_empty("test-end-to-end_sorted.json")); + REQUIRE(false == std::filesystem::is_empty(cTestEndToEndOutputSortedJson)); result = std::system("command -v diff >/dev/null 2>&1"); REQUIRE(0 == result); - std::string const command = "diff -u test-end-to-end_sorted.json " + get_test_input_local_path() - + " > diff_out.txt"; + command = ""; + command = ((((command +="diff -u ") += cTestEndToEndOutputSortedJson) += " ") += get_test_input_local_path()) += " > /dev/null"; result = std::system(command.c_str()); - REQUIRE((0 == WEXITSTATUS(result) || 1 == WEXITSTATUS(result))); - - REQUIRE(std::filesystem::is_empty("diff_out.txt")); + REQUIRE(0 == WEXITSTATUS(result)); } From a039bf975ee22655391ce8255cf1953eaf013d8b Mon Sep 17 00:00:00 2001 From: Abigail Matthews Date: Wed, 20 Nov 2024 17:44:12 -0500 Subject: [PATCH 10/18] Fix compilation error form merging msot recent commit form main, remove excess JSON file --- components/core/tests/test-end_to_end.cpp | 50 +++++++++---------- .../tests/test_log_files/test_sorted.json | 4 -- 2 files changed, 24 insertions(+), 30 deletions(-) delete mode 100644 components/core/tests/test_log_files/test_sorted.json diff --git a/components/core/tests/test-end_to_end.cpp b/components/core/tests/test-end_to_end.cpp index 571842b9d..4d00fad64 100644 --- a/components/core/tests/test-end_to_end.cpp +++ b/components/core/tests/test-end_to_end.cpp @@ -1,8 +1,8 @@ +#include + #include #include #include -#include -#include #include #include @@ -10,34 +10,35 @@ #include "../src/clp_s/JsonConstructor.hpp" #include "../src/clp_s/JsonParser.hpp" -auto const cDefaultTargetEncodedSize = 8ULL * 1024 * 1024 * 1024; // 8 GB -auto const cDefaultMaxDocumentSize = 512ULL * 1024 * 1024; // 512 MB -auto const cDefaultMinTableSize = 1ULL * 1024 * 1024; // 1 MB +auto const cDefaultTargetEncodedSize = 8ULL * 1024 * 1024 * 1024; // 8 GB +auto const cDefaultMaxDocumentSize = 512ULL * 1024 * 1024; // 512 MB +auto const cDefaultMinTableSize = 1ULL * 1024 * 1024; // 1 MB auto const cDefaultCompressionLevel = 3; auto const cDefaultPrintArchiveStats = false; +auto const cDefaultOrdered = false; +auto const cDefaultTargetOrderedChunkSize = 0; -constexpr char cTestEndToEndArchiveDirectory[] = "test-end-to-end-archive"; -constexpr char cTestEndToEndOutputDirectory[] = "test-end-to-end-out"; -constexpr char cTestEndToEndOutputSortedJson[] = "test-end-to-end_sorted.json"; -constexpr char cTestEndToEndInputFileDirectory[] = "test_log_files"; -constexpr char cTestEndToEndInputFile[] = "test_no_floats_sorted.json"; - +std::string const cTestEndToEndArchiveDirectory{"test-end-to-end-archive"}; +std::string const cTestEndToEndOutputDirectory = "test-end-to-end-out"; +std::string const cTestEndToEndOutputSortedJson = "test-end-to-end_sorted.json"; +std::string const cTestEndToEndInputFileDirectory = "test_log_files"; +std::string const cTestEndToEndInputFile = "test_no_floats_sorted.json"; namespace { class Cleanup { - public: +public: Cleanup() { std::filesystem::remove_all(cTestEndToEndArchiveDirectory); std::filesystem::remove_all(cTestEndToEndOutputDirectory); std::filesystem::remove(cTestEndToEndOutputSortedJson); } + ~Cleanup() { std::filesystem::remove_all(cTestEndToEndArchiveDirectory); std::filesystem::remove_all(cTestEndToEndOutputDirectory); std::filesystem::remove(cTestEndToEndOutputSortedJson); } - }; auto get_test_input_path_relative_to_tests_dir() -> std::filesystem::path { @@ -49,15 +50,13 @@ auto get_test_input_local_path() -> std::string { auto const tests_dir{current_file_path.parent_path()}; return (tests_dir / get_test_input_path_relative_to_tests_dir()).string(); } -} +} // namespace // NOLINTNEXTLINE(readability-function-cognitive-complexity) -TEST_CASE("clp-s_compression_and_extraction_no_floats", - "[clp-s][end-to-end]") { - +TEST_CASE("clp-s_compression_and_extraction_no_floats", "[clp-s][end-to-end]") { auto structurize_arrays = GENERATE(true, false); - Cleanup test_cleanup; + Cleanup const test_cleanup; std::filesystem::create_directory(cTestEndToEndArchiveDirectory); REQUIRE(std::filesystem::is_directory(cTestEndToEndArchiveDirectory)); @@ -82,10 +81,10 @@ TEST_CASE("clp-s_compression_and_extraction_no_floats", REQUIRE(std::filesystem::is_directory(cTestEndToEndOutputDirectory)); clp_s::JsonConstructorOption constructor_option{}; + constructor_option.archives_dir = parser_option.archives_dir; constructor_option.output_dir = cTestEndToEndOutputDirectory; constructor_option.ordered = false; - constructor_option.archives_dir = parser_option.archives_dir; - constructor_option.ordered_chunk_size = 0; + constructor_option.target_ordered_chunk_size = 0; for (auto const& entry : std::filesystem::directory_iterator(constructor_option.archives_dir)) { if (false == entry.is_directory()) { // Skip non-directories @@ -97,14 +96,13 @@ TEST_CASE("clp-s_compression_and_extraction_no_floats", constructor.store(); } - std::string command = cTestEndToEndOutputDirectory; - command += "/original"; + std::string command = cTestEndToEndOutputDirectory + "/original"; REQUIRE(std::filesystem::exists(command.c_str())); int result = std::system("command -v jq >/dev/null 2>&1"); REQUIRE(0 == result); - command = ""; - command = (((command += "jq -S -c '.' ") += cTestEndToEndOutputDirectory) += "/original | sort > ") += cTestEndToEndOutputSortedJson; + command = "jq -S -c '.' " + cTestEndToEndOutputDirectory + "/original | sort > " + + cTestEndToEndOutputSortedJson; result = std::system(command.c_str()); REQUIRE(0 == result); @@ -112,8 +110,8 @@ TEST_CASE("clp-s_compression_and_extraction_no_floats", result = std::system("command -v diff >/dev/null 2>&1"); REQUIRE(0 == result); - command = ""; - command = ((((command +="diff -u ") += cTestEndToEndOutputSortedJson) += " ") += get_test_input_local_path()) += " > /dev/null"; + command = "diff -u " + cTestEndToEndOutputSortedJson + " " + get_test_input_local_path() + + " > /dev/null"; result = std::system(command.c_str()); REQUIRE(0 == WEXITSTATUS(result)); } diff --git a/components/core/tests/test_log_files/test_sorted.json b/components/core/tests/test_log_files/test_sorted.json deleted file mode 100644 index 50804bb98..000000000 --- a/components/core/tests/test_log_files/test_sorted.json +++ /dev/null @@ -1,4 +0,0 @@ -{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"empty_object":{},"false":false,"float_neg":-1.01,"float_pos":1.01,"float_zero":0.0,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max_supported":9223372036854776,"int64_min_supported":-9223372036854776,"int8_max":127,"int8_min":-128,"null":null,"string":"short_string","true":true} -{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"false":false,"float_neg":-1.01,"float_pos":1.01,"float_zero":0.0,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max_supported":9223372036854776,"int64_min_supported":-9223372036854776,"int8_max":127,"int8_min":-128,"nonempty_object":{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"empty_object":{},"false":false,"float_neg":-1.01,"float_pos":1.01,"float_zero":0.0,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max_supported":9223372036854776,"int64_min_supported":-9223372036854776,"int8_max":127,"int8_min":-128,"null":null,"string":"short_string","true":true},"null":null,"string":"short_string","true":true} -{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"false":false,"float_neg":-1.01,"float_pos":1.01,"float_zero":0.0,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max_supported":9223372036854776,"int64_min_supported":-9223372036854776,"int8_max":127,"int8_min":-128,"nonempty_object":{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"false":false,"float_neg":-1.01,"float_pos":1.01,"float_zero":0.0,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max_supported":9223372036854776,"int64_min_supported":-9223372036854776,"int8_max":127,"int8_min":-128,"non_empty_object2":{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"empty_object":{},"false":false,"float_neg":-1.01,"float_pos":1.01,"float_zero":0.0,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max_supported":9223372036854776,"int64_min_supported":-9223372036854776,"int8_max":127,"int8_min":-128,"null":null,"string":"short_string","true":true},"null":null,"string":"short_string","true":true},"null":null,"string":"short_string","true":true} -{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_object":{},"false":false,"float_neg":-1.01,"float_pos":1.01,"float_zero":0.0,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max_supported":9223372036854776,"int64_min_supported":-9223372036854776,"int8_max":127,"int8_min":-128,"nonempty_array":[1,2,3,4,5],"null":null,"string":"short_string","true":true} From ad18256c2127d941323281e110eb7e2d18710128 Mon Sep 17 00:00:00 2001 From: Abigail Matthews Date: Fri, 22 Nov 2024 02:41:12 -0500 Subject: [PATCH 11/18] brace init, move variables, doc string, command construction --- components/core/tests/test-end_to_end.cpp | 76 +++++++++++-------- .../ubuntu-focal/install-prebuilt-packages.sh | 1 - .../ubuntu-jammy/install-prebuilt-packages.sh | 1 - 3 files changed, 43 insertions(+), 35 deletions(-) diff --git a/components/core/tests/test-end_to_end.cpp b/components/core/tests/test-end_to_end.cpp index 4d00fad64..28271e213 100644 --- a/components/core/tests/test-end_to_end.cpp +++ b/components/core/tests/test-end_to_end.cpp @@ -2,7 +2,9 @@ #include #include +#include #include +#include #include #include @@ -10,31 +12,25 @@ #include "../src/clp_s/JsonConstructor.hpp" #include "../src/clp_s/JsonParser.hpp" -auto const cDefaultTargetEncodedSize = 8ULL * 1024 * 1024 * 1024; // 8 GB -auto const cDefaultMaxDocumentSize = 512ULL * 1024 * 1024; // 512 MB -auto const cDefaultMinTableSize = 1ULL * 1024 * 1024; // 1 MB -auto const cDefaultCompressionLevel = 3; -auto const cDefaultPrintArchiveStats = false; -auto const cDefaultOrdered = false; -auto const cDefaultTargetOrderedChunkSize = 0; - -std::string const cTestEndToEndArchiveDirectory{"test-end-to-end-archive"}; -std::string const cTestEndToEndOutputDirectory = "test-end-to-end-out"; -std::string const cTestEndToEndOutputSortedJson = "test-end-to-end_sorted.json"; -std::string const cTestEndToEndInputFileDirectory = "test_log_files"; -std::string const cTestEndToEndInputFile = "test_no_floats_sorted.json"; +constexpr std::string_view cTestEndToEndArchiveDirectory{"test-end-to-end-archive"}; +constexpr std::string_view cTestEndToEndOutputDirectory{"test-end-to-end-out"}; +constexpr std::string_view cTestEndToEndOutputSortedJson{"test-end-to-end_sorted.json"}; +constexpr std::string_view cTestEndToEndInputFileDirectory{"test_log_files"}; +constexpr std::string_view cTestEndToEndInputFile{"test_no_floats_sorted.json"}; namespace { - +/** + * Class with no data members who's contruction and destruction is soley + * for cleanup up files and directorys created by the test case. + */ class Cleanup { public: - Cleanup() { - std::filesystem::remove_all(cTestEndToEndArchiveDirectory); - std::filesystem::remove_all(cTestEndToEndOutputDirectory); - std::filesystem::remove(cTestEndToEndOutputSortedJson); - } + Cleanup() { delete_files(); } + + ~Cleanup() { delete_files(); } - ~Cleanup() { +private: + static void delete_files() { std::filesystem::remove_all(cTestEndToEndArchiveDirectory); std::filesystem::remove_all(cTestEndToEndOutputDirectory); std::filesystem::remove(cTestEndToEndOutputSortedJson); @@ -54,6 +50,13 @@ auto get_test_input_local_path() -> std::string { // NOLINTNEXTLINE(readability-function-cognitive-complexity) TEST_CASE("clp-s_compression_and_extraction_no_floats", "[clp-s][end-to-end]") { + auto const default_target_encoded_size = 8ULL * 1024 * 1024 * 1024; // 8 GiB + auto const default_max_document_size = 512ULL * 1024 * 1024; // 512 MiB + auto const default_min_table_size = 1ULL * 1024 * 1024; // 1 MiB + auto const default_compression_level = 3; + auto const default_print_archive_stats = false; + auto const default_ordered = false; + auto const default_target_ordered_chunk_size = 0; auto structurize_arrays = GENERATE(true, false); Cleanup const test_cleanup; @@ -64,14 +67,14 @@ TEST_CASE("clp-s_compression_and_extraction_no_floats", "[clp-s][end-to-end]") { clp_s::JsonParserOption parser_option{}; parser_option.file_paths.push_back(get_test_input_local_path()); parser_option.archives_dir = cTestEndToEndArchiveDirectory; - parser_option.target_encoded_size = cDefaultTargetEncodedSize; - parser_option.max_document_size = cDefaultMaxDocumentSize; - parser_option.min_table_size = cDefaultMinTableSize; - parser_option.compression_level = cDefaultCompressionLevel; - parser_option.print_archive_stats = cDefaultPrintArchiveStats; + parser_option.target_encoded_size = default_target_encoded_size; + parser_option.max_document_size = default_max_document_size; + parser_option.min_table_size = default_min_table_size; + parser_option.compression_level = default_compression_level; + parser_option.print_archive_stats = default_print_archive_stats; parser_option.structurize_arrays = structurize_arrays; - clp_s::JsonParser parser(parser_option); + clp_s::JsonParser parser{parser_option}; REQUIRE(parser.parse()); parser.store(); @@ -92,17 +95,21 @@ TEST_CASE("clp-s_compression_and_extraction_no_floats", "[clp-s][end-to-end]") { } constructor_option.archive_id = entry.path().filename(); - clp_s::JsonConstructor constructor(constructor_option); + clp_s::JsonConstructor constructor{constructor_option}; constructor.store(); } - std::string command = cTestEndToEndOutputDirectory + "/original"; - REQUIRE(std::filesystem::exists(command.c_str())); + std::filesystem::path extracted_json_path{cTestEndToEndOutputDirectory}; + extracted_json_path /= "original"; + REQUIRE(std::filesystem::exists(extracted_json_path)); int result = std::system("command -v jq >/dev/null 2>&1"); REQUIRE(0 == result); - command = "jq -S -c '.' " + cTestEndToEndOutputDirectory + "/original | sort > " - + cTestEndToEndOutputSortedJson; + std::string command = std::format( + "jq --sort-keys --compact-output '.' {}/original | sort > {}", + cTestEndToEndOutputDirectory, + cTestEndToEndOutputSortedJson + ); result = std::system(command.c_str()); REQUIRE(0 == result); @@ -110,8 +117,11 @@ TEST_CASE("clp-s_compression_and_extraction_no_floats", "[clp-s][end-to-end]") { result = std::system("command -v diff >/dev/null 2>&1"); REQUIRE(0 == result); - command = "diff -u " + cTestEndToEndOutputSortedJson + " " + get_test_input_local_path() - + " > /dev/null"; + command = std::format( + "diff -u {} {} > /dev/null", + cTestEndToEndOutputSortedJson, + get_test_input_local_path() + ); result = std::system(command.c_str()); REQUIRE(0 == WEXITSTATUS(result)); } diff --git a/components/core/tools/scripts/lib_install/ubuntu-focal/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/ubuntu-focal/install-prebuilt-packages.sh index 9e278468d..8997ffe01 100755 --- a/components/core/tools/scripts/lib_install/ubuntu-focal/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/ubuntu-focal/install-prebuilt-packages.sh @@ -12,7 +12,6 @@ DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ checkinstall \ cmake \ curl \ - diffutils \ g++ \ g++-10 \ gcc \ diff --git a/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh index ffde62210..9ed6b9b10 100755 --- a/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh @@ -13,7 +13,6 @@ DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ cmake \ curl \ build-essential \ - diffutils \ git \ jq \ libboost-filesystem-dev \ From ea375b2e1d345f18936ddf1074946df62ac7780f Mon Sep 17 00:00:00 2001 From: Abigail Matthews Date: Fri, 22 Nov 2024 03:20:20 -0500 Subject: [PATCH 12/18] remove std::format due to lack of support --- components/core/tests/test-end_to_end.cpp | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/components/core/tests/test-end_to_end.cpp b/components/core/tests/test-end_to_end.cpp index 28271e213..288f47cdd 100644 --- a/components/core/tests/test-end_to_end.cpp +++ b/components/core/tests/test-end_to_end.cpp @@ -105,11 +105,8 @@ TEST_CASE("clp-s_compression_and_extraction_no_floats", "[clp-s][end-to-end]") { int result = std::system("command -v jq >/dev/null 2>&1"); REQUIRE(0 == result); - std::string command = std::format( - "jq --sort-keys --compact-output '.' {}/original | sort > {}", - cTestEndToEndOutputDirectory, - cTestEndToEndOutputSortedJson - ); + std::string command = "jq -S -c '.' " + std::string(cTestEndToEndOutputDirectory) + + "/original | sort > " + std::string(cTestEndToEndOutputSortedJson); result = std::system(command.c_str()); REQUIRE(0 == result); @@ -117,11 +114,8 @@ TEST_CASE("clp-s_compression_and_extraction_no_floats", "[clp-s][end-to-end]") { result = std::system("command -v diff >/dev/null 2>&1"); REQUIRE(0 == result); - command = std::format( - "diff -u {} {} > /dev/null", - cTestEndToEndOutputSortedJson, - get_test_input_local_path() - ); + command = "diff -u " + std::string(cTestEndToEndOutputSortedJson) + " " + + get_test_input_local_path() + " > /dev/null"; result = std::system(command.c_str()); REQUIRE(0 == WEXITSTATUS(result)); } From 90c6cef7b360173352ca95dcb50e9090ca675111 Mon Sep 17 00:00:00 2001 From: Abigail Matthews Date: Fri, 22 Nov 2024 03:43:10 -0500 Subject: [PATCH 13/18] remove format include --- components/core/tests/test-end_to_end.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/components/core/tests/test-end_to_end.cpp b/components/core/tests/test-end_to_end.cpp index 288f47cdd..ea0bf521f 100644 --- a/components/core/tests/test-end_to_end.cpp +++ b/components/core/tests/test-end_to_end.cpp @@ -2,7 +2,6 @@ #include #include -#include #include #include #include From 01ca9ca5ff9dac087d044153af9fcec2f3a6a15c Mon Sep 17 00:00:00 2001 From: Abigail Matthews Date: Fri, 22 Nov 2024 16:35:53 -0500 Subject: [PATCH 14/18] fmt:format for command string building --- components/core/tests/test-end_to_end.cpp | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/components/core/tests/test-end_to_end.cpp b/components/core/tests/test-end_to_end.cpp index ea0bf521f..6b1648df3 100644 --- a/components/core/tests/test-end_to_end.cpp +++ b/components/core/tests/test-end_to_end.cpp @@ -7,6 +7,7 @@ #include #include +#include #include "../src/clp_s/JsonConstructor.hpp" #include "../src/clp_s/JsonParser.hpp" @@ -104,8 +105,11 @@ TEST_CASE("clp-s_compression_and_extraction_no_floats", "[clp-s][end-to-end]") { int result = std::system("command -v jq >/dev/null 2>&1"); REQUIRE(0 == result); - std::string command = "jq -S -c '.' " + std::string(cTestEndToEndOutputDirectory) - + "/original | sort > " + std::string(cTestEndToEndOutputSortedJson); + std::string command = fmt::format( + "jq --sort-keys --compact-output '.' {}/original | sort > {}", + cTestEndToEndOutputDirectory, + cTestEndToEndOutputSortedJson + ); result = std::system(command.c_str()); REQUIRE(0 == result); @@ -113,8 +117,11 @@ TEST_CASE("clp-s_compression_and_extraction_no_floats", "[clp-s][end-to-end]") { result = std::system("command -v diff >/dev/null 2>&1"); REQUIRE(0 == result); - command = "diff -u " + std::string(cTestEndToEndOutputSortedJson) + " " - + get_test_input_local_path() + " > /dev/null"; + command = fmt::format( + "diff -u {} {} > /dev/null", + cTestEndToEndOutputSortedJson, + get_test_input_local_path() + ); result = std::system(command.c_str()); REQUIRE(0 == WEXITSTATUS(result)); } From 03d65109b2ee7cf53e39c23f41a3e301d597f5cc Mon Sep 17 00:00:00 2001 From: Abigail Matthews Date: Fri, 29 Nov 2024 17:33:09 -0500 Subject: [PATCH 15/18] restructure into compress, extract, compare --- components/core/CMakeLists.txt | 2 +- .../core/tests/test-clp_s-end_to_end.cpp | 159 ++++++++++++++++++ components/core/tests/test-end_to_end.cpp | 127 -------------- ...orted.json => test_no_floats_sorted.jsonl} | 0 4 files changed, 160 insertions(+), 128 deletions(-) create mode 100644 components/core/tests/test-clp_s-end_to_end.cpp delete mode 100644 components/core/tests/test-end_to_end.cpp rename components/core/tests/test_log_files/{test_no_floats_sorted.json => test_no_floats_sorted.jsonl} (100%) diff --git a/components/core/CMakeLists.txt b/components/core/CMakeLists.txt index d4f9ff918..1b4fdb1be 100644 --- a/components/core/CMakeLists.txt +++ b/components/core/CMakeLists.txt @@ -548,9 +548,9 @@ set(SOURCE_FILES_unitTest tests/LogSuppressor.hpp tests/test-Array.cpp tests/test-BufferedFileReader.cpp + tests/test-clp_s-end_to_end.cpp tests/test-EncodedVariableInterpreter.cpp tests/test-encoding_methods.cpp - tests/test-end_to_end.cpp tests/test-ffi_IrUnitHandlerInterface.cpp tests/test-ffi_KeyValuePairLogEvent.cpp tests/test-ffi_SchemaTree.cpp diff --git a/components/core/tests/test-clp_s-end_to_end.cpp b/components/core/tests/test-clp_s-end_to_end.cpp new file mode 100644 index 000000000..1009111b4 --- /dev/null +++ b/components/core/tests/test-clp_s-end_to_end.cpp @@ -0,0 +1,159 @@ +#include + +#include +#include +#include +#include +#include + +#include +#include + +#include "../src/clp_s/JsonConstructor.hpp" +#include "../src/clp_s/JsonParser.hpp" + +constexpr std::string_view cTestEndToEndArchiveDirectory{"test-end-to-end-archive"}; +constexpr std::string_view cTestEndToEndOutputDirectory{"test-end-to-end-out"}; +constexpr std::string_view cTestEndToEndOutputSortedJson{"test-end-to-end_sorted.jsonl"}; +constexpr std::string_view cTestEndToEndInputFileDirectory{"test_log_files"}; +constexpr std::string_view cTestEndToEndInputFile{"test_no_floats_sorted.jsonl"}; + +namespace { +/** + * A class that deletes the directories and files created by test cases, both before and after each + * test case where the class is instantiated. + */ +class TestOutputCleaner { +public: + TestOutputCleaner() { delete_files(); } + + ~TestOutputCleaner() { delete_files(); } + + // Delete copy & move constructors and assignment operators + TestOutputCleaner(TestOutputCleaner const&) = delete; + TestOutputCleaner(TestOutputCleaner&&) = delete; + auto operator=(TestOutputCleaner const&) -> TestOutputCleaner& = delete; + auto operator=(TestOutputCleaner&&) -> TestOutputCleaner& = delete; + +private: + static void delete_files() { + std::filesystem::remove_all(cTestEndToEndArchiveDirectory); + std::filesystem::remove_all(cTestEndToEndOutputDirectory); + std::filesystem::remove(cTestEndToEndOutputSortedJson); + } +}; + +auto get_test_input_path_relative_to_tests_dir() -> std::filesystem::path; +auto get_test_input_local_path() -> std::string; +void compress(bool structurize_arrays); +auto extract() -> std::filesystem::path; +void compare(std::filesystem::path const& extracted_json_path); + +auto get_test_input_path_relative_to_tests_dir() -> std::filesystem::path { + return std::filesystem::path{cTestEndToEndInputFileDirectory} / cTestEndToEndInputFile; +} + +auto get_test_input_local_path() -> std::string { + std::filesystem::path const current_file_path{__FILE__}; + auto const tests_dir{current_file_path.parent_path()}; + return (tests_dir / get_test_input_path_relative_to_tests_dir()).string(); +} + +void compress(bool structurize_arrays) { + constexpr auto cDefaultTargetEncodedSize = 8ULL * 1024 * 1024 * 1024; // 8 GiB + constexpr auto cDefaultMaxDocumentSize = 512ULL * 1024 * 1024; // 512 MiB + constexpr auto cDefaultMinTableSize = 1ULL * 1024 * 1024; // 1 MiB + constexpr auto cDefaultCompressionLevel = 3; + constexpr auto cDefaultPrintArchiveStats = false; + + std::filesystem::create_directory(cTestEndToEndArchiveDirectory); + REQUIRE((std::filesystem::is_directory(cTestEndToEndArchiveDirectory))); + + clp_s::JsonParserOption parser_option{}; + parser_option.file_paths.push_back(get_test_input_local_path()); + parser_option.archives_dir = cTestEndToEndArchiveDirectory; + parser_option.target_encoded_size = cDefaultTargetEncodedSize; + parser_option.max_document_size = cDefaultMaxDocumentSize; + parser_option.min_table_size = cDefaultMinTableSize; + parser_option.compression_level = cDefaultCompressionLevel; + parser_option.print_archive_stats = cDefaultPrintArchiveStats; + parser_option.structurize_arrays = structurize_arrays; + + clp_s::JsonParser parser{parser_option}; + REQUIRE(parser.parse()); + parser.store(); + + REQUIRE((false == std::filesystem::is_empty(cTestEndToEndArchiveDirectory))); +} + +auto extract() -> std::filesystem::path { + constexpr auto cDefaultOrdered = false; + constexpr auto cDefaultTargetOrderedChunkSize = 0; + + std::filesystem::create_directory(cTestEndToEndOutputDirectory); + REQUIRE(std::filesystem::is_directory(cTestEndToEndOutputDirectory)); + + clp_s::JsonConstructorOption constructor_option{}; + constructor_option.archives_dir = cTestEndToEndArchiveDirectory; + constructor_option.output_dir = cTestEndToEndOutputDirectory; + constructor_option.ordered = cDefaultOrdered; + constructor_option.target_ordered_chunk_size = cDefaultTargetOrderedChunkSize; + for (auto const& entry : std::filesystem::directory_iterator(constructor_option.archives_dir)) { + if (false == entry.is_directory()) { + // Skip non-directories + continue; + } + + constructor_option.archive_id = entry.path().filename(); + clp_s::JsonConstructor constructor{constructor_option}; + constructor.store(); + } + std::filesystem::path extracted_json_path{cTestEndToEndOutputDirectory}; + extracted_json_path /= "original"; + REQUIRE(std::filesystem::exists(extracted_json_path)); + + return extracted_json_path; +} + +// Silence the checks below since our use of `std::system` is safe in the context of testing. +// NOLINTBEGIN(cert-env33-c,concurrency-mt-unsafe) +void compare(std::filesystem::path const& extracted_json_path) { + int result{std::system("command -v jq >/dev/null 2>&1")}; + REQUIRE((0 == result)); + std::string command = fmt::format( + "jq --sort-keys --compact-output '.' {} | sort > {}", + extracted_json_path.string(), + cTestEndToEndOutputSortedJson + ); + result = std::system(command.c_str()); + REQUIRE((0 == result)); + + REQUIRE((false == std::filesystem::is_empty(cTestEndToEndOutputSortedJson))); + + result = std::system("command -v diff >/dev/null 2>&1"); + REQUIRE((0 == result)); + command = fmt::format( + "diff --unified {} {} > /dev/null", + cTestEndToEndOutputSortedJson, + get_test_input_local_path() + ); + result = std::system(command.c_str()); + REQUIRE((true == WIFEXITED(result))); + REQUIRE((0 == WEXITSTATUS(result))); +} + +// NOLINTEND(cert-env33-c,concurrency-mt-unsafe) +} // namespace + +// NOLINTNEXTLINE(readability-function-cognitive-complexity) +TEST_CASE("clp-s-compress-extract-no-floats", "[clp-s][end-to-end]") { + auto structurize_arrays = GENERATE(true, false); + + TestOutputCleaner const test_cleanup; + + compress(structurize_arrays); + + std::filesystem::path extracted_json_path = extract(); + + compare(extracted_json_path); +} diff --git a/components/core/tests/test-end_to_end.cpp b/components/core/tests/test-end_to_end.cpp deleted file mode 100644 index 6b1648df3..000000000 --- a/components/core/tests/test-end_to_end.cpp +++ /dev/null @@ -1,127 +0,0 @@ -#include - -#include -#include -#include -#include -#include - -#include -#include - -#include "../src/clp_s/JsonConstructor.hpp" -#include "../src/clp_s/JsonParser.hpp" - -constexpr std::string_view cTestEndToEndArchiveDirectory{"test-end-to-end-archive"}; -constexpr std::string_view cTestEndToEndOutputDirectory{"test-end-to-end-out"}; -constexpr std::string_view cTestEndToEndOutputSortedJson{"test-end-to-end_sorted.json"}; -constexpr std::string_view cTestEndToEndInputFileDirectory{"test_log_files"}; -constexpr std::string_view cTestEndToEndInputFile{"test_no_floats_sorted.json"}; - -namespace { -/** - * Class with no data members who's contruction and destruction is soley - * for cleanup up files and directorys created by the test case. - */ -class Cleanup { -public: - Cleanup() { delete_files(); } - - ~Cleanup() { delete_files(); } - -private: - static void delete_files() { - std::filesystem::remove_all(cTestEndToEndArchiveDirectory); - std::filesystem::remove_all(cTestEndToEndOutputDirectory); - std::filesystem::remove(cTestEndToEndOutputSortedJson); - } -}; - -auto get_test_input_path_relative_to_tests_dir() -> std::filesystem::path { - return std::filesystem::path{cTestEndToEndInputFileDirectory} / cTestEndToEndInputFile; -} - -auto get_test_input_local_path() -> std::string { - std::filesystem::path const current_file_path{__FILE__}; - auto const tests_dir{current_file_path.parent_path()}; - return (tests_dir / get_test_input_path_relative_to_tests_dir()).string(); -} -} // namespace - -// NOLINTNEXTLINE(readability-function-cognitive-complexity) -TEST_CASE("clp-s_compression_and_extraction_no_floats", "[clp-s][end-to-end]") { - auto const default_target_encoded_size = 8ULL * 1024 * 1024 * 1024; // 8 GiB - auto const default_max_document_size = 512ULL * 1024 * 1024; // 512 MiB - auto const default_min_table_size = 1ULL * 1024 * 1024; // 1 MiB - auto const default_compression_level = 3; - auto const default_print_archive_stats = false; - auto const default_ordered = false; - auto const default_target_ordered_chunk_size = 0; - auto structurize_arrays = GENERATE(true, false); - - Cleanup const test_cleanup; - - std::filesystem::create_directory(cTestEndToEndArchiveDirectory); - REQUIRE(std::filesystem::is_directory(cTestEndToEndArchiveDirectory)); - - clp_s::JsonParserOption parser_option{}; - parser_option.file_paths.push_back(get_test_input_local_path()); - parser_option.archives_dir = cTestEndToEndArchiveDirectory; - parser_option.target_encoded_size = default_target_encoded_size; - parser_option.max_document_size = default_max_document_size; - parser_option.min_table_size = default_min_table_size; - parser_option.compression_level = default_compression_level; - parser_option.print_archive_stats = default_print_archive_stats; - parser_option.structurize_arrays = structurize_arrays; - - clp_s::JsonParser parser{parser_option}; - REQUIRE(parser.parse()); - parser.store(); - - REQUIRE(false == std::filesystem::is_empty(cTestEndToEndArchiveDirectory)); - - std::filesystem::create_directory(cTestEndToEndOutputDirectory); - REQUIRE(std::filesystem::is_directory(cTestEndToEndOutputDirectory)); - - clp_s::JsonConstructorOption constructor_option{}; - constructor_option.archives_dir = parser_option.archives_dir; - constructor_option.output_dir = cTestEndToEndOutputDirectory; - constructor_option.ordered = false; - constructor_option.target_ordered_chunk_size = 0; - for (auto const& entry : std::filesystem::directory_iterator(constructor_option.archives_dir)) { - if (false == entry.is_directory()) { - // Skip non-directories - continue; - } - - constructor_option.archive_id = entry.path().filename(); - clp_s::JsonConstructor constructor{constructor_option}; - constructor.store(); - } - - std::filesystem::path extracted_json_path{cTestEndToEndOutputDirectory}; - extracted_json_path /= "original"; - REQUIRE(std::filesystem::exists(extracted_json_path)); - - int result = std::system("command -v jq >/dev/null 2>&1"); - REQUIRE(0 == result); - std::string command = fmt::format( - "jq --sort-keys --compact-output '.' {}/original | sort > {}", - cTestEndToEndOutputDirectory, - cTestEndToEndOutputSortedJson - ); - result = std::system(command.c_str()); - REQUIRE(0 == result); - - REQUIRE(false == std::filesystem::is_empty(cTestEndToEndOutputSortedJson)); - - result = std::system("command -v diff >/dev/null 2>&1"); - REQUIRE(0 == result); - command = fmt::format( - "diff -u {} {} > /dev/null", - cTestEndToEndOutputSortedJson, - get_test_input_local_path() - ); - result = std::system(command.c_str()); - REQUIRE(0 == WEXITSTATUS(result)); -} diff --git a/components/core/tests/test_log_files/test_no_floats_sorted.json b/components/core/tests/test_log_files/test_no_floats_sorted.jsonl similarity index 100% rename from components/core/tests/test_log_files/test_no_floats_sorted.json rename to components/core/tests/test_log_files/test_no_floats_sorted.jsonl From c362b224f1f7b89b1905fe087370a50e825afa33 Mon Sep 17 00:00:00 2001 From: Abigail Matthews Date: Fri, 29 Nov 2024 17:51:03 -0500 Subject: [PATCH 16/18] small declaration/assignment changes --- components/core/tests/test-clp_s-end_to_end.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/core/tests/test-clp_s-end_to_end.cpp b/components/core/tests/test-clp_s-end_to_end.cpp index 1009111b4..a09ea32e7 100644 --- a/components/core/tests/test-clp_s-end_to_end.cpp +++ b/components/core/tests/test-clp_s-end_to_end.cpp @@ -120,7 +120,7 @@ auto extract() -> std::filesystem::path { void compare(std::filesystem::path const& extracted_json_path) { int result{std::system("command -v jq >/dev/null 2>&1")}; REQUIRE((0 == result)); - std::string command = fmt::format( + auto command = fmt::format( "jq --sort-keys --compact-output '.' {} | sort > {}", extracted_json_path.string(), cTestEndToEndOutputSortedJson From d013e465859e0b1260170dadcb116901585e501d Mon Sep 17 00:00:00 2001 From: Abigail Matthews Date: Sat, 30 Nov 2024 13:24:10 -0500 Subject: [PATCH 17/18] remove no lint line and change type to auto --- components/core/tests/test-clp_s-end_to_end.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/components/core/tests/test-clp_s-end_to_end.cpp b/components/core/tests/test-clp_s-end_to_end.cpp index a09ea32e7..3f138b472 100644 --- a/components/core/tests/test-clp_s-end_to_end.cpp +++ b/components/core/tests/test-clp_s-end_to_end.cpp @@ -145,7 +145,6 @@ void compare(std::filesystem::path const& extracted_json_path) { // NOLINTEND(cert-env33-c,concurrency-mt-unsafe) } // namespace -// NOLINTNEXTLINE(readability-function-cognitive-complexity) TEST_CASE("clp-s-compress-extract-no-floats", "[clp-s][end-to-end]") { auto structurize_arrays = GENERATE(true, false); @@ -153,7 +152,7 @@ TEST_CASE("clp-s-compress-extract-no-floats", "[clp-s][end-to-end]") { compress(structurize_arrays); - std::filesystem::path extracted_json_path = extract(); + auto extracted_json_path = extract(); compare(extracted_json_path); } From 1d187b3eeb41f202c3f257118572e36ca9785653 Mon Sep 17 00:00:00 2001 From: Abigail Matthews Date: Sat, 30 Nov 2024 18:17:00 -0500 Subject: [PATCH 18/18] update jsonl test file --- .../core/tests/test_log_files/test_no_floats_sorted.jsonl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/components/core/tests/test_log_files/test_no_floats_sorted.jsonl b/components/core/tests/test_log_files/test_no_floats_sorted.jsonl index 8d82aea33..8dfcd85f6 100644 --- a/components/core/tests/test_log_files/test_no_floats_sorted.jsonl +++ b/components/core/tests/test_log_files/test_no_floats_sorted.jsonl @@ -1,4 +1,4 @@ -{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"empty_object":{},"false":false,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max_supported":9223372036854776,"int64_min_supported":-9223372036854776,"int8_max":127,"int8_min":-128,"null":null,"string":"short_string","true":true} -{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"false":false,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max_supported":9223372036854776,"int64_min_supported":-9223372036854776,"int8_max":127,"int8_min":-128,"nonempty_object":{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"empty_object":{},"false":false,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max_supported":9223372036854776,"int64_min_supported":-9223372036854776,"int8_max":127,"int8_min":-128,"null":null,"string":"short_string","true":true},"null":null,"string":"short_string","true":true} -{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"false":false,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max_supported":9223372036854776,"int64_min_supported":-9223372036854776,"int8_max":127,"int8_min":-128,"nonempty_object":{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"false":false,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max_supported":9223372036854776,"int64_min_supported":-9223372036854776,"int8_max":127,"int8_min":-128,"non_empty_object2":{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"empty_object":{},"false":false,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max_supported":9223372036854776,"int64_min_supported":-9223372036854776,"int8_max":127,"int8_min":-128,"null":null,"string":"short_string","true":true},"null":null,"string":"short_string","true":true},"null":null,"string":"short_string","true":true} -{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_object":{},"false":false,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max_supported":9223372036854776,"int64_min_supported":-9223372036854776,"int8_max":127,"int8_min":-128,"nonempty_array":[1,2,3,4,5],"null":null,"string":"short_string","true":true} +{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"empty_object":{},"false":false,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max_jq_losslessly_represents":9824299763229016,"int64_min_jq_losslessly_represents":-9007199254740992,"int8_max":127,"int8_min":-128,"null":null,"string":"short_string","true":true} +{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"false":false,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max_jq_losslessly_represents":9824299763229016,"int64_min_jq_losslessly_represents":-9007199254740992,"int8_max":127,"int8_min":-128,"nonempty_object":{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"empty_object":{},"false":false,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max_jq_losslessly_represents":9824299763229016,"int64_min_jq_losslessly_represents":-9007199254740992,"int8_max":127,"int8_min":-128,"null":null,"string":"short_string","true":true},"null":null,"string":"short_string","true":true} +{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"false":false,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max_jq_losslessly_represents":9824299763229016,"int64_min_jq_losslessly_represents":-9007199254740992,"int8_max":127,"int8_min":-128,"nonempty_object":{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"false":false,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max_jq_losslessly_represents":9824299763229016,"int64_min_jq_losslessly_represents":-9007199254740992,"int8_max":127,"int8_min":-128,"non_empty_object2":{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"empty_object":{},"false":false,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max_jq_losslessly_represents":9824299763229016,"int64_min_jq_losslessly_represents":-9007199254740992,"int8_max":127,"int8_min":-128,"null":null,"string":"short_string","true":true},"null":null,"string":"short_string","true":true},"null":null,"string":"short_string","true":true} +{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_object":{},"false":false,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max_jq_losslessly_represents":9824299763229016,"int64_min_jq_losslessly_represents":-9007199254740992,"int8_max":127,"int8_min":-128,"nonempty_array":[1,2,3,4,5],"null":null,"string":"short_string","true":true}