diff --git a/.jenkins/Dockerfile b/.jenkins/Dockerfile index d481ff641..20c4489b0 100644 --- a/.jenkins/Dockerfile +++ b/.jenkins/Dockerfile @@ -13,6 +13,7 @@ RUN set -eux; \ rm -rf /var/lib/apt/lists/*; COPY test_index_btree /usr/local/bin/test_index_btree +COPY test_index_crash_recovery /usr/local/bin/test_index_crash_recovery COPY test_meta_blk_mgr /usr/local/bin/test_meta_blk_mgr COPY test_log_store /usr/local/bin/test_log_store COPY test_home_raft_logstore /usr/local/bin/test_home_raft_logstore diff --git a/.jenkins/jenkinsfile_nightly b/.jenkins/jenkinsfile_nightly index 7ef305257..7efd9b935 100644 --- a/.jenkins/jenkinsfile_nightly +++ b/.jenkins/jenkinsfile_nightly @@ -42,6 +42,7 @@ pipeline { steps { sh "conan create --build missing -o homestore:sanitize=True -pr debug . ${PROJECT}/${VER}@" sh "find ${CONAN_USER_HOME} -type f -wholename '*tests/test_index_btree' -exec cp {} .jenkins/test_index_btree \\;" + sh "find ${CONAN_USER_HOME} -type f -wholename '*tests/test_index_crash_recovery' -exec cp {} .jenkins/test_index_crash_recovery \\;" sh "find ${CONAN_USER_HOME} -type f -wholename '*tests/test_meta_blk_mgr' -exec cp {} .jenkins/test_meta_blk_mgr \\;" sh "find ${CONAN_USER_HOME} -type f -wholename '*tests/test_log_store' -exec cp {} .jenkins/test_log_store \\;" sh "find ${CONAN_USER_HOME} -type f -wholename '*tests/test_home_raft_logstore' -exec cp {} .jenkins/test_home_raft_logstore \\;" diff --git a/conanfile.py b/conanfile.py index d6d24c8c0..64d35df54 100644 --- a/conanfile.py +++ b/conanfile.py @@ -9,7 +9,7 @@ class HomestoreConan(ConanFile): name = "homestore" - version = "6.4.55" + version = "6.4.56" homepage = "https://github.com/eBay/Homestore" description = "HomeStore Storage Engine" diff --git a/src/include/homestore/btree/detail/btree_internal.hpp b/src/include/homestore/btree/detail/btree_internal.hpp index a77bfe5ac..67b33b089 100644 --- a/src/include/homestore/btree/detail/btree_internal.hpp +++ b/src/include/homestore/btree/detail/btree_internal.hpp @@ -244,6 +244,9 @@ struct BtreeConfig { uint8_t m_suggested_min_pct{30}; uint8_t m_split_pct{50}; uint32_t m_max_merge_nodes{3}; +#ifdef _PRERELEASE + uint64_t m_max_keys_in_node{0}; +#endif bool m_rebalance_turned_on{false}; bool m_merge_turned_on{true}; diff --git a/src/include/homestore/btree/detail/btree_node.hpp b/src/include/homestore/btree/detail/btree_node.hpp index 84f70aa05..a3285ef35 100644 --- a/src/include/homestore/btree/detail/btree_node.hpp +++ b/src/include/homestore/btree/detail/btree_node.hpp @@ -36,6 +36,7 @@ struct transient_hdr_t { /* these variables are accessed without taking lock and are not expected to change after init */ uint8_t leaf_node{0}; + uint64_t max_keys_in_node{0}; bool is_leaf() const { return (leaf_node != 0); } }; @@ -113,6 +114,10 @@ class BtreeNode : public sisl::ObjLifeCounter< BtreeNode > { DEBUG_ASSERT_EQ(version(), BTREE_NODE_VERSION); } m_trans_hdr.leaf_node = is_leaf; +#ifdef _PRERELEASE + m_trans_hdr.max_keys_in_node = cfg.m_max_keys_in_node; +#endif + } virtual ~BtreeNode() = default; @@ -327,6 +332,7 @@ class BtreeNode : public sisl::ObjLifeCounter< BtreeNode > { uint16_t level() const { return get_persistent_header_const()->level; } // uint32_t total_entries() const { return (has_valid_edge() ? total_entries() + 1 : total_entries()); } + uint64_t max_keys_in_node() const { return m_trans_hdr.max_keys_in_node; } void lock(locktype_t l) const { if (l == locktype_t::READ) { diff --git a/src/include/homestore/btree/detail/simple_node.hpp b/src/include/homestore/btree/detail/simple_node.hpp index 20f8b264d..1f4c30e32 100644 --- a/src/include/homestore/btree/detail/simple_node.hpp +++ b/src/include/homestore/btree/detail/simple_node.hpp @@ -42,6 +42,7 @@ class SimpleNode : public VariantNode< K, V > { using BtreeNode::get_nth_value_size; using BtreeNode::to_string; using VariantNode< K, V >::get_nth_value; + using VariantNode< K, V >::max_keys_in_node; // Insert the key and value in provided index // Assumption: Node lock is already taken @@ -202,7 +203,8 @@ class SimpleNode : public VariantNode< K, V > { bool has_room_for_put(btree_put_type put_type, uint32_t key_size, uint32_t value_size) const override { #ifdef _PRERELEASE - // return (this->total_entries() <= 3); + auto max_keys = max_keys_in_node(); + if(max_keys) {return (this->total_entries() < max_keys);} #endif return ((put_type == btree_put_type::UPSERT) || (put_type == btree_put_type::INSERT)) ? (get_available_entries() > 0) diff --git a/src/lib/index/wb_cache.cpp b/src/lib/index/wb_cache.cpp index 6c0c722ae..cf484eef9 100644 --- a/src/lib/index/wb_cache.cpp +++ b/src/lib/index/wb_cache.cpp @@ -551,9 +551,10 @@ folly::Future< bool > IndexWBCache::async_cp_flush(IndexCPContext* cp_ctx) { void IndexWBCache::do_flush_one_buf(IndexCPContext* cp_ctx, IndexBufferPtr const& buf, bool part_of_batch) { #ifdef _PRERELEASE if (buf->m_crash_flag_on) { - std::string filename = "crash_buf_" + std::to_string(cp_ctx->id()) + ".dot"; - LOGINFOMOD(wbcache, "Simulating crash while writing buffer {}, stored in file {}", buf->to_string(), filename); - cp_ctx->to_string_dot(filename); +// std::string filename = "crash_buf_" + std::to_string(cp_ctx->id()) + ".dot"; +// LOGINFOMOD(wbcache, "Simulating crash while writing buffer {}, stored in file {}", buf->to_string(), filename); +// cp_ctx->to_string_dot(filename); + LOGINFOMOD(wbcache, "Simulating crash while writing buffer {}", buf->to_string()); hs()->crash_simulator().crash(); cp_ctx->complete(true); return; @@ -581,6 +582,7 @@ void IndexWBCache::do_flush_one_buf(IndexCPContext* cp_ctx, IndexBufferPtr const BtreeNode::to_string_buf(buf->raw_buffer())); m_vdev->async_write(r_cast< const char* >(buf->raw_buffer()), m_node_size, buf->m_blkid, part_of_batch) .thenValue([buf, cp_ctx](auto) { + // TODO: crash may cause wb_cache() to be destroyed and return null pointer auto& pthis = s_cast< IndexWBCache& >(wb_cache()); // Avoiding more than 16 bytes capture pthis.process_write_completion(cp_ctx, buf); }); diff --git a/src/tests/btree_helpers/btree_test_helper.hpp b/src/tests/btree_helpers/btree_test_helper.hpp index 298235068..6c00975ea 100644 --- a/src/tests/btree_helpers/btree_test_helper.hpp +++ b/src/tests/btree_helpers/btree_test_helper.hpp @@ -217,7 +217,7 @@ struct BtreeTestHelper { } ////////////////////// All remove operation variants /////////////////////////////// - void remove_one(uint32_t k) { + void remove_one(uint32_t k, bool care_success = true) { auto existing_v = std::make_unique< V >(); auto pk = std::make_unique< K >(k); @@ -225,10 +225,16 @@ struct BtreeTestHelper { rreq.enable_route_tracing(); bool removed = (m_bt->remove(rreq) == btree_status_t::success); - ASSERT_EQ(removed, m_shadow_map.exists(*pk)) - << "Removal of key " << pk->key() << " status doesn't match with shadow"; + if(care_success) { + ASSERT_EQ(removed, m_shadow_map.exists(*pk)) + << "Removal of key " << pk->key() << " status doesn't match with shadow"; + if (removed) { m_shadow_map.remove_and_check(*pk, *existing_v); } + }else { + // Do not care if the key is not present in the btree, just cleanup the shadow map + m_shadow_map.erase(*pk); + } + - if (removed) { m_shadow_map.remove_and_check(*pk, *existing_v); } } void remove_random() { diff --git a/src/tests/test_index_crash_recovery.cpp b/src/tests/test_index_crash_recovery.cpp index 2cc932315..65c1e64f3 100644 --- a/src/tests/test_index_crash_recovery.cpp +++ b/src/tests/test_index_crash_recovery.cpp @@ -34,23 +34,25 @@ SISL_LOGGING_DECL(test_index_crash_recovery) // TODO Add tests to do write,remove after recovery. // TODO Test with var len key with io mgr page size is 512. -SISL_OPTION_GROUP( - test_index_crash_recovery, - (num_iters, "", "num_iters", "number of iterations for rand ops", - ::cxxopts::value< uint32_t >()->default_value("500"), "number"), - (num_entries, "", "num_entries", "number of entries to test with", - ::cxxopts::value< uint32_t >()->default_value("5000"), "number"), - (run_time, "", "run_time", "run time for io", ::cxxopts::value< uint32_t >()->default_value("360000"), "seconds"), - (disable_merge, "", "disable_merge", "disable_merge", ::cxxopts::value< bool >()->default_value("0"), ""), - (operation_list, "", "operation_list", "operation list instead of default created following by percentage", - ::cxxopts::value< std::vector< std::string > >(), "operations [...]"), - (preload_size, "", "preload_size", "number of entries to preload tree with", - ::cxxopts::value< uint32_t >()->default_value("1000"), "number"), - (init_device, "", "init_device", "init device", ::cxxopts::value< bool >()->default_value("1"), ""), - (cleanup_after_shutdown, "", "cleanup_after_shutdown", "cleanup after shutdown", - ::cxxopts::value< bool >()->default_value("1"), ""), - (seed, "", "seed", "random engine seed, use random if not defined", - ::cxxopts::value< uint64_t >()->default_value("0"), "number")) +SISL_OPTION_GROUP(test_index_crash_recovery, + (num_iters, "", "num_iters", "number of iterations for rand ops", + ::cxxopts::value< uint32_t >()->default_value("500"), "number"), + (num_entries, "", "num_entries", "number of entries to test with", + ::cxxopts::value< uint32_t >()->default_value("5000"), "number"), + (run_time, "", "run_time", "run time for io", ::cxxopts::value< uint32_t >()->default_value("360000"), + "seconds"), + (max_keys_in_node, "", "max_keys_in_node", "max_keys_in_node", + ::cxxopts::value< uint32_t >()->default_value("0"), ""), + (operation_list, "", "operation_list", + "operation list instead of default created following by percentage", + ::cxxopts::value< std::vector< std::string > >(), "operations [...]"), + (preload_size, "", "preload_size", "number of entries to preload tree with", + ::cxxopts::value< uint32_t >()->default_value("1000"), "number"), + (init_device, "", "init_device", "init device", ::cxxopts::value< bool >()->default_value("1"), ""), + (cleanup_after_shutdown, "", "cleanup_after_shutdown", "cleanup after shutdown", + ::cxxopts::value< bool >()->default_value("1"), ""), + (seed, "", "seed", "random engine seed, use random if not defined", + ::cxxopts::value< uint64_t >()->default_value("0"), "number")) void log_obj_life_counter() { std::string str; @@ -60,6 +62,121 @@ void log_obj_life_counter() { LOGINFO("Object Life Counter\n:{}", str); } +enum class OperationType { + Put, + Remove, +}; + +using Operation = std::pair< uint64_t, OperationType >; +using OperationList = std::vector< Operation >; + +class SequenceGenerator { +public: + SequenceGenerator(int putFreq, int removeFreq, uint64_t start_range, uint64_t end_range) : + putFreq_(putFreq), removeFreq_(removeFreq), start_range_(start_range), end_range_(end_range) { + std::random_device rd; + gen_ = std::mt19937(rd()); + keyDist_ = std::uniform_int_distribution<>(start_range_, end_range_); + updateOperationTypeDistribution(); + } + + void setPutFrequency(int putFreq) { + putFreq_ = putFreq; + updateOperationTypeDistribution(); + } + + void setRemoveFrequency(int removeFreq) { + removeFreq_ = removeFreq; + updateOperationTypeDistribution(); + } + + void setRange(uint64_t start_range, uint64_t end_range) { + start_range_ = start_range; + end_range_ = end_range; + keyDist_ = std::uniform_int_distribution<>(start_range_, end_range_); + } + + OperationList generateOperations(size_t numOperations, bool reset = false) { + std::vector< Operation > operations; + if (reset) { this->reset(); } + for (size_t i = 0; i < numOperations; ++i) { + uint32_t key = keyDist_(gen_); + auto [it, inserted] = keyStates.try_emplace(key, false); + auto& inUse = it->second; + + OperationType operation = static_cast< OperationType >(opTypeDist_(gen_)); + + if (operation == OperationType::Put && !inUse) { + operations.emplace_back(key, OperationType::Put); + inUse = true; + } else if (operation == OperationType::Remove && inUse) { + operations.emplace_back(key, OperationType::Remove); + inUse = false; + } + } + + return operations; + } + __attribute__((noinline)) std::string showKeyState(uint64_t key) const { + auto it = keyStates.find(key); + if (it != keyStates.end()) { return it->second ? "Put" : "Remove"; } + return "Not in keyStates"; + } + + __attribute__((noinline)) static OperationList inspect(const OperationList& operations, uint32_t key) { + OperationList occurrences; + for (size_t i = 0; i < operations.size(); ++i) { + const auto& [opKey, opType] = operations[i]; + if (opKey == key) { occurrences.emplace_back(i, opType); } + } + return occurrences; + } + __attribute__((noinline)) std::string printOperations(const OperationList& operations) const { + std::ostringstream oss; + for (const auto& [key, opType] : operations) { + std::string opTypeStr = (opType == OperationType::Put) ? "Put" : "Remove"; + oss << "{" << key << ", " << opTypeStr << "}\n"; + } + return oss.str(); + } + __attribute__((noinline)) std::string printKeyOccurrences(const OperationList& operations) const { + std::set< uint64_t > keys = collectUniqueKeys(operations); + std::ostringstream oss; + for (auto key : keys) { + auto keyOccurrences = inspect(operations, key); + oss << "Occurrences of key " << key << ":\n"; + for (const auto& [index, operation] : keyOccurrences) { + std::string opTypeStr = (operation == OperationType::Put) ? "Put" : "Remove"; + oss << "Index: " << index << ", Operation: " << opTypeStr << "\n"; + } + } + return oss.str(); + } + void reset() { keyStates.clear(); } + +private: + int putFreq_; + int removeFreq_; + uint64_t start_range_; + uint64_t end_range_; + std::mt19937 gen_; + std::uniform_int_distribution<> keyDist_; + std::discrete_distribution<> opTypeDist_; + std::map< uint64_t, bool > keyStates; + + void updateOperationTypeDistribution() { + opTypeDist_ = + std::discrete_distribution<>({static_cast< double >(putFreq_), static_cast< double >(removeFreq_)}); + } + + std::set< uint64_t > collectUniqueKeys(const OperationList& operations) const { + std::set< uint64_t > keys; + for (const auto& [key, _] : operations) { + keys.insert(key); + } + return keys; + } +}; #ifdef _PRERELEASE template < typename TestType > struct IndexCrashTest : public test_common::HSTestHelper, BtreeTestHelper< TestType >, public ::testing::Test { @@ -72,9 +189,11 @@ struct IndexCrashTest : public test_common::HSTestHelper, BtreeTestHelper< TestT std::shared_ptr< IndexTableBase > on_index_table_found(superblk< index_table_sb >&& sb) override { LOGINFO("Index table recovered, root bnode_id {} version {}", sb->root_node, sb->root_link_version); + m_test->m_cfg = BtreeConfig(hs()->index_service().node_size()); m_test->m_cfg.m_leaf_node_type = T::leaf_node_type; m_test->m_cfg.m_int_node_type = T::interior_node_type; + m_test->m_cfg.m_max_keys_in_node = SISL_OPTIONS["max_keys_in_node"].as< uint32_t >(); m_test->m_bt = std::make_shared< typename T::BtreeType >(std::move(sb), m_test->m_cfg); return m_test->m_bt; } @@ -102,7 +221,7 @@ struct IndexCrashTest : public test_common::HSTestHelper, BtreeTestHelper< TestT LOGINFO("Node size {} ", hs()->index_service().node_size()); this->m_cfg = BtreeConfig(hs()->index_service().node_size()); - + this->m_cfg.m_max_keys_in_node = SISL_OPTIONS["max_keys_in_node"].as< uint32_t >(); auto uuid = boost::uuids::random_generator()(); auto parent_uuid = boost::uuids::random_generator()(); @@ -127,10 +246,18 @@ struct IndexCrashTest : public test_common::HSTestHelper, BtreeTestHelper< TestT this->get_all(); } + void reset_btree() { + this->m_bt->destroy(); + auto uuid = boost::uuids::random_generator()(); + auto parent_uuid = boost::uuids::random_generator()(); + this->m_bt = std::make_shared< typename T::BtreeType >(uuid, parent_uuid, 0, this->m_cfg); + hs()->index_service().add_index_table(this->m_bt); + this->m_shadow_map.range_erase(0, SISL_OPTIONS["num_entries"].as< uint32_t >() - 1); + } + void restart_homestore(uint32_t shutdown_delay_sec = 3) override { this->params(HS_SERVICE::INDEX).index_svc_cbs = new TestIndexServiceCallbacks(this); LOGINFO("\n\n\n\n\n\n shutdown homestore for index service Test\n\n\n\n\n"); - // this->m_shadow_map.save(this->m_shadow_filename); test_common::HSTestHelper::restart_homestore(shutdown_delay_sec); } @@ -159,6 +286,21 @@ struct IndexCrashTest : public test_common::HSTestHelper, BtreeTestHelper< TestT test_common::HSTestHelper::trigger_cp(true); this->m_shadow_map.save(m_shadow_filename); } + void reapply_after_crash(OperationList& operations) { + for (const auto& [key, opType] : operations) { + switch (opType) { + case OperationType::Put: + LOGDEBUG("Reapply: Inserting key {}", key); + this->force_upsert(key); + break; + case OperationType::Remove: + LOGDEBUG("Reapply: Removing key {}", key); + this->remove_one(key, false); + break; + } + } + test_common::HSTestHelper::trigger_cp(true); + } void TearDown() override { bool cleanup = SISL_OPTIONS["cleanup_after_shutdown"].as< bool >(); @@ -177,6 +319,7 @@ struct IndexCrashTest : public test_common::HSTestHelper, BtreeTestHelper< TestT BtreeTestHelper< TestType >::TearDown(); this->shutdown_homestore(false); } + void crash_and_recover(uint32_t s_key, uint32_t e_key) { this->print_keys("Btree prior to CP and susbsequent simulated crash: "); test_common::HSTestHelper::trigger_cp(false); @@ -187,10 +330,33 @@ struct IndexCrashTest : public test_common::HSTestHelper, BtreeTestHelper< TestT this->reapply_after_crash(); this->get_all(); - LOGINFO(" except to have [{},{}) in tree and it is actually{} ", s_key, e_key, tree_key_count()); + LOGINFO("Expect to have [{},{}) in tree and it is actually{} ", s_key, e_key, tree_key_count()); ASSERT_EQ(this->m_shadow_map.size(), this->m_bt->count_keys(this->m_bt->root_node_id())) << "shadow map size and tree size mismatch"; } + + void crash_and_recover(OperationList& operations, std::string filename = "") { + // this->print_keys("Btree prior to CP and susbsequent simulated crash: "); + test_common::HSTestHelper::trigger_cp(false); + this->wait_for_crash_recovery(); + // this->print_keys("Post crash and recovery, btree structure:"); + + if (!filename.empty()) { + LOGINFO("Visualize the tree file {}", filename); + this->visualize_keys(filename); + } + + this->reapply_after_crash(operations); + + // this->print_keys("\n\nafter reapply keys"); + if (!filename.empty()) { + LOGINFO("Visualize the tree file after_reapply__{}", filename); + this->visualize_keys("after_reapply__" + filename); + } + + this->get_all(); + } + uint32_t tree_key_count() { return this->m_bt->count_keys(this->m_bt->root_node_id()); } protected: @@ -240,7 +406,6 @@ TYPED_TEST(IndexCrashTest, SplitOnLeftEdge) { LOGINFO("Step 4: Crash and reapply the missing entries to tree"); this->crash_and_recover(num_entries / 2, num_entries); - // TODO: Uncomment this once we do a fix for the inconsistent query results LOGINFO("Step 5: Fill the 2nd quarter of the tree, to make sure left child is split and we crash on flush of the " "left child"); this->set_basic_flip("crash_flush_on_split_at_left_child"); @@ -251,14 +416,14 @@ TYPED_TEST(IndexCrashTest, SplitOnLeftEdge) { this->put(k, btree_put_type::INSERT, true /* expect_success */); } this->visualize_keys("tree_before_crash.dot"); - this->dump_to_file("tree_before_crash.txt"); + this->dump_to_file("tree_before_crash.dot"); LOGINFO("Step 6: Simulate crash and then recover, reapply keys to tree"); this->crash_and_recover(num_entries / 4, num_entries); LOGINFO("Step 7: Fill the 1st quarter of the tree, to make sure left child is split and we crash on flush of the " "parent node"); this->set_basic_flip("crash_flush_on_split_at_parent"); - for (auto k = 0u; k <= num_entries / 4; ++k) { + for (auto k = 0u; k < num_entries / 4; ++k) { this->put(k, btree_put_type::INSERT, true /* expect_success */); } LOGINFO("Step 8: Post crash we reapply the missing entries to tree"); @@ -266,10 +431,151 @@ TYPED_TEST(IndexCrashTest, SplitOnLeftEdge) { LOGINFO("Step 9: Query all entries and validate with pagination of 80 entries"); this->query_all_paginate(80); } + +/* +TYPED_TEST(IndexCrashTest, ManualMergeCrash){ + // Define the lambda function + const uint32_t num_entries = 30; + + auto initTree = [this, num_entries]() { + for (uint64_t k = 0u; k < num_entries; ++k) { + this->force_upsert(k); + } + test_common::HSTestHelper::trigger_cp(true); + this->m_shadow_map.save(this->m_shadow_filename); + }; + + std::vector< OperationList > removing_scenarios = { + {{29, OperationType::Remove}, + {28, OperationType::Remove}, + {27, OperationType::Remove}, + {26, OperationType::Remove}, + {25, OperationType::Remove}, + {24, OperationType::Remove}} + }; + + auto scenario = removing_scenarios[0]; + + LOGINFO("Step 1-1: Populate some keys and flush"); + initTree(); + this->visualize_keys("tree_init.dot"); + LOGINFO("Step 2-1: Set crash flag, remove some keys in reverse order"); + this->set_basic_flip("crash_flush_on_merge_at_parent"); + + for (auto [k, _] : scenario) { + LOGINFO("\n\n\t\t\t\t\t\t\t\t\t\t\t\t\tRemoving entry {}", k); + this->remove_one(k); + } + this->visualize_keys("tree_before_crash.dot"); + + LOGINFO("Step 3-1: Trigger cp to crash"); + this->crash_and_recover(scenario, "recover_tree_crash_1.dot"); + test_common::HSTestHelper::trigger_cp(true); + this->get_all(); + + LOGINFO("Step 1-2: Populate some keys and flush"); + initTree(); + this->visualize_keys("tree_init_02.dot"); + LOGINFO("Step 2-2: Set crash flag, remove some keys in reverse order"); + this->set_basic_flip("crash_flush_on_merge_at_left_child"); + for (auto [k, _] : scenario) { + LOGINFO("\n\n\t\t\t\t\t\t\t\t\t\t\t\t\tRemoving entry {}", k); + this->remove_one(k); + } + this->visualize_keys("tree_before_crash_2.dot"); + + LOGINFO("Step 3-2: Trigger cp to crash"); + this->crash_and_recover(scenario, "recover_tree_crash_2.dot"); + test_common::HSTestHelper::trigger_cp(true); + this->get_all(); + + LOGINFO("Step 1-3: Populate some keys and flush"); + initTree(); + this->visualize_keys("tree_init_03.dot"); + LOGINFO("Step 2-3: Set crash flag, remove some keys in reverse order"); + this->set_basic_flip("crash_flush_on_freed_child"); + for (auto [k, _] : scenario) { + LOGINFO("\n\n\t\t\t\t\t\t\t\t\t\t\t\t\tRemoving entry {}", k); + this->remove_one(k); + } + LOGINFO("Step 2-3: Set crash flag, remove some keys in reverse order"); + this->visualize_keys("tree_before_crash_3.dot"); + + LOGINFO("Step 3-3: Trigger cp to crash"); + this->crash_and_recover(scenario, "recover_tree_crash_3.dot"); + test_common::HSTestHelper::trigger_cp(true); + this->get_all(); +} +*/ + +TYPED_TEST(IndexCrashTest, SplitCrash1) { + // Define the lambda function + auto const num_entries = SISL_OPTIONS["num_entries"].as< uint32_t >(); + SequenceGenerator generator(100 /*putFreq*/, 0 /* removeFreq*/, 0 /*start_range*/, num_entries - 1 /*end_range*/); + vector< std::string > flips = {"crash_flush_on_split_at_parent", "crash_flush_on_split_at_left_child", + "crash_flush_on_split_at_right_child"}; + OperationList operations; + for (size_t i = 0; i < flips.size(); ++i) { + this->reset_btree(); + LOGINFO("Step 1-{}: Set flag {}", i + 1, flips[i]); + this->set_basic_flip(flips[i]); + operations = generator.generateOperations(num_entries -1 , true /* reset */); + // LOGINFO("Batch {} Operations:\n {} \n ", i + 1, generator.printOperations(operations)); + // LOGINFO("Detailed Key Occurrences for Batch {}:\n {} \n ", i + 1, + // generator.printKeyOccurrences(operations)); + for (auto [k, _] : operations) { + // LOGINFO("\t\t\t\t\t\t\t\t\t\t\t\t\tupserting entry {}", k); + this->put(k, btree_put_type::INSERT, true /* expect_success */); + } + this->crash_and_recover(operations, fmt::format("recover_tree_crash_{}.dot", i + 1)); + } +} + +TYPED_TEST(IndexCrashTest, long_running_put_crash) { + // Define the lambda function + auto const num_entries = SISL_OPTIONS["num_entries"].as< uint32_t >(); + SequenceGenerator generator(100 /*putFreq*/, 0 /* removeFreq*/, 0 /*start_range*/, num_entries - 1 /*end_range*/); + vector< std::string > flips = {"crash_flush_on_split_at_parent", "crash_flush_on_split_at_left_child", + "crash_flush_on_split_at_right_child"}; + OperationList operations; + auto m_start_time = Clock::now(); + auto time_to_stop = [this, m_start_time]() { return (get_elapsed_time_sec(m_start_time) > this->m_run_time); }; + double elapsed_time, progress_percent, last_progress_time = 0; + for (size_t i = 0; !time_to_stop(); ++i) { + bool print_time = false; + elapsed_time = get_elapsed_time_sec(m_start_time); + + this->reset_btree(); + auto flip = flips[i % flips.size()]; + LOGINFO("Step 1-{}: Set flag {}", i + 1, flip); + + this->set_basic_flip(flip, 1, 10); + operations = generator.generateOperations(num_entries -1, true /* reset */); + // operations = generator.generateOperations(num_entries/10, false /* reset */); + // LOGINFO("Batch {} Operations:\n {} \n ", i + 1, generator.printOperations(operations)); + // LOGINFO("Detailed Key Occurrences for Batch {}:\n {} \n ", i + 1, + // generator.printKeyOccurrences(operations)); + for (auto [k, _] : operations) { + // LOGINFO("\t\t\t\t\t\t\t\t\t\t\t\t\tupserting entry {}", k); + this->put(k, btree_put_type::INSERT, true /* expect_success */); + } + this->crash_and_recover(operations/*, fmt::format("recover_tree_crash_{}.dot", i + 1)*/); + if (elapsed_time - last_progress_time > 30) { + last_progress_time = elapsed_time; + print_time = true; + } + if (print_time) { + LOGINFO("\n\n\n\t\t\tProgress: {} iterations completed - Elapsed time: {:.0f} seconds of total " + "{} ({:.2f}%)\n\n\n", + i, elapsed_time, this->m_run_time, elapsed_time * 100.0 / this->m_run_time); + } + } +} #endif int main(int argc, char* argv[]) { int parsed_argc{argc}; + ::testing::GTEST_FLAG(filter) = "-*long_running*"; ::testing::InitGoogleTest(&parsed_argc, argv); SISL_OPTIONS_LOAD(parsed_argc, argv, logging, test_index_crash_recovery, iomgr, test_common_setup); sisl::logging::SetLogger("test_index_crash_recovery"); diff --git a/src/tests/test_index_recovery.cpp b/src/tests/test_index_recovery.cpp deleted file mode 100644 index c70bb0cab..000000000 --- a/src/tests/test_index_recovery.cpp +++ /dev/null @@ -1,278 +0,0 @@ -/********************************************************************************* - * Modifications Copyright 2017-2019 eBay Inc. - * - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed - * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR - * CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - *********************************************************************************/ -#include -#include - -#include -#include -#include "common/homestore_config.hpp" -#include "common/resource_mgr.hpp" -#include "test_common/homestore_test_common.hpp" -#include "test_common/range_scheduler.hpp" -#include "btree_helpers/btree_test_helper.hpp" -#include "btree_helpers/btree_test_kvs.hpp" -#include "btree_helpers/btree_decls.h" - -using namespace homestore; - -SISL_LOGGING_INIT(HOMESTORE_LOG_MODS) -SISL_OPTIONS_ENABLE(test_index_recovery, logging, iomgr, test_common_setup) -SISL_LOGGING_DECL(test_index_recovery) - -std::vector< std::string > test_common::HSTestHelper::s_dev_names; - -SISL_OPTION_GROUP( - test_index_recovery, - (num_iters, "", "num_iters", "number of iterations for rand ops", - ::cxxopts::value< uint32_t >()->default_value("100"), "number"), - (num_entries, "", "num_entries", "number of entries to test with", - ::cxxopts::value< uint32_t >()->default_value("20"), "number"), - (run_time, "", "run_time", "run time for io", ::cxxopts::value< uint32_t >()->default_value("360000"), "seconds"), - (operation_list, "", "operation_list", "operation list instead of default created following by percentage", - ::cxxopts::value< std::vector< std::string > >(), "operations [...]"), - // (flip_list, "", "flip_list", "btree flip list", - // ::cxxopts::value< std::vector< std::string > >(), "flips [...]"), - (preload_size, "", "preload_size", "number of entries to preload tree with", - ::cxxopts::value< uint32_t >()->default_value("10"), "number"), - (init_device, "", "init_device", "init device", ::cxxopts::value< bool >()->default_value("1"), ""), - (cleanup_after_shutdown, "", "cleanup_after_shutdown", "cleanup after shutdown", - ::cxxopts::value< bool >()->default_value("1"), ""), - // (enable_crash, "", "enable_crash", "enable crash", ::cxxopts::value< bool >()->default_value("0"), ""), - (seed, "", "seed", "random engine seed, use random if not defined", - ::cxxopts::value< uint64_t >()->default_value("0"), "number")) - -static void change_cp_time(uint64_t value_us) { - HS_SETTINGS_FACTORY().modifiable_settings([value_us](auto& s) { - s.generic.cp_timer_us = value_us; - HS_SETTINGS_FACTORY().save(); - }); - LOGINFO("\n\n\nCP TIMER changed to {}", value_us); -} - -template < typename TestType > -struct BtreeRecoveryTest : public BtreeTestHelper< TestType >, public ::testing::Test { - using T = TestType; - using K = typename TestType::KeyType; - using V = typename TestType::ValueType; - class TestIndexServiceCallbacks : public IndexServiceCallbacks { - public: - TestIndexServiceCallbacks(BtreeRecoveryTest* test) : m_test(test) {} - - std::shared_ptr< IndexTableBase > on_index_table_found(superblk< index_table_sb >&& sb) override { - LOGINFO("Index table recovered"); - LOGINFO("root bnode_id {} version {}", sb->root_node, sb->root_link_version); - m_test->m_cfg = BtreeConfig(hs()->index_service().node_size()); - m_test->m_cfg.m_leaf_node_type = T::leaf_node_type; - m_test->m_cfg.m_int_node_type = T::interior_node_type; - m_test->m_bt = std::make_shared< typename T::BtreeType >(std::move(sb), m_test->m_cfg); - return m_test->m_bt; - } - - private: - BtreeRecoveryTest* m_test; - }; - - BtreeRecoveryTest() : testing::Test() {} - - void restart_homestore() { - m_token.params(HS_SERVICE::INDEX).index_svc_cbs = new TestIndexServiceCallbacks(this); - test_common::HSTestHelper::restart_homestore(m_token); - } - - void destroy_btree() { - auto cpg = hs()->cp_mgr().cp_guard(); - auto op_context = (void*)cpg.context(cp_consumer_t::INDEX_SVC); - const auto [ret, free_node_cnt] = this->m_bt->destroy_btree(op_context); - ASSERT_EQ(ret, btree_status_t::success) << "btree destroy failed"; - this->m_bt.reset(); - } - - void SetUp() override { - m_token = test_common::HSTestHelper::start_homestore( - "test_index_recovery", - {{HS_SERVICE::META, {.size_pct = 10.0}}, - {HS_SERVICE::INDEX, {.size_pct = 70.0, .index_svc_cbs = new TestIndexServiceCallbacks(this)}}}, - nullptr, {}, SISL_OPTIONS["init_device"].as< bool >()); - - LOGINFO("Node size {} ", hs()->index_service().node_size()); - this->m_cfg = BtreeConfig(hs()->index_service().node_size()); - - auto uuid = boost::uuids::random_generator()(); - auto parent_uuid = boost::uuids::random_generator()(); - - // Test cp flush of write back. - // not here - // set it as 0 HS_DYNAMIC_CONFIG(generic.cp_timer_us) to make the CP to fail - HS_SETTINGS_FACTORY().modifiable_settings([](auto& s) { - s.generic.cache_max_throttle_cnt = 100000; - HS_SETTINGS_FACTORY().save(); - }); - homestore::hs()->resource_mgr().reset_dirty_buf_qd(); - homestore::hs()->resource_mgr().register_dirty_buf_exceed_cb( - [this]([[maybe_unused]] int64_t dirty_buf_count, bool critical) {}); - // Create index table and attach to index service. - BtreeTestHelper< TestType >::SetUp(); - if (this->m_bt == nullptr) { - this->m_bt = std::make_shared< typename T::BtreeType >(uuid, parent_uuid, 0, this->m_cfg); - } else { - // this->m_bt->retrieve_root_node(); - LOGINFO("root bnode_id {} version {}", this->m_bt->root_node_id(), this->m_bt->root_link_version()); - test_common::HSTestHelper::trigger_cp(true /* wait */); - populate_shadow_map(); - } - - hs()->index_service().add_index_table(this->m_bt); - LOGINFO("Added index table to index service"); - } - void populate_shadow_map() { - this->m_shadow_map.load(m_shadow_filename); - ASSERT_EQ(this->m_shadow_map.size(), this->m_bt->count_keys(this->m_bt->root_node_id())) - << "shadow map size and tree size mismatch"; - this->get_all(); - } - - void TearDown() override { - bool cleanup = SISL_OPTIONS["cleanup_after_shutdown"].as< bool >(); - LOGINFO("cleanup the dump map and index data? {}", cleanup); - if (!cleanup) { - this->m_shadow_map.save(m_shadow_filename); - } else { - if (std::filesystem::remove(m_shadow_filename)) { - LOGINFO("File {} removed successfully", m_shadow_filename); - } else { - LOGINFO("Error: failed to remove {}", m_shadow_filename); - } - } - LOGINFO("Teardown with Root bnode_id {} tree size: {}", this->m_bt->root_node_id(), - this->m_bt->count_keys(this->m_bt->root_node_id())); - BtreeTestHelper< TestType >::TearDown(); - // test_common::HSTestHelper::shutdown_homestore(cleanup); - test_common::HSTestHelper::shutdown_homestore(false); - } - -private: - const std::string m_shadow_filename = "/tmp/shadow_map.txt"; - test_common::HSTestHelper::test_token m_token; -}; -using BtreeTypes = testing::Types< FixedLenBtree /*, VarKeySizeBtree, VarValueSizeBtree, VarObjSizeBtree */ >; - -TYPED_TEST_SUITE(BtreeRecoveryTest, BtreeTypes); -TYPED_TEST(BtreeRecoveryTest, Test1) { - uint32_t i = 0; - auto num_entries = SISL_OPTIONS["num_entries"].as< uint32_t >(); - // auto num_entries = 30; - // uint32_t part1 = num_entries - 1000; - // uint32_t part2 = num_entries; - uint32_t part1 = num_entries - 5; - uint32_t part2 = num_entries; - LOGINFO("\n\n\n \t\t\t\t\t\t\t\t\t\t\t\t\t\t 1- Do Forward sequential insert for [0, {}] entries", part1); - for (; i < part1; ++i) { - this->put(i, btree_put_type::INSERT); - } - LOGINFO("\n\n\n\n\n \t\t\t\t\t\t\t\t\t\t\t\t\t\t 1- flushing {} entries", part1); - // this->visualize_keys("tree1.dot"); - - test_common::HSTestHelper::trigger_cp(true /* wait */); - LOGINFO("\n\n\n \t\t\t\t\t\t\t\t\t\t\t\t\t\t 1 - flushing part 1 done"); - this->print_keys(); - - if (SISL_OPTIONS.count("enable_crash")) { -#ifdef _PRERELEASE - if (SISL_OPTIONS.count("flip_list")) { - auto flips = SISL_OPTIONS["flip_list"].as< std::vector< std::string > >(); - for (const auto& flip : flips) { - this->set_flip_point(flip); - } - } - LOGINFO(" enabled flips {}", this->m_bt->flip_list()); -#endif - } - // LOGINFO( "\n print before crash ") - // this->print_keys(); - LOGINFO("\n\n\n\n \t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t 2- enable crash"); - LOGINFO( - "\n\n\n\n\n \t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t 2- Do Forward sequential insert [{}, {}) entries - size {}", - part1, part2, this->m_bt->count_keys(this->m_bt->root_node_id())); - for (i = part1; i < part2; ++i) { - this->put(i, btree_put_type::INSERT); - } - LOGINFO("\n\n\n\n PART2 \t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t insert done\n \t\t trigger cp flush\n\n"); - LOGINFO("\n\n\n\n \t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t 2- print keys before flushing crashes"); - this->print_keys(); - // this->visualize_keys("tree2.dot"); - test_common::HSTestHelper::trigger_cp(true /* wait */); - // - // LOGINFO("Query {} entries and validate with pagination of 75 entries", num_entries); - // this->do_query(0, num_entries - 1, 75); - // - // this->print(std::string("before_recovery.txt")); - // - // this->destroy_btree(); - // Restart homestore. m_bt is updated by the TestIndexServiceCallback. -#ifdef _PRERELEASE - if (SISL_OPTIONS.count("flip_list")) { - auto flips = SISL_OPTIONS["flip_list"].as< std::vector< std::string > >(); - for (const auto& flip : flips) { - this->reset_flip_point(flip); - } - } - LOGINFO(" reseted flips {}", this->m_bt->flip_list()); -#endif - this->restart_homestore(); - LOGINFO("\n\n\n\n \t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t PART3 RETRY put {} - {} entries", part1, part2); - LOGINFO("\n\n\n\n\n \t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t 3-Root bnode_id {} version {} ", - this->m_bt->root_node_id(), this->m_bt->root_link_version()); - LOGINFO("\n\n\n\n \t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t 3- print keys BEFORE recovery"); - // LOGINFO("Query {} entries", num_entries); - // this->do_query(0, num_entries - 1, 1000); - this->print_keys(); - // this->print(std::string("after_recovery.txt")); - - // LOGINFO("Query {} entries", num_entries); - // this->do_query(0, num_entries - 1, 1000); - - // this->compare_files("before_recovery.txt", "after_recovery.txt"); - // LOGINFO("CpFlush test end"); - // this->print_keys(true,0 ); - // this->visualize_keys("tree3.dot"); - - LOGINFO("\n\n\n\n\n \t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t retry Do Forward sequential insert [{}, {}) entries", - part1, part2); - for (i = part1; i < part2; ++i) { - LOGINFO("\n put retry {}", i); - // this->visualize_keys("tree3_"+std::to_string(i)+".dot"); - this->put(i, btree_put_type::INSERT, false); - } - - // this->print_keys(false,0); - LOGINFO("test done") -} - -int main(int argc, char* argv[]) { - int parsed_argc{argc}; - ::testing::InitGoogleTest(&parsed_argc, argv); - SISL_OPTIONS_LOAD(parsed_argc, argv, test_index_recovery, logging, iomgr, test_common_setup); - sisl::logging::SetLogger("test_index_recovery"); - spdlog::set_pattern("[%D %T%z] [%^%L%$] [%t] %v"); - change_cp_time(90000000000); - if (SISL_OPTIONS.count("seed")) { - auto seed = SISL_OPTIONS["seed"].as< uint64_t >(); - LOGINFO("Using seed {} to sow the random generation", seed); - g_re.seed(seed); - } - auto ret = RUN_ALL_TESTS(); - return ret; -} diff --git a/src/tests/test_scripts/index_test.py b/src/tests/test_scripts/index_test.py index fe31f9d23..60b751c3d 100755 --- a/src/tests/test_scripts/index_test.py +++ b/src/tests/test_scripts/index_test.py @@ -20,6 +20,16 @@ def run_test(options, type): raise TestFailedError(f"Test failed for type {type}") print("Test completed") +def run_test(options): + cmd_opts = f"--gtest_filter=IndexCrashTest/0.long_running_put_crash --gtest_break_on_failure --max_keys_in_node={options['max_keys_in_node']} --init_device={options['init_device']} {options['log_mods']} --run_time={options['run_time']} --num_entries={options['num_entries']} {options['dev_list']}" + # print(f"Running test with options: {cmd_opts}") + try: + subprocess.check_call(f"{options['dirpath']}test_index_crash_recovery {cmd_opts}", stderr=subprocess.STDOUT, shell=True) + except subprocess.CalledProcessError as e: + print(f"Test failed: {e}") + raise TestFailedError(f"Test failed for type {type}") + print("Crash Test completed") + def parse_arguments(): # Create the parser @@ -39,6 +49,7 @@ def parse_arguments(): parser.add_argument('--dev_list', help='Device list', default='') parser.add_argument('--cleanup_after_shutdown', help='Cleanup after shutdown', type=bool, default=False) parser.add_argument('--init_device', help='Initialize device', type=bool, default=True) + parser.add_argument('--max_keys_in_node', help='Maximum num of keys in btree nodes', type=int, default=5) # Parse the known arguments and ignore any unknown arguments args, unknown = parser.parse_known_args() @@ -76,6 +87,12 @@ def long_running_clean_shutdown(options, type=0): raise print("Long running clean shutdown completed") +def long_running_crash_put(options): + print("Long running crash put started") + options['num_entries'] = 20480 # 20K + print(f"options: {options}") + run_test(options) + print("Long running crash put completed") def main(): options = parse_arguments() @@ -96,6 +113,7 @@ def long_running(*args): options = parse_arguments() long_runnig_index(options) long_running_clean_shutdown(options) + long_running_crash_put(options) if __name__ == "__main__":