Skip to content

Commit

Permalink
Lazy superblock flush on colocated table creation
Browse files Browse the repository at this point in the history 
Test Plan: TODO

Subscribers: bogdan

Differential Revision: https://phabricator.dev.yugabyte.com/D23349
  • Loading branch information
@arpang
arpang committed Mar 6, 2023
1 parent b9e4ce3 commit 22c8d6c
Show file tree
Hide file tree
Showing 10 changed files with 34 additions and 11 deletions.
6 changes: 5 additions & 1 deletion src/yb/consensus/consensus_peers-test.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,8 @@

using namespace std::chrono_literals;

DECLARE_bool(lazily_flush_superblock);

METRIC_DECLARE_entity(tablet);

namespace yb {
Expand Down Expand Up @@ -92,6 +94,7 @@ class ConsensusPeersTest : public YBTest {
ASSERT_OK(ThreadPoolBuilder("log").Build(&log_thread_pool_));
fs_manager_.reset(new FsManager(env_.get(), GetTestPath("fs_root"), "tserver_test"));

FLAGS_lazily_flush_superblock = false;
ASSERT_OK(fs_manager_->CreateInitialFileSystemLayout());
ASSERT_OK(fs_manager_->CheckAndOpenFileSystemRoots());
ASSERT_OK(Log::Open(options_,
Expand All @@ -106,7 +109,8 @@ class ConsensusPeersTest : public YBTest {
log_thread_pool_.get(),
log_thread_pool_.get(),
std::numeric_limits<int64_t>::max(), // cdc_min_replicated_index
&log_));
&log_,
nullptr));
clock_.reset(new server::HybridClock());
ASSERT_OK(clock_->Init());

Expand Down
4 changes: 3 additions & 1 deletion src/yb/consensus/consensus_queue-test.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,7 @@ class ConsensusQueueTest : public YBTest {

void SetUp() override {
YBTest::SetUp();
FLAGS_lazily_flush_superblock = false;
fs_manager_.reset(new FsManager(env_.get(), GetTestPath("fs_root"), "tserver_test"));
ASSERT_OK(fs_manager_->CreateInitialFileSystemLayout());
ASSERT_OK(fs_manager_->CheckAndOpenFileSystemRoots());
Expand All @@ -97,7 +98,8 @@ class ConsensusQueueTest : public YBTest {
log_thread_pool_.get(),
log_thread_pool_.get(),
std::numeric_limits<int64_t>::max(), // cdc_min_replicated_index
&log_));
&log_,
nullptr));
clock_.reset(new server::HybridClock());
ASSERT_OK(clock_->Init());

Expand Down
6 changes: 5 additions & 1 deletion src/yb/consensus/log-dump.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,8 @@ DEFINE_UNKNOWN_int64(max_op_index_to_omit, yb::OpId::Invalid().index,

DEFINE_UNKNOWN_string(output_wal_dir, "", "WAL directory for the output of --filter_log_segment");

DECLARE_bool(lazily_flush_superblock);

namespace yb {
namespace log {

Expand Down Expand Up @@ -406,6 +408,7 @@ Status FilterLogSegment(const string& segment_path) {
LOG(INFO) << "Will include all records of the source WAL in the output";
}

FLAGS_lazily_flush_superblock = false;
scoped_refptr<Log> log;
RETURN_NOT_OK(Log::Open(
log_options,
Expand All @@ -420,7 +423,8 @@ Status FilterLogSegment(const string& segment_path) {
log_thread_pool.get(),
log_thread_pool.get(),
/* cdc_min_replicated_index */ 0,
&log));
&log,
nullptr));

auto read_entries = segment->ReadEntries();
RETURN_NOT_OK(read_entries.status);
Expand Down
5 changes: 4 additions & 1 deletion src/yb/consensus/log-test-base.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,7 @@
METRIC_DECLARE_entity(table);
METRIC_DECLARE_entity(tablet);

DECLARE_bool(lazily_flush_superblock);
DECLARE_int32(log_min_seconds_to_retain);

namespace yb {
Expand Down Expand Up @@ -183,6 +184,7 @@ class LogTestBase : public YBTest {

void BuildLog() {
Schema schema_with_ids = SchemaBuilder(schema_).Build();
FLAGS_lazily_flush_superblock = false;
ASSERT_OK(Log::Open(options_,
kTestTablet,
tablet_wal_path_,
Expand All @@ -195,7 +197,8 @@ class LogTestBase : public YBTest {
log_thread_pool_.get(),
log_thread_pool_.get(),
std::numeric_limits<int64_t>::max(), // cdc_min_replicated_index
&log_));
&log_,
nullptr));
LOG(INFO) << "Sucessfully opened the log at " << tablet_wal_path_;
}

Expand Down
2 changes: 1 addition & 1 deletion src/yb/consensus/log.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -153,7 +153,7 @@ class Log : public RefCountedThreadSafe<Log> {
ThreadPool* background_sync_threadpool,
int64_t cdc_min_replicated_index,
scoped_refptr<Log> *log,
tablet::RaftGroupMetadata* metadata = nullptr, // TODO
tablet::RaftGroupMetadata* metadata,
CreateNewSegment create_new_segment = CreateNewSegment::kTrue);

~Log();
Expand Down
5 changes: 4 additions & 1 deletion src/yb/consensus/log_cache-test.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@ using std::atomic;
using std::shared_ptr;
using std::thread;

DECLARE_bool(lazily_flush_superblock);
DECLARE_int32(log_cache_size_limit_mb);
DECLARE_int32(global_log_cache_size_limit_mb);
DECLARE_int32(global_log_cache_size_limit_percentage);
Expand Down Expand Up @@ -102,6 +103,7 @@ class LogCacheTest : public YBTest {

void SetUp() override {
YBTest::SetUp();
FLAGS_lazily_flush_superblock = false;
fs_manager_.reset(new FsManager(env_.get(), GetTestPath("fs_root"), "tserver_test"));
ASSERT_OK(fs_manager_->CreateInitialFileSystemLayout());
ASSERT_OK(fs_manager_->CheckAndOpenFileSystemRoots());
Expand All @@ -118,7 +120,8 @@ class LogCacheTest : public YBTest {
log_thread_pool_.get(),
log_thread_pool_.get(),
std::numeric_limits<int64_t>::max(), // cdc_min_replicated_index
&log_));
&log_,
nullptr));

CloseAndReopenCache(MinimumOpId());
clock_.reset(new server::HybridClock());
Expand Down
6 changes: 4 additions & 2 deletions src/yb/consensus/raft_consensus-test.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@
#include "yb/util/test_macros.h"
#include "yb/util/test_util.h"

DECLARE_bool(lazily_flush_superblock);
DECLARE_bool(enable_leader_failure_detection);
DECLARE_bool(never_fsync);

Expand Down Expand Up @@ -223,7 +224,7 @@ class RaftConsensusTest : public YBTest {

void SetUp() override {
YBTest::SetUp();

FLAGS_lazily_flush_superblock = false;
LogOptions options;
string test_path = GetTestPath("test-peer-root");

Expand All @@ -246,7 +247,8 @@ class RaftConsensusTest : public YBTest {
log_thread_pool_.get(),
log_thread_pool_.get(),
std::numeric_limits<int64_t>::max(), // cdc_min_replicated_index
&log_));
&log_,
nullptr));

log_->TEST_SetAllOpIdsSafe(true);

Expand Down
5 changes: 4 additions & 1 deletion src/yb/consensus/raft_consensus_quorum-test.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@

DECLARE_int32(raft_heartbeat_interval_ms);
DECLARE_bool(enable_leader_failure_detection);
DECLARE_bool(lazily_flush_superblock);

METRIC_DECLARE_entity(table);
METRIC_DECLARE_entity(tablet);
Expand Down Expand Up @@ -134,6 +135,7 @@ class RaftConsensusQuorumTest : public YBTest {
RETURN_NOT_OK(fs_manager->CreateInitialFileSystemLayout());
RETURN_NOT_OK(fs_manager->CheckAndOpenFileSystemRoots());

FLAGS_lazily_flush_superblock = false;
scoped_refptr<Log> log;
RETURN_NOT_OK(Log::Open(LogOptions(),
kTestTablet,
Expand All @@ -147,7 +149,8 @@ class RaftConsensusQuorumTest : public YBTest {
log_thread_pool_.get(),
log_thread_pool_.get(),
std::numeric_limits<int64_t>::max(), // cdc_min_replicated_index
&log));
&log,
nullptr));
logs_.push_back(log.get());
fs_managers_.push_back(fs_manager.release());
}
Expand Down
3 changes: 2 additions & 1 deletion src/yb/tablet/tablet_peer-test.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -185,7 +185,8 @@ class TabletPeerTest : public YBTabletTest {
*tablet()->schema(), tablet()->metadata()->schema_version(),
table_metric_entity_.get(), tablet_metric_entity_.get(),
log_thread_pool_.get(), log_thread_pool_.get(), log_thread_pool_.get(),
tablet()->metadata()->cdc_min_replicated_index(), &log));
tablet()->metadata()->cdc_min_replicated_index(), &log,
tablet()->metadata()));

ASSERT_OK(tablet_peer_->SetBootstrapping());
ASSERT_OK(tablet_peer_->InitTabletPeer(tablet(),
Expand Down
3 changes: 2 additions & 1 deletion src/yb/tserver/remote_bootstrap_session-test.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,8 @@ void RemoteBootstrapSessionTest::SetUpTabletPeer() {
log_thread_pool_.get(),
log_thread_pool_.get(),
std::numeric_limits<int64_t>::max(), // cdc_min_replicated_index
&log));
&log,
tablet()->metadata()));

scoped_refptr<MetricEntity> table_metric_entity =
METRIC_ENTITY_table.Instantiate(&metric_registry_, Format("table-$0", CURRENT_TEST_NAME()));
Expand Down

0 comments on commit 22c8d6c

Please sign in to comment.