Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[prototype] automatic DB migation on nodeos upgrades #394

Draft
wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

[prototype] automatic DB migation on nodeos upgrades #394

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,7 @@ if(ENABLE_WEXTRA)
set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wextra" )
endif()

set(CMAKE_POSITION_INDEPENDENT_CODE ON)
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is actually not as much of a change as it might look -- while gcc defaults to non-PIC, most (nearly all?) distros tweak their gcc to produce PIC by default (since it's needed for ASLR) including Ubuntu. Our pinned builds are all PIC.

So it most cases this is really a no-op change.

At one point clang didn't have the PIC tweak in Ubuntu but it appears they do it to clang now too.


option(EOSIO_ENABLE_DEVELOPER_OPTIONS "enable developer options for Spring" OFF)

Expand Down
1 change: 1 addition & 0 deletions libraries/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,3 +35,4 @@ if(eos-vm IN_LIST EOSIO_WASM_RUNTIMES OR eos-vm-jit IN_LIST EOSIO_WASM_RUNTIMES)
add_subdirectory( eos-vm )
endif()
add_subdirectory( prometheus )
add_subdirectory( snapshotter )
14 changes: 7 additions & 7 deletions libraries/chain/authorization_manager.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,8 @@ namespace eosio { namespace chain {
authorization_manager::authorization_manager(controller& c, database& d)
:_control(c),_db(d){}

void authorization_manager::add_indices() {
authorization_index_set::add_indices(_db);
void authorization_manager::add_indices(chainbase::database& db) {
authorization_index_set::add_indices(db);
}

void authorization_manager::initialize_database() {
Expand Down Expand Up @@ -93,18 +93,18 @@ namespace eosio { namespace chain {
};
}

void authorization_manager::add_to_snapshot( const snapshot_writer_ptr& snapshot ) const {
authorization_index_set::walk_indices([this, &snapshot]( auto utils ){
void authorization_manager::add_to_snapshot( const snapshot_writer_ptr& snapshot, chainbase::database& db ) {
authorization_index_set::walk_indices([&db, &snapshot]( auto utils ){
using section_t = typename decltype(utils)::index_t::value_type;

// skip the permission_usage_index as its inlined with permission_index
if (std::is_same<section_t, permission_usage_object>::value) {
return;
}

snapshot->write_section<section_t>([this]( auto& section ){
decltype(utils)::walk(_db, [this, &section]( const auto &row ) {
section.add_row(row, _db);
snapshot->write_section<section_t>([&db]( auto& section ){
decltype(utils)::walk(db, [&db, &section]( const auto &row ) {
section.add_row(row, db);
});
});
});
Expand Down
94 changes: 54 additions & 40 deletions libraries/chain/controller.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1205,11 +1205,11 @@ struct controller_impl {
controller_impl( const controller::config& cfg, controller& s, protocol_feature_set&& pfs, const chain_id_type& chain_id )
:rnh(),
self(s),
db( cfg.state_dir,
db( cfg.state_dir / config::chainbasedb_filename,
cfg.read_only ? database::read_only : database::read_write,
cfg.state_size, false, cfg.db_map_mode ),
blog( cfg.blocks_dir, cfg.blog ),
fork_db(cfg.blocks_dir / config::reversible_blocks_dir_name),
fork_db(cfg.blocks_dir / config::reversible_blocks_dir_name / config::forkdb_filename),
resource_limits( db, [&s](bool is_trx_transient) { return s.get_deep_mind_logger(is_trx_transient); }),
authorization( s, db ),
protocol_features( std::move(pfs), [&s](bool is_trx_transient) { return s.get_deep_mind_logger(is_trx_transient); } ),
Expand Down Expand Up @@ -2022,27 +2022,27 @@ struct controller_impl {
ilog( "chain database stopped with hash: ${hash}", ("hash", calculate_integrity_hash()) );
}

void add_indices() {
static void add_indices(chainbase::database& db) {
controller_index_set::add_indices(db);
contract_database_index_set::add_indices(db);

authorization.add_indices();
resource_limits.add_indices();
authorization_manager::add_indices(db);
resource_limits_manager::add_indices(db);
}

void clear_all_undo() {
// Rewind the database to the last irreversible block
db.undo_all();
}

void add_contract_tables_to_snapshot( const snapshot_writer_ptr& snapshot ) const {
snapshot->write_section("contract_tables", [this]( auto& section ) {
index_utils<table_id_multi_index>::walk(db, [this, &section]( const table_id_object& table_row ){
static void add_contract_tables_to_snapshot( const snapshot_writer_ptr& snapshot, chainbase::database& db ) {
snapshot->write_section("contract_tables", [&db]( auto& section ) {
index_utils<table_id_multi_index>::walk(db, [&db, &section]( const table_id_object& table_row ){
// add a row for the table
section.add_row(table_row, db);

// followed by a size row and then N data rows for each type of table
contract_database_index_set::walk_indices([this, &section, &table_row]( auto utils ) {
contract_database_index_set::walk_indices([&db, &section, &table_row]( auto utils ) {
using utils_t = decltype(utils);
using value_t = typename decltype(utils)::index_t::value_type;
using by_table_id = object_to_table_id_tag_t<value_t>;
Expand All @@ -2053,7 +2053,7 @@ struct controller_impl {
unsigned_int size = utils_t::template size_range<by_table_id>(db, tid_key, next_tid_key);
section.add_row(size, db);

utils_t::template walk_range<by_table_id>(db, tid_key, next_tid_key, [this, &section]( const auto &row ) {
utils_t::template walk_range<by_table_id>(db, tid_key, next_tid_key, [&db, &section]( const auto &row ) {
section.add_row(row, db);
});
});
Expand Down Expand Up @@ -2090,35 +2090,28 @@ struct controller_impl {
});
}

block_state_pair get_block_state_to_snapshot() const
{
return block_handle_accessor::apply<block_state_pair>(chain_head, overloaded{
[&](const block_state_legacy_ptr& head) -> block_state_pair {
if (head->header.contains_header_extension(finality_extension::extension_id())) {
// During transition to Savanna, we need to build Transition Savanna block
// from Savanna Genesis block
return { head, get_transition_savanna_block(head) };
}
return block_state_pair{ head, {} };
},
[](const block_state_ptr& head) {
return block_state_pair{ {}, head };
}});
block_state_pair get_block_state_to_snapshot() const {
return controller::block_state_for_snapshot(fork_db, protocol_features, chain_head);
}

void add_to_snapshot( const snapshot_writer_ptr& snapshot ) {
// clear in case the previous call to clear did not finish in time of deadline
clear_expired_input_transactions( fc::time_point::maximum() );

snapshot->write_section<chain_snapshot_header>([this]( auto &section ){
add_to_snapshot_no_expired(snapshot, db, get_block_state_to_snapshot());
}

///XXX we'll assume there are no expired transactions in this case: should purge all expired txns on shutdown
static void add_to_snapshot_no_expired( const snapshot_writer_ptr& snapshot, database& db, const block_state_pair& bstate_pair) {
snapshot->write_section<chain_snapshot_header>([&db]( auto &section ){
section.add_row(chain_snapshot_header(), db);
});

snapshot->write_section("eosio::chain::block_state", [&]( auto& section ) {
section.add_row(snapshot_detail::snapshot_block_state_data_v7(get_block_state_to_snapshot()), db);
section.add_row(snapshot_detail::snapshot_block_state_data_v7(bstate_pair), db);
});

controller_index_set::walk_indices([this, &snapshot]( auto utils ){
controller_index_set::walk_indices([&db, &snapshot]( auto utils ){
using value_t = typename decltype(utils)::index_t::value_type;

// skip the table_id_object as its inlined with contract tables section
Expand All @@ -2131,17 +2124,17 @@ struct controller_impl {
return;
}

snapshot->write_section<value_t>([this]( auto& section ){
decltype(utils)::walk(db, [this, &section]( const auto &row ) {
snapshot->write_section<value_t>([&db]( auto& section ){
decltype(utils)::walk(db, [&db, &section]( const auto &row ) {
section.add_row(row, db);
});
});
});

add_contract_tables_to_snapshot(snapshot);
add_contract_tables_to_snapshot(snapshot, db);

authorization.add_to_snapshot(snapshot);
resource_limits.add_to_snapshot(snapshot);
authorization_manager::add_to_snapshot(snapshot, db);
resource_limits_manager::add_to_snapshot(snapshot, db);
}

static std::optional<genesis_state> extract_legacy_genesis_state( snapshot_reader& snapshot, uint32_t version ) {
Expand Down Expand Up @@ -3313,7 +3306,7 @@ struct controller_impl {
block_handle_accessor::apply<void>(chain_head,
[&](const block_state_legacy_ptr& head) {
if (head->block->contains_header_extension(finality_extension::extension_id())) {
auto bsp = get_transition_savanna_block(head);
auto bsp = get_transition_savanna_block(fork_db, protocol_features, head);
assert(bsp);
assert(bsp->active_finalizer_policy);
dm_logger->on_accepted_block_v2(head->id(), fork_db_root_block_num(), head->block,
Expand Down Expand Up @@ -4542,7 +4535,7 @@ struct controller_impl {
}

// Returns corresponding Transition Savanna block for a given Legacy block.
block_state_ptr get_transition_savanna_block(const block_state_legacy_ptr& head) const {
static block_state_ptr get_transition_savanna_block(const fork_database& fork_db, const protocol_feature_manager& protocol_features, const block_state_legacy_ptr& head) {
fork_database_legacy_t::branch_t legacy_branch;
block_state_legacy_ptr legacy_root;
fork_db.apply_l<void>([&](const auto& forkdb) {
Expand Down Expand Up @@ -4576,7 +4569,7 @@ struct controller_impl {
const bool skip_validate_signee = true; // validated already

for (; bitr != legacy_branch.rend(); ++bitr) {
assert(read_mode == db_read_mode::IRREVERSIBLE || (*bitr)->action_mroot_savanna.has_value());
///assert(read_mode == db_read_mode::IRREVERSIBLE || (*bitr)->action_mroot_savanna.has_value());
auto new_bsp = block_state::create_transition_block(
*prev,
(*bitr)->block,
Expand All @@ -4593,7 +4586,7 @@ struct controller_impl {
}

std::optional<finality_data_t> get_transition_block_finality_data(const block_state_legacy_ptr& head) const {
return get_transition_savanna_block(head)->get_finality_data();
return get_transition_savanna_block(fork_db, protocol_features, head)->get_finality_data();
}

std::optional<finality_data_t> head_finality_data() const {
Expand Down Expand Up @@ -4819,8 +4812,8 @@ controller::~controller() {
my->thread_pool.stop();
}

void controller::add_indices() {
my->add_indices();
void controller::add_indices(chainbase::database& db) {
controller_impl::add_indices(db);
}

void controller::startup( std::function<void()> shutdown, std::function<bool()> check_shutdown, const snapshot_reader_ptr& snapshot ) {
Expand All @@ -4835,7 +4828,8 @@ void controller::startup(std::function<void()> shutdown, std::function<bool()> c
my->startup(shutdown, check_shutdown);
}

const chainbase::database& controller::db()const { return my->db; }
//XXX restore const somehow
chainbase::database& controller::db() const { return my->db; }

chainbase::database& controller::mutable_db()const { return my->db; }

Expand Down Expand Up @@ -5300,6 +5294,26 @@ void controller::write_snapshot( const snapshot_writer_ptr& snapshot ) {
my->add_to_snapshot(snapshot);
}

void controller::write_nonlive_snapshot( const snapshot_writer_ptr& snapshot, chainbase::database& db, const protocol_feature_manager& protocol_features, const fork_database& forkdb, const block_handle& chain_head ) {
controller_impl::add_to_snapshot_no_expired(snapshot, db, block_state_for_snapshot(forkdb, protocol_features, chain_head));
}

block_state_pair controller::block_state_for_snapshot( const fork_database& fork_db, const protocol_feature_manager& protocol_features, const block_handle& chain_head ) {
return block_handle_accessor::apply<block_state_pair>(chain_head, overloaded{
[&](const block_state_legacy_ptr& head) -> block_state_pair {
if (head->header.contains_header_extension(finality_extension::extension_id())) {
// During transition to Savanna, we need to build Transition Savanna block
// from Savanna Genesis block
return { head, controller_impl::get_transition_savanna_block(fork_db, protocol_features, head) };
}
return block_state_pair{ head, {} };
},
[](const block_state_ptr& head) {
return block_state_pair{ {}, head };
}
});
}

bool controller::is_writing_snapshot() const {
return my->writing_snapshot.load(std::memory_order_acquire);
}
Expand Down Expand Up @@ -5737,7 +5751,7 @@ chain_id_type controller::extract_chain_id(snapshot_reader& snapshot) {

std::optional<chain_id_type> controller::extract_chain_id_from_db( const path& state_dir ) {
try {
chainbase::database db( state_dir, chainbase::database::read_only );
chainbase::database db( state_dir / chain::config::chainbasedb_filename, chainbase::database::read_only );

db.add_index<database_header_multi_index>();
db.add_index<global_property_multi_index>();
Expand Down
32 changes: 15 additions & 17 deletions libraries/chain/fork_database.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -611,8 +611,8 @@ namespace eosio::chain {

// ------------------ fork_database -------------------------

fork_database::fork_database(const std::filesystem::path& data_dir)
: data_dir(data_dir)
fork_database::fork_database(const std::filesystem::path& path)
: forkdb_path(path)
{
}

Expand All @@ -621,7 +621,6 @@ namespace eosio::chain {
}

void fork_database::close() {
auto fork_db_file {data_dir / config::forkdb_filename};
bool legacy_valid = fork_db_l.is_valid();
bool savanna_valid = fork_db_s.is_valid();

Expand All @@ -637,8 +636,8 @@ namespace eosio::chain {
(savanna_valid && (in_use_value == in_use_t::savanna)) ||
(legacy_valid && savanna_valid && (in_use_value == in_use_t::both)) );

ilog("Persisting to fork_database file: ${f}", ("f", fork_db_file));
std::ofstream out( fork_db_file.generic_string().c_str(), std::ios::out | std::ios::binary | std::ofstream::trunc );
ilog("Persisting to fork_database file: ${f}", ("f", forkdb_path));
std::ofstream out( forkdb_path.generic_string().c_str(), std::ios::out | std::ios::binary | std::ofstream::trunc );

fc::raw::pack( out, magic_number );
fc::raw::pack( out, max_supported_version ); // write out current version which is always max_supported_version
Expand All @@ -658,16 +657,15 @@ namespace eosio::chain {
}

void fork_database::open( validator_t& validator ) {
if (!std::filesystem::is_directory(data_dir))
std::filesystem::create_directories(data_dir);
if (!std::filesystem::is_directory(forkdb_path.parent_path()))
std::filesystem::create_directories(forkdb_path.parent_path());

assert(!fork_db_l.is_valid() && !fork_db_s.is_valid());

auto fork_db_file = data_dir / config::forkdb_filename;
if( std::filesystem::exists( fork_db_file ) ) {
if( std::filesystem::exists( forkdb_path ) ) {
try {
fc::cfile f;
f.set_file_path(fork_db_file);
f.set_file_path(forkdb_path);
f.open("rb");

fc::cfile_datastream ds(f);
Expand All @@ -677,22 +675,22 @@ namespace eosio::chain {
fc::raw::unpack( ds, totem );
EOS_ASSERT( totem == magic_number, fork_database_exception,
"Fork database file '${filename}' has unexpected magic number: ${actual_totem}. Expected ${t}",
("filename", fork_db_file)("actual_totem", totem)("t", magic_number));
("filename", forkdb_path)("actual_totem", totem)("t", magic_number));

uint32_t version = 0;
fc::raw::unpack( ds, version );
EOS_ASSERT( version >= fork_database::min_supported_version && version <= fork_database::max_supported_version,
fork_database_exception,
"Unsupported version of fork database file '${filename}'. "
"Fork database version is ${version} while code supports version(s) [${min},${max}]",
("filename", fork_db_file)("version", version)("min", min_supported_version)("max", max_supported_version));
("filename", forkdb_path)("version", version)("min", min_supported_version)("max", max_supported_version));

switch(version) {
case 1:
{
// ---------- pre-Savanna format. Just a single fork_database_l ----------------
in_use = in_use_t::legacy;
fork_db_l.open("legacy", fork_db_file, ds, validator);
fork_db_l.open("legacy", forkdb_path, ds, validator);
break;
}

Expand All @@ -706,13 +704,13 @@ namespace eosio::chain {
bool legacy_valid { false };
fc::raw::unpack( ds, legacy_valid );
if (legacy_valid) {
fork_db_l.open("legacy", fork_db_file, ds, validator);
fork_db_l.open("legacy", forkdb_path, ds, validator);
}

bool savanna_valid { false };
fc::raw::unpack( ds, savanna_valid );
if (savanna_valid) {
fork_db_s.open("savanna", fork_db_file, ds, validator);
fork_db_s.open("savanna", forkdb_path, ds, validator);
}
break;
}
Expand All @@ -721,8 +719,8 @@ namespace eosio::chain {
assert(0);
break;
}
} FC_CAPTURE_AND_RETHROW( (fork_db_file) );
std::filesystem::remove( fork_db_file );
} FC_CAPTURE_AND_RETHROW( (forkdb_path) );
std::filesystem::remove( forkdb_path );
}
}

Expand Down
4 changes: 2 additions & 2 deletions libraries/chain/include/eosio/chain/authorization_manager.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,9 @@ namespace eosio { namespace chain {

explicit authorization_manager(controller& c, chainbase::database& d);

void add_indices();
static void add_indices(chainbase::database& db);
void initialize_database();
void add_to_snapshot( const snapshot_writer_ptr& snapshot ) const;
static void add_to_snapshot( const snapshot_writer_ptr& snapshot, chainbase::database& db );
void read_from_snapshot( const snapshot_reader_ptr& snapshot );

const permission_object& create_permission( account_name account,
Expand Down
3 changes: 2 additions & 1 deletion libraries/chain/include/eosio/chain/chain_id_type.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,9 +41,10 @@ namespace chain {
static chain_id_type empty_chain_id() {
return {};
}
chain_id_type() = default;

private:
chain_id_type() = default;


// Some exceptions are unfortunately necessary:
template<typename T>
Expand Down
Loading
Loading