From efffdb2276ece22577334e3d1fc02ffafbf5806f Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Fri, 23 Aug 2024 12:17:18 -0500 Subject: [PATCH 1/3] Add new checksum datastream --- libraries/libfc/include/fc/io/cfile.hpp | 10 ++ .../libfc/include/fc/io/datastream_crc.hpp | 109 ++++++++++++++++++ 2 files changed, 119 insertions(+) create mode 100644 libraries/libfc/include/fc/io/datastream_crc.hpp diff --git a/libraries/libfc/include/fc/io/cfile.hpp b/libraries/libfc/include/fc/io/cfile.hpp index f3cd05ded1..5ca5c90461 100644 --- a/libraries/libfc/include/fc/io/cfile.hpp +++ b/libraries/libfc/include/fc/io/cfile.hpp @@ -329,6 +329,16 @@ class datastream : public fc::cfile { return true; } + bool read( char* d, size_t s ) { + cfile::read( d, s ); + return true; + } + + bool write(const char* d, size_t s) { + cfile::write(d, s); + return true; + } + fc::cfile& storage() { return *this; } const fc::cfile& storage() const { return *this; } diff --git a/libraries/libfc/include/fc/io/datastream_crc.hpp b/libraries/libfc/include/fc/io/datastream_crc.hpp new file mode 100644 index 0000000000..d447cc6379 --- /dev/null +++ b/libraries/libfc/include/fc/io/datastream_crc.hpp @@ -0,0 +1,109 @@ +#pragma once +#include +#include + +namespace fc { + +/** +* Provides a datasteam wrapper around another datastream for calculation of checksum. +* Example use: +* fc::datastream persist_cfile; +* fc::datastream_crc> persist_stream{persist_cfile}; +* +* persist_stream.seekp(0); +* fc::raw::pack(persist_stream, 'a'); +* uint32_t cs = persist_stream.check_sum(); +* fc::raw::pack(persist_stream, cs); // write checksum to file +* // ... +* persist_stream.seekp(0); +* char c; +* fc::raw::unpack(persist_stream, c); +* uint32_t calc_cs = persist_stream.check_sum(); +* uint32_t cs; +* fc::raw::unpack(persist_stream, cs); +* FC_ASSERT(calc_cs == cs, "checksum not equal"); +*/ +template +class datastream_crc { +public: + // ds must outlive datasteam_crc + explicit datastream_crc( DS& ds ) : ds_(ds) {} + + void skip(size_t s) { + ds_.skip(s); + } + + bool read(char* d, size_t s) { + bool r = ds_.read(d, s); + crc_.process_bytes(d, s); + return r; + } + + bool write(const char* d, size_t s) { + crc_.process_bytes(d, s); + return ds_.write(d, s); + } + + bool put(char c) { + crc_.process_byte(c); + return ds_.put(c); + } + + bool get(unsigned char& c) { + bool r = ds_.get(c); + crc_.process_byte(c); + return r; + } + + bool get(char& c) { + bool r = ds_.get(c); + crc_.process_byte(c); + return r; + } + + auto pos() const { + return ds_.pos(); + } + + bool valid() const { + return ds_.valid(); + } + + // only use with p==0, otherwise use seekp() below + bool seekp(size_t p) { + if (p == 0) { + crc_.reset(); + return ds_.seekp(0); + } + return false; + } + + size_t tellp() const { + return ds_.tellp(); + } + size_t remaining() const { + return ds_.remaining(); + } + + // extension to datastream + + bool seekp(size_t p, const CRC& crc) { + crc_ = crc; + return ds_.seekp(p); + } + + uint32_t checksum() const { + return crc_.checksum(); + } + + CRC crc() const { + return crc_; + } + +private: + DS& ds_; + CRC crc_; +}; + + +} // namespace fc From 7f8bd4e9fa1df194cc4618d3892b1ddd04ff5769 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Fri, 23 Aug 2024 12:32:37 -0500 Subject: [PATCH 2/3] GH-617 Add version 1 of safety file. Uses checksum to verify contents. --- libraries/chain/finality/finalizer.cpp | 108 +++++++++++++----- .../eosio/chain/finality/finalizer.hpp | 27 ++++- unittests/test-data/fsi/safety_v1.dat | Bin 0 -> 730 bytes 3 files changed, 100 insertions(+), 35 deletions(-) create mode 100644 unittests/test-data/fsi/safety_v1.dat diff --git a/libraries/chain/finality/finalizer.cpp b/libraries/chain/finality/finalizer.cpp index c88153accf..5aeaf95cb5 100644 --- a/libraries/chain/finality/finalizer.cpp +++ b/libraries/chain/finality/finalizer.cpp @@ -175,47 +175,80 @@ void my_finalizers_t::maybe_update_fsi(const block_state_ptr& bsp, const qc_t& r void my_finalizers_t::save_finalizer_safety_info() const { - if (!persist_file.is_open()) { + if (!cfile_ds.is_open()) { EOS_ASSERT(!persist_file_path.empty(), finalizer_safety_exception, "path for storing finalizer safety information file not specified"); if (!std::filesystem::exists(persist_file_path.parent_path())) std::filesystem::create_directories(persist_file_path.parent_path()); - persist_file.set_file_path(persist_file_path); - persist_file.open(fc::cfile::truncate_rw_mode); + cfile_ds.set_file_path(persist_file_path); + cfile_ds.open(fc::cfile::truncate_rw_mode); } try { - static_assert(sizeof(finalizer_safety_information) == 152, "If size changes then need to handle loading old files"); - static_assert(sizeof(decltype(bls_public_key{}.affine_non_montgomery_le())) == 96, - "If size changes then need to handle loading old files, fc::pack uses affine_non_montgomery_le()"); - - persist_file.seek(0); - fc::raw::pack(persist_file, fsi_t::magic); - fc::raw::pack(persist_file, current_safety_file_version); - fc::raw::pack(persist_file, (uint64_t)(finalizers.size() + inactive_safety_info.size())); - for (const auto& [pub_key, f] : finalizers) { - fc::raw::pack(persist_file, pub_key); - fc::raw::pack(persist_file, f.fsi); - } - if (!inactive_safety_info_written) { + // optimize by only calculating crc for inactive once + if (inactive_safety_info_written_pos == 0) { + persist_file.seekp(0); + fc::raw::pack(persist_file, fsi_t::magic); + fc::raw::pack(persist_file, current_safety_file_version); + uint64_t size = finalizers.size() + inactive_safety_info.size(); + fc::raw::pack(persist_file, size); + // save also the fsi that was originally present in the file, but which applied to // finalizers not configured anymore. for (const auto& [pub_key, fsi] : inactive_safety_info) { fc::raw::pack(persist_file, pub_key); fc::raw::pack(persist_file, fsi); } - inactive_safety_info_written = true; + inactive_safety_info_written_pos = persist_file.tellp(); + inactive_crc32 = persist_file.crc(); + } else { + persist_file.seekp(inactive_safety_info_written_pos, inactive_crc32); + } + + // active finalizers + for (const auto& [pub_key, f] : finalizers) { + fc::raw::pack(persist_file, pub_key); + fc::raw::pack(persist_file, f.fsi); } - persist_file.flush(); + + uint32_t cs = persist_file.checksum(); + fc::raw::pack(persist_file, cs); + + cfile_ds.flush(); } FC_LOG_AND_RETHROW() } // ---------------------------------------------------------------------------------------- + +void my_finalizers_t::load_finalizer_safety_info_v0(fsi_map& res) { + uint64_t num_finalizers {0}; + fc::raw::unpack(persist_file, num_finalizers); + for (size_t i=0; i #include #include +#include #include #include #include @@ -70,21 +71,34 @@ namespace eosio::chain { // ---------------------------------------------------------------------------------------- struct my_finalizers_t { public: - static constexpr uint64_t current_safety_file_version = 0; + /// + /// Version 0: Spring 1.0.0 RC2 - File has fixed packed sizes with inactive safety info written to the end + /// of the file. Consists of [finalizer public_key, FSI].. + /// Version 1: Spring 1.0.0 RC3 - Variable length FSI with inactive safety info written at the beginning of + /// the file. Uses crc32 checksum to verify data on read. + /// + static constexpr uint64_t safety_file_version_0 = 0; + static constexpr uint64_t safety_file_version_1 = 1; + static constexpr uint64_t current_safety_file_version = safety_file_version_1; using fsi_t = finalizer_safety_information; using fsi_map = std::map; using vote_t = std::tuple; private: + using persist_file_t = fc::datastream_crc>; + using finalizer_map_t = std::map; + const std::filesystem::path persist_file_path; // where we save the safety data std::atomic has_voted{false}; // true if this node has voted and updated safety info mutable std::mutex mtx; - mutable fc::datastream persist_file; // we want to keep the file open for speed - std::map finalizers; // the active finalizers for this node, loaded at startup, not mutated afterwards + mutable fc::datastream cfile_ds; // we want to keep the file open for speed + mutable persist_file_t persist_file{cfile_ds};// we want to calculate checksum + finalizer_map_t finalizers; // the active finalizers for this node, loaded at startup, not mutated afterwards fsi_map inactive_safety_info; // loaded at startup, not mutated afterwards fsi_t default_fsi = fsi_t::unset_fsi(); // default provided at spring startup - mutable bool inactive_safety_info_written{false}; + mutable long inactive_safety_info_written_pos{0}; + mutable boost::crc_32_type inactive_crc32; // cached value public: explicit my_finalizers_t(const std::filesystem::path& persist_file_path) @@ -167,6 +181,11 @@ namespace eosio::chain { // for testing purposes only, not thread safe const fsi_t& get_fsi(const bls_public_key& k) { return finalizers[k].fsi; } void set_fsi(const bls_public_key& k, const fsi_t& fsi) { finalizers[k].fsi = fsi; } + + private: + void load_finalizer_safety_info_v0(fsi_map& res); + void load_finalizer_safety_info_v1(fsi_map& res); + }; } diff --git a/unittests/test-data/fsi/safety_v1.dat b/unittests/test-data/fsi/safety_v1.dat new file mode 100644 index 0000000000000000000000000000000000000000..58c22860ca01910b4e2c6c0ff93e9f5fcfcac52d GIT binary patch literal 730 zcmV<00ww(s5&l{c5&l{M0000000003000000001BK*&7%09Avcbu^tsiuL%v3=VL< z21S2S8<(4L`i*-wxREd8jE2wRVwv{pLf*;|&mo?cG*2iW9s0lARXNOqWE@@f@R|ga z09pd!afyB%i%W7K1PqkD`~#Vy&EhH)$jSffwk@eD*inS5`3Zp!(<4EL$*6_*uMN|d z&D~l1%K!iX>R7=MV06WC!8xGPA-$MGZ3=Ja+e-4(hbnu8kRM|kys}18S>m4&quK>T z7%i$rQ!HjdcllxyJ>7Vbj*DU-$N&HUHw5nq%4(%e+eQL(CdF)I9FpIDy&ZL8VE71o zGoct?0AP|#F2yNb0s>tVg>DY$$EccbuBK~5dPQUJCYSLZ6H0I1u4;RL- zcLM(A;+Q|FfpITJb`G4vVACdLm`xTLdA}CT-SiJmZ08WL(}uq$4X3x?wCbK>d=7PG z=rJ8*!>fDv<5K=&I4sY5q?sTuELo8<)K2SH?-2t4000b(?^*zBw(fK)!NggT_Py)e zNgI4px;y}kR>a5J0e~WFWgw`rPO`hKmXVppF26Z>PGuQCODVAc6Lnwe&ANEUx&AgRcAyP%J M(hlVH03dkuwU{$fDgXcg literal 0 HcmV?d00001 From 5315ce679a5f3519552582a688c02c1f62d434c8 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Fri, 23 Aug 2024 13:03:45 -0500 Subject: [PATCH 3/3] Add Boost::crc --- libraries/libfc/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/libfc/CMakeLists.txt b/libraries/libfc/CMakeLists.txt index 18b1b2e7a2..c9c88f83a8 100644 --- a/libraries/libfc/CMakeLists.txt +++ b/libraries/libfc/CMakeLists.txt @@ -103,7 +103,7 @@ if(APPLE) find_library(corefoundation_framework CoreFoundation) endif() target_link_libraries( fc PUBLIC Boost::date_time Boost::chrono Boost::iostreams Boost::interprocess Boost::multi_index Boost::dll - Boost::multiprecision Boost::beast Boost::asio Boost::thread Threads::Threads + Boost::multiprecision Boost::beast Boost::asio Boost::thread Boost::crc Threads::Threads boringssl ZLIB::ZLIB ${PLATFORM_SPECIFIC_LIBS} ${CMAKE_DL_LIBS} secp256k1 bls12-381 ${security_framework} ${corefoundation_framework}) add_subdirectory( test )