[3.2] Implement JSON Snapshot Reader

.gitmodules

@@ -18,6 +18,9 @@
 [submodule "libraries/eos-vm"]
 	path = libraries/eos-vm
 	url = https://github.com/eosnetworkfoundation/mandel-eos-vm
+[submodule "libraries/rapidjson"]
+	path = libraries/rapidjson
+	url = https://github.com/Tencent/rapidjson/
 [submodule "eosio-wasm-spec-tests"]
 	path = eosio-wasm-spec-tests
 	url = https://github.com/eosnetworkfoundation/mandel-wasm-spec-tests

libraries/CMakeLists.txt

@@ -20,6 +20,7 @@ add_subdirectory( chain )
 add_subdirectory( testing )
 add_subdirectory( version )
 add_subdirectory( state_history )
+add_subdirectory( rapidjson )
 
 set(USE_EXISTING_SOFTFLOAT ON CACHE BOOL "use pre-exisiting softfloat lib")
 set(ENABLE_TOOLS OFF CACHE BOOL "Build tools")

libraries/chain/include/eosio/chain/snapshot.hpp

@@ -5,6 +5,7 @@
 #include <fc/variant_object.hpp>
 #include <boost/core/demangle.hpp>
 #include <ostream>
+#include "../rapidjson/include/rapidjson/document.h"
 
 namespace eosio { namespace chain {
    /**
@@ -272,19 +273,13 @@ namespace eosio { namespace chain {
          read_section(detail::snapshot_section_traits<T>::section_name(), f);
       }
 
-      template<typename T>
-      bool has_section(const std::string& suffix = std::string()) {
-         return has_section(suffix + detail::snapshot_section_traits<T>::section_name());
-      }
-
       virtual void validate() const = 0;
 
       virtual void return_to_header() = 0;
 
       virtual ~snapshot_reader(){};
 
       protected:
-         virtual bool has_section( const std::string& section_name ) = 0;
          virtual void set_section( const std::string& section_name ) = 0;
          virtual bool read_row( detail::abstract_snapshot_row_reader& row_reader ) = 0;
          virtual bool empty( ) = 0;
@@ -313,7 +308,6 @@ namespace eosio { namespace chain {
          explicit variant_snapshot_reader(const fc::variant& snapshot);
 
          void validate() const override;
-         bool has_section( const string& section_name ) override;
          void set_section( const string& section_name ) override;
          bool read_row( detail::abstract_snapshot_row_reader& row_reader ) override;
          bool empty ( ) override;
@@ -365,7 +359,6 @@ namespace eosio { namespace chain {
          explicit istream_snapshot_reader(std::istream& snapshot);
 
          void validate() const override;
-         bool has_section( const string& section_name ) override;
          void set_section( const string& section_name ) override;
          bool read_row( detail::abstract_snapshot_row_reader& row_reader ) override;
          bool empty ( ) override;
@@ -381,6 +374,26 @@ namespace eosio { namespace chain {
          uint64_t       cur_row;
    };
 
+   class istream_json_snapshot_reader : public snapshot_reader {
+      public:
+         explicit istream_json_snapshot_reader(const fc::path& p);
+
+         void validate() const override;
+         void set_section( const string& section_name ) override;
+         bool read_row( detail::abstract_snapshot_row_reader& row_reader ) override;
+         bool empty ( ) override;
+         void clear_section() override;
+         void return_to_header() override;
+
+      private:
+         bool validate_section() const;
+
+         uint64_t            num_rows;
+         uint64_t            cur_row;
+         rapidjson::Document doc;
+         std::string         sec_name;
+   };
+
    class integrity_hash_snapshot_writer : public snapshot_writer {
       public:
          explicit integrity_hash_snapshot_writer(fc::sha256::encoder&  enc);

libraries/chain/snapshot.cpp

@@ -2,7 +2,9 @@
 #include <eosio/chain/exceptions.hpp>
 #include <fc/scoped_exit.hpp>
 #include <fc/io/json.hpp>
-
+#include "../rapidjson/include/rapidjson/filereadstream.h"
+#include "../rapidjson/include/rapidjson/stringbuffer.h"
+#include "../rapidjson/include/rapidjson/writer.h"
 namespace eosio { namespace chain {
 
 variant_snapshot_writer::variant_snapshot_writer(fc::mutable_variant_object& snapshot)
@@ -77,17 +79,6 @@ void variant_snapshot_reader::validate() const {
    }
 }
 
-bool variant_snapshot_reader::has_section( const string& section_name ) {
-   const auto& sections = snapshot["sections"].get_array();
-   for( const auto& section: sections ) {
-      if (section["name"].as_string() == section_name) {
-         return true;
-      }
-   }
-
-   return false;
-}
-
 void variant_snapshot_reader::set_section( const string& section_name ) {
    const auto& sections = snapshot["sections"].get_array();
    for( const auto& section: sections ) {
@@ -286,44 +277,6 @@ bool istream_snapshot_reader::validate_section() const {
    return true;
 }
 
-bool istream_snapshot_reader::has_section( const string& section_name ) {
-   auto restore_pos = fc::make_scoped_exit([this,pos=snapshot.tellg()](){
-      snapshot.seekg(pos);
-   });
-
-   const std::streamoff header_size = sizeof(ostream_snapshot_writer::magic_number) + sizeof(current_snapshot_version);
-
-   auto next_section_pos = header_pos + header_size;
-
-   while (true) {
-      snapshot.seekg(next_section_pos);
-      uint64_t section_size = 0;
-      snapshot.read((char*)&section_size,sizeof(section_size));
-      if (section_size == std::numeric_limits<uint64_t>::max()) {
-         break;
-      }
-
-      next_section_pos = snapshot.tellg() + std::streamoff(section_size);
-
-      uint64_t ignore = 0;
-      snapshot.read((char*)&ignore,sizeof(ignore));
-
-      bool match = true;
-      for(auto c : section_name) {
-         if(snapshot.get() != c) {
-            match = false;
-            break;
-         }
-      }
-
-      if (match && snapshot.get() == 0) {
-         return true;
-      }
-   }
-
-   return false;
-}
-
 void istream_snapshot_reader::set_section( const string& section_name ) {
    auto restore_pos = fc::make_scoped_exit([this,pos=snapshot.tellg()](){
       snapshot.seekg(pos);
@@ -386,6 +339,81 @@ void istream_snapshot_reader::return_to_header() {
    clear_section();
 }
 
+istream_json_snapshot_reader::istream_json_snapshot_reader(const fc::path& p)
+   :num_rows(0)
+   ,cur_row(0)
+{
+   FILE* fp = fopen(p.string().c_str(), "rb");
+   char readBuffer[65536];
+   rapidjson::FileReadStream is(fp, readBuffer, sizeof(readBuffer));
+   doc.ParseStream(is);
+   fclose(fp);
+}
+
+void istream_json_snapshot_reader::validate() const {
+   try {
+      // validate totem
+      auto expected_totem = ostream_json_snapshot_writer::magic_number;
+      decltype(expected_totem) actual_totem;
+      if (doc.HasMember("magic_number")) {
+         actual_totem = doc["magic_number"].GetInt();
+         EOS_ASSERT(actual_totem == expected_totem, snapshot_exception,
+                    "JSON snapshot has unexpected magic number!");
+      }
+
+      // validate version
+      auto expected_version = current_snapshot_version;
+      decltype(expected_version) actual_version;
+      if (doc.HasMember("version")) {
+         actual_version = doc["version"].GetInt();
+         EOS_ASSERT(actual_version == expected_version, snapshot_exception,
+                    "JSON snapshot is an unsuppored version.  Expected : ${expected}, Got: ${actual}",
+                    ("expected", expected_version)("actual", actual_version));
+      }
+   } catch( const std::exception& e ) {  \
+      snapshot_exception fce(FC_LOG_MESSAGE( warn, "JSON snapshot validation threw IO exception (${what})",("what",e.what())));
+      throw fce;
+   }
+}
+
+bool istream_json_snapshot_reader::validate_section() const {
+   return true;
+}
+
+void istream_json_snapshot_reader::set_section( const string& section_name ) {
+   if (doc.HasMember(section_name.c_str())) {
+      sec_name = section_name.c_str();
+      cur_row = 0;
+      num_rows = doc[section_name.c_str()].Size();
+   } else {
+      EOS_THROW(snapshot_exception, "JSON snapshot has no section named ${n}", ("n", section_name));
+   }
+}
+
+bool istream_json_snapshot_reader::read_row( detail::abstract_snapshot_row_reader& row_reader ) {
+   std::string row_name = "row_" + std::to_string(cur_row);
+   rapidjson::StringBuffer buffer;
+   rapidjson::Writer<rapidjson::StringBuffer> writer(buffer);
+   doc[sec_name.c_str()][row_name.c_str()].Accept(writer);
+   const auto& row = fc::json::from_string(buffer.GetString());
+   row_reader.provide(row);
+   return ++cur_row < num_rows;
+}
+
+bool istream_json_snapshot_reader::empty ( ) {
+   return num_rows == 0;
+}
+
+void istream_json_snapshot_reader::clear_section() {
+   num_rows = 0;
+   cur_row = 0;
+   sec_name = "";
+}
+
+void istream_json_snapshot_reader::return_to_header() {
+   clear_section();
+}
+
 integrity_hash_snapshot_writer::integrity_hash_snapshot_writer(fc::sha256::encoder& enc)
 :enc(enc)
 {

libraries/testing/include/eosio/testing/snapshot_suites.hpp

@@ -106,5 +106,62 @@ struct buffered_snapshot_suite {
    }
 };
 
-using snapshot_suites = boost::mpl::list<variant_snapshot_suite, buffered_snapshot_suite>;
+
+struct json_snapshot_suite {
+   using writer_t = ostream_json_snapshot_writer;
+   using reader_t = istream_json_snapshot_reader;
+   using write_storage_t = std::ostringstream;
+   using snapshot_t = std::string;
+   using read_storage_t = std::istringstream;
+
+   struct writer : public writer_t {
+      writer( const std::shared_ptr<write_storage_t>& storage )
+      :writer_t(*storage)
+      ,storage(storage)
+      {
+
+      }
+
+      std::shared_ptr<write_storage_t> storage;
+   };
+
+   struct reader : public reader_t {
+      explicit reader(const fc::path& p)
+      :reader_t(p)
+      {}
+      ~reader() {
+         remove("temp.bin.json");
+      }
+   };
+
+
+   static auto get_writer() {
+      return std::make_shared<writer>(std::make_shared<write_storage_t>());
+   }
+
+   static auto finalize(const std::shared_ptr<writer>& w) {
+      w->finalize();
+      return w->storage->str();
+   }
+
+   static auto get_reader( const snapshot_t& buffer) {
+      std::ofstream fs("temp.bin.json");
+      fs << buffer;
+      fs.close();
+      fc::path p("temp.bin.json");
+      return std::make_shared<reader>(p);
+   }
+
+   static snapshot_t load_from_file(const std::string& filename) {
+      snapshot_input_file<snapshot::json_snapshot> file(filename);
+      return file.read_as_string();
+   }
+
+   static void write_to_file( const std::string& basename, const snapshot_t& snapshot ) {
+      snapshot_output_file<snapshot::json_snapshot> file(basename);
+      file.write<snapshot_t>(snapshot);
+   }
+};
+
+using snapshot_suites = boost::mpl::list<variant_snapshot_suite, buffered_snapshot_suite, json_snapshot_suite>;
 

plugins/chain_plugin/chain_plugin.cpp

@@ -161,6 +161,7 @@ class chain_plugin_impl {
    std::optional<vm_type>            wasm_runtime;
    fc::microseconds                  abi_serializer_max_time_us;
    std::optional<bfs::path>          snapshot_path;
+   std::optional<bfs::path>          json_snapshot_path;
 
 
    // retained references to channels for easy publication
@@ -377,6 +378,7 @@ void chain_plugin::set_program_options(options_description& cli, options_descrip
          ("terminate-at-block", bpo::value<uint32_t>()->default_value(0),
           "terminate after reaching this block number (if set to a non-zero number)")
          ("snapshot", bpo::value<bfs::path>(), "File to read Snapshot State from")
+         ("json-snapshot", bpo::value<bfs::path>(), "JSON File to read Snapshot State from")
          ;
 
 }
@@ -941,30 +943,47 @@ void chain_plugin::plugin_initialize(const variables_map& options) {
          wlog( "The --truncate-at-block option can only be used with --hard-replay-blockchain." );
       }
 
-      if (options.count( "snapshot" )) {
-         my->snapshot_path = options.at( "snapshot" ).as<bfs::path>();
-         EOS_ASSERT( fc::exists(*my->snapshot_path), plugin_config_exception,
-                     "Cannot load snapshot, ${name} does not exist", ("name", my->snapshot_path->generic_string()) );
+      if (options.count( "snapshot" ) || options.count( "json-snapshot")) {
+         bfs::path snap_path;
+         std::string command;
+         bool is_json_snapshot = false;
+         if (options.count( "snapshot" )) {
+            snap_path = options.at( "snapshot" ).as<bfs::path>();
+            my->snapshot_path = snap_path;
+            command = "--snapshot";
+         } else {
+            snap_path = options.at( "json-snapshot" ).as<bfs::path>();
+            is_json_snapshot = true;
+            my->json_snapshot_path = snap_path;
+            command = "--json-snapshot";
+         }
+         EOS_ASSERT( fc::exists(snap_path), plugin_config_exception,
+                     "Cannot load snapshot, ${name} does not exist", ("name", snap_path.generic_string()) );
 
          // recover genesis information from the snapshot
          // used for validation code below
-         auto infile = std::ifstream(my->snapshot_path->generic_string(), (std::ios::in | std::ios::binary));
-         istream_snapshot_reader reader(infile);
-         reader.validate();
-         chain_id = controller::extract_chain_id(reader);
+         auto infile = std::ifstream(snap_path.generic_string(), (std::ios::in | std::ios::binary));
+         shared_ptr<snapshot_reader> reader_ptr;
+         if (is_json_snapshot) {
+            reader_ptr = shared_ptr<snapshot_reader>(new istream_json_snapshot_reader(snap_path));
+         } else {
+            reader_ptr = shared_ptr<snapshot_reader>(new istream_snapshot_reader(infile));
+         }
+         reader_ptr->validate();
+         chain_id = controller::extract_chain_id(*reader_ptr);
          infile.close();
 
          EOS_ASSERT( options.count( "genesis-timestamp" ) == 0,
-                 plugin_config_exception,
-                 "--snapshot is incompatible with --genesis-timestamp as the snapshot contains genesis information");
+                     plugin_config_exception,
+                     command + " is incompatible with --genesis-timestamp as the snapshot contains genesis information");
          EOS_ASSERT( options.count( "genesis-json" ) == 0,
                      plugin_config_exception,
-                     "--snapshot is incompatible with --genesis-json as the snapshot contains genesis information");
+                     command + " is incompatible with --genesis-json as the snapshot contains genesis information");
 
          auto shared_mem_path = my->chain_config->state_dir / "shared_memory.bin";
          EOS_ASSERT( !fc::is_regular_file(shared_mem_path),
                  plugin_config_exception,
-                 "Snapshot can only be used to initialize an empty database." );
+                 "A snapshot can only be used to initialize an empty database." );
 
          if( fc::is_regular_file( my->blocks_dir / "blocks.log" )) {
             auto block_log_genesis = block_log::extract_genesis_state(my->blocks_dir);
@@ -1309,6 +1328,9 @@ void chain_plugin::plugin_startup()
          auto reader = std::make_shared<istream_snapshot_reader>(infile);
          my->chain->startup(shutdown, check_shutdown, reader);
          infile.close();
+      } else if (my->json_snapshot_path) {
+         auto reader = std::make_shared<istream_json_snapshot_reader>(fc::path(*my->json_snapshot_path));
+         my->chain->startup(shutdown, check_shutdown, reader);
       } else if( my->genesis ) {
          my->chain->startup(shutdown, check_shutdown, *my->genesis);
       } else {