Merge pull request paritytech#148 from subspace/remove-pre-genesis-ob…

…jects

Remove pre-genesis objects

crates/pallet-feeds/Cargo.toml

@@ -23,7 +23,7 @@ sp-std = { version = "4.0.0-dev", default-features = false, git = "https://githu
 subspace-core-primitives = { version = "0.1.0", default-features = false, path = "../subspace-core-primitives" }
 
 [dev-dependencies]
-serde = "1.0.127"
+serde = "1.0.130"
 sp-io = { version = "4.0.0-dev", default-features = false, git = "https://github.com/paritytech/substrate", rev = "26d69bcbe26f6b463e9374e1b1c54c3067fb6131" }
 sp-runtime = { version = "4.0.0-dev", default-features = false, git = "https://github.com/paritytech/substrate", rev = "26d69bcbe26f6b463e9374e1b1c54c3067fb6131" }
 

crates/pallet-object-store/Cargo.toml

@@ -23,7 +23,7 @@ sp-std = { version = "4.0.0-dev", default-features = false, git = "https://githu
 subspace-core-primitives = { version = "0.1.0", default-features = false, path = "../subspace-core-primitives" }
 
 [dev-dependencies]
-serde = "1.0.127"
+serde = "1.0.130"
 sp-core = { version = "4.0.0-dev", git = "https://github.com/paritytech/substrate", rev = "26d69bcbe26f6b463e9374e1b1c54c3067fb6131" }
 sp-io = { version = "4.0.0-dev", git = "https://github.com/paritytech/substrate", rev = "26d69bcbe26f6b463e9374e1b1c54c3067fb6131" }
 sp-runtime = { version = "4.0.0-dev", git = "https://github.com/paritytech/substrate", rev = "26d69bcbe26f6b463e9374e1b1c54c3067fb6131" }

crates/pallet-subspace/src/lib.rs

@@ -199,26 +199,6 @@ mod pallet {
         #[pallet::constant]
         type RecordedHistorySegmentSize: Get<u32>;
 
-        // TODO: This is not the only way to bootstrap the network, we might want to use multiple
-        //  objects from file system instead (like Bitcoin history)
-        /// This constant defines the size (in bytes) of one pre-genesis object.
-        ///
-        /// Pre-genesis objects are needed to solve chicken-and-egg problem where in order to
-        /// produce a solution (and farm a block) a plot is needed, but plot is supposed to contain
-        /// some archived blocks. Pre-genesis history allows network bootstrapping.
-        #[pallet::constant]
-        type PreGenesisObjectSize: Get<u32>;
-
-        /// This constant defines the number of a pre-genesis objects that will bootstrap the
-        /// history.
-        #[pallet::constant]
-        type PreGenesisObjectCount: Get<u32>;
-
-        /// This constant defines the seed used for deriving pre-genesis objects that will bootstrap
-        /// the history.
-        #[pallet::constant]
-        type PreGenesisObjectSeed: Get<&'static [u8]>;
-
         /// Subspace requires some logic to be triggered on every block to query for whether an epoch
         /// has ended and to perform the transition to the next epoch.
         ///

crates/pallet-subspace/src/mock.rs

@@ -166,9 +166,6 @@ impl Config for Test {
     type ConfirmationDepthK = ConfirmationDepthK;
     type RecordSize = RecordSize;
     type RecordedHistorySegmentSize = RecordedHistorySegmentSize;
-    type PreGenesisObjectSize = PreGenesisObjectSize;
-    type PreGenesisObjectCount = PreGenesisObjectCount;
-    type PreGenesisObjectSeed = PreGenesisObjectSeed;
     type EpochChangeTrigger = NormalEpochChange;
     type EraChangeTrigger = NormalEraChange;
     type EonChangeTrigger = NormalEonChange;

crates/sc-consensus-subspace-rpc/src/lib.rs

@@ -194,12 +194,6 @@ where
                     record_size: runtime_api.record_size(&best_block_hash)?,
                     recorded_history_segment_size: runtime_api
                         .recorded_history_segment_size(&best_block_hash)?,
-                    pre_genesis_object_size: runtime_api
-                        .pre_genesis_object_size(&best_block_hash)?,
-                    pre_genesis_object_count: runtime_api
-                        .pre_genesis_object_count(&best_block_hash)?,
-                    pre_genesis_object_seed: runtime_api
-                        .pre_genesis_object_seed(&best_block_hash)?,
                 }
             };
 

crates/sc-consensus-subspace/Cargo.toml

@@ -31,7 +31,7 @@ sc-telemetry = { version = "4.0.0-dev", git = "https://github.com/paritytech/sub
 sc-client-api = { version = "4.0.0-dev", git = "https://github.com/paritytech/substrate", rev = "26d69bcbe26f6b463e9374e1b1c54c3067fb6131" }
 sc-consensus-epochs = { version = "0.10.0-dev", git = "https://github.com/paritytech/substrate", rev = "26d69bcbe26f6b463e9374e1b1c54c3067fb6131" }
 sc-utils = { version = "4.0.0-dev", git = "https://github.com/paritytech/substrate", rev = "26d69bcbe26f6b463e9374e1b1c54c3067fb6131" }
-serde = { version = "1.0.127", features = ["derive"] }
+serde = { version = "1.0.130", features = ["derive"] }
 sp-api = { version = "4.0.0-dev", git = "https://github.com/paritytech/substrate", rev = "26d69bcbe26f6b463e9374e1b1c54c3067fb6131" }
 sp-blockchain = { version = "4.0.0-dev", git = "https://github.com/paritytech/substrate", rev = "26d69bcbe26f6b463e9374e1b1c54c3067fb6131" }
 sp-block-builder = { version = "4.0.0-dev", git = "https://github.com/paritytech/substrate", rev = "26d69bcbe26f6b463e9374e1b1c54c3067fb6131" }

crates/sc-consensus-subspace/src/archiver.rs

@@ -25,8 +25,7 @@ use sp_consensus_subspace::SubspaceApi;
 use sp_runtime::generic::BlockId;
 use sp_runtime::traits::{Block as BlockT, CheckedSub, Header, One, Saturating, Zero};
 use std::sync::Arc;
-use subspace_archiving::archiver::{ArchivedSegment, BlockArchiver, ObjectArchiver};
-use subspace_archiving::pre_genesis_data;
+use subspace_archiving::archiver::{ArchivedSegment, Archiver};
 use subspace_core_primitives::RootBlock;
 
 fn find_last_root_block<Block: BlockT, Client>(client: &Client) -> Option<RootBlock>
@@ -101,15 +100,7 @@ pub fn start_subspace_archiver<Block: BlockT, Client>(
         .recorded_history_segment_size(&genesis_block_id)
         .expect("Failed to get `recorded_history_segment_size` from runtime API");
 
-    let maybe_last_root_block = find_last_root_block(client.as_ref()).and_then(|last_root_block| {
-        // At least one non-genesis block needs to be archived for restarts
-        // TODO: This check can be removed when we no longer have pre-genesis objects
-        if last_root_block.last_archived_block().number == 0 {
-            None
-        } else {
-            Some(last_root_block)
-        }
-    });
+    let maybe_last_root_block = find_last_root_block(client.as_ref());
 
     let mut archiver = if let Some(last_root_block) = maybe_last_root_block {
         // Continuing from existing initial state
@@ -121,8 +112,8 @@ pub fn start_subspace_archiver<Block: BlockT, Client>(
         );
         let last_archived_block = client
             .block(&BlockId::Number(last_archived_block_number.into()))
-            .expect("Older blocks should always exist")
-            .expect("Older blocks should always exist");
+            .expect("Older blocks must always exist")
+            .expect("Older blocks must always exist");
 
         let block_object_mapping = client
             .runtime_api()
@@ -132,7 +123,7 @@ pub fn start_subspace_archiver<Block: BlockT, Client>(
             )
             .expect("Must be able to make runtime call");
 
-        BlockArchiver::with_initial_state(
+        Archiver::with_initial_state(
             record_size as usize,
             recorded_history_segment_size as usize,
             last_root_block,
@@ -141,50 +132,10 @@ pub fn start_subspace_archiver<Block: BlockT, Client>(
         )
         .expect("Incorrect parameters for archiver")
     } else {
-        // Starting from genesis
-        let runtime_api = client.runtime_api();
-
-        let mut object_archiver =
-            ObjectArchiver::new(record_size as usize, recorded_history_segment_size as usize)
-                .expect("Incorrect parameters for archiver");
-
-        let pre_genesis_object_size = runtime_api
-            .pre_genesis_object_size(&genesis_block_id)
-            .expect("Failed to get `pre_genesis_object_size` from runtime API");
-        let pre_genesis_object_count = runtime_api
-            .pre_genesis_object_count(&genesis_block_id)
-            .expect("Failed to get `pre_genesis_object_count` from runtime API");
-        let pre_genesis_object_seed = runtime_api
-            .pre_genesis_object_seed(&genesis_block_id)
-            .expect("Failed to get `pre_genesis_object_seed` from runtime API");
-
-        // These archived segments are a part of the public parameters of network setup, thus do not
-        // need to be sent to farmers
-        info!(target: "subspace", "Processing pre-genesis objects");
-        let new_root_blocks: Vec<RootBlock> = (0..pre_genesis_object_count)
-            .map(|index| {
-                object_archiver
-                    .add_object(pre_genesis_data::from_seed(
-                        &pre_genesis_object_seed,
-                        index,
-                        pre_genesis_object_size,
-                    ))
-                    .into_iter()
-            })
-            .flatten()
-            .map(|archived_segment| archived_segment.root_block)
-            .collect();
-
-        // Set list of expected root blocks for the next block after genesis (we can't have
-        // extrinsics in genesis block, at least not right now)
-        subspace_link
-            .root_blocks
-            .lock()
-            .put(One::one(), new_root_blocks);
+        info!(target: "subspace", "Starting archiving from genesis");
 
-        info!(target: "subspace", "Finished processing pre-genesis objects");
-
-        object_archiver.into_block_archiver()
+        Archiver::new(record_size as usize, recorded_history_segment_size as usize)
+            .expect("Incorrect parameters for archiver")
     };
 
     // Process blocks since last fully archived block (or genesis) up to the current head minus K
@@ -201,7 +152,15 @@ pub fn start_subspace_archiver<Block: BlockT, Client>(
                     best_number,
                 );
             })
-            .checked_sub(confirmation_depth_k);
+            .checked_sub(confirmation_depth_k)
+            .or_else(|| {
+                if maybe_last_root_block.is_none() {
+                    // If not continuation, archive genesis block
+                    Some(0)
+                } else {
+                    None
+                }
+            });
 
         if let Some(blocks_to_archive_to) = blocks_to_archive_to {
             info!(
@@ -244,7 +203,13 @@ pub fn start_subspace_archiver<Block: BlockT, Client>(
                     // Set list of expected root blocks for the block where we expect root block
                     // extrinsic to be included
                     subspace_link.root_blocks.lock().put(
-                        (block_to_archive + confirmation_depth_k + 1).into(),
+                        if block_to_archive.is_zero() {
+                            // Special case for genesis block whose root block should be included in
+                            // the first block in order for further validation to work properly.
+                            One::one()
+                        } else {
+                            (block_to_archive + confirmation_depth_k + 1).into()
+                        },
                         new_root_blocks,
                     );
                 }

crates/sc-consensus-subspace/src/authorship.rs

@@ -232,8 +232,8 @@ where
             .records_root(&parent_block_id, segment_index)
             .ok()?;
 
-        // TODO: This is not a very nice hack due to the fact that at the time first block is
-        //  produced extrinsics with root blocks are not yet in runtime
+        // This is not a very nice hack due to the fact that at the time first block is produced
+        // extrinsics with root blocks are not yet in runtime.
         if maybe_records_root.is_none() && parent_header.number().is_zero() {
             maybe_records_root = worker.subspace_link.root_blocks.lock().iter().find_map(
                 |(_block_number, root_blocks)| {

crates/sc-consensus-subspace/src/lib.rs

@@ -1033,8 +1033,8 @@ where
                     );
                 });
 
-            // TODO: This is not a very nice hack due to the fact that at the time first block is
-            //  produced extrinsics with root blocks are not yet in runtime
+            // This is not a very nice hack due to the fact that at the time first block is produced
+            // extrinsics with root blocks are not yet in runtime.
             if maybe_records_root.is_none() && block.header.number().is_one() {
                 maybe_records_root =
                     self.root_blocks

crates/sc-consensus-subspace/src/tests.rs

@@ -30,7 +30,7 @@ use parking_lot::Mutex;
 use rand::prelude::*;
 use sc_block_builder::{BlockBuilder, BlockBuilderProvider};
 use sc_client_api::backend::TransactionFor;
-use sc_client_api::BlockchainEvents;
+use sc_client_api::{BlockBackend, BlockchainEvents};
 use sc_consensus::block_import::ForkChoiceStrategy;
 use sc_consensus::{
     BlockCheckParams, BlockImport, BlockImportParams, BoxBlockImport, BoxJustificationImport,
@@ -71,8 +71,8 @@ use std::future::Future;
 use std::pin::Pin;
 use std::sync::Arc;
 use std::{cell::RefCell, task::Poll, time::Duration};
-use subspace_archiving::archiver::ObjectArchiver;
-use subspace_archiving::pre_genesis_data;
+use subspace_archiving::archiver::Archiver;
+use subspace_core_primitives::objects::BlockObjectMapping;
 use subspace_core_primitives::{LocalChallenge, Piece, Signature, Tag};
 use subspace_solving::{SubspaceCodec, SOLUTION_SIGNING_CONTEXT};
 use substrate_test_runtime::{Block as TestBlock, Hash};
@@ -473,33 +473,15 @@ fn get_archived_pieces(client: &TestClient) -> Vec<Piece> {
     let recorded_history_segment_size = runtime_api
         .recorded_history_segment_size(&genesis_block_id)
         .unwrap();
-    let pre_genesis_object_size = runtime_api
-        .pre_genesis_object_size(&genesis_block_id)
-        .unwrap();
-    let pre_genesis_object_count = runtime_api
-        .pre_genesis_object_count(&genesis_block_id)
-        .unwrap();
-    let pre_genesis_object_seed = runtime_api
-        .pre_genesis_object_seed(&genesis_block_id)
-        .unwrap();
 
-    let mut object_archiver =
-        ObjectArchiver::new(record_size as usize, recorded_history_segment_size as usize)
-            .expect("Incorrect parameters for archiver");
-
-    (0..pre_genesis_object_count)
-        .map(|index| {
-            object_archiver
-                .add_object(pre_genesis_data::from_seed(
-                    &pre_genesis_object_seed,
-                    index,
-                    pre_genesis_object_size,
-                ))
-                .into_iter()
-        })
-        .flatten()
-        .map(|archived_segment| archived_segment.pieces.into_iter())
-        .flatten()
+    let mut archiver = Archiver::new(record_size as usize, recorded_history_segment_size as usize)
+        .expect("Incorrect parameters for archiver");
+
+    let genesis_block = client.block(&genesis_block_id).unwrap().unwrap();
+    archiver
+        .add_block(genesis_block.encode(), BlockObjectMapping::default())
+        .into_iter()
+        .flat_map(|archived_segment| archived_segment.pieces.into_iter())
         .collect()
 }
 

crates/sp-consensus-subspace/Cargo.toml

@@ -16,7 +16,7 @@ targets = ["x86_64-unknown-linux-gnu"]
 async-trait = { version = "0.1.51", optional = true }
 codec = { package = "parity-scale-codec", version = "2.3.0", default-features = false }
 scale-info = { version = "1.0", default-features = false, features = ["derive"] }
-serde = { version = "1.0.127", features = ["derive"], optional = true }
+serde = { version = "1.0.130", features = ["derive"], optional = true }
 sp-api = { version = "4.0.0-dev", default-features = false, git = "https://github.com/paritytech/substrate", rev = "26d69bcbe26f6b463e9374e1b1c54c3067fb6131" }
 sp-application-crypto = { version = "4.0.0-dev", default-features = false, git = "https://github.com/paritytech/substrate", rev = "26d69bcbe26f6b463e9374e1b1c54c3067fb6131" }
 sp-consensus = { version = "0.10.0-dev", optional = true, git = "https://github.com/paritytech/substrate", rev = "26d69bcbe26f6b463e9374e1b1c54c3067fb6131" }

crates/sp-consensus-subspace/src/lib.rs

@@ -255,17 +255,6 @@ sp_api::decl_runtime_apis! {
         /// Recorded history is encoded and plotted in segments of this size (in bytes).
         fn recorded_history_segment_size() -> u32;
 
-        /// This constant defines the size (in bytes) of one pre-genesis object.
-        fn pre_genesis_object_size() -> u32;
-
-        /// This constant defines the number of a pre-genesis objects that will bootstrap the
-        /// history.
-        fn pre_genesis_object_count() -> u32;
-
-        /// This constant defines the seed used for deriving pre-genesis objects that will bootstrap
-        /// the history.
-        fn pre_genesis_object_seed() -> Vec<u8>;
-
         /// Return the genesis configuration for Subspace. The configuration is only read on genesis.
         fn configuration() -> SubspaceGenesisConfiguration;