From 8210dae2d4092e4637a5a423c25d8ed5facfc4fe Mon Sep 17 00:00:00 2001 From: Nazar Mokrynskyi Date: Tue, 23 Nov 2021 21:08:43 +0200 Subject: [PATCH 1/6] Customize new block creation to inject digest item into genesis block to expand it in size and allow plotting without pre-genesis objects in the future --- Cargo.lock | 5 ++ crates/pallet-feeds/Cargo.toml | 2 +- crates/pallet-object-store/Cargo.toml | 2 +- crates/sc-consensus-subspace/Cargo.toml | 2 +- crates/sp-consensus-subspace/Cargo.toml | 2 +- crates/subspace-farmer/Cargo.toml | 2 +- crates/subspace-rpc-primitives/Cargo.toml | 2 +- crates/subspace-runtime-primitives/Cargo.toml | 10 +++ crates/subspace-runtime-primitives/src/lib.rs | 84 +++++++++++++++++-- crates/subspace-runtime/src/lib.rs | 43 +++------- substrate/substrate-test-runtime/Cargo.toml | 2 +- 11 files changed, 111 insertions(+), 45 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 05994631c4b16..c0b3bca92c68c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7417,8 +7417,13 @@ dependencies = [ name = "subspace-runtime-primitives" version = "0.1.0" dependencies = [ + "parity-scale-codec", + "parity-util-mem", + "serde", "sp-core", "sp-runtime", + "sp-std", + "subspace-core-primitives", ] [[package]] diff --git a/crates/pallet-feeds/Cargo.toml b/crates/pallet-feeds/Cargo.toml index 0af2b82fe4d3c..96accd9ff5556 100644 --- a/crates/pallet-feeds/Cargo.toml +++ b/crates/pallet-feeds/Cargo.toml @@ -23,7 +23,7 @@ sp-std = { version = "4.0.0-dev", default-features = false, git = "https://githu subspace-core-primitives = { version = "0.1.0", default-features = false, path = "../subspace-core-primitives" } [dev-dependencies] -serde = "1.0.127" +serde = "1.0.130" sp-io = { version = "4.0.0-dev", default-features = false, git = "https://github.com/paritytech/substrate", rev = "26d69bcbe26f6b463e9374e1b1c54c3067fb6131" } sp-runtime = { version = "4.0.0-dev", default-features = false, git = "https://github.com/paritytech/substrate", rev = "26d69bcbe26f6b463e9374e1b1c54c3067fb6131" } diff --git a/crates/pallet-object-store/Cargo.toml b/crates/pallet-object-store/Cargo.toml index 3a149c7465b68..537f8b1dc7c12 100644 --- a/crates/pallet-object-store/Cargo.toml +++ b/crates/pallet-object-store/Cargo.toml @@ -23,7 +23,7 @@ sp-std = { version = "4.0.0-dev", default-features = false, git = "https://githu subspace-core-primitives = { version = "0.1.0", default-features = false, path = "../subspace-core-primitives" } [dev-dependencies] -serde = "1.0.127" +serde = "1.0.130" sp-core = { version = "4.0.0-dev", git = "https://github.com/paritytech/substrate", rev = "26d69bcbe26f6b463e9374e1b1c54c3067fb6131" } sp-io = { version = "4.0.0-dev", git = "https://github.com/paritytech/substrate", rev = "26d69bcbe26f6b463e9374e1b1c54c3067fb6131" } sp-runtime = { version = "4.0.0-dev", git = "https://github.com/paritytech/substrate", rev = "26d69bcbe26f6b463e9374e1b1c54c3067fb6131" } diff --git a/crates/sc-consensus-subspace/Cargo.toml b/crates/sc-consensus-subspace/Cargo.toml index 9ee762c4d91b3..8e663b46d392d 100644 --- a/crates/sc-consensus-subspace/Cargo.toml +++ b/crates/sc-consensus-subspace/Cargo.toml @@ -31,7 +31,7 @@ sc-telemetry = { version = "4.0.0-dev", git = "https://github.com/paritytech/sub sc-client-api = { version = "4.0.0-dev", git = "https://github.com/paritytech/substrate", rev = "26d69bcbe26f6b463e9374e1b1c54c3067fb6131" } sc-consensus-epochs = { version = "0.10.0-dev", git = "https://github.com/paritytech/substrate", rev = "26d69bcbe26f6b463e9374e1b1c54c3067fb6131" } sc-utils = { version = "4.0.0-dev", git = "https://github.com/paritytech/substrate", rev = "26d69bcbe26f6b463e9374e1b1c54c3067fb6131" } -serde = { version = "1.0.127", features = ["derive"] } +serde = { version = "1.0.130", features = ["derive"] } sp-api = { version = "4.0.0-dev", git = "https://github.com/paritytech/substrate", rev = "26d69bcbe26f6b463e9374e1b1c54c3067fb6131" } sp-blockchain = { version = "4.0.0-dev", git = "https://github.com/paritytech/substrate", rev = "26d69bcbe26f6b463e9374e1b1c54c3067fb6131" } sp-block-builder = { version = "4.0.0-dev", git = "https://github.com/paritytech/substrate", rev = "26d69bcbe26f6b463e9374e1b1c54c3067fb6131" } diff --git a/crates/sp-consensus-subspace/Cargo.toml b/crates/sp-consensus-subspace/Cargo.toml index 01235c60701f9..399b9e6a5e112 100644 --- a/crates/sp-consensus-subspace/Cargo.toml +++ b/crates/sp-consensus-subspace/Cargo.toml @@ -16,7 +16,7 @@ targets = ["x86_64-unknown-linux-gnu"] async-trait = { version = "0.1.51", optional = true } codec = { package = "parity-scale-codec", version = "2.3.0", default-features = false } scale-info = { version = "1.0", default-features = false, features = ["derive"] } -serde = { version = "1.0.127", features = ["derive"], optional = true } +serde = { version = "1.0.130", features = ["derive"], optional = true } sp-api = { version = "4.0.0-dev", default-features = false, git = "https://github.com/paritytech/substrate", rev = "26d69bcbe26f6b463e9374e1b1c54c3067fb6131" } sp-application-crypto = { version = "4.0.0-dev", default-features = false, git = "https://github.com/paritytech/substrate", rev = "26d69bcbe26f6b463e9374e1b1c54c3067fb6131" } sp-consensus = { version = "0.10.0-dev", optional = true, git = "https://github.com/paritytech/substrate", rev = "26d69bcbe26f6b463e9374e1b1c54c3067fb6131" } diff --git a/crates/subspace-farmer/Cargo.toml b/crates/subspace-farmer/Cargo.toml index dbe85dd9f833c..78d8ef5d75406 100644 --- a/crates/subspace-farmer/Cargo.toml +++ b/crates/subspace-farmer/Cargo.toml @@ -30,7 +30,7 @@ lru = "0.6.6" parity-scale-codec = "2.3.0" rayon = "1.5.0" schnorrkel = "0.9.1" -serde = { version = "1.0.125", features = ["derive"] } +serde = { version = "1.0.130", features = ["derive"] } serde_json = "1.0.64" subspace-archiving = { version = "0.1.0", path = "../subspace-archiving" } subspace-solving = { version = "0.1.0", path = "../subspace-solving" } diff --git a/crates/subspace-rpc-primitives/Cargo.toml b/crates/subspace-rpc-primitives/Cargo.toml index f695d0e2175dd..eba84bedbf506 100644 --- a/crates/subspace-rpc-primitives/Cargo.toml +++ b/crates/subspace-rpc-primitives/Cargo.toml @@ -14,5 +14,5 @@ include = [ [dependencies] hex-buffer-serde = "0.3.0" -serde = { version = "1.0.125", features = ["derive"] } +serde = { version = "1.0.130", features = ["derive"] } subspace-core-primitives = { version = "0.1.0", path = "../subspace-core-primitives" } diff --git a/crates/subspace-runtime-primitives/Cargo.toml b/crates/subspace-runtime-primitives/Cargo.toml index 7d7305fca11d1..ed79d93537cd3 100644 --- a/crates/subspace-runtime-primitives/Cargo.toml +++ b/crates/subspace-runtime-primitives/Cargo.toml @@ -16,12 +16,22 @@ include = [ targets = ["x86_64-unknown-linux-gnu"] [dependencies] +parity-scale-codec = { version = "2.3.0", default-features = false, features = ["derive"] } +parity-util-mem = { version = "0.10.0", optional = true, default-features = false, features = ["primitive-types"] } +serde = { version = "1.0.130", optional = true, features = ["derive"] } sp-core = { version = "4.0.0-dev", default-features = false, git = "https://github.com/paritytech/substrate", rev = "26d69bcbe26f6b463e9374e1b1c54c3067fb6131" } sp-runtime = { version = "4.0.0-dev", default-features = false, git = "https://github.com/paritytech/substrate", rev = "26d69bcbe26f6b463e9374e1b1c54c3067fb6131" } +sp-std = { version = "4.0.0-dev", default-features = false, git = "https://github.com/paritytech/substrate", rev = "26d69bcbe26f6b463e9374e1b1c54c3067fb6131" } +subspace-core-primitives = { version = "0.1.0", default-features = false, path = "../subspace-core-primitives" } [features] default = ["std"] std = [ + "parity-scale-codec/std", + "parity-util-mem/std", + "serde/std", "sp-core/std", "sp-runtime/std", + "sp-std/std", + "subspace-core-primitives/std", ] diff --git a/crates/subspace-runtime-primitives/src/lib.rs b/crates/subspace-runtime-primitives/src/lib.rs index 00ec7255b459f..16562c6cebafc 100644 --- a/crates/subspace-runtime-primitives/src/lib.rs +++ b/crates/subspace-runtime-primitives/src/lib.rs @@ -17,9 +17,35 @@ //! Runtime primitives for Subspace Network. #![cfg_attr(not(feature = "std"), no_std)] +#![feature(int_log)] use sp_runtime::traits::{IdentifyAccount, Verify}; use sp_runtime::MultiSignature; +use subspace_core_primitives::{PIECE_SIZE, SHA256_HASH_SIZE}; + +// TODO: Proper value here +pub const CONFIRMATION_DEPTH_K: u32 = 100; +/// 128 data records and 128 parity records (as a result of erasure coding) together form a perfect +/// Merkle Tree and will result in witness size of `log2(MERKLE_NUM_LEAVES) * SHA256_HASH_SIZE`. +/// +/// This number is a tradeoff: +/// * as this number goes up, fewer [`RootBlock`]s are required to be stored for verifying archival +/// history of the network, which makes sync quicker and more efficient, but also more data in +/// each [`Piece`] will be occupied with witness, thus wasting space that otherwise could have +/// been used for storing data (record part of a Piece) +/// * as this number goes down, witness get smaller leading to better piece utilization, but the +/// number of root blocks goes up making sync less efficient and less records are needed to be +/// lost before part of the archived history become unrecoverable, reducing reliability of the +/// data stored on the network +const MERKLE_NUM_LEAVES: u32 = 256; +/// Size of witness for a segment record (in bytes). +const WITNESS_SIZE: u32 = SHA256_HASH_SIZE as u32 * MERKLE_NUM_LEAVES.log2(); +/// Size of a segment record given the global piece size (in bytes). +pub const RECORD_SIZE: u32 = PIECE_SIZE as u32 - WITNESS_SIZE; +/// Recorded History Segment Size includes half of the records (just data records) that will later +/// be erasure coded and together with corresponding witnesses will result in `MERKLE_NUM_LEAVES` +/// pieces of archival history. +pub const RECORDED_HISTORY_SEGMENT_SIZE: u32 = RECORD_SIZE * MERKLE_NUM_LEAVES / 2; /// An index to a block. pub type BlockNumber = u32; @@ -48,15 +74,63 @@ pub type Moment = u64; /// of data like extrinsics, allowing for them to continue syncing the network through upgrades /// to even the core data structures. pub mod opaque { - use super::BlockNumber; - use sp_runtime::generic; - use sp_runtime::traits::BlakeTwo256; - pub use sp_runtime::OpaqueExtrinsic as UncheckedExtrinsic; + use super::{BlockNumber, RECORDED_HISTORY_SEGMENT_SIZE}; + use parity_scale_codec::{Decode, Encode}; + #[cfg(feature = "std")] + use serde::{Deserialize, Serialize}; + use sp_core::RuntimeDebug; + use sp_runtime::traits::{BlakeTwo256, Block as BlockT, Header as HeaderT}; + use sp_runtime::{generic, DigestItem, OpaqueExtrinsic}; + use sp_std::prelude::*; /// Opaque block header type. pub type Header = generic::Header; /// Opaque block type. - pub type Block = generic::Block; + + /// Abstraction over a substrate block. + #[derive(PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug)] + #[cfg_attr( + feature = "std", + derive(Serialize, Deserialize, parity_util_mem::MallocSizeOf) + )] + #[cfg_attr(feature = "std", serde(rename_all = "camelCase"))] + #[cfg_attr(feature = "std", serde(deny_unknown_fields))] + pub struct Block { + /// The block header. + pub header: Header, + /// The accompanying extrinsics. + pub extrinsics: Vec, + } + + impl BlockT for Block { + type Extrinsic = OpaqueExtrinsic; + type Header = Header; + type Hash =
::Hash; + + fn header(&self) -> &Self::Header { + &self.header + } + fn extrinsics(&self) -> &[Self::Extrinsic] { + &self.extrinsics[..] + } + fn deconstruct(self) -> (Self::Header, Vec) { + (self.header, self.extrinsics) + } + fn new(mut header: Self::Header, extrinsics: Vec) -> Self { + if header.number == 0 { + // We fill genesis block with extra data such that the very first archived segment + // can be produced right away, bootstrapping the farming process. + let ballast = vec![0; RECORDED_HISTORY_SEGMENT_SIZE as usize]; + header.digest.logs.push(DigestItem::Other(ballast)); + Block { header, extrinsics } + } else { + Block { header, extrinsics } + } + } + fn encode_from(header: &Self::Header, extrinsics: &[Self::Extrinsic]) -> Vec { + (header, extrinsics).encode() + } + } /// Opaque block identifier type. pub type BlockId = generic::BlockId; } diff --git a/crates/subspace-runtime/src/lib.rs b/crates/subspace-runtime/src/lib.rs index 673f6f3fc4af4..674c244478d33 100644 --- a/crates/subspace-runtime/src/lib.rs +++ b/crates/subspace-runtime/src/lib.rs @@ -17,7 +17,6 @@ #![cfg_attr(not(feature = "std"), no_std)] // `construct_runtime!` does a lot of recursion and requires us to increase the limit to 256. #![recursion_limit = "256"] -#![feature(int_log)] // Make the WASM binary available. #[cfg(feature = "std")] @@ -47,8 +46,11 @@ use sp_std::prelude::*; use sp_version::NativeVersion; use sp_version::RuntimeVersion; use subspace_core_primitives::objects::{BlockObject, BlockObjectMapping}; -use subspace_core_primitives::{RootBlock, Sha256Hash, PIECE_SIZE, SHA256_HASH_SIZE}; -pub use subspace_runtime_primitives::*; +use subspace_core_primitives::{RootBlock, Sha256Hash, PIECE_SIZE}; +pub use subspace_runtime_primitives::{ + opaque, AccountId, Balance, BlockNumber, Hash, Index, Moment, Signature, CONFIRMATION_DEPTH_K, + RECORDED_HISTORY_SEGMENT_SIZE, RECORD_SIZE, +}; sp_runtime::impl_opaque_keys! { pub struct SessionKeys { @@ -131,39 +133,14 @@ pub const SUBSPACE_GENESIS_EPOCH_CONFIG: sp_consensus_subspace::SubspaceEpochCon c: SLOT_PROBABILITY, }; -// TODO: Proper value here -const CONFIRMATION_DEPTH_K: u32 = 100; -/// 128 data records and 128 parity records (as a result of erasure coding) together form a perfect -/// Merkle Tree and will result in witness size of `log2(MERKLE_NUM_LEAVES) * SHA256_HASH_SIZE`. -/// -/// This number is a tradeoff: -/// * as this number goes up, fewer [`RootBlock`]s are required to be stored for verifying archival -/// history of the network, which makes sync quicker and more efficient, but also more data in -/// each [`Piece`] will be occupied with witness, thus wasting space that otherwise could have -/// been used for storing data (record part of a Piece) -/// * as this number goes down, witness get smaller leading to better piece utilization, but the -/// number of root blocks goes up making sync less efficient and less records are needed to be -/// lost before part of the archived history become unrecoverable, reducing reliability of the -/// data stored on the network -const MERKLE_NUM_LEAVES: u32 = 256; -/// Size of witness for a segment record (in bytes). -const WITNESS_SIZE: u32 = SHA256_HASH_SIZE as u32 * MERKLE_NUM_LEAVES.log2(); -/// Size of a segment record given the global piece size (in bytes). -const RECORD_SIZE: u32 = PIECE_SIZE as u32 - WITNESS_SIZE; -/// Recorded History Segment Size includes half of the records (just data records) that will later -/// be erasure coded and together with corresponding witnesses will result in `MERKLE_NUM_LEAVES` -/// pieces of archival history. -const RECORDED_HISTORY_SEGMENT_SIZE: u32 = RECORD_SIZE * MERKLE_NUM_LEAVES / 2; const PRE_GENESIS_OBJECT_SIZE: u32 = RECORDED_HISTORY_SEGMENT_SIZE; const PRE_GENESIS_OBJECT_COUNT: u32 = 10; const PRE_GENESIS_OBJECT_SEED: &[u8] = b"subspace"; - -// We assume initial plot size starts with the size of pre-genesis history (roughly, there is some -// overhead in archiving process) -const INITIAL_SOLUTION_RANGE: u64 = u64::MAX - / (PRE_GENESIS_OBJECT_SIZE * PRE_GENESIS_OBJECT_COUNT / PIECE_SIZE as u32) as u64 // number of total pieces in pre-genesis. - * SLOT_PROBABILITY.0 - / SLOT_PROBABILITY.1; +// We assume initial plot size starts with the a single recorded history segment (which is erasure +// coded of course, hence `*2`). +const INITIAL_SOLUTION_RANGE: u64 = + u64::MAX / (RECORDED_HISTORY_SEGMENT_SIZE * 2 / PIECE_SIZE as u32) as u64 * SLOT_PROBABILITY.0 + / SLOT_PROBABILITY.1; /// A ratio of `Normal` dispatch class within block, for `BlockWeight` and `BlockLength`. const NORMAL_DISPATCH_RATIO: Perbill = Perbill::from_percent(75); diff --git a/substrate/substrate-test-runtime/Cargo.toml b/substrate/substrate-test-runtime/Cargo.toml index f1acc891f3809..ab817e6b17d8d 100644 --- a/substrate/substrate-test-runtime/Cargo.toml +++ b/substrate/substrate-test-runtime/Cargo.toml @@ -51,7 +51,7 @@ subspace-core-primitives = { version = "0.1.0", default-features = false, path = # 3rd party cfg-if = "1.0" log = { version = "0.4.14", default-features = false } -serde = { version = "1.0.126", optional = true, features = ["derive"] } +serde = { version = "1.0.130", optional = true, features = ["derive"] } [dev-dependencies] sc-block-builder = { version = "0.10.0-dev", git = "https://github.com/paritytech/substrate", rev = "26d69bcbe26f6b463e9374e1b1c54c3067fb6131" } From 1a2ce27dc7f1479e9e8bf4eaf063f1ebaa95e060 Mon Sep 17 00:00:00 2001 From: Nazar Mokrynskyi Date: Tue, 23 Nov 2021 21:29:30 +0200 Subject: [PATCH 2/6] Stop using object archiving as such in the codebase (archiver itself isn't changed yet) --- crates/pallet-subspace/src/lib.rs | 20 --- crates/pallet-subspace/src/mock.rs | 3 - crates/sc-consensus-subspace-rpc/src/lib.rs | 6 - crates/sc-consensus-subspace/src/archiver.rs | 71 ++++------- crates/sp-consensus-subspace/src/lib.rs | 11 -- crates/subspace-archiving/src/lib.rs | 2 - .../src/pre_genesis_data.rs | 33 ----- .../tests/integration/main.rs | 1 - .../tests/integration/pre_genesis_data.rs | 23 ---- crates/subspace-farmer/src/plotting.rs | 114 +++--------------- crates/subspace-rpc-primitives/src/lib.rs | 8 -- crates/subspace-runtime-primitives/src/lib.rs | 11 +- crates/subspace-runtime/src/lib.rs | 21 ---- substrate/substrate-test-runtime/src/lib.rs | 27 ----- 14 files changed, 44 insertions(+), 307 deletions(-) delete mode 100644 crates/subspace-archiving/src/pre_genesis_data.rs delete mode 100644 crates/subspace-archiving/tests/integration/pre_genesis_data.rs diff --git a/crates/pallet-subspace/src/lib.rs b/crates/pallet-subspace/src/lib.rs index ee8aa30c0e758..1fd1c966589df 100644 --- a/crates/pallet-subspace/src/lib.rs +++ b/crates/pallet-subspace/src/lib.rs @@ -199,26 +199,6 @@ mod pallet { #[pallet::constant] type RecordedHistorySegmentSize: Get; - // TODO: This is not the only way to bootstrap the network, we might want to use multiple - // objects from file system instead (like Bitcoin history) - /// This constant defines the size (in bytes) of one pre-genesis object. - /// - /// Pre-genesis objects are needed to solve chicken-and-egg problem where in order to - /// produce a solution (and farm a block) a plot is needed, but plot is supposed to contain - /// some archived blocks. Pre-genesis history allows network bootstrapping. - #[pallet::constant] - type PreGenesisObjectSize: Get; - - /// This constant defines the number of a pre-genesis objects that will bootstrap the - /// history. - #[pallet::constant] - type PreGenesisObjectCount: Get; - - /// This constant defines the seed used for deriving pre-genesis objects that will bootstrap - /// the history. - #[pallet::constant] - type PreGenesisObjectSeed: Get<&'static [u8]>; - /// Subspace requires some logic to be triggered on every block to query for whether an epoch /// has ended and to perform the transition to the next epoch. /// diff --git a/crates/pallet-subspace/src/mock.rs b/crates/pallet-subspace/src/mock.rs index e16378dd42b3f..1e184bf88128c 100644 --- a/crates/pallet-subspace/src/mock.rs +++ b/crates/pallet-subspace/src/mock.rs @@ -166,9 +166,6 @@ impl Config for Test { type ConfirmationDepthK = ConfirmationDepthK; type RecordSize = RecordSize; type RecordedHistorySegmentSize = RecordedHistorySegmentSize; - type PreGenesisObjectSize = PreGenesisObjectSize; - type PreGenesisObjectCount = PreGenesisObjectCount; - type PreGenesisObjectSeed = PreGenesisObjectSeed; type EpochChangeTrigger = NormalEpochChange; type EraChangeTrigger = NormalEraChange; type EonChangeTrigger = NormalEonChange; diff --git a/crates/sc-consensus-subspace-rpc/src/lib.rs b/crates/sc-consensus-subspace-rpc/src/lib.rs index 50102ab4d3771..636e16670126c 100644 --- a/crates/sc-consensus-subspace-rpc/src/lib.rs +++ b/crates/sc-consensus-subspace-rpc/src/lib.rs @@ -194,12 +194,6 @@ where record_size: runtime_api.record_size(&best_block_hash)?, recorded_history_segment_size: runtime_api .recorded_history_segment_size(&best_block_hash)?, - pre_genesis_object_size: runtime_api - .pre_genesis_object_size(&best_block_hash)?, - pre_genesis_object_count: runtime_api - .pre_genesis_object_count(&best_block_hash)?, - pre_genesis_object_seed: runtime_api - .pre_genesis_object_seed(&best_block_hash)?, } }; diff --git a/crates/sc-consensus-subspace/src/archiver.rs b/crates/sc-consensus-subspace/src/archiver.rs index f9389a6730f14..cb90c47e050bd 100644 --- a/crates/sc-consensus-subspace/src/archiver.rs +++ b/crates/sc-consensus-subspace/src/archiver.rs @@ -26,7 +26,6 @@ use sp_runtime::generic::BlockId; use sp_runtime::traits::{Block as BlockT, CheckedSub, Header, One, Saturating, Zero}; use std::sync::Arc; use subspace_archiving::archiver::{ArchivedSegment, BlockArchiver, ObjectArchiver}; -use subspace_archiving::pre_genesis_data; use subspace_core_primitives::RootBlock; fn find_last_root_block(client: &Client) -> Option @@ -121,8 +120,8 @@ pub fn start_subspace_archiver( ); let last_archived_block = client .block(&BlockId::Number(last_archived_block_number.into())) - .expect("Older blocks should always exist") - .expect("Older blocks should always exist"); + .expect("Older blocks must always exist") + .expect("Older blocks must always exist"); let block_object_mapping = client .runtime_api() @@ -141,50 +140,10 @@ pub fn start_subspace_archiver( ) .expect("Incorrect parameters for archiver") } else { - // Starting from genesis - let runtime_api = client.runtime_api(); - - let mut object_archiver = - ObjectArchiver::new(record_size as usize, recorded_history_segment_size as usize) - .expect("Incorrect parameters for archiver"); - - let pre_genesis_object_size = runtime_api - .pre_genesis_object_size(&genesis_block_id) - .expect("Failed to get `pre_genesis_object_size` from runtime API"); - let pre_genesis_object_count = runtime_api - .pre_genesis_object_count(&genesis_block_id) - .expect("Failed to get `pre_genesis_object_count` from runtime API"); - let pre_genesis_object_seed = runtime_api - .pre_genesis_object_seed(&genesis_block_id) - .expect("Failed to get `pre_genesis_object_seed` from runtime API"); - - // These archived segments are a part of the public parameters of network setup, thus do not - // need to be sent to farmers - info!(target: "subspace", "Processing pre-genesis objects"); - let new_root_blocks: Vec = (0..pre_genesis_object_count) - .map(|index| { - object_archiver - .add_object(pre_genesis_data::from_seed( - &pre_genesis_object_seed, - index, - pre_genesis_object_size, - )) - .into_iter() - }) - .flatten() - .map(|archived_segment| archived_segment.root_block) - .collect(); - - // Set list of expected root blocks for the next block after genesis (we can't have - // extrinsics in genesis block, at least not right now) - subspace_link - .root_blocks - .lock() - .put(One::one(), new_root_blocks); - - info!(target: "subspace", "Finished processing pre-genesis objects"); - - object_archiver.into_block_archiver() + info!(target: "subspace", "Starting archiving from genesis"); + ObjectArchiver::new(record_size as usize, recorded_history_segment_size as usize) + .expect("Incorrect parameters for archiver") + .into_block_archiver() }; // Process blocks since last fully archived block (or genesis) up to the current head minus K @@ -201,7 +160,15 @@ pub fn start_subspace_archiver( best_number, ); }) - .checked_sub(confirmation_depth_k); + .checked_sub(confirmation_depth_k) + .or_else(|| { + if maybe_last_root_block.is_none() { + // If not continuation, archive genesis block + Some(0) + } else { + None + } + }); if let Some(blocks_to_archive_to) = blocks_to_archive_to { info!( @@ -244,7 +211,13 @@ pub fn start_subspace_archiver( // Set list of expected root blocks for the block where we expect root block // extrinsic to be included subspace_link.root_blocks.lock().put( - (block_to_archive + confirmation_depth_k + 1).into(), + if block_to_archive.is_zero() { + // Special case for genesis block whose root block should be included in + // the first block in order for further validation to work properly. + One::one() + } else { + (block_to_archive + confirmation_depth_k + 1).into() + }, new_root_blocks, ); } diff --git a/crates/sp-consensus-subspace/src/lib.rs b/crates/sp-consensus-subspace/src/lib.rs index 95faddc17ded8..64bb15041645c 100644 --- a/crates/sp-consensus-subspace/src/lib.rs +++ b/crates/sp-consensus-subspace/src/lib.rs @@ -255,17 +255,6 @@ sp_api::decl_runtime_apis! { /// Recorded history is encoded and plotted in segments of this size (in bytes). fn recorded_history_segment_size() -> u32; - /// This constant defines the size (in bytes) of one pre-genesis object. - fn pre_genesis_object_size() -> u32; - - /// This constant defines the number of a pre-genesis objects that will bootstrap the - /// history. - fn pre_genesis_object_count() -> u32; - - /// This constant defines the seed used for deriving pre-genesis objects that will bootstrap - /// the history. - fn pre_genesis_object_seed() -> Vec; - /// Return the genesis configuration for Subspace. The configuration is only read on genesis. fn configuration() -> SubspaceGenesisConfiguration; diff --git a/crates/subspace-archiving/src/lib.rs b/crates/subspace-archiving/src/lib.rs index 6380b252318a7..c1837d02f6e91 100644 --- a/crates/subspace-archiving/src/lib.rs +++ b/crates/subspace-archiving/src/lib.rs @@ -22,5 +22,3 @@ pub mod archiver; #[cfg(feature = "std")] pub mod merkle_tree; -#[cfg(feature = "std")] -pub mod pre_genesis_data; diff --git a/crates/subspace-archiving/src/pre_genesis_data.rs b/crates/subspace-archiving/src/pre_genesis_data.rs deleted file mode 100644 index 7d5ea27eed5b1..0000000000000 --- a/crates/subspace-archiving/src/pre_genesis_data.rs +++ /dev/null @@ -1,33 +0,0 @@ -use sha2::{Digest, Sha256}; -use subspace_core_primitives::{crypto, Sha256Hash, SHA256_HASH_SIZE}; - -/// Derives a single object blob of a given size from given seed and index, which is intended to be -/// used as pre-genesis object (blockchain seed data) -pub fn from_seed>(seed: S, index: u32, size: u32) -> Vec { - let size = size as usize; - let mut object = Vec::with_capacity(size); - let mut acc: Sha256Hash = { - let mut hasher = Sha256::new(); - hasher.update(seed.as_ref()); - hasher.update(index.to_le_bytes().as_ref()); - - hasher - .finalize() - .as_slice() - .try_into() - .expect("Sha256 output is always 32 bytes; qed") - }; - for _ in 0..size / SHA256_HASH_SIZE { - object.extend_from_slice(&acc); - acc = crypto::sha256_hash(&acc); - } - - let remainder = size % SHA256_HASH_SIZE; - if remainder > 0 { - object.extend_from_slice(&acc[..remainder]); - } - - assert_eq!(object.len(), size); - - object -} diff --git a/crates/subspace-archiving/tests/integration/main.rs b/crates/subspace-archiving/tests/integration/main.rs index 229df51b31763..607cee8c46791 100644 --- a/crates/subspace-archiving/tests/integration/main.rs +++ b/crates/subspace-archiving/tests/integration/main.rs @@ -3,4 +3,3 @@ mod archiver; mod merkle_tree; -mod pre_genesis_data; diff --git a/crates/subspace-archiving/tests/integration/pre_genesis_data.rs b/crates/subspace-archiving/tests/integration/pre_genesis_data.rs deleted file mode 100644 index 9c306f4e08e2f..0000000000000 --- a/crates/subspace-archiving/tests/integration/pre_genesis_data.rs +++ /dev/null @@ -1,23 +0,0 @@ -use subspace_archiving::pre_genesis_data; - -#[test] -fn pre_genesis_data() { - { - // Below 1 Sha256 block - let object = pre_genesis_data::from_seed(b"subspace", 0, 10); - assert_eq!(object.len(), 10); - assert!(object.iter().find(|byte| **byte != 0).is_some()); - } - { - // Exactly 1 Sha256 block - let object = pre_genesis_data::from_seed(b"subspace", 0, 32); - assert_eq!(object.len(), 32); - assert!(object.iter().find(|byte| **byte != 0).is_some()); - } - { - // Over 1 Sha256 block - let object = pre_genesis_data::from_seed(b"subspace", 0, 40); - assert_eq!(object.len(), 40); - assert!(object.iter().find(|byte| **byte != 0).is_some()); - } -} diff --git a/crates/subspace-farmer/src/plotting.rs b/crates/subspace-farmer/src/plotting.rs index f857139909f33..62cd601b6d86f 100644 --- a/crates/subspace-farmer/src/plotting.rs +++ b/crates/subspace-farmer/src/plotting.rs @@ -6,7 +6,6 @@ use log::{debug, error, info}; use std::sync::atomic::{AtomicU32, Ordering}; use std::sync::Arc; use subspace_archiving::archiver::{ArchivedSegment, BlockArchiver, ObjectArchiver}; -use subspace_archiving::pre_genesis_data; use subspace_core_primitives::objects::{GlobalObject, PieceObject, PieceObjectMapping}; use subspace_core_primitives::Sha256Hash; use subspace_rpc_primitives::{EncodedBlockWithObjectMapping, FarmerMetadata}; @@ -29,12 +28,6 @@ pub enum PlottingError { JoinTask(tokio::task::JoinError), #[error("Archiver instantiation error: {0}")] Archiver(subspace_archiving::archiver::ArchiverInstantiationError), - #[error("Object mapping store error: {0}")] - ObjectMapping(crate::object_mappings::ObjectMappingError), - #[error("Commitment creation error: {0}")] - Commitment(crate::commitments::CommitmentError), - #[error("Plot write error: {0}")] - PlotWrite(std::io::Error), } /// `Plotting` struct is the abstraction of the plotting process /// @@ -112,19 +105,17 @@ async fn background_plotting( confirmation_depth_k, record_size, recorded_history_segment_size, - pre_genesis_object_size, - pre_genesis_object_count, - pre_genesis_object_seed, } = farmer_metadata; // TODO: This assumes fixed size segments, which might not be the case let merkle_num_leaves = u64::from(recorded_history_segment_size / record_size * 2); - let mut archiver = if let Some(last_root_block) = plot + let maybe_last_root_block = plot .get_last_root_block() .await - .map_err(PlottingError::LastBlock)? - { + .map_err(PlottingError::LastBlock)?; + + let mut archiver = if let Some(last_root_block) = maybe_last_root_block { // Continuing from existing initial state if plot.is_empty() { return Err(PlottingError::ContinueError); @@ -165,96 +156,14 @@ async fn background_plotting( drop(plot); - let mut object_archiver = - ObjectArchiver::new(record_size as usize, recorded_history_segment_size as usize) - .map_err(PlottingError::Archiver)?; - - // Erasure coding in archiver and piece encoding are a CPU-intensive operations - let maybe_block_archiver_handle = tokio::task::spawn_blocking({ - let weak_plot = weak_plot.clone(); - let commitments = commitments.clone(); - let object_mappings = object_mappings.clone(); - - move || -> Result, PlottingError> { - let runtime_handle = tokio::runtime::Handle::current(); - info!("Plotting pre-genesis objects"); - - // These archived segments are a part of the public parameters of network setup - for index in 0..pre_genesis_object_count { - let archived_segments = - object_archiver.add_object(pre_genesis_data::from_seed( - &pre_genesis_object_seed, - index, - pre_genesis_object_size, - )); - - for archived_segment in archived_segments { - let ArchivedSegment { - root_block, - mut pieces, - object_mapping, - } = archived_segment; - let piece_index_offset = merkle_num_leaves * root_block.segment_index(); - - let object_mapping = - create_global_object_mapping(piece_index_offset, object_mapping); - - // TODO: Batch encoding - for (position, piece) in pieces.iter_mut().enumerate() { - if let Err(error) = - subspace_codec.encode(piece_index_offset + position as u64, piece) - { - error!("Failed to encode a piece: error: {}", error); - continue; - } - } - - if let Some(plot) = weak_plot.upgrade() { - let pieces = Arc::new(pieces); - // TODO: There is no internal mapping between pieces and their indexes yet - // TODO: Then we might want to send indexes as a separate vector - runtime_handle - .block_on(plot.write_many(Arc::clone(&pieces), piece_index_offset)) - .map_err(PlottingError::PlotWrite)?; - runtime_handle - .block_on( - commitments.create_for_pieces(&pieces, piece_index_offset), - ) - .map_err(PlottingError::Commitment)?; - object_mappings - .store(&object_mapping) - .map_err(PlottingError::ObjectMapping)?; - info!( - "Archived segment {} at object {}", - root_block.segment_index(), - index - ); - } else { - return Ok(None); - } - } - } - - info!("Finished plotting pre-genesis objects"); - - Ok(Some(object_archiver.into_block_archiver())) - } - }); - - match maybe_block_archiver_handle - .await - .map_err(PlottingError::JoinTask)?? - { - Some(block_archiver) => block_archiver, - None => { - // Plot was dropped, time to exit already - return Ok(()); - } - } + ObjectArchiver::new(record_size as usize, recorded_history_segment_size as usize) + .map_err(PlottingError::Archiver)? + .into_block_archiver() }; let (new_block_to_archive_sender, new_block_to_archive_receiver) = std::sync::mpsc::sync_channel::>(0); + // Process blocks since last fully archived block (or genesis) up to the current head minus K let mut blocks_to_archive_from = archiver .last_archived_block_number() @@ -380,6 +289,13 @@ async fn background_plotting( let block_to_archive = Arc::new(AtomicU32::default()); + if maybe_last_root_block.is_none() { + // If not continuation, archive genesis block + new_block_to_archive_sender + .send(Arc::clone(&block_to_archive)) + .expect("Failed to send genesis block archiving message"); + } + // Listen for new blocks produced on the network // receiver.fuse(); loop { diff --git a/crates/subspace-rpc-primitives/src/lib.rs b/crates/subspace-rpc-primitives/src/lib.rs index 9c9bd48c2cb16..7a4eca9716c5d 100644 --- a/crates/subspace-rpc-primitives/src/lib.rs +++ b/crates/subspace-rpc-primitives/src/lib.rs @@ -45,14 +45,6 @@ pub struct FarmerMetadata { pub record_size: u32, /// Recorded history is encoded and plotted in segments of this size (in bytes). pub recorded_history_segment_size: u32, - /// This constant defines the size (in bytes) of one pre-genesis object. - pub pre_genesis_object_size: u32, - /// This constant defines the number of a pre-genesis objects that will bootstrap the - /// history. - pub pre_genesis_object_count: u32, - /// This constant defines the seed used for deriving pre-genesis objects that will bootstrap - /// the history. - pub pre_genesis_object_seed: Vec, } /// Information about new slot that just arrived diff --git a/crates/subspace-runtime-primitives/src/lib.rs b/crates/subspace-runtime-primitives/src/lib.rs index 16562c6cebafc..ff0708199fec5 100644 --- a/crates/subspace-runtime-primitives/src/lib.rs +++ b/crates/subspace-runtime-primitives/src/lib.rs @@ -118,10 +118,13 @@ pub mod opaque { } fn new(mut header: Self::Header, extrinsics: Vec) -> Self { if header.number == 0 { - // We fill genesis block with extra data such that the very first archived segment - // can be produced right away, bootstrapping the farming process. - let ballast = vec![0; RECORDED_HISTORY_SEGMENT_SIZE as usize]; - header.digest.logs.push(DigestItem::Other(ballast)); + // This check is necessary in case block was deconstructed and constructed again. + if header.digest.logs.is_empty() { + // We fill genesis block with extra data such that the very first archived + // segment can be produced right away, bootstrapping the farming process. + let ballast = vec![0; RECORDED_HISTORY_SEGMENT_SIZE as usize]; + header.digest.logs.push(DigestItem::Other(ballast)); + } Block { header, extrinsics } } else { Block { header, extrinsics } diff --git a/crates/subspace-runtime/src/lib.rs b/crates/subspace-runtime/src/lib.rs index 674c244478d33..abfc840683290 100644 --- a/crates/subspace-runtime/src/lib.rs +++ b/crates/subspace-runtime/src/lib.rs @@ -133,9 +133,6 @@ pub const SUBSPACE_GENESIS_EPOCH_CONFIG: sp_consensus_subspace::SubspaceEpochCon c: SLOT_PROBABILITY, }; -const PRE_GENESIS_OBJECT_SIZE: u32 = RECORDED_HISTORY_SEGMENT_SIZE; -const PRE_GENESIS_OBJECT_COUNT: u32 = 10; -const PRE_GENESIS_OBJECT_SEED: &[u8] = b"subspace"; // We assume initial plot size starts with the a single recorded history segment (which is erasure // coded of course, hence `*2`). const INITIAL_SOLUTION_RANGE: u64 = @@ -221,9 +218,6 @@ parameter_types! { pub const ConfirmationDepthK: u32 = CONFIRMATION_DEPTH_K; pub const RecordSize: u32 = RECORD_SIZE; pub const RecordedHistorySegmentSize: u32 = RECORDED_HISTORY_SEGMENT_SIZE; - pub const PreGenesisObjectSize: u32 = PRE_GENESIS_OBJECT_SIZE; - pub const PreGenesisObjectCount: u32 = PRE_GENESIS_OBJECT_COUNT; - pub const PreGenesisObjectSeed: &'static [u8] = PRE_GENESIS_OBJECT_SEED; pub const ReportLongevity: u64 = EPOCH_DURATION_IN_BLOCKS as u64; } @@ -238,9 +232,6 @@ impl pallet_subspace::Config for Runtime { type ConfirmationDepthK = ConfirmationDepthK; type RecordSize = RecordSize; type RecordedHistorySegmentSize = RecordedHistorySegmentSize; - type PreGenesisObjectSize = PreGenesisObjectSize; - type PreGenesisObjectCount = PreGenesisObjectCount; - type PreGenesisObjectSeed = PreGenesisObjectSeed; type EpochChangeTrigger = pallet_subspace::NormalEpochChange; type EraChangeTrigger = pallet_subspace::NormalEraChange; type EonChangeTrigger = pallet_subspace::NormalEonChange; @@ -632,18 +623,6 @@ impl_runtime_apis! { RecordedHistorySegmentSize::get() } - fn pre_genesis_object_size() -> u32 { - PreGenesisObjectSize::get() - } - - fn pre_genesis_object_count() -> u32 { - PreGenesisObjectCount::get() - } - - fn pre_genesis_object_seed() -> Vec { - Vec::from(PreGenesisObjectSeed::get()) - } - fn configuration() -> sp_consensus_subspace::SubspaceGenesisConfiguration { // The choice of `c` parameter (where `1 - c` represents the // probability of a slot being empty), is done in accordance to the diff --git a/substrate/substrate-test-runtime/src/lib.rs b/substrate/substrate-test-runtime/src/lib.rs index 9d80b6fd38fa6..ac660f0821a7e 100644 --- a/substrate/substrate-test-runtime/src/lib.rs +++ b/substrate/substrate-test-runtime/src/lib.rs @@ -668,9 +668,6 @@ impl pallet_subspace::Config for Runtime { type ConfirmationDepthK = ConfirmationDepthK; type RecordSize = RecordSize; type RecordedHistorySegmentSize = RecordedHistorySegmentSize; - type PreGenesisObjectSize = PreGenesisObjectSize; - type PreGenesisObjectCount = PreGenesisObjectCount; - type PreGenesisObjectSeed = PreGenesisObjectSeed; type EpochChangeTrigger = pallet_subspace::NormalEpochChange; type EraChangeTrigger = pallet_subspace::NormalEraChange; type EonChangeTrigger = pallet_subspace::NormalEonChange; @@ -945,18 +942,6 @@ cfg_if! { RecordedHistorySegmentSize::get() } - fn pre_genesis_object_size() -> u32 { - PreGenesisObjectSize::get() - } - - fn pre_genesis_object_count() -> u32 { - PreGenesisObjectCount::get() - } - - fn pre_genesis_object_seed() -> Vec { - Vec::from(PreGenesisObjectSeed::get()) - } - fn configuration() -> sp_consensus_subspace::SubspaceGenesisConfiguration { sp_consensus_subspace::SubspaceGenesisConfiguration { slot_duration: 1000, @@ -1283,18 +1268,6 @@ cfg_if! { RecordedHistorySegmentSize::get() } - fn pre_genesis_object_size() -> u32 { - PreGenesisObjectSize::get() - } - - fn pre_genesis_object_count() -> u32 { - PreGenesisObjectCount::get() - } - - fn pre_genesis_object_seed() -> Vec { - Vec::from(PreGenesisObjectSeed::get()) - } - fn configuration() -> sp_consensus_subspace::SubspaceGenesisConfiguration { sp_consensus_subspace::SubspaceGenesisConfiguration { slot_duration: 1000, From ef3e0728a48435642f70098f36cd2a911d2d5c9d Mon Sep 17 00:00:00 2001 From: Nazar Mokrynskyi Date: Tue, 23 Nov 2021 22:34:52 +0200 Subject: [PATCH 3/6] Remove object archiving from the codebase --- crates/sc-consensus-subspace/src/archiver.rs | 8 +- crates/sc-consensus-subspace/src/tests.rs | 40 +++------ crates/subspace-archiving/src/archiver.rs | 84 +------------------ .../tests/integration/archiver.rs | 73 ++++------------ crates/subspace-farmer/src/plotting.rs | 7 +- 5 files changed, 35 insertions(+), 177 deletions(-) diff --git a/crates/sc-consensus-subspace/src/archiver.rs b/crates/sc-consensus-subspace/src/archiver.rs index cb90c47e050bd..376fa1638cdb2 100644 --- a/crates/sc-consensus-subspace/src/archiver.rs +++ b/crates/sc-consensus-subspace/src/archiver.rs @@ -25,7 +25,7 @@ use sp_consensus_subspace::SubspaceApi; use sp_runtime::generic::BlockId; use sp_runtime::traits::{Block as BlockT, CheckedSub, Header, One, Saturating, Zero}; use std::sync::Arc; -use subspace_archiving::archiver::{ArchivedSegment, BlockArchiver, ObjectArchiver}; +use subspace_archiving::archiver::{ArchivedSegment, Archiver}; use subspace_core_primitives::RootBlock; fn find_last_root_block(client: &Client) -> Option @@ -131,7 +131,7 @@ pub fn start_subspace_archiver( ) .expect("Must be able to make runtime call"); - BlockArchiver::with_initial_state( + Archiver::with_initial_state( record_size as usize, recorded_history_segment_size as usize, last_root_block, @@ -141,9 +141,9 @@ pub fn start_subspace_archiver( .expect("Incorrect parameters for archiver") } else { info!(target: "subspace", "Starting archiving from genesis"); - ObjectArchiver::new(record_size as usize, recorded_history_segment_size as usize) + + Archiver::new(record_size as usize, recorded_history_segment_size as usize) .expect("Incorrect parameters for archiver") - .into_block_archiver() }; // Process blocks since last fully archived block (or genesis) up to the current head minus K diff --git a/crates/sc-consensus-subspace/src/tests.rs b/crates/sc-consensus-subspace/src/tests.rs index 9ad18e8feca65..423048dbd8e22 100644 --- a/crates/sc-consensus-subspace/src/tests.rs +++ b/crates/sc-consensus-subspace/src/tests.rs @@ -30,7 +30,7 @@ use parking_lot::Mutex; use rand::prelude::*; use sc_block_builder::{BlockBuilder, BlockBuilderProvider}; use sc_client_api::backend::TransactionFor; -use sc_client_api::BlockchainEvents; +use sc_client_api::{BlockBackend, BlockchainEvents}; use sc_consensus::block_import::ForkChoiceStrategy; use sc_consensus::{ BlockCheckParams, BlockImport, BlockImportParams, BoxBlockImport, BoxJustificationImport, @@ -71,8 +71,8 @@ use std::future::Future; use std::pin::Pin; use std::sync::Arc; use std::{cell::RefCell, task::Poll, time::Duration}; -use subspace_archiving::archiver::ObjectArchiver; -use subspace_archiving::pre_genesis_data; +use subspace_archiving::archiver::Archiver; +use subspace_core_primitives::objects::BlockObjectMapping; use subspace_core_primitives::{LocalChallenge, Piece, Signature, Tag}; use subspace_solving::{SubspaceCodec, SOLUTION_SIGNING_CONTEXT}; use substrate_test_runtime::{Block as TestBlock, Hash}; @@ -473,33 +473,15 @@ fn get_archived_pieces(client: &TestClient) -> Vec { let recorded_history_segment_size = runtime_api .recorded_history_segment_size(&genesis_block_id) .unwrap(); - let pre_genesis_object_size = runtime_api - .pre_genesis_object_size(&genesis_block_id) - .unwrap(); - let pre_genesis_object_count = runtime_api - .pre_genesis_object_count(&genesis_block_id) - .unwrap(); - let pre_genesis_object_seed = runtime_api - .pre_genesis_object_seed(&genesis_block_id) - .unwrap(); - let mut object_archiver = - ObjectArchiver::new(record_size as usize, recorded_history_segment_size as usize) - .expect("Incorrect parameters for archiver"); - - (0..pre_genesis_object_count) - .map(|index| { - object_archiver - .add_object(pre_genesis_data::from_seed( - &pre_genesis_object_seed, - index, - pre_genesis_object_size, - )) - .into_iter() - }) - .flatten() - .map(|archived_segment| archived_segment.pieces.into_iter()) - .flatten() + let mut archiver = Archiver::new(record_size as usize, recorded_history_segment_size as usize) + .expect("Incorrect parameters for archiver"); + + let genesis_block = client.block(&genesis_block_id).unwrap().unwrap(); + archiver + .add_block(genesis_block.encode(), BlockObjectMapping::default()) + .into_iter() + .flat_map(|archived_segment| archived_segment.pieces.into_iter()) .collect() } diff --git a/crates/subspace-archiving/src/archiver.rs b/crates/subspace-archiving/src/archiver.rs index 48f703af8dcf2..9a69f6538cd86 100644 --- a/crates/subspace-archiving/src/archiver.rs +++ b/crates/subspace-archiving/src/archiver.rs @@ -22,7 +22,6 @@ use std::cmp::Ordering; use std::collections::VecDeque; use std::io::Write; use std::iter; -use std::marker::PhantomData; use subspace_core_primitives::objects::{ BlockObject, BlockObjectMapping, PieceObject, PieceObjectMapping, }; @@ -157,35 +156,6 @@ pub enum ArchiverInstantiationError { NoBlocksInvalidInitialState, } -mod private { - pub trait ArchiverState {} - - /// Marker struct used in archiver to define a state where archiver is used for archiving object - /// before genesis block - #[derive(Debug)] - pub struct ObjectArchiverState; - - impl ArchiverState for ObjectArchiverState {} - - /// Marker struct used in archiver to define a state where archiver is used for archiving blocks - /// at or after genesis block - #[derive(Debug)] - pub struct BlockArchiverState; - - impl ArchiverState for BlockArchiverState {} -} - -/// Object archiver for Subspace blockchain. -/// -/// It takes pre-genesis objects and concatenates them into a buffer, buffer is -/// sliced into segments of `RECORDED_HISTORY_SEGMENT_SIZE` size, segments are sliced into source -/// records of `RECORD_SIZE`, records are erasure coded, Merkle Tree is built over them, and -/// with Merkle Proofs appended records become pieces that are returned alongside corresponding root -/// block header. -/// -/// Object archiver is only used pre-genesis and should be turned into block archiver for later use. -pub type ObjectArchiver = Archiver; - /// Block archiver for Subspace blockchain. /// /// It takes new confirmed (at `K` depth) blocks and concatenates them into a buffer, buffer is @@ -193,13 +163,8 @@ pub type ObjectArchiver = Archiver; /// records of `RECORD_SIZE`, records are erasure coded, Merkle Tree is built over them, and /// with Merkle Proofs appended records become pieces that are returned alongside corresponding root /// block header. -pub type BlockArchiver = Archiver; - -/// Generic archiver for Subspace blockchain. -/// -/// Shouldn't be used directly, but rather through `ObjectArchiver` or `BlockArchiver` type aliases. #[derive(Debug)] -pub struct Archiver { +pub struct Archiver { /// Buffer containing blocks and other buffered items that are pending to be included into the /// next segment buffer: VecDeque, @@ -216,11 +181,9 @@ pub struct Archiver { prev_root_block_hash: Sha256Hash, /// Last archived block last_archived_block: LastArchivedBlock, - /// Just a marker for otherwise unused generic parameter - state_marker: PhantomData, } -impl Archiver { +impl Archiver { /// Create a new instance with specified record size and recorded history segment size. /// /// Note: this is the only way to instantiate object archiver, while block archiver can be @@ -270,7 +233,6 @@ impl Archiver { segment_index: 0, prev_root_block_hash: Sha256Hash::default(), last_archived_block: INITIAL_LAST_ARCHIVED_BLOCK, - state_marker: PhantomData::default(), }) } @@ -687,49 +649,7 @@ impl Archiver { object_mapping, } } -} - -impl ObjectArchiver { - /// Adds new object to internal buffer, potentially producing pieces and root block headers - pub fn add_object(&mut self, object: Vec) -> Vec { - // Append new block to the buffer - self.buffer.push_back(SegmentItem::Object(object)); - - let mut archived_segments = Vec::new(); - - while let Some(segment) = self.produce_segment() { - archived_segments.push(self.produce_archived_segment(segment)); - } - - archived_segments - } - - pub fn into_block_archiver(self) -> BlockArchiver { - let Self { - buffer, - record_size, - segment_size, - reed_solomon, - segment_index, - prev_root_block_hash, - last_archived_block, - .. - } = self; - - Archiver { - buffer, - record_size, - segment_size, - reed_solomon, - segment_index, - prev_root_block_hash, - last_archived_block, - state_marker: PhantomData::default(), - } - } -} -impl BlockArchiver { /// Create a new instance of block archiver with initial state in case of restart. /// /// `block` corresponds to `last_archived_block` and will be processed accordingly to its state. diff --git a/crates/subspace-archiving/tests/integration/archiver.rs b/crates/subspace-archiving/tests/integration/archiver.rs index 4067a9f40dfd2..2eaaba2d10e16 100644 --- a/crates/subspace-archiving/tests/integration/archiver.rs +++ b/crates/subspace-archiving/tests/integration/archiver.rs @@ -1,10 +1,9 @@ use parity_scale_codec::{Compact, Decode, Encode}; -use rand::Rng; use std::assert_matches::assert_matches; use std::io::Write; use std::iter; use subspace_archiving::archiver; -use subspace_archiving::archiver::{ArchiverInstantiationError, BlockArchiver, ObjectArchiver}; +use subspace_archiving::archiver::{Archiver, ArchiverInstantiationError}; use subspace_core_primitives::objects::{BlockObject, BlockObjectMapping, PieceObject}; use subspace_core_primitives::{ ArchivedBlockProgress, LastArchivedBlock, RootBlock, Sha256Hash, PIECE_SIZE, SHA256_HASH_SIZE, @@ -38,7 +37,7 @@ fn compare_block_objects_to_piece_objects<'a>( #[test] fn archiver() { - let mut archiver = BlockArchiver::new(RECORD_SIZE, SEGMENT_SIZE).unwrap(); + let mut archiver = Archiver::new(RECORD_SIZE, SEGMENT_SIZE).unwrap(); let (block_0, block_0_object_mapping) = { let mut block = rand::random::<[u8; SEGMENT_SIZE / 2]>().to_vec(); @@ -158,7 +157,7 @@ fn archiver() { // Check that initializing archiver with initial state before last block results in the same // archived segments once last block is added { - let mut archiver_with_initial_state = BlockArchiver::with_initial_state( + let mut archiver_with_initial_state = Archiver::with_initial_state( RECORD_SIZE, SEGMENT_SIZE, first_archived_segment.root_block, @@ -249,7 +248,7 @@ fn archiver() { // Check that initializing archiver with initial state before last block results in the same // archived segments once last block is added { - let mut archiver_with_initial_state = BlockArchiver::with_initial_state( + let mut archiver_with_initial_state = Archiver::with_initial_state( RECORD_SIZE, SEGMENT_SIZE, last_root_block, @@ -282,78 +281,36 @@ fn archiver() { } } -#[test] -fn object_archiver() { - let mut archiver = ObjectArchiver::new(RECORD_SIZE, SEGMENT_SIZE).unwrap(); - - let mut rng = rand::thread_rng(); - { - let mut object = vec![0u8; SEGMENT_SIZE]; - - for _ in 0..10 { - rng.fill(object.as_mut_slice()); - - assert_eq!(archiver.add_object(object.to_vec()).len(), 1); - } - } - - let mut archiver = archiver.into_block_archiver(); - - let block_0 = rand::random::<[u8; SEGMENT_SIZE]>(); - - let root_block = archiver - .add_block(block_0.to_vec(), BlockObjectMapping::default()) - .into_iter() - .next() - .unwrap() - .root_block; - - let mut archiver_with_initial_state = BlockArchiver::with_initial_state( - RECORD_SIZE, - SEGMENT_SIZE, - root_block, - block_0.to_vec(), - BlockObjectMapping::default(), - ) - .unwrap(); - - let block_1 = rand::random::<[u8; SEGMENT_SIZE]>(); - assert_eq!( - archiver.add_block(block_1.to_vec(), BlockObjectMapping::default()), - archiver_with_initial_state.add_block(block_1.to_vec(), BlockObjectMapping::default()), - ); -} - // TODO: Tests for block to piece object translation that crosses piece boundary #[test] fn archiver_invalid_usage() { assert_matches!( - BlockArchiver::new(5, SEGMENT_SIZE), + Archiver::new(5, SEGMENT_SIZE), Err(ArchiverInstantiationError::RecordSizeTooSmall), ); assert_matches!( - BlockArchiver::new(10, 9), + Archiver::new(10, 9), Err(ArchiverInstantiationError::SegmentSizeTooSmall), ); assert_matches!( - BlockArchiver::new(SEGMENT_SIZE, SEGMENT_SIZE), + Archiver::new(SEGMENT_SIZE, SEGMENT_SIZE), Err(ArchiverInstantiationError::SegmentSizeTooSmall), ); assert_matches!( - BlockArchiver::new(17, SEGMENT_SIZE), + Archiver::new(17, SEGMENT_SIZE), Err(ArchiverInstantiationError::SegmentSizesNotMultipleOfRecordSize), ); assert_matches!( - BlockArchiver::new(17, 34), + Archiver::new(17, 34), Err(ArchiverInstantiationError::WrongRecordAndSegmentCombination), ); { - let result = BlockArchiver::with_initial_state( + let result = Archiver::with_initial_state( RECORD_SIZE, SEGMENT_SIZE, RootBlock::V0 { @@ -380,7 +337,7 @@ fn archiver_invalid_usage() { } { - let result = BlockArchiver::with_initial_state( + let result = Archiver::with_initial_state( RECORD_SIZE, SEGMENT_SIZE, RootBlock::V0 { @@ -412,7 +369,7 @@ fn archiver_invalid_usage() { } { - let result = BlockArchiver::with_initial_state( + let result = Archiver::with_initial_state( RECORD_SIZE, SEGMENT_SIZE, RootBlock::V0 { @@ -437,7 +394,7 @@ fn archiver_invalid_usage() { #[test] fn one_byte_smaller_segment() { - let mut archiver = BlockArchiver::new(RECORD_SIZE, SEGMENT_SIZE).unwrap(); + let mut archiver = Archiver::new(RECORD_SIZE, SEGMENT_SIZE).unwrap(); // Carefully compute the block size such that there is just 2 bytes left to fill the segment, // but this should already produce archived segment since just enum variant and smallest compact @@ -453,7 +410,7 @@ fn one_byte_smaller_segment() { #[test] fn spill_over_edge_case() { - let mut archiver = BlockArchiver::new(RECORD_SIZE, SEGMENT_SIZE).unwrap(); + let mut archiver = Archiver::new(RECORD_SIZE, SEGMENT_SIZE).unwrap(); // Carefully compute the block size such that there is just 3 byte left to fill the segment assert!(archiver @@ -474,7 +431,7 @@ fn spill_over_edge_case() { #[test] fn object_on_the_edge_of_segment() { - let mut archiver = BlockArchiver::new(RECORD_SIZE, SEGMENT_SIZE).unwrap(); + let mut archiver = Archiver::new(RECORD_SIZE, SEGMENT_SIZE).unwrap(); assert_eq!( archiver .add_block(vec![0u8; SEGMENT_SIZE], BlockObjectMapping::default()) diff --git a/crates/subspace-farmer/src/plotting.rs b/crates/subspace-farmer/src/plotting.rs index 62cd601b6d86f..5b111e0682aab 100644 --- a/crates/subspace-farmer/src/plotting.rs +++ b/crates/subspace-farmer/src/plotting.rs @@ -5,7 +5,7 @@ use crate::rpc::RpcClient; use log::{debug, error, info}; use std::sync::atomic::{AtomicU32, Ordering}; use std::sync::Arc; -use subspace_archiving::archiver::{ArchivedSegment, BlockArchiver, ObjectArchiver}; +use subspace_archiving::archiver::{ArchivedSegment, Archiver}; use subspace_core_primitives::objects::{GlobalObject, PieceObject, PieceObjectMapping}; use subspace_core_primitives::Sha256Hash; use subspace_rpc_primitives::{EncodedBlockWithObjectMapping, FarmerMetadata}; @@ -135,7 +135,7 @@ async fn background_plotting( Some(EncodedBlockWithObjectMapping { block, object_mapping, - }) => BlockArchiver::with_initial_state( + }) => Archiver::with_initial_state( record_size as usize, recorded_history_segment_size as usize, last_root_block, @@ -156,9 +156,8 @@ async fn background_plotting( drop(plot); - ObjectArchiver::new(record_size as usize, recorded_history_segment_size as usize) + Archiver::new(record_size as usize, recorded_history_segment_size as usize) .map_err(PlottingError::Archiver)? - .into_block_archiver() }; let (new_block_to_archive_sender, new_block_to_archive_receiver) = From f36660f33e1c6d0320488d455d419939134dd3b7 Mon Sep 17 00:00:00 2001 From: Nazar Mokrynskyi Date: Tue, 23 Nov 2021 22:46:27 +0200 Subject: [PATCH 4/6] Remove various todos and make some cleanups --- crates/sc-consensus-subspace/src/archiver.rs | 10 +------- .../sc-consensus-subspace/src/authorship.rs | 4 +-- crates/sc-consensus-subspace/src/lib.rs | 4 +-- crates/subspace-archiving/src/archiver.rs | 8 ------ .../tests/integration/archiver.rs | 25 ------------------- crates/subspace-farmer/src/object_mappings.rs | 2 -- 6 files changed, 5 insertions(+), 48 deletions(-) diff --git a/crates/sc-consensus-subspace/src/archiver.rs b/crates/sc-consensus-subspace/src/archiver.rs index 376fa1638cdb2..d24440a27a132 100644 --- a/crates/sc-consensus-subspace/src/archiver.rs +++ b/crates/sc-consensus-subspace/src/archiver.rs @@ -100,15 +100,7 @@ pub fn start_subspace_archiver( .recorded_history_segment_size(&genesis_block_id) .expect("Failed to get `recorded_history_segment_size` from runtime API"); - let maybe_last_root_block = find_last_root_block(client.as_ref()).and_then(|last_root_block| { - // At least one non-genesis block needs to be archived for restarts - // TODO: This check can be removed when we no longer have pre-genesis objects - if last_root_block.last_archived_block().number == 0 { - None - } else { - Some(last_root_block) - } - }); + let maybe_last_root_block = find_last_root_block(client.as_ref()); let mut archiver = if let Some(last_root_block) = maybe_last_root_block { // Continuing from existing initial state diff --git a/crates/sc-consensus-subspace/src/authorship.rs b/crates/sc-consensus-subspace/src/authorship.rs index cce6e2c968743..2e538aa9159b1 100644 --- a/crates/sc-consensus-subspace/src/authorship.rs +++ b/crates/sc-consensus-subspace/src/authorship.rs @@ -232,8 +232,8 @@ where .records_root(&parent_block_id, segment_index) .ok()?; - // TODO: This is not a very nice hack due to the fact that at the time first block is - // produced extrinsics with root blocks are not yet in runtime + // This is not a very nice hack due to the fact that at the time first block is produced + // extrinsics with root blocks are not yet in runtime. if maybe_records_root.is_none() && parent_header.number().is_zero() { maybe_records_root = worker.subspace_link.root_blocks.lock().iter().find_map( |(_block_number, root_blocks)| { diff --git a/crates/sc-consensus-subspace/src/lib.rs b/crates/sc-consensus-subspace/src/lib.rs index 04833b5e20846..2b6fca0c5ce93 100644 --- a/crates/sc-consensus-subspace/src/lib.rs +++ b/crates/sc-consensus-subspace/src/lib.rs @@ -1033,8 +1033,8 @@ where ); }); - // TODO: This is not a very nice hack due to the fact that at the time first block is - // produced extrinsics with root blocks are not yet in runtime + // This is not a very nice hack due to the fact that at the time first block is produced + // extrinsics with root blocks are not yet in runtime. if maybe_records_root.is_none() && block.header.number().is_one() { maybe_records_root = self.root_blocks diff --git a/crates/subspace-archiving/src/archiver.rs b/crates/subspace-archiving/src/archiver.rs index 9a69f6538cd86..09416f821525f 100644 --- a/crates/subspace-archiving/src/archiver.rs +++ b/crates/subspace-archiving/src/archiver.rs @@ -151,9 +151,6 @@ pub enum ArchiverInstantiationError { /// Already archived portion of the block archived_block_bytes: u32, }, - /// No archived blocks, invalid initial state - #[error("No archived blocks, invalid initial state")] - NoBlocksInvalidInitialState, } /// Block archiver for Subspace blockchain. @@ -660,11 +657,6 @@ impl Archiver { encoded_block: B, mut object_mapping: BlockObjectMapping, ) -> Result { - if root_block.last_archived_block() == INITIAL_LAST_ARCHIVED_BLOCK { - // TODO: This check can be removed when we no longer have pre-genesis objects - return Err(ArchiverInstantiationError::NoBlocksInvalidInitialState); - } - let mut archiver = Self::new(record_size, segment_size)?; archiver.segment_index = root_block.segment_index() + 1; diff --git a/crates/subspace-archiving/tests/integration/archiver.rs b/crates/subspace-archiving/tests/integration/archiver.rs index 2eaaba2d10e16..eec458e2da276 100644 --- a/crates/subspace-archiving/tests/integration/archiver.rs +++ b/crates/subspace-archiving/tests/integration/archiver.rs @@ -281,8 +281,6 @@ fn archiver() { } } -// TODO: Tests for block to piece object translation that crosses piece boundary - #[test] fn archiver_invalid_usage() { assert_matches!( @@ -367,29 +365,6 @@ fn archiver_invalid_usage() { assert_eq!(archived_block_bytes, 10); } } - - { - let result = Archiver::with_initial_state( - RECORD_SIZE, - SEGMENT_SIZE, - RootBlock::V0 { - segment_index: 0, - records_root: Sha256Hash::default(), - prev_root_block_hash: Sha256Hash::default(), - last_archived_block: LastArchivedBlock { - number: 0, - archived_progress: ArchivedBlockProgress::Partial(0), - }, - }, - &vec![0u8; 5], - BlockObjectMapping::default(), - ); - - assert_matches!( - result, - Err(ArchiverInstantiationError::NoBlocksInvalidInitialState), - ); - } } #[test] diff --git a/crates/subspace-farmer/src/object_mappings.rs b/crates/subspace-farmer/src/object_mappings.rs index bedbbae3016ce..b893d2e21ebe4 100644 --- a/crates/subspace-farmer/src/object_mappings.rs +++ b/crates/subspace-farmer/src/object_mappings.rs @@ -29,8 +29,6 @@ impl ObjectMappings { Ok(Self { db: Arc::new(db) }) } - // TODO: Remove suppression once we start using this - #[allow(dead_code)] /// Retrieve mapping for object pub(super) fn retrieve( &self, From d6f12ab51ea2e2c1073532572252ff784c9986b6 Mon Sep 17 00:00:00 2001 From: Nazar Mokrynskyi Date: Tue, 23 Nov 2021 22:54:50 +0200 Subject: [PATCH 5/6] Rearrange methods for better readability --- crates/subspace-archiving/src/archiver.rs | 186 +++++++++++----------- 1 file changed, 93 insertions(+), 93 deletions(-) diff --git a/crates/subspace-archiving/src/archiver.rs b/crates/subspace-archiving/src/archiver.rs index 09416f821525f..dc8abfd9e2a2f 100644 --- a/crates/subspace-archiving/src/archiver.rs +++ b/crates/subspace-archiving/src/archiver.rs @@ -233,6 +233,99 @@ impl Archiver { }) } + /// Create a new instance of the archiver with initial state in case of restart. + /// + /// `block` corresponds to `last_archived_block` and will be processed accordingly to its state. + pub fn with_initial_state>( + record_size: usize, + segment_size: usize, + root_block: RootBlock, + encoded_block: B, + mut object_mapping: BlockObjectMapping, + ) -> Result { + let mut archiver = Self::new(record_size, segment_size)?; + + archiver.segment_index = root_block.segment_index() + 1; + archiver.prev_root_block_hash = root_block.hash(); + archiver.last_archived_block = root_block.last_archived_block(); + + // The first thing in the buffer should be root block + archiver + .buffer + .push_back(SegmentItem::RootBlock(root_block)); + + if let Some(archived_block_bytes) = archiver.last_archived_block.partial_archived() { + let encoded_block = encoded_block.as_ref(); + let encoded_block_bytes = u32::try_from(encoded_block.len()) + .expect("Blocks length is never bigger than u32; qed"); + + match encoded_block_bytes.cmp(&archived_block_bytes) { + Ordering::Less => { + return Err(ArchiverInstantiationError::InvalidBlockSmallSize { + block_bytes: encoded_block_bytes, + archived_block_bytes, + }); + } + Ordering::Equal => { + return Err(ArchiverInstantiationError::InvalidLastArchivedBlock( + encoded_block_bytes, + )); + } + Ordering::Greater => { + // Take part of the encoded block that wasn't archived yet and push to the + // buffer and block continuation + object_mapping + .objects + .drain_filter(|block_object: &mut BlockObject| { + let current_offset = block_object.offset(); + if current_offset >= archived_block_bytes { + block_object.set_offset(current_offset - archived_block_bytes); + false + } else { + true + } + }); + archiver.buffer.push_back(SegmentItem::BlockContinuation { + bytes: encoded_block[(archived_block_bytes as usize)..].to_vec(), + object_mapping, + }); + } + } + } + + Ok(archiver) + } + + /// Get last archived block if there was any + pub fn last_archived_block_number(&self) -> Option { + if self.last_archived_block != INITIAL_LAST_ARCHIVED_BLOCK { + Some(self.last_archived_block.number) + } else { + None + } + } + + /// Adds new block to internal buffer, potentially producing pieces and root block headers + pub fn add_block( + &mut self, + bytes: Vec, + object_mapping: BlockObjectMapping, + ) -> Vec { + // Append new block to the buffer + self.buffer.push_back(SegmentItem::Block { + bytes, + object_mapping, + }); + + let mut archived_segments = Vec::new(); + + while let Some(segment) = self.produce_segment() { + archived_segments.push(self.produce_archived_segment(segment)); + } + + archived_segments + } + /// Try to slice buffer contents into segments if there is enough data, producing one segment at /// a time fn produce_segment(&mut self) -> Option { @@ -646,99 +739,6 @@ impl Archiver { object_mapping, } } - - /// Create a new instance of block archiver with initial state in case of restart. - /// - /// `block` corresponds to `last_archived_block` and will be processed accordingly to its state. - pub fn with_initial_state>( - record_size: usize, - segment_size: usize, - root_block: RootBlock, - encoded_block: B, - mut object_mapping: BlockObjectMapping, - ) -> Result { - let mut archiver = Self::new(record_size, segment_size)?; - - archiver.segment_index = root_block.segment_index() + 1; - archiver.prev_root_block_hash = root_block.hash(); - archiver.last_archived_block = root_block.last_archived_block(); - - // The first thing in the buffer should be root block - archiver - .buffer - .push_back(SegmentItem::RootBlock(root_block)); - - if let Some(archived_block_bytes) = archiver.last_archived_block.partial_archived() { - let encoded_block = encoded_block.as_ref(); - let encoded_block_bytes = u32::try_from(encoded_block.len()) - .expect("Blocks length is never bigger than u32; qed"); - - match encoded_block_bytes.cmp(&archived_block_bytes) { - Ordering::Less => { - return Err(ArchiverInstantiationError::InvalidBlockSmallSize { - block_bytes: encoded_block_bytes, - archived_block_bytes, - }); - } - Ordering::Equal => { - return Err(ArchiverInstantiationError::InvalidLastArchivedBlock( - encoded_block_bytes, - )); - } - Ordering::Greater => { - // Take part of the encoded block that wasn't archived yet and push to the - // buffer and block continuation - object_mapping - .objects - .drain_filter(|block_object: &mut BlockObject| { - let current_offset = block_object.offset(); - if current_offset >= archived_block_bytes { - block_object.set_offset(current_offset - archived_block_bytes); - false - } else { - true - } - }); - archiver.buffer.push_back(SegmentItem::BlockContinuation { - bytes: encoded_block[(archived_block_bytes as usize)..].to_vec(), - object_mapping, - }); - } - } - } - - Ok(archiver) - } - - /// Get last archived block if there was any - pub fn last_archived_block_number(&self) -> Option { - if self.last_archived_block != INITIAL_LAST_ARCHIVED_BLOCK { - Some(self.last_archived_block.number) - } else { - None - } - } - - /// Adds new block to internal buffer, potentially producing pieces and root block headers - pub fn add_block( - &mut self, - bytes: Vec, - object_mapping: BlockObjectMapping, - ) -> Vec { - // Append new block to the buffer - self.buffer.push_back(SegmentItem::Block { - bytes, - object_mapping, - }); - - let mut archived_segments = Vec::new(); - - while let Some(segment) = self.produce_segment() { - archived_segments.push(self.produce_archived_segment(segment)); - } - - archived_segments - } } /// Validate witness embedded within a piece produced by archiver From d44fc152a79e7a1aad9f9fe39994251048cd4d6e Mon Sep 17 00:00:00 2001 From: Nazar Mokrynskyi Date: Wed, 24 Nov 2021 03:49:52 +0200 Subject: [PATCH 6/6] Remove objects from segment items --- crates/subspace-archiving/src/archiver.rs | 51 ----------------------- 1 file changed, 51 deletions(-) diff --git a/crates/subspace-archiving/src/archiver.rs b/crates/subspace-archiving/src/archiver.rs index dc8abfd9e2a2f..2b4651058a871 100644 --- a/crates/subspace-archiving/src/archiver.rs +++ b/crates/subspace-archiving/src/archiver.rs @@ -68,15 +68,6 @@ impl Segment { /// Kinds of items that are contained within a segment #[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Encode, Decode)] pub enum SegmentItem { - /// Contains full object inside - #[codec(index = 0)] - Object(Vec), - /// Contains the beginning of the object inside, remainder will be found in subsequent segments - #[codec(index = 1)] - ObjectStart(Vec), - /// Continuation of the partial object spilled over into the next segment - #[codec(index = 2)] - ObjectContinuation(Vec), /// Contains full block inside #[codec(index = 3)] Block { @@ -369,11 +360,6 @@ impl Archiver { // last segment item insertion needs to be skipped to avoid out of range panic when // trying to cut segment item internal bytes. let inner_bytes_size = match &segment_item { - SegmentItem::Object(bytes) => bytes.len(), - SegmentItem::ObjectStart(_) => { - unreachable!("Buffer never contains SegmentItem::ObjectStart; qed"); - } - SegmentItem::ObjectContinuation(bytes) => bytes.len(), SegmentItem::Block { bytes, .. } => bytes.len(), SegmentItem::BlockStart { .. } => { unreachable!("Buffer never contains SegmentItem::BlockStart; qed"); @@ -394,11 +380,6 @@ impl Archiver { } match &segment_item { - SegmentItem::Object(_) - | SegmentItem::ObjectStart(_) - | SegmentItem::ObjectContinuation(_) => { - // We are not interested in object here - } SegmentItem::Block { .. } => { // Skip block number increase in case of the very first block if last_archived_block != INITIAL_LAST_ARCHIVED_BLOCK { @@ -446,33 +427,6 @@ impl Archiver { .expect("Segment over segment size always has at least one item; qed"); let segment_item = match segment_item { - SegmentItem::Object(mut bytes) => { - let split_point = bytes.len() - spill_over; - let continuation_bytes = bytes[split_point..].to_vec(); - - bytes.truncate(split_point); - - // Push continuation element back into the buffer where removed segment item was - self.buffer - .push_front(SegmentItem::ObjectContinuation(continuation_bytes)); - - SegmentItem::ObjectStart(bytes) - } - SegmentItem::ObjectStart(_) => { - unreachable!("Buffer never contains SegmentItem::ObjectStart; qed"); - } - SegmentItem::ObjectContinuation(mut bytes) => { - let split_point = bytes.len() - spill_over; - let continuation_bytes = bytes[split_point..].to_vec(); - - bytes.truncate(split_point); - - // Push continuation element back into the buffer where removed segment item was - self.buffer - .push_front(SegmentItem::ObjectContinuation(continuation_bytes)); - - SegmentItem::ObjectContinuation(bytes) - } SegmentItem::Block { mut bytes, mut object_mapping, @@ -596,11 +550,6 @@ impl Archiver { let mut base_offset_in_segment = 1 + Compact::compact_len(&(items.len() as u32)); for segment_item in items { match segment_item { - SegmentItem::Object(_) - | SegmentItem::ObjectStart(_) - | SegmentItem::ObjectContinuation(_) => { - // Ignore, no objects mappings here - } SegmentItem::Block { bytes, object_mapping,