From ab816ec0dc2d435aba39da1541ac74491740e689 Mon Sep 17 00:00:00 2001 From: Aaryamann Challani <43716372+rymnc@users.noreply.github.com> Date: Wed, 15 Jan 2025 16:58:57 +0530 Subject: [PATCH] fault_proving(compression): include block_id in da compressed block headers (#2551) ## Linked Issues/PRs - closes https://github.com/FuelLabs/fuel-core/issues/2567 ## Description Defines a new version of the `CompressedBlockPayload` which has a header that contains the block_id, relevant for fault proving. Also refactored the proptest strategy generation and moved the macro internally so that tests can be ran from IDEs easily. ## Checklist - [x] Breaking changes are clearly marked as such in the PR description and changelog - [x] New behavior is reflected in tests - [x] [The specification](https://github.com/FuelLabs/fuel-specs/) matches the implemented behavior (link update PR if changes are needed) ### Before requesting review - [x] I have reviewed the code myself - [x] I have created follow-up issues caused by this PR and linked them here ### After merging, notify other teams [Add or remove entries as needed] - [ ] [Rust SDK](https://github.com/FuelLabs/fuels-rs/) - [ ] [Sway compiler](https://github.com/FuelLabs/sway/) - [ ] [Platform documentation](https://github.com/FuelLabs/devrel-requests/issues/new?assignees=&labels=new+request&projects=&template=NEW-REQUEST.yml&title=%5BRequest%5D%3A+) (for out-of-organization contributors, the person merging the PR will do this) - [ ] Someone else? --- CHANGELOG.md | 3 + Cargo.lock | 13 + bin/fuel-core/Cargo.toml | 1 + crates/compression/Cargo.toml | 2 + crates/compression/src/compress.rs | 7 +- .../src/compressed_block_payload/mod.rs | 3 + .../src/compressed_block_payload/v0.rs | 70 +++++ .../src/compressed_block_payload/v1.rs | 122 ++++++++ crates/compression/src/decompress.rs | 15 +- crates/compression/src/lib.rs | 289 +++++++++++++----- crates/fuel-core/Cargo.toml | 1 + 11 files changed, 442 insertions(+), 84 deletions(-) create mode 100644 crates/compression/src/compressed_block_payload/mod.rs create mode 100644 crates/compression/src/compressed_block_payload/v0.rs create mode 100644 crates/compression/src/compressed_block_payload/v1.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 9385bfd3875..0513c5eb1e0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ## [Unreleased] +### Added +- [2551](https://github.com/FuelLabs/fuel-core/pull/2551): Enhanced the DA compressed block header to include block id. + ## [Version 0.41.0] ### Added diff --git a/Cargo.lock b/Cargo.lock index c4deb52f448..9dd90e6a94b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2793,6 +2793,18 @@ dependencies = [ "syn 2.0.96", ] +[[package]] +name = "enum_dispatch" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa18ce2bc66555b3218614519ac839ddb759a7d6720732f979ef8d13be147ecd" +dependencies = [ + "once_cell", + "proc-macro2", + "quote", + "syn 2.0.96", +] + [[package]] name = "equivalent" version = "1.0.1" @@ -3567,6 +3579,7 @@ name = "fuel-core-compression" version = "0.41.0" dependencies = [ "anyhow", + "enum_dispatch", "fuel-core-compression", "fuel-core-types 0.41.0", "paste", diff --git a/bin/fuel-core/Cargo.toml b/bin/fuel-core/Cargo.toml index 3ffda465384..a2624d582a7 100644 --- a/bin/fuel-core/Cargo.toml +++ b/bin/fuel-core/Cargo.toml @@ -82,3 +82,4 @@ production = [ "parquet", "aws-kms", ] +fault-proving = ["fuel-core-compression/fault-proving"] diff --git a/crates/compression/Cargo.toml b/crates/compression/Cargo.toml index 40e092d1e00..c47b24e941e 100644 --- a/crates/compression/Cargo.toml +++ b/crates/compression/Cargo.toml @@ -18,6 +18,7 @@ description = "Compression and decompression of Fuel blocks for DA storage." [dependencies] anyhow = { workspace = true } +enum_dispatch = "0.3.13" fuel-core-types = { workspace = true, features = [ "alloc", "serde", @@ -42,3 +43,4 @@ test-helpers = [ "fuel-core-types/random", "fuel-core-types/std", ] +fault-proving = [] diff --git a/crates/compression/src/compress.rs b/crates/compression/src/compress.rs index 0ad14e39f55..0330121daa4 100644 --- a/crates/compression/src/compress.rs +++ b/crates/compression/src/compress.rs @@ -12,7 +12,6 @@ use crate::{ RegistrationsPerTable, TemporalRegistryAll, }, - CompressedBlockPayloadV0, VersionedCompressedBlock, }; use anyhow::Context; @@ -70,11 +69,11 @@ where let transactions = target.compress_with(&mut ctx).await?; let registrations: RegistrationsPerTable = ctx.finalize()?; - Ok(VersionedCompressedBlock::V0(CompressedBlockPayloadV0 { + Ok(VersionedCompressedBlock::new( + block.header(), registrations, - header: block.header().into(), transactions, - })) + )) } /// Preparation pass through the block to collect all keys accessed during compression. diff --git a/crates/compression/src/compressed_block_payload/mod.rs b/crates/compression/src/compressed_block_payload/mod.rs new file mode 100644 index 00000000000..eeebe11bfb5 --- /dev/null +++ b/crates/compression/src/compressed_block_payload/mod.rs @@ -0,0 +1,3 @@ +pub mod v0; + +pub mod v1; diff --git a/crates/compression/src/compressed_block_payload/v0.rs b/crates/compression/src/compressed_block_payload/v0.rs new file mode 100644 index 00000000000..ea1555e8e24 --- /dev/null +++ b/crates/compression/src/compressed_block_payload/v0.rs @@ -0,0 +1,70 @@ +use crate::{ + registry::RegistrationsPerTable, + VersionedBlockPayload, +}; +use fuel_core_types::{ + blockchain::{ + header::{ + ApplicationHeader, + BlockHeader, + ConsensusHeader, + PartialBlockHeader, + }, + primitives::Empty, + }, + fuel_tx::CompressedTransaction, + fuel_types::BlockHeight, +}; + +/// Compressed block, without the preceding version byte. +#[derive(Debug, Default, Clone, PartialEq, serde::Serialize, serde::Deserialize)] +pub struct CompressedBlockPayloadV0 { + /// Temporal registry insertions + pub registrations: RegistrationsPerTable, + /// Compressed block header + pub header: PartialBlockHeader, + /// Compressed transactions + pub transactions: Vec, +} + +impl VersionedBlockPayload for CompressedBlockPayloadV0 { + fn height(&self) -> &BlockHeight { + self.header.height() + } + + fn consensus_header(&self) -> &ConsensusHeader { + &self.header.consensus + } + + fn application_header(&self) -> &ApplicationHeader { + &self.header.application + } + + fn registrations(&self) -> &RegistrationsPerTable { + &self.registrations + } + + fn transactions(&self) -> Vec { + self.transactions.clone() + } + + fn partial_block_header(&self) -> PartialBlockHeader { + self.header + } +} + +impl CompressedBlockPayloadV0 { + /// Create a new compressed block payload V0. + #[allow(unused)] + pub(crate) fn new( + header: &BlockHeader, + registrations: RegistrationsPerTable, + transactions: Vec, + ) -> Self { + Self { + header: PartialBlockHeader::from(header), + registrations, + transactions, + } + } +} diff --git a/crates/compression/src/compressed_block_payload/v1.rs b/crates/compression/src/compressed_block_payload/v1.rs new file mode 100644 index 00000000000..51e0012a615 --- /dev/null +++ b/crates/compression/src/compressed_block_payload/v1.rs @@ -0,0 +1,122 @@ +use crate::{ + registry::RegistrationsPerTable, + VersionedBlockPayload, +}; +use fuel_core_types::{ + blockchain::{ + header::{ + ApplicationHeader, + BlockHeader, + ConsensusHeader, + PartialBlockHeader, + }, + primitives::{ + BlockId, + Empty, + }, + }, + fuel_tx::CompressedTransaction, + fuel_types::BlockHeight, +}; + +/// A partially complete fuel block header that does not +/// have any generated fields because it has not been executed yet. +#[derive( + Copy, Clone, Debug, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize, +)] +pub struct CompressedBlockHeader { + /// The application header. + pub application: ApplicationHeader, + /// The consensus header. + pub consensus: ConsensusHeader, + // The block id. + pub block_id: BlockId, +} + +impl From<&BlockHeader> for CompressedBlockHeader { + fn from(header: &BlockHeader) -> Self { + let ConsensusHeader { + prev_root, + height, + time, + .. + } = *header.consensus(); + CompressedBlockHeader { + application: ApplicationHeader { + da_height: header.da_height, + consensus_parameters_version: header.consensus_parameters_version, + state_transition_bytecode_version: header + .state_transition_bytecode_version, + generated: Empty {}, + }, + consensus: ConsensusHeader { + prev_root, + height, + time, + generated: Empty {}, + }, + block_id: header.id(), + } + } +} + +impl From<&CompressedBlockHeader> for PartialBlockHeader { + fn from(value: &CompressedBlockHeader) -> Self { + PartialBlockHeader { + application: value.application, + consensus: value.consensus, + } + } +} + +#[derive(Debug, Default, Clone, PartialEq, serde::Serialize, serde::Deserialize)] +pub struct CompressedBlockPayloadV1 { + /// Temporal registry insertions + pub registrations: RegistrationsPerTable, + /// Compressed block header + pub header: CompressedBlockHeader, + /// Compressed transactions + pub transactions: Vec, +} + +impl VersionedBlockPayload for CompressedBlockPayloadV1 { + fn height(&self) -> &BlockHeight { + &self.header.consensus.height + } + + fn consensus_header(&self) -> &ConsensusHeader { + &self.header.consensus + } + + fn application_header(&self) -> &ApplicationHeader { + &self.header.application + } + + fn registrations(&self) -> &RegistrationsPerTable { + &self.registrations + } + + fn transactions(&self) -> Vec { + self.transactions.clone() + } + + fn partial_block_header(&self) -> PartialBlockHeader { + PartialBlockHeader::from(&self.header) + } +} + +impl CompressedBlockPayloadV1 { + /// Create a new compressed block payload V1. + #[allow(unused)] + pub(crate) fn new( + header: &BlockHeader, + registrations: RegistrationsPerTable, + transactions: Vec, + ) -> Self { + Self { + header: CompressedBlockHeader::from(header), + registrations, + transactions, + } + } +} diff --git a/crates/compression/src/decompress.rs b/crates/compression/src/decompress.rs index 15565ce8433..082124e9161 100644 --- a/crates/compression/src/decompress.rs +++ b/crates/compression/src/decompress.rs @@ -5,6 +5,7 @@ use crate::{ TemporalRegistry, }, registry::TemporalRegistryAll, + VersionedBlockPayload, VersionedCompressedBlock, }; use fuel_core_types::{ @@ -55,28 +56,26 @@ pub async fn decompress( where D: DecompressDb, { - let VersionedCompressedBlock::V0(compressed) = block; - // TODO: merkle root verification: https://github.com/FuelLabs/fuel-core/issues/2232 - compressed - .registrations - .write_to_registry(&mut db, compressed.header.consensus.time)?; + block + .registrations() + .write_to_registry(&mut db, block.consensus_header().time)?; let ctx = DecompressCtx { config, - timestamp: compressed.header.consensus.time, + timestamp: block.consensus_header().time, db, }; let transactions = as DecompressibleBy<_>>::decompress_with( - compressed.transactions, + block.transactions(), &ctx, ) .await?; Ok(PartialFuelBlock { - header: compressed.header, + header: block.partial_block_header(), transactions, }) } diff --git a/crates/compression/src/lib.rs b/crates/compression/src/lib.rs index d41deccefa1..681a466e121 100644 --- a/crates/compression/src/lib.rs +++ b/crates/compression/src/lib.rs @@ -4,6 +4,7 @@ #![deny(warnings)] pub mod compress; +mod compressed_block_payload; pub mod config; pub mod decompress; mod eviction_policy; @@ -11,49 +12,79 @@ pub mod ports; mod registry; pub use config::Config; +use enum_dispatch::enum_dispatch; pub use registry::RegistryKeyspace; +use crate::compressed_block_payload::v0::CompressedBlockPayloadV0; +#[cfg(feature = "fault-proving")] +use crate::compressed_block_payload::v1::CompressedBlockPayloadV1; use fuel_core_types::{ - blockchain::header::PartialBlockHeader, + blockchain::{ + header::{ + ApplicationHeader, + BlockHeader, + ConsensusHeader, + PartialBlockHeader, + }, + primitives::Empty, + }, fuel_tx::CompressedTransaction, fuel_types::BlockHeight, }; use registry::RegistrationsPerTable; -/// Compressed block, without the preceding version byte. -#[derive(Debug, Default, Clone, PartialEq, serde::Serialize, serde::Deserialize)] -pub struct CompressedBlockPayloadV0 { - /// Temporal registry insertions - pub registrations: RegistrationsPerTable, - /// Compressed block header - pub header: PartialBlockHeader, - /// Compressed transactions - pub transactions: Vec, +/// A compressed block payload MUST implement this trait +/// It is used to provide a convenient interface for usage within +/// compression +#[enum_dispatch] +pub trait VersionedBlockPayload { + fn height(&self) -> &BlockHeight; + fn consensus_header(&self) -> &ConsensusHeader; + fn application_header(&self) -> &ApplicationHeader; + fn registrations(&self) -> &RegistrationsPerTable; + fn transactions(&self) -> Vec; + fn partial_block_header(&self) -> PartialBlockHeader; } /// Versioned compressed block. #[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] +#[enum_dispatch(VersionedBlockPayload)] pub enum VersionedCompressedBlock { V0(CompressedBlockPayloadV0), + #[cfg(feature = "fault-proving")] + V1(CompressedBlockPayloadV1), } -impl Default for VersionedCompressedBlock { - fn default() -> Self { - Self::V0(Default::default()) +impl VersionedCompressedBlock { + fn new( + header: &BlockHeader, + registrations: RegistrationsPerTable, + transactions: Vec, + ) -> Self { + #[cfg(not(feature = "fault-proving"))] + return Self::V0(CompressedBlockPayloadV0::new( + header, + registrations, + transactions, + )); + #[cfg(feature = "fault-proving")] + Self::V1(CompressedBlockPayloadV1::new( + header, + registrations, + transactions, + )) } } -impl VersionedCompressedBlock { - /// Returns the height of the compressed block. - pub fn height(&self) -> &BlockHeight { - match self { - VersionedCompressedBlock::V0(block) => block.header.height(), - } +impl Default for VersionedCompressedBlock { + fn default() -> Self { + Self::V0(Default::default()) } } #[cfg(test)] mod tests { + use super::*; use fuel_core_compression as _; use fuel_core_types::{ blockchain::{ @@ -68,8 +99,6 @@ mod tests { }; use proptest::prelude::*; - use super::*; - fn keyspace() -> impl Strategy { prop_oneof![ Just(RegistryKeyspace::Address), @@ -80,46 +109,96 @@ mod tests { ] } - proptest! { - /// Serialization for compressed transactions is already tested in fuel-vm, - /// but the rest of the block de/serialization is tested here. - #[test] - fn postcard_roundtrip( - da_height in 0..=u64::MAX, - prev_root in prop::array::uniform32(0..=u8::MAX), - height in 0..=u32::MAX, - consensus_parameters_version in 0..=u32::MAX, - state_transition_bytecode_version in 0..=u32::MAX, - registration_inputs in prop::collection::vec( - (keyspace(), prop::num::u16::ANY, prop::array::uniform32(0..=u8::MAX)).prop_map(|(ks, rk, arr)| { - let k = RegistryKey::try_from(rk as u32).unwrap(); - (ks, k, arr) - }), - 0..123 + #[derive(Debug)] + struct PostcardRoundtripStrategy { + da_height: u64, + prev_root: [u8; 32], + height: u32, + consensus_parameters_version: u32, + state_transition_bytecode_version: u32, + registrations: RegistrationsPerTable, + } + + fn postcard_roundtrip_strategy() -> impl Strategy { + ( + 0..=u64::MAX, + prop::array::uniform32(0..=u8::MAX), + 0..=u32::MAX, + 0..=u32::MAX, + 0..=u32::MAX, + prop::collection::vec( + ( + keyspace(), + prop::num::u16::ANY, + prop::array::uniform32(0..=u8::MAX), + ) + .prop_map(|(ks, rk, arr)| { + let k = RegistryKey::try_from(rk as u32).unwrap(); + (ks, k, arr) + }), + 0..123, ), - ) { - let mut registrations: RegistrationsPerTable = Default::default(); - - for (ks, key, arr) in registration_inputs { - let value_len_limit = (key.as_u32() % 32) as usize; - match ks { - RegistryKeyspace::Address => { - registrations.address.push((key, arr.into())); - } - RegistryKeyspace::AssetId => { - registrations.asset_id.push((key, arr.into())); - } - RegistryKeyspace::ContractId => { - registrations.contract_id.push((key, arr.into())); - } - RegistryKeyspace::ScriptCode => { - registrations.script_code.push((key, arr[..value_len_limit].to_vec().into())); + ) + .prop_map( + |( + da_height, + prev_root, + height, + consensus_parameters_version, + state_transition_bytecode_version, + registration_inputs, + )| { + let mut registrations: RegistrationsPerTable = Default::default(); + for (ks, key, arr) in registration_inputs { + let value_len_limit = (key.as_u32() % 32) as usize; + match ks { + RegistryKeyspace::Address => { + registrations.address.push((key, arr.into())); + } + RegistryKeyspace::AssetId => { + registrations.asset_id.push((key, arr.into())); + } + RegistryKeyspace::ContractId => { + registrations.contract_id.push((key, arr.into())); + } + RegistryKeyspace::ScriptCode => { + registrations + .script_code + .push((key, arr[..value_len_limit].to_vec().into())); + } + RegistryKeyspace::PredicateCode => { + registrations + .predicate_code + .push((key, arr[..value_len_limit].to_vec().into())); + } + } } - RegistryKeyspace::PredicateCode => { - registrations.predicate_code.push((key, arr[..value_len_limit].to_vec().into())); + + PostcardRoundtripStrategy { + da_height, + prev_root, + height, + consensus_parameters_version, + state_transition_bytecode_version, + registrations, } - } - } + }, + ) + } + + /// Serialization for compressed transactions is already tested in fuel-vm, + /// but the rest of the block de/serialization is tested here. + #[test] + fn postcard_roundtrip_v0() { + proptest!(|(strategy in postcard_roundtrip_strategy())| { + let PostcardRoundtripStrategy { + da_height, + prev_root, + height, + consensus_parameters_version, + state_transition_bytecode_version, + registrations, + } = strategy; let header = PartialBlockHeader { application: ApplicationHeader { @@ -135,31 +214,97 @@ mod tests { generated: Empty } }; - let original = CompressedBlockPayloadV0 { + + let original = VersionedCompressedBlock::V0(CompressedBlockPayloadV0 { registrations, header, transactions: vec![], - }; + }); let compressed = postcard::to_allocvec(&original).unwrap(); - let decompressed: CompressedBlockPayloadV0 = + let decompressed: VersionedCompressedBlock = postcard::from_bytes(&compressed).unwrap(); - let CompressedBlockPayloadV0 { + let consensus_header = decompressed.consensus_header(); + let application_header = decompressed.application_header(); + + assert_eq!(decompressed.registrations(), original.registrations()); + + assert_eq!(application_header.da_height, da_height.into()); + assert_eq!(consensus_header.prev_root, prev_root.into()); + assert_eq!(consensus_header.height, height.into()); + assert_eq!(application_header.consensus_parameters_version, consensus_parameters_version); + assert_eq!(application_header.state_transition_bytecode_version, state_transition_bytecode_version); + + assert!(decompressed.transactions().is_empty()); + }); + } + + #[cfg(feature = "fault-proving")] + #[test] + fn postcard_roundtrip_v1() { + use compressed_block_payload::v1::{ + CompressedBlockHeader, + CompressedBlockPayloadV1, + }; + use fuel_core_types::blockchain::primitives::BlockId; + use std::str::FromStr; + + proptest!(|(strategy in postcard_roundtrip_strategy())| { + let PostcardRoundtripStrategy { + da_height, + prev_root, + height, + consensus_parameters_version, + state_transition_bytecode_version, registrations, + } = strategy; + + let header = CompressedBlockHeader { + application: ApplicationHeader { + da_height: da_height.into(), + consensus_parameters_version, + state_transition_bytecode_version, + generated: Empty, + }, + consensus: ConsensusHeader { + prev_root: prev_root.into(), + height: height.into(), + time: Tai64::UNIX_EPOCH, + generated: Empty, + }, + block_id: BlockId::from_str("0xecea85c17070bc2e65f911310dbd01198f4436052ebba96cded9ddf30c58dd1a").unwrap(), + }; + + + let original = VersionedCompressedBlock::V1(CompressedBlockPayloadV1 { header, - transactions, - } = decompressed; + registrations, + transactions: vec![] + }); + + let compressed = postcard::to_allocvec(&original).unwrap(); + let decompressed: VersionedCompressedBlock = + postcard::from_bytes(&compressed).unwrap(); - assert_eq!(registrations, original.registrations); + let consensus_header = decompressed.consensus_header(); + let application_header = decompressed.application_header(); - assert_eq!(header.da_height, da_height.into()); - assert_eq!(*header.prev_root(), prev_root.into()); - assert_eq!(*header.height(), height.into()); - assert_eq!(header.consensus_parameters_version, consensus_parameters_version); - assert_eq!(header.state_transition_bytecode_version, state_transition_bytecode_version); + assert_eq!(decompressed.registrations(), original.registrations()); - assert!(transactions.is_empty()); - } + assert_eq!(application_header.da_height, da_height.into()); + assert_eq!(consensus_header.prev_root, prev_root.into()); + assert_eq!(consensus_header.height, height.into()); + assert_eq!(application_header.consensus_parameters_version, consensus_parameters_version); + assert_eq!(application_header.state_transition_bytecode_version, state_transition_bytecode_version); + + assert!(decompressed.transactions().is_empty()); + + if let VersionedCompressedBlock::V1(block) = decompressed { + assert_eq!(block.header.block_id, header.block_id); + } else { + panic!("Expected V1 block, got {:?}", decompressed); + } + }); } } diff --git a/crates/fuel-core/Cargo.toml b/crates/fuel-core/Cargo.toml index 34a36fd6c20..31f087d72ac 100644 --- a/crates/fuel-core/Cargo.toml +++ b/crates/fuel-core/Cargo.toml @@ -117,3 +117,4 @@ test-helpers = [ # features to enable in production, but increase build times rocksdb-production = ["rocksdb", "rocksdb/jemalloc"] wasm-executor = ["fuel-core-upgradable-executor/wasm-executor"] +fault-proving = ["fuel-core-compression/fault-proving"]