From ed725002a61264046ca51742d2c89938f20a3b30 Mon Sep 17 00:00:00 2001 From: Mirko <48352201+Mirko-von-Leipzig@users.noreply.github.com> Date: Mon, 7 Oct 2024 13:05:25 +0200 Subject: [PATCH 01/50] feat: block producer redesign skeleton (#502) Initial outline of the block-producer redesign. Includes a new mempool which can track transaction and batch dependencies as a graph. This enables more complex mempool strategies. --- .../block-producer/src/batch_builder/batch.rs | 3 +- .../block-producer/src/batch_builder/mod.rs | 100 +++++- .../block-producer/src/block_builder/mod.rs | 34 +- crates/block-producer/src/errors.rs | 28 ++ crates/block-producer/src/lib.rs | 4 + .../block-producer/src/mempool/batch_graph.rs | 201 +++++++++++ .../src/mempool/inflight_state.rs | 314 ++++++++++++++++++ crates/block-producer/src/mempool/mod.rs | 275 +++++++++++++++ .../src/mempool/transaction_graph.rs | 170 ++++++++++ crates/block-producer/src/server/mod.rs | 136 +++++++- 10 files changed, 1250 insertions(+), 15 deletions(-) create mode 100644 crates/block-producer/src/mempool/batch_graph.rs create mode 100644 crates/block-producer/src/mempool/inflight_state.rs create mode 100644 crates/block-producer/src/mempool/mod.rs create mode 100644 crates/block-producer/src/mempool/transaction_graph.rs diff --git a/crates/block-producer/src/batch_builder/batch.rs b/crates/block-producer/src/batch_builder/batch.rs index e8a13fb36..6490a95a7 100644 --- a/crates/block-producer/src/batch_builder/batch.rs +++ b/crates/block-producer/src/batch_builder/batch.rs @@ -22,8 +22,7 @@ pub type BatchId = Blake3Digest<32>; // TRANSACTION BATCH // ================================================================================================ -/// A batch of transactions that share a common proof. For any given account, at most 1 transaction -/// in the batch must be addressing that account (issue: #186). +/// A batch of transactions that share a common proof. /// /// Note: Until recursive proofs are available in the Miden VM, we don't include the common proof. #[derive(Debug, Clone, PartialEq, Eq)] diff --git a/crates/block-producer/src/batch_builder/mod.rs b/crates/block-producer/src/batch_builder/mod.rs index 513321f7e..db4d78079 100644 --- a/crates/block-producer/src/batch_builder/mod.rs +++ b/crates/block-producer/src/batch_builder/mod.rs @@ -1,11 +1,18 @@ -use std::{cmp::min, collections::BTreeSet, sync::Arc, time::Duration}; - -use async_trait::async_trait; -use miden_objects::{notes::NoteId, transaction::OutputNote}; -use tokio::time; +use std::{cmp::min, collections::BTreeSet, num::NonZeroUsize, sync::Arc, time::Duration}; + +use miden_objects::{ + notes::NoteId, + transaction::{OutputNote, TransactionId}, +}; +use tokio::{sync::Mutex, time}; +use tonic::async_trait; use tracing::{debug, info, instrument, Span}; -use crate::{block_builder::BlockBuilder, ProvenTransaction, SharedRwVec, COMPONENT}; +use crate::{ + block_builder::BlockBuilder, + mempool::{BatchJobId, Mempool}, + ProvenTransaction, SharedRwVec, COMPONENT, +}; #[cfg(test)] mod tests; @@ -206,3 +213,84 @@ where Ok(()) } } + +pub struct BatchProducer { + pub batch_interval: Duration, + pub workers: NonZeroUsize, + pub mempool: Arc>, + pub tx_per_batch: usize, +} + +type BatchResult = Result; + +/// Wrapper around tokio's JoinSet that remains pending if the set is empty, +/// instead of returning None. +struct WorkerPool(tokio::task::JoinSet); + +impl WorkerPool { + async fn join_next(&mut self) -> Result { + if self.0.is_empty() { + std::future::pending().await + } else { + // Cannot be None as its not empty. + self.0.join_next().await.unwrap() + } + } + + fn len(&self) -> usize { + self.0.len() + } + + fn spawn(&mut self, id: BatchJobId, transactions: Vec) { + self.0.spawn(async move { + todo!("Do actual work like aggregating transaction data"); + }); + } +} + +impl BatchProducer { + pub async fn run(self) { + let mut interval = tokio::time::interval(self.batch_interval); + interval.set_missed_tick_behavior(time::MissedTickBehavior::Delay); + + let mut inflight = WorkerPool(tokio::task::JoinSet::new()); + + loop { + tokio::select! { + _ = interval.tick() => { + if inflight.len() >= self.workers.get() { + tracing::info!("All batch workers occupied."); + continue; + } + + // Transactions available? + let Some((batch_id, transactions)) = + self.mempool.lock().await.select_batch() + else { + tracing::info!("No transactions available for batch."); + continue; + }; + + inflight.spawn(batch_id, transactions); + }, + result = inflight.join_next() => { + let mut mempool = self.mempool.lock().await; + match result { + Err(err) => { + tracing::warn!(%err, "Batch job panic'd.") + // TODO: somehow embed the batch ID into the join error, though this doesn't seem possible? + // mempool.batch_failed(batch_id); + }, + Ok(Err((batch_id, err))) => { + tracing::warn!(%batch_id, %err, "Batch job failed."); + mempool.batch_failed(batch_id); + }, + Ok(Ok(batch_id)) => { + mempool.batch_proved(batch_id); + } + } + } + } + } + } +} diff --git a/crates/block-producer/src/block_builder/mod.rs b/crates/block-producer/src/block_builder/mod.rs index 0522d1140..111c7c3d1 100644 --- a/crates/block-producer/src/block_builder/mod.rs +++ b/crates/block-producer/src/block_builder/mod.rs @@ -8,12 +8,14 @@ use miden_objects::{ notes::{NoteHeader, Nullifier}, transaction::InputNoteCommitment, }; +use tokio::sync::Mutex; use tracing::{debug, info, instrument}; use crate::{ batch_builder::batch::TransactionBatch, errors::BuildBlockError, - store::{ApplyBlock, Store}, + mempool::{BatchJobId, Mempool}, + store::{ApplyBlock, DefaultStore, Store}, COMPONENT, }; @@ -143,3 +145,33 @@ where Ok(()) } } + +struct BlockProducer { + pub mempool: Arc>, + pub block_interval: tokio::time::Duration, +} + +impl BlockProducer { + pub async fn run(self) { + let mut interval = tokio::time::interval(self.block_interval); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + + loop { + interval.tick().await; + + let (block_number, batches) = self.mempool.lock().await.select_block(); + + let result = self.build_and_commit_block(batches).await; + let mut mempool = self.mempool.lock().await; + + match result { + Ok(_) => mempool.block_committed(block_number), + Err(_) => mempool.block_failed(block_number), + } + } + } + + async fn build_and_commit_block(&self, batches: BTreeSet) -> Result<(), ()> { + todo!("Aggregate, prove and commit block"); + } +} diff --git a/crates/block-producer/src/errors.rs b/crates/block-producer/src/errors.rs index 717a29268..a963236d4 100644 --- a/crates/block-producer/src/errors.rs +++ b/crates/block-producer/src/errors.rs @@ -1,3 +1,5 @@ +use std::collections::BTreeSet; + use miden_node_proto::errors::ConversionError; use miden_node_utils::formatting::format_opt; use miden_objects::{ @@ -9,8 +11,11 @@ use miden_objects::{ MAX_BATCHES_PER_BLOCK, MAX_INPUT_NOTES_PER_BATCH, MAX_OUTPUT_NOTES_PER_BATCH, }; use miden_processor::ExecutionError; +use miden_tx::TransactionVerifierError; use thiserror::Error; +use crate::mempool::BlockNumber; + // Transaction verification errors // ================================================================================================= @@ -58,6 +63,29 @@ pub enum AddTransactionError { VerificationFailed(#[from] VerifyTxError), } +#[derive(thiserror::Error, Debug, PartialEq)] +pub enum AddTransactionErrorRework { + #[error("Transaction's initial account state {expected} did not match the current account state {current}.")] + InvalidAccountState { current: Digest, expected: Digest }, + #[error("Transaction input data is stale. Required data fresher than {stale_limit} but inputs are from {input_block}.")] + StaleInputs { + input_block: BlockNumber, + stale_limit: BlockNumber, + }, + #[error("Authenticated note nullifier {0} not found.")] + AuthenticatedNoteNotFound(Nullifier), + #[error("Unauthenticated note {0} not found.")] + UnauthenticatedNoteNotFound(NoteId), + #[error("Note nullifiers already consumed: {0:?}")] + NotesAlreadyConsumed(BTreeSet), + #[error(transparent)] + TxInputsError(#[from] TxInputsError), + #[error(transparent)] + ProofVerificationFailed(#[from] TransactionVerifierError), + #[error("Failed to deserialize transaction: {0}.")] + DeserializationError(String), +} + // Batch building errors // ================================================================================================= diff --git a/crates/block-producer/src/lib.rs b/crates/block-producer/src/lib.rs index dbbdd9803..d40ce14c8 100644 --- a/crates/block-producer/src/lib.rs +++ b/crates/block-producer/src/lib.rs @@ -1,3 +1,6 @@ +// TODO: remove once block-producer rework is complete +#![allow(unused)] + use std::{sync::Arc, time::Duration}; use batch_builder::batch::TransactionBatch; @@ -10,6 +13,7 @@ pub mod test_utils; mod batch_builder; mod block_builder; mod errors; +mod mempool; mod state_view; mod store; mod txqueue; diff --git a/crates/block-producer/src/mempool/batch_graph.rs b/crates/block-producer/src/mempool/batch_graph.rs new file mode 100644 index 000000000..fe963b02e --- /dev/null +++ b/crates/block-producer/src/mempool/batch_graph.rs @@ -0,0 +1,201 @@ +use std::collections::{BTreeMap, BTreeSet}; + +use miden_objects::transaction::TransactionId; +use miden_tx::utils::collections::KvMap; + +use super::BatchJobId; + +#[derive(Default, Clone)] +pub struct BatchGraph { + nodes: BTreeMap, + roots: BTreeSet, + + /// Allows for reverse lookup of transaction -> batch. + transactions: BTreeMap, +} + +impl BatchGraph { + pub fn insert( + &mut self, + id: BatchJobId, + transactions: Vec, + parents: BTreeSet, + ) { + // Reverse lookup parent transaction batches. + let parents = parents + .into_iter() + .map(|tx| self.transactions.get(&tx).expect("Parent transaction must be in a batch")) + .copied() + .collect(); + + // Inform parents of their new child. + for parent in &parents { + self.nodes + .get_mut(parent) + .expect("Parent batch must be present") + .children + .insert(id); + } + + // Insert transactions for reverse lookup. + for tx in &transactions { + self.transactions.insert(*tx, id); + } + + // Insert the new node into the graph. + let batch = Node { + status: Status::InFlight, + transactions, + parents, + children: Default::default(), + }; + self.nodes.insert(id, batch); + + // New node might be a root. + // + // This could be optimised by inlining this inside the parent loop. This would prevent the + // double iteration over parents, at the cost of some code duplication. + self.try_make_root(id); + } + + /// Removes the batches and all their descendents from the graph. + /// + /// Returns all removed batches and their transactions. + pub fn purge_subgraphs( + &mut self, + batches: BTreeSet, + ) -> BTreeMap> { + let mut removed = BTreeMap::new(); + + let mut to_process = batches; + + while let Some(node_id) = to_process.pop_first() { + // Its possible for a node to already have been removed as part of this subgraph + // removal. + let Some(node) = self.nodes.remove(&node_id) else { + continue; + }; + + // All the child batches are also removed so no need to chec + // for new roots. No new roots are possible as a result of this subgraph removal. + self.roots.remove(&node_id); + + for transaction in &node.transactions { + self.transactions.remove(transaction); + } + + // Inform parent that this child no longer exists. + // + // The same is not required for children of this batch as we will + // be removing those as well. + for parent in &node.parents { + // Parent could already be removed as part of this subgraph removal. + if let Some(parent) = self.nodes.get_mut(parent) { + parent.children.remove(&node_id); + } + } + + to_process.extend(node.children); + removed.insert(node_id, node.transactions); + } + + removed + } + + /// Removes a set of batches from the graph without removing any descendents. + /// + /// This is intended to cull completed batches from stale blocs. + pub fn remove_committed(&mut self, batches: BTreeSet) -> Vec { + let mut transactions = Vec::new(); + + for batch in batches { + let node = self.nodes.remove(&batch).expect("Node must be in graph"); + assert_eq!(node.status, Status::InBlock); + + // Remove batch from graph. No need to update parents as they should be removed in this + // call as well. + for child in node.children { + // Its possible for the child to part of this same set of batches and therefore + // already removed. + if let Some(child) = self.nodes.get_mut(&child) { + child.parents.remove(&batch); + } + } + + transactions.extend_from_slice(&node.transactions); + } + + transactions + } + + /// Mark a batch as proven if it exists. + pub fn mark_proven(&mut self, id: BatchJobId) { + // Its possible for inflight batches to have been removed as part + // of another batches failure. + if let Some(node) = self.nodes.get_mut(&id) { + node.status = Status::Proven; + self.try_make_root(id); + } + } + + /// Returns at most `count` __indepedent__ batches which are ready for inclusion in a block. + pub fn select_block(&mut self, count: usize) -> BTreeSet { + let mut batches = BTreeSet::new(); + + // Track children so we can evaluate them for root afterwards. + let mut children = BTreeSet::new(); + + for batch in &self.roots { + let mut node = self.nodes.get_mut(batch).expect("Root node must be in graph"); + + // Filter out batches which have dependencies in our selection so far. + if batches.union(&node.parents).next().is_some() { + continue; + } + + batches.insert(*batch); + node.status = Status::Proven; + + if batches.len() == count { + break; + } + } + + // Performing this outside the main loop has two benefits: + // 1. We avoid checking the same child multiple times. + // 2. We avoid evaluating children for selection (they're dependents). + for child in children { + self.try_make_root(child); + } + + batches + } + + fn try_make_root(&mut self, id: BatchJobId) { + let node = self.nodes.get_mut(&id).expect("Node must be in graph"); + + for parent in node.parents.clone() { + let parent = self.nodes.get(&parent).expect("Parent must be in pool"); + + if parent.status != Status::InBlock { + return; + } + } + self.roots.insert(id); + } +} + +#[derive(Clone, Debug)] +struct Node { + status: Status, + transactions: Vec, + parents: BTreeSet, + children: BTreeSet, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Status { + InFlight, + Proven, + InBlock, +} diff --git a/crates/block-producer/src/mempool/inflight_state.rs b/crates/block-producer/src/mempool/inflight_state.rs new file mode 100644 index 000000000..d2df8d2b0 --- /dev/null +++ b/crates/block-producer/src/mempool/inflight_state.rs @@ -0,0 +1,314 @@ +use std::{ + collections::{BTreeMap, BTreeSet, VecDeque}, + sync::Arc, +}; + +use miden_objects::{ + accounts::AccountId, + notes::Nullifier, + transaction::{ProvenTransaction, TransactionId}, + Digest, +}; + +use crate::{errors::AddTransactionErrorRework, store::TransactionInputs}; + +/// Tracks the inflight state of the mempool. This includes recently committed blocks. +/// +/// Allows appending and reverting transactions as well as marking them +/// as part of a committed block. Committed state can also be pruned once the +/// state is considered past the stale threshold. +#[derive(Default)] +pub struct InflightState { + /// Non-empty inflight account states. + /// + /// Accounts which are [AccountStatus::Empty] are immedietely pruned. + accounts: BTreeMap, + + /// Nullifiers emitted by inflight transactions and recently committed blocks. + nullifiers: BTreeSet, +} + +/// Describes the impact that a set of transactions had on the state. +/// +/// TODO: this is a terrible name. +pub struct StateDiff { + /// The number of transactions that affected each account. + account_transactions: BTreeMap, + + /// The nullifiers that were emitted by the transactions. + nullifiers: BTreeSet, + // TODO: input/output notes +} + +impl StateDiff { + pub fn new(txs: &[Arc]) -> Self { + let mut account_transactions = BTreeMap::::new(); + let mut nullifiers = BTreeSet::new(); + + for tx in txs { + *account_transactions.entry(tx.account_id()).or_default() += 1; + nullifiers.extend(tx.get_nullifiers()); + } + + Self { account_transactions, nullifiers } + } +} + +impl InflightState { + /// Appends the transaction to the inflight state. + /// + /// This operation is atomic i.e. a rejected transaction has no impact of the state. + pub fn add_transaction( + &mut self, + tx: &ProvenTransaction, + inputs: &TransactionInputs, + ) -> Result, AddTransactionErrorRework> { + // Separate verification and state mutation so that a rejected transaction + // does not impact the state (atomicity). + self.verify_transaction(tx, inputs)?; + + let parents = self.insert_transaction(tx); + + Ok(parents) + } + + fn verify_transaction( + &mut self, + tx: &ProvenTransaction, + inputs: &TransactionInputs, + ) -> Result<(), AddTransactionErrorRework> { + // Ensure current account state is correct. + let current = self + .accounts + .get(&tx.account_id()) + .and_then(|account_state| account_state.latest_state()) + .copied() + .or(inputs.account_hash) + .unwrap_or_default(); + let expected = tx.account_update().init_state_hash(); + + if expected != current { + return Err(AddTransactionErrorRework::InvalidAccountState { current, expected }); + } + + // Ensure nullifiers aren't already present. + // TODO: Verifying the inputs nullifiers should be done externally already. + // TODO: The above should cause a change in type for inputs indicating this. + let input_nullifiers = tx.get_nullifiers().collect::>(); + let double_spend = + self.nullifiers.union(&input_nullifiers).copied().collect::>(); + if !double_spend.is_empty() { + return Err(AddTransactionErrorRework::NotesAlreadyConsumed(double_spend)); + } + + // TODO: additional input and output note checks, depends on the transaction type changes. + + Ok(()) + } + + /// Aggregate the transaction into the state, returning its parent transactions. + fn insert_transaction(&mut self, tx: &ProvenTransaction) -> BTreeSet { + let account_parent = self + .accounts + .entry(tx.account_id()) + .or_default() + .insert(tx.account_update().final_state_hash(), tx.id()); + + self.nullifiers.extend(tx.get_nullifiers()); + + // TODO: input and output notes + + account_parent.into_iter().collect() + } + + /// Reverts the given state diff. + /// + /// # Panics + /// + /// Panics if any part of the diff isn't present in the state. Callers should take + /// care to only revert transaction sets who's ancestors are all either committed or reverted. + pub fn revert_transactions(&mut self, diff: StateDiff) { + for (account, count) in diff.account_transactions { + let status = self.accounts.get_mut(&account).expect("Account must exist").revert(count); + + // Prune empty accounts. + if status.is_empty() { + self.accounts.remove(&account); + } + } + + for nullifier in diff.nullifiers { + assert!(self.nullifiers.remove(&nullifier), "Nullifier must exist"); + } + + // TODO: input and output notes + } + + /// Marks the given state diff as committed. + /// + /// These transactions are no longer considered inflight. Callers should take care to only + /// commit transactions who's ancestors are all committed. + /// + /// # Panics + /// + /// Panics if the accounts don't have enough inflight transactions to commit. + pub fn commit_transactions(&mut self, diff: &StateDiff) { + for (account, count) in &diff.account_transactions { + self.accounts.get_mut(account).expect("Account must exist").commit(*count); + } + } + + /// Drops the given state diff from memory. + /// + /// # Panics + /// + /// Panics if the accounts don't have enough inflight transactions to commit. + pub fn prune_committed_state(&mut self, diff: StateDiff) { + for (account, count) in diff.account_transactions { + let status = self + .accounts + .get_mut(&account) + .expect("Account must exist") + .remove_commited(count); + + // Prune empty accounts. + if status.is_empty() { + self.accounts.remove(&account); + } + } + + for nullifier in diff.nullifiers { + self.nullifiers.remove(&nullifier); + } + } +} + +/// Tracks the state of a single account. +#[derive(Default)] +struct AccountState { + /// State progression of this account over all uncommitted inflight transactions. + /// + /// Ordering is chronological from front (oldest) to back (newest). + inflight: VecDeque<(Digest, TransactionId)>, + + /// The latest committed state. + /// + /// Only valid if the committed count is greater than zero. + committed_state: Digest, + + /// The number of committed transactions. + /// + /// If this is zero then the committed state is meaningless. + committed_count: usize, +} + +impl AccountState { + /// Inserts a new transaction and its state, returning the previous inflight transaction, if + /// any. + pub fn insert(&mut self, state: Digest, tx: TransactionId) -> Option { + let previous = self.inflight.back().map(|(_, tx)| tx).copied(); + + self.inflight.push_back((state, tx)); + + previous + } + + /// The latest state of this account. + pub fn latest_state(&self) -> Option<&Digest> { + self.inflight + .back() + .map(|(state, _)| state) + // A count of zero indicates no committed state. + .or((self.committed_count > 1).then_some(&self.committed_state)) + } + + /// Reverts the most recent `n` inflight transactions. + /// + /// # Returns + /// + /// Returns the emptyness state of the account. + /// + /// # Panics + /// + /// Panics if there are less than `n` inflight transactions. + pub fn revert(&mut self, n: usize) -> AccountStatus { + let length = self.inflight.len(); + assert!( + length >= n, "Attempted to revert {n} transactions which is more than the {length} which are inflight.", + ); + + self.inflight.drain(length - n..); + + self.emptyness() + } + + /// Commits the first `n` inflight transactions. + /// + /// # Panics + /// + /// Panics if there are less than `n` inflight transactions. + pub fn commit(&mut self, n: usize) { + if n == 0 { + return; + } + + let length = self.inflight.len(); + assert!( + length >= n, "Attempted to revert {n} transactions which is more than the {length} which are inflight.", + ); + + self.committed_state = self + .inflight + .drain(..n) + .last() + .map(|(state, _)| state) + .expect("Must be Some as n > 0"); + self.committed_count += n; + } + + /// Removes `n` committed transactions. + /// + /// # Returns + /// + /// Returns the emptyness state of the account. + /// + /// # Panics + /// + /// Panics if there are less than `n` committed transactions. + pub fn remove_commited(&mut self, n: usize) -> AccountStatus { + assert!( + n <= self.committed_count, + "Attempted to remove {n} committed transactions, but only {} are present.", + self.committed_count + ); + self.committed_count -= n; + + self.emptyness() + } + + fn emptyness(&self) -> AccountStatus { + if self.inflight.is_empty() && self.committed_count == 0 { + AccountStatus::Empty + } else { + AccountStatus::NonEmpty + } + } +} + +/// Describes the emptyness of an [AccountState]. +/// +/// Is marked as #[must_use] so that callers handle prune empty accounts. +#[must_use] +#[derive(Clone, Copy, PartialEq, Eq)] +enum AccountStatus { + /// [AccountState] contains no state and should be pruned. + Empty, + /// [AccountState] contains state and should be kept. + NonEmpty, +} + +impl AccountStatus { + fn is_empty(&self) -> bool { + *self == Self::Empty + } +} diff --git a/crates/block-producer/src/mempool/mod.rs b/crates/block-producer/src/mempool/mod.rs new file mode 100644 index 000000000..67081009a --- /dev/null +++ b/crates/block-producer/src/mempool/mod.rs @@ -0,0 +1,275 @@ +use std::{ + collections::{btree_map::Entry, BTreeMap, BTreeSet, VecDeque}, + fmt::Display, + ops::Sub, + sync::Arc, +}; + +use batch_graph::BatchGraph; +use inflight_state::{InflightState, StateDiff}; +use miden_objects::{ + accounts::AccountId, + notes::{NoteId, Nullifier}, + transaction::{ProvenTransaction, TransactionId}, + Digest, +}; +use miden_tx::{utils::collections::KvMap, TransactionVerifierError}; +use transaction_graph::TransactionGraph; + +use crate::{ + errors::AddTransactionErrorRework, + store::{TransactionInputs, TxInputsError}, +}; + +mod batch_graph; +mod inflight_state; +mod transaction_graph; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub struct BatchJobId(u64); + +impl Display for BatchJobId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.0.fmt(f) + } +} + +impl BatchJobId { + pub fn increment(mut self) { + self.0 += 1; + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub struct BlockNumber(u32); + +impl Display for BlockNumber { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.0.fmt(f) + } +} + +impl BlockNumber { + pub fn next(&self) -> Self { + let mut ret = *self; + ret.increment(); + + ret + } + + pub fn prev(&self) -> Option { + self.checked_sub(Self(1)) + } + + pub fn increment(mut self) { + self.0 += 1; + } + + pub fn checked_sub(&self, rhs: Self) -> Option { + self.0.checked_sub(rhs.0).map(Self) + } +} + +pub struct Mempool { + /// The latest inflight state of each account. + /// + /// Accounts without inflight transactions are not stored. + state: InflightState, + + /// Note's consumed by inflight transactions. + nullifiers: BTreeSet, + + /// Notes produced by inflight transactions. + /// + /// It is possible for these to already be consumed - check nullifiers. + notes: BTreeMap, + + /// Inflight transactions. + transactions: TransactionGraph, + + /// Inflight batches. + batches: BatchGraph, + + /// The next batches ID. + next_batch_id: BatchJobId, + + /// Blocks which have been committed but are not yet considered stale. + committed_diffs: VecDeque, + + /// The current block height of the chain. + chain_tip: BlockNumber, + + block_in_progress: Option>, + + /// Number of blocks before transaction input is considered stale. + staleness: BlockNumber, + + batch_transaction_limit: usize, + block_batch_limit: usize, +} + +impl Mempool { + /// Adds a transaction to the mempool. + /// + /// # Returns + /// + /// Returns the current block height. + /// + /// # Errors + /// + /// Returns an error if the transaction's initial conditions don't match the current state. + pub fn add_transaction( + &mut self, + transaction: ProvenTransaction, + inputs: TransactionInputs, + ) -> Result { + // Ensure inputs aren't stale. + if let Some(stale_block) = self.stale_block() { + if inputs.current_block_height <= stale_block.0 { + return Err(AddTransactionErrorRework::StaleInputs { + input_block: BlockNumber(inputs.current_block_height), + stale_limit: stale_block, + }); + } + } + + // Add transaction to inflight state. + let parents = self.state.add_transaction(&transaction, &inputs)?; + + self.transactions.insert(transaction, parents); + + Ok(self.chain_tip.0) + } + + /// Returns a set of transactions for the next batch. + /// + /// Transactions are returned in a valid execution ordering. + /// + /// Returns `None` if no transactions are available. + pub fn select_batch(&mut self) -> Option<(BatchJobId, Vec)> { + let mut parents = BTreeSet::new(); + let mut batch = Vec::with_capacity(self.batch_transaction_limit); + + for _ in 0..self.batch_transaction_limit { + // Select transactions according to some strategy here. For now its just arbitrary. + let Some((tx, tx_parents)) = self.transactions.pop_for_batching() else { + break; + }; + batch.push(tx); + parents.extend(tx_parents); + } + + // Update the depedency graph to reflect parents at the batch level by removing + // all edges within this batch. + for tx in &batch { + parents.remove(tx); + } + + let batch_id = self.next_batch_id; + self.next_batch_id.increment(); + + self.batches.insert(batch_id, batch.clone(), parents); + + Some((batch_id, batch)) + } + + /// Drops the failed batch and all of its descendents. + /// + /// Transactions are placed back in the queue. + pub fn batch_failed(&mut self, batch: BatchJobId) { + let removed_batches = self.batches.purge_subgraphs([batch].into()); + + // Its possible to receive failures for batches which were already removed + // as part of a prior failure. Early exit to prevent logging these no-ops. + if removed_batches.is_empty() { + return; + } + + let batches = removed_batches.keys().copied().collect::>(); + let transactions = removed_batches.into_values().flatten().collect(); + + self.transactions.requeue_transactions(transactions); + + tracing::warn!(%batch, descendents=?batches, "Batch failed, dropping all inflight descendent batches, impacted transactions are back in queue."); + } + + /// Marks a batch as proven if it exists. + pub fn batch_proved(&mut self, batch_id: BatchJobId) { + self.batches.mark_proven(batch_id); + } + + /// Select batches for the next block. + /// + /// May return an empty set if no batches are ready. + /// + /// # Panics + /// + /// Panics if there is already a block in flight. + pub fn select_block(&mut self) -> (BlockNumber, BTreeSet) { + assert!(self.block_in_progress.is_none(), "Cannot have two blocks inflight."); + + let batches = self.batches.select_block(self.block_batch_limit); + self.block_in_progress = Some(batches.clone()); + + (self.chain_tip.next(), batches) + } + + /// Notify the pool that the block was succesfully completed. + /// + /// # Panics + /// + /// Panics if blocks are completed out-of-order or if there is no block in flight. + pub fn block_committed(&mut self, block_number: BlockNumber) { + assert_eq!(block_number, self.chain_tip.next(), "Blocks must be submitted sequentially"); + + // Remove committed batches and transactions from graphs. + let batches = self.block_in_progress.take().expect("No block in progress to commit"); + let transactions = self.batches.remove_committed(batches); + let transactions = self.transactions.remove_committed(&transactions); + + // Inform inflight state about committed data. + let diff = StateDiff::new(&transactions); + self.state.commit_transactions(&diff); + + self.committed_diffs.push_back(diff); + if self.committed_diffs.len() > self.staleness.0 as usize { + // SAFETY: just checked that length is non-zero. + let stale_block = self.committed_diffs.pop_front().unwrap(); + + self.state.prune_committed_state(stale_block); + } + + self.chain_tip.increment(); + } + + /// Block and all of its contents and dependents are purged from the mempool. + /// + /// # Panics + /// + /// Panics if there is no block in flight or if the block number does not match the current + /// inflight block. + pub fn block_failed(&mut self, block_number: BlockNumber) { + assert_eq!(block_number, self.chain_tip.next(), "Blocks must be submitted sequentially"); + + let batches = self.block_in_progress.take().expect("No block in progress to be failed"); + + // Remove all transactions from the graphs. + let purged = self.batches.purge_subgraphs(batches); + let batches = purged.keys().collect::>(); + let transactions = purged.into_values().flatten().collect(); + + let transactions = self.transactions.purge_subgraphs(transactions); + + // Rollback state. + let impact = StateDiff::new(&transactions); + self.state.revert_transactions(impact); + // TODO: revert nullifiers and notes. + } + + /// The highest block height we consider stale. + /// + /// Returns [None] if the blockchain is so short that all blocks are considered fresh. + fn stale_block(&self) -> Option { + self.chain_tip.checked_sub(self.staleness) + } +} diff --git a/crates/block-producer/src/mempool/transaction_graph.rs b/crates/block-producer/src/mempool/transaction_graph.rs new file mode 100644 index 000000000..2384924b1 --- /dev/null +++ b/crates/block-producer/src/mempool/transaction_graph.rs @@ -0,0 +1,170 @@ +use std::{ + collections::{BTreeMap, BTreeSet}, + sync::Arc, +}; + +use miden_objects::transaction::{ProvenTransaction, TransactionId}; + +use super::BatchJobId; + +#[derive(Default, Clone, Debug)] +pub struct TransactionGraph { + /// All transactions currently inflight. + nodes: BTreeMap, + + /// Transactions ready for inclusion in a batch. + /// + /// aka transactions whose parent transactions are already included in batches. + roots: BTreeSet, +} +impl TransactionGraph { + pub fn insert(&mut self, transaction: ProvenTransaction, parents: BTreeSet) { + let id = transaction.id(); + + // Inform parent's of their new child. + for parent in &parents { + self.nodes.get_mut(parent).expect("Parent must be in pool").children.insert(id); + } + + let transaction = Node::new(transaction, parents); + if self.nodes.insert(id, transaction).is_some() { + panic!("Transaction already exists in pool"); + } + + // This could be optimised by inlining this inside the parent loop. This would prevent the + // double iteration over parents, at the cost of some code duplication. + self.try_make_root(id); + } + + pub fn pop_for_batching(&mut self) -> Option<(TransactionId, BTreeSet)> { + let tx_id = self.roots.pop_first()?; + let node = self.nodes.get_mut(&tx_id).expect("Root transaction must be in graph"); + node.status = Status::Processed; + + // Work around multiple mutable borrows of self. + let parents = node.parents.clone(); + let children = node.children.clone(); + + for child in children { + self.try_make_root(child); + } + + Some((tx_id, parents)) + } + + /// Marks the given transactions as being back inqueue. + pub fn requeue_transactions(&mut self, transactions: BTreeSet) { + for tx in &transactions { + self.nodes.get_mut(tx).expect("Node must exist").status = Status::InQueue; + } + + // All requeued transactions are potential roots, and current roots may have been + // invalidated. + let mut potential_roots = transactions; + potential_roots.extend(&self.roots); + self.roots.clear(); + for tx in potential_roots { + self.try_make_root(tx); + } + } + + pub fn remove_committed(&mut self, tx_ids: &[TransactionId]) -> Vec> { + let mut transactions = Vec::with_capacity(tx_ids.len()); + for transaction in tx_ids { + let node = self.nodes.remove(transaction).expect("Node must be in graph"); + assert_eq!(node.status, Status::Processed); + + transactions.push(node.data); + + // Remove node from graph. No need to update parents as they should be removed in this + // call as well. + for child in node.children { + // Its possible for the child to part of this same set of batches and therefore + // already removed. + if let Some(child) = self.nodes.get_mut(&child) { + child.parents.remove(transaction); + } + } + } + + transactions + } + + /// Removes the transactions and all their descendents from the graph. + /// + /// Returns all transactions removed. + pub fn purge_subgraphs( + &mut self, + transactions: Vec, + ) -> Vec> { + let mut removed = Vec::new(); + + let mut to_process = transactions; + + while let Some(node_id) = to_process.pop() { + // Its possible for a node to already have been removed as part of this subgraph + // removal. + let Some(node) = self.nodes.remove(&node_id) else { + continue; + }; + + // All the child batches are also removed so no need to check + // for new roots. No new roots are possible as a result of this subgraph removal. + self.roots.remove(&node_id); + + // Inform parent that this child no longer exists. + // + // The same is not required for children of this batch as we will + // be removing those as well. + for parent in &node.parents { + // Parent could already be removed as part of this subgraph removal. + if let Some(parent) = self.nodes.get_mut(parent) { + parent.children.remove(&node_id); + } + } + + to_process.extend(node.children); + removed.push(node.data); + } + + removed + } + + fn try_make_root(&mut self, tx_id: TransactionId) { + let tx = self.nodes.get_mut(&tx_id).expect("Transaction must be in graph"); + + for parent in tx.parents.clone() { + let parent = self.nodes.get(&parent).expect("Parent must be in pool"); + + if parent.status != Status::Processed { + return; + } + } + self.roots.insert(tx_id); + } +} + +#[derive(Clone, Debug)] +struct Node { + status: Status, + data: Arc, + parents: BTreeSet, + children: BTreeSet, +} + +impl Node { + fn new(tx: ProvenTransaction, parents: BTreeSet) -> Self { + Self { + status: Status::InQueue, + data: Arc::new(tx), + parents, + children: Default::default(), + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Status { + InQueue, + Processed, +} diff --git a/crates/block-producer/src/server/mod.rs b/crates/block-producer/src/server/mod.rs index e81471804..56114be7e 100644 --- a/crates/block-producer/src/server/mod.rs +++ b/crates/block-producer/src/server/mod.rs @@ -1,17 +1,34 @@ -use std::{net::ToSocketAddrs, sync::Arc}; +use std::{ + collections::{BTreeMap, BTreeSet}, + net::ToSocketAddrs, + sync::Arc, +}; -use miden_node_proto::generated::{block_producer::api_server, store::api_client as store_client}; -use miden_node_utils::errors::ApiError; -use tokio::net::TcpListener; +use miden_node_proto::generated::{ + block_producer::api_server, requests::SubmitProvenTransactionRequest, + responses::SubmitProvenTransactionResponse, store::api_client as store_client, +}; +use miden_node_utils::{ + errors::ApiError, + formatting::{format_input_notes, format_output_notes}, +}; +use miden_objects::{ + transaction::ProvenTransaction, utils::serde::Deserializable, MIN_PROOF_SECURITY_LEVEL, +}; +use miden_tx::TransactionVerifier; +use tokio::{net::TcpListener, sync::Mutex}; use tokio_stream::wrappers::TcpListenerStream; -use tracing::info; +use tonic::Status; +use tracing::{debug, info, instrument}; use crate::{ batch_builder::{DefaultBatchBuilder, DefaultBatchBuilderOptions}, block_builder::DefaultBlockBuilder, config::BlockProducerConfig, + errors::AddTransactionErrorRework, + mempool::Mempool, state_view::DefaultStateView, - store::DefaultStore, + store::{DefaultStore, Store}, txqueue::{TransactionQueue, TransactionQueueOptions}, COMPONENT, SERVER_BATCH_SIZE, SERVER_BLOCK_FREQUENCY, SERVER_BUILD_BATCH_FREQUENCY, SERVER_MAX_BATCHES_PER_BLOCK, @@ -105,3 +122,110 @@ impl BlockProducer { .map_err(ApiError::ApiServeFailed) } } + +pub struct Server { + /// This outer mutex enforces that the incoming transactions won't crowd out more important + /// mempool locks. + /// + /// The inner mutex will be abstracted away once we are happy with the api. + mempool: Mutex>>, + + store: DefaultStore, +} + +// FIXME: remove the allow when the upstream clippy issue is fixed: +// https://github.com/rust-lang/rust-clippy/issues/12281 +#[allow(clippy::blocks_in_conditions)] +#[tonic::async_trait] +impl api_server::Api for Server { + async fn submit_proven_transaction( + &self, + request: tonic::Request, + ) -> Result, Status> { + self.submit_proven_transaction(request.into_inner()) + .await + .map(tonic::Response::new) + .map_err(|err| match err { + AddTransactionErrorRework::InvalidAccountState { .. } + | AddTransactionErrorRework::AuthenticatedNoteNotFound(_) + | AddTransactionErrorRework::UnauthenticatedNoteNotFound(_) + | AddTransactionErrorRework::NotesAlreadyConsumed(_) + | AddTransactionErrorRework::DeserializationError(_) + | AddTransactionErrorRework::ProofVerificationFailed(_) => { + Status::invalid_argument(err.to_string()) + }, + // Internal errors. + AddTransactionErrorRework::StaleInputs { .. } + | AddTransactionErrorRework::TxInputsError(_) => Status::internal("Internal error"), + }) + } +} + +impl Server { + #[instrument( + target = "miden-block-producer", + name = "block_producer:submit_proven_transaction", + skip_all, + err + )] + async fn submit_proven_transaction( + &self, + request: SubmitProvenTransactionRequest, + ) -> Result { + debug!(target: COMPONENT, ?request); + + let tx = ProvenTransaction::read_from_bytes(&request.transaction) + .map_err(|err| AddTransactionErrorRework::DeserializationError(err.to_string()))?; + + let tx_id = tx.id(); + + info!( + target: COMPONENT, + tx_id = %tx_id.to_hex(), + account_id = %tx.account_id().to_hex(), + initial_account_hash = %tx.account_update().init_state_hash(), + final_account_hash = %tx.account_update().final_state_hash(), + input_notes = %format_input_notes(tx.input_notes()), + output_notes = %format_output_notes(tx.output_notes()), + block_ref = %tx.block_ref(), + "Deserialized transaction" + ); + debug!(target: COMPONENT, proof = ?tx.proof()); + + let mut inputs = self.store.get_tx_inputs(&tx).await?; + + let mut authenticated_notes = BTreeSet::new(); + let mut unauthenticated_notes = BTreeMap::new(); + + for note in tx.input_notes() { + match note.header() { + Some(header) => { + unauthenticated_notes.insert(header.id(), note.nullifier()); + }, + None => { + authenticated_notes.insert(note.nullifier()); + }, + } + } + + // Authenticated note nullifiers must be present in the store and must be unconsumed. + for nullifier in &authenticated_notes { + let nullifier_state = inputs + .nullifiers + .remove(nullifier) + .ok_or(AddTransactionErrorRework::AuthenticatedNoteNotFound(*nullifier))?; + + if nullifier_state.is_some() { + return Err(AddTransactionErrorRework::NotesAlreadyConsumed([*nullifier].into())); + } + } + + self.mempool + .lock() + .await + .lock() + .await + .add_transaction(tx, inputs) + .map(|block_height| SubmitProvenTransactionResponse { block_height }) + } +} From 34a45d9d0e18ce6982812245e9b76287dab96a5f Mon Sep 17 00:00:00 2001 From: Mirko <48352201+Mirko-von-Leipzig@users.noreply.github.com> Date: Tue, 22 Oct 2024 16:02:16 +0200 Subject: [PATCH 02/50] feat(block-producer): implement the skeleton (#515) Co-authored-by: Bobbin Threadbare --- Cargo.lock | 1 + crates/block-producer/Cargo.toml | 1 + .../block-producer/src/batch_builder/batch.rs | 3 +- .../block-producer/src/batch_builder/mod.rs | 68 +- .../block-producer/src/block_builder/mod.rs | 17 +- crates/block-producer/src/domain/mod.rs | 1 + .../block-producer/src/domain/transaction.rs | 151 ++++ crates/block-producer/src/errors.rs | 45 +- crates/block-producer/src/lib.rs | 1 + .../block-producer/src/mempool/batch_graph.rs | 63 +- .../src/mempool/inflight_state.rs | 314 --------- .../mempool/inflight_state/account_state.rs | 336 +++++++++ .../src/mempool/inflight_state/mod.rs | 653 ++++++++++++++++++ crates/block-producer/src/mempool/mod.rs | 101 +-- .../src/mempool/transaction_graph.rs | 506 ++++++++++++-- crates/block-producer/src/server/mod.rs | 64 +- crates/block-producer/src/test_utils/mod.rs | 37 +- crates/block-producer/src/test_utils/store.rs | 1 + 18 files changed, 1826 insertions(+), 537 deletions(-) create mode 100644 crates/block-producer/src/domain/mod.rs create mode 100644 crates/block-producer/src/domain/transaction.rs delete mode 100644 crates/block-producer/src/mempool/inflight_state.rs create mode 100644 crates/block-producer/src/mempool/inflight_state/account_state.rs create mode 100644 crates/block-producer/src/mempool/inflight_state/mod.rs diff --git a/Cargo.lock b/Cargo.lock index cf34633e8..357910349 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1978,6 +1978,7 @@ dependencies = [ "miden-processor", "miden-stdlib", "miden-tx", + "rand", "rand_chacha", "serde", "thiserror", diff --git a/crates/block-producer/Cargo.toml b/crates/block-producer/Cargo.toml index 1d1476fdf..5355d30fb 100644 --- a/crates/block-producer/Cargo.toml +++ b/crates/block-producer/Cargo.toml @@ -41,6 +41,7 @@ miden-lib = { workspace = true, features = ["testing"] } miden-node-test-macro = { path = "../test-macro" } miden-objects = { workspace = true, features = ["testing"] } miden-tx = { workspace = true, features = ["testing"] } +rand = { version = "0.8.5" } rand_chacha = { version = "0.3", default-features = false } tokio = { workspace = true, features = ["test-util"] } winterfell = { version = "0.9" } diff --git a/crates/block-producer/src/batch_builder/batch.rs b/crates/block-producer/src/batch_builder/batch.rs index 6490a95a7..e8a13fb36 100644 --- a/crates/block-producer/src/batch_builder/batch.rs +++ b/crates/block-producer/src/batch_builder/batch.rs @@ -22,7 +22,8 @@ pub type BatchId = Blake3Digest<32>; // TRANSACTION BATCH // ================================================================================================ -/// A batch of transactions that share a common proof. +/// A batch of transactions that share a common proof. For any given account, at most 1 transaction +/// in the batch must be addressing that account (issue: #186). /// /// Note: Until recursive proofs are available in the Miden VM, we don't include the common proof. #[derive(Debug, Clone, PartialEq, Eq)] diff --git a/crates/block-producer/src/batch_builder/mod.rs b/crates/block-producer/src/batch_builder/mod.rs index db4d78079..65f693006 100644 --- a/crates/block-producer/src/batch_builder/mod.rs +++ b/crates/block-producer/src/batch_builder/mod.rs @@ -1,15 +1,24 @@ -use std::{cmp::min, collections::BTreeSet, num::NonZeroUsize, sync::Arc, time::Duration}; +use std::{ + cmp::min, + collections::{BTreeMap, BTreeSet}, + num::NonZeroUsize, + sync::Arc, + time::Duration, +}; use miden_objects::{ + accounts::AccountId, notes::NoteId, transaction::{OutputNote, TransactionId}, + Digest, }; -use tokio::{sync::Mutex, time}; +use tokio::{sync::Mutex, task::JoinSet, time}; use tonic::async_trait; use tracing::{debug, info, instrument, Span}; use crate::{ block_builder::BlockBuilder, + domain::transaction::AuthenticatedTransaction, mempool::{BatchJobId, Mempool}, ProvenTransaction, SharedRwVec, COMPONENT, }; @@ -219,41 +228,74 @@ pub struct BatchProducer { pub workers: NonZeroUsize, pub mempool: Arc>, pub tx_per_batch: usize, + pub simulated_proof_time: Duration, } -type BatchResult = Result; +type BatchResult = Result<(BatchJobId, TransactionBatch), (BatchJobId, BuildBatchError)>; /// Wrapper around tokio's JoinSet that remains pending if the set is empty, /// instead of returning None. -struct WorkerPool(tokio::task::JoinSet); +struct WorkerPool { + in_progress: JoinSet, + simulated_proof_time: Duration, +} impl WorkerPool { + fn new(simulated_proof_time: Duration) -> Self { + Self { + simulated_proof_time, + in_progress: JoinSet::new(), + } + } + async fn join_next(&mut self) -> Result { - if self.0.is_empty() { + if self.in_progress.is_empty() { std::future::pending().await } else { // Cannot be None as its not empty. - self.0.join_next().await.unwrap() + self.in_progress.join_next().await.unwrap() } } fn len(&self) -> usize { - self.0.len() + self.in_progress.len() } - fn spawn(&mut self, id: BatchJobId, transactions: Vec) { - self.0.spawn(async move { - todo!("Do actual work like aggregating transaction data"); + fn spawn(&mut self, id: BatchJobId, transactions: Vec) { + self.in_progress.spawn({ + let simulated_proof_time = self.simulated_proof_time; + async move { + tracing::debug!("Begin proving batch."); + + let transactions = + transactions.into_iter().map(AuthenticatedTransaction::into_raw).collect(); + + let batch = TransactionBatch::new(transactions, Default::default()) + .map_err(|err| (id, err))?; + + tokio::time::sleep(simulated_proof_time).await; + tracing::debug!("Batch proof completed."); + + Ok((id, batch)) + } }); } } impl BatchProducer { + /// Starts the [BlockProducer], infinitely producing blocks at the configured interval. + /// + /// Block production is sequential and consists of + /// + /// 1. Pulling the next set of batches from the [Mempool] + /// 2. Compiling these batches into the next block + /// 3. Proving the block (this is not yet implemented) + /// 4. Committing the block to the store pub async fn run(self) { let mut interval = tokio::time::interval(self.batch_interval); interval.set_missed_tick_behavior(time::MissedTickBehavior::Delay); - let mut inflight = WorkerPool(tokio::task::JoinSet::new()); + let mut inflight = WorkerPool::new(self.simulated_proof_time); loop { tokio::select! { @@ -285,8 +327,8 @@ impl BatchProducer { tracing::warn!(%batch_id, %err, "Batch job failed."); mempool.batch_failed(batch_id); }, - Ok(Ok(batch_id)) => { - mempool.batch_proved(batch_id); + Ok(Ok((batch_id, batch))) => { + mempool.batch_proved(batch_id, batch); } } } diff --git a/crates/block-producer/src/block_builder/mod.rs b/crates/block-producer/src/block_builder/mod.rs index 111c7c3d1..9d9deda00 100644 --- a/crates/block-producer/src/block_builder/mod.rs +++ b/crates/block-producer/src/block_builder/mod.rs @@ -1,4 +1,7 @@ -use std::{collections::BTreeSet, sync::Arc}; +use std::{ + collections::{BTreeMap, BTreeSet}, + sync::Arc, +}; use async_trait::async_trait; use miden_node_utils::formatting::{format_array, format_blake3_digest}; @@ -146,12 +149,13 @@ where } } -struct BlockProducer { +struct BlockProducer { pub mempool: Arc>, pub block_interval: tokio::time::Duration, + pub block_builder: BB, } -impl BlockProducer { +impl BlockProducer { pub async fn run(self) { let mut interval = tokio::time::interval(self.block_interval); interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); @@ -160,8 +164,9 @@ impl BlockProducer { interval.tick().await; let (block_number, batches) = self.mempool.lock().await.select_block(); + let batches = batches.into_values().collect::>(); - let result = self.build_and_commit_block(batches).await; + let result = self.block_builder.build_block(&batches).await; let mut mempool = self.mempool.lock().await; match result { @@ -170,8 +175,4 @@ impl BlockProducer { } } } - - async fn build_and_commit_block(&self, batches: BTreeSet) -> Result<(), ()> { - todo!("Aggregate, prove and commit block"); - } } diff --git a/crates/block-producer/src/domain/mod.rs b/crates/block-producer/src/domain/mod.rs new file mode 100644 index 000000000..37f08066e --- /dev/null +++ b/crates/block-producer/src/domain/mod.rs @@ -0,0 +1 @@ +pub mod transaction; diff --git a/crates/block-producer/src/domain/transaction.rs b/crates/block-producer/src/domain/transaction.rs new file mode 100644 index 000000000..756af859a --- /dev/null +++ b/crates/block-producer/src/domain/transaction.rs @@ -0,0 +1,151 @@ +use std::collections::BTreeSet; + +use miden_objects::{ + accounts::AccountId, + notes::{NoteId, Nullifier}, + transaction::{ProvenTransaction, TransactionId, TxAccountUpdate}, + Digest, +}; + +use crate::{errors::VerifyTxError, mempool::BlockNumber, store::TransactionInputs}; + +/// A transaction who's proof has been verified, and which has been authenticated against the store. +/// +/// Authentication ensures that all nullifiers are unspent, and additionally authenticates some +/// previously unauthenticated input notes. +/// +/// Note that this is of course only valid for the chain height of the authentication. +#[derive(Clone, PartialEq)] +pub struct AuthenticatedTransaction { + inner: ProvenTransaction, + /// The account state provided by the store [inputs](TransactionInputs). + /// + /// This does not necessarily have to match the transaction's initial state + /// as this may still be modified by inflight transactions. + store_account_state: Option, + /// Unauthenticates notes that have now been authenticated by the store + /// [inputs](TransactionInputs). + /// + /// In other words, notes which were unauthenticated at the time the transaction was proven, + /// but which have since been committed to, and authenticated by the store. + notes_authenticated_by_store: BTreeSet, + /// Chain height that the authentication took place at. + authentication_height: BlockNumber, +} + +impl AuthenticatedTransaction { + /// Verifies the transaction against the inputs, enforcing that all nullifiers are unspent. + /// + /// __No__ proof verification is peformed. The caller takes responsibility for ensuring + /// that the proof is valid. + /// + /// # Errors + /// + /// Returns an error if any of the transaction's nullifiers are marked as spent by the inputs. + pub fn new( + tx: ProvenTransaction, + inputs: TransactionInputs, + ) -> Result { + let nullifiers_already_spent = tx + .get_nullifiers() + .filter(|nullifier| inputs.nullifiers.get(nullifier).cloned().flatten().is_some()) + .collect::>(); + if !nullifiers_already_spent.is_empty() { + return Err(VerifyTxError::InputNotesAlreadyConsumed(nullifiers_already_spent)); + } + + // Invert the missing notes; i.e. we now know the rest were actually found. + let authenticated_notes = tx + .get_unauthenticated_notes() + .map(|header| header.id()) + .filter(|note_id| !inputs.missing_unauthenticated_notes.contains(note_id)) + .collect(); + + Ok(AuthenticatedTransaction { + inner: tx, + notes_authenticated_by_store: authenticated_notes, + authentication_height: BlockNumber::new(inputs.current_block_height), + store_account_state: inputs.account_hash, + }) + } + + pub fn id(&self) -> TransactionId { + self.inner.id() + } + + pub fn account_id(&self) -> AccountId { + self.inner.account_id() + } + + pub fn account_update(&self) -> &TxAccountUpdate { + self.inner.account_update() + } + + pub fn store_account_state(&self) -> Option { + self.store_account_state + } + + pub fn authentication_height(&self) -> BlockNumber { + self.authentication_height + } + + pub fn nullifiers(&self) -> impl Iterator + '_ { + self.inner.get_nullifiers() + } + + pub fn output_notes(&self) -> impl Iterator + '_ { + self.inner.output_notes().iter().map(|note| note.id()) + } + + /// Notes which were unauthenticate in the transaction __and__ which were + /// not authenticated by the store inputs. + pub fn unauthenticated_notes(&self) -> impl Iterator + '_ { + self.inner + .get_unauthenticated_notes() + .cloned() + .map(|header| header.id()) + .filter(|note_id| !self.notes_authenticated_by_store.contains(note_id)) + } + + pub fn into_raw(self) -> ProvenTransaction { + self.inner + } +} + +#[cfg(test)] +impl AuthenticatedTransaction { + //! Builder methods intended for easier test setup. + + /// Short-hand for `Self::new` where the input's are setup to match the transaction's initial + /// account state. + pub fn from_inner(inner: ProvenTransaction) -> Self { + let store_account_state = match inner.account_update().init_state_hash() { + zero if zero == Digest::default() => None, + non_zero => Some(non_zero), + }; + Self { + inner, + store_account_state, + notes_authenticated_by_store: Default::default(), + authentication_height: Default::default(), + } + } + + /// Overrides the authentication height with the given value. + pub fn with_authentication_height(mut self, height: u32) -> Self { + self.authentication_height = BlockNumber::new(height); + self + } + + /// Overrides the store state with the given value. + pub fn with_store_state(mut self, state: Digest) -> Self { + self.store_account_state = Some(state); + self + } + + /// Unsets the store state. + pub fn with_empty_store_state(mut self) -> Self { + self.store_account_state = None; + self + } +} diff --git a/crates/block-producer/src/errors.rs b/crates/block-producer/src/errors.rs index a963236d4..4791971a6 100644 --- a/crates/block-producer/src/errors.rs +++ b/crates/block-producer/src/errors.rs @@ -1,5 +1,3 @@ -use std::collections::BTreeSet; - use miden_node_proto::errors::ConversionError; use miden_node_utils::formatting::format_opt; use miden_objects::{ @@ -11,7 +9,6 @@ use miden_objects::{ MAX_BATCHES_PER_BLOCK, MAX_INPUT_NOTES_PER_BATCH, MAX_OUTPUT_NOTES_PER_BATCH, }; use miden_processor::ExecutionError; -use miden_tx::TransactionVerifierError; use thiserror::Error; use crate::mempool::BlockNumber; @@ -32,6 +29,9 @@ pub enum VerifyTxError { )] UnauthenticatedNotesNotFound(Vec), + #[error("Output note IDs already used: {0:?}")] + OutputNotesAlreadyExist(Vec), + /// The account's initial hash did not match the current account's hash #[error("Incorrect account's initial hash ({tx_initial_account_hash}, current: {})", format_opt(.current_account_hash.as_ref()))] IncorrectAccountInitialHash { @@ -61,31 +61,36 @@ pub enum VerifyTxError { pub enum AddTransactionError { #[error("Transaction verification failed: {0}")] VerificationFailed(#[from] VerifyTxError), -} -#[derive(thiserror::Error, Debug, PartialEq)] -pub enum AddTransactionErrorRework { - #[error("Transaction's initial account state {expected} did not match the current account state {current}.")] - InvalidAccountState { current: Digest, expected: Digest }, - #[error("Transaction input data is stale. Required data fresher than {stale_limit} but inputs are from {input_block}.")] + #[error("Transaction input data is stale. Required data from {stale_limit} or newer, but inputs are from {input_block}.")] StaleInputs { input_block: BlockNumber, stale_limit: BlockNumber, }, - #[error("Authenticated note nullifier {0} not found.")] - AuthenticatedNoteNotFound(Nullifier), - #[error("Unauthenticated note {0} not found.")] - UnauthenticatedNoteNotFound(NoteId), - #[error("Note nullifiers already consumed: {0:?}")] - NotesAlreadyConsumed(BTreeSet), - #[error(transparent)] - TxInputsError(#[from] TxInputsError), - #[error(transparent)] - ProofVerificationFailed(#[from] TransactionVerifierError), - #[error("Failed to deserialize transaction: {0}.")] + + #[error("Deserialization failed: {0}")] DeserializationError(String), } +impl From for tonic::Status { + fn from(value: AddTransactionError) -> Self { + use AddTransactionError::*; + match value { + VerificationFailed(VerifyTxError::InputNotesAlreadyConsumed(_)) + | VerificationFailed(VerifyTxError::UnauthenticatedNotesNotFound(_)) + | VerificationFailed(VerifyTxError::OutputNotesAlreadyExist(_)) + | VerificationFailed(VerifyTxError::IncorrectAccountInitialHash { .. }) + | VerificationFailed(VerifyTxError::InvalidTransactionProof(_)) + | DeserializationError(_) => Self::invalid_argument(value.to_string()), + + // Internal errors which should not be communicated to the user. + VerificationFailed(VerifyTxError::TransactionInputError(_)) + | VerificationFailed(VerifyTxError::StoreConnectionFailed(_)) + | StaleInputs { .. } => Self::internal("Internal error"), + } + } +} + // Batch building errors // ================================================================================================= diff --git a/crates/block-producer/src/lib.rs b/crates/block-producer/src/lib.rs index d40ce14c8..421718ce9 100644 --- a/crates/block-producer/src/lib.rs +++ b/crates/block-producer/src/lib.rs @@ -12,6 +12,7 @@ pub mod test_utils; mod batch_builder; mod block_builder; +mod domain; mod errors; mod mempool; mod state_view; diff --git a/crates/block-producer/src/mempool/batch_graph.rs b/crates/block-producer/src/mempool/batch_graph.rs index fe963b02e..a69849279 100644 --- a/crates/block-producer/src/mempool/batch_graph.rs +++ b/crates/block-producer/src/mempool/batch_graph.rs @@ -4,6 +4,10 @@ use miden_objects::transaction::TransactionId; use miden_tx::utils::collections::KvMap; use super::BatchJobId; +use crate::batch_builder::batch::TransactionBatch; + +// BATCH GRAPH +// ================================================================================================ #[derive(Default, Clone)] pub struct BatchGraph { @@ -44,7 +48,7 @@ impl BatchGraph { // Insert the new node into the graph. let batch = Node { - status: Status::InFlight, + status: Status::Queued, transactions, parents, children: Default::default(), @@ -53,12 +57,12 @@ impl BatchGraph { // New node might be a root. // - // This could be optimised by inlining this inside the parent loop. This would prevent the + // This could be optimized by inlining this inside the parent loop. This would prevent the // double iteration over parents, at the cost of some code duplication. self.try_make_root(id); } - /// Removes the batches and all their descendents from the graph. + /// Removes the batches and all their descendants from the graph. /// /// Returns all removed batches and their transactions. pub fn purge_subgraphs( @@ -76,8 +80,8 @@ impl BatchGraph { continue; }; - // All the child batches are also removed so no need to chec - // for new roots. No new roots are possible as a result of this subgraph removal. + // All the child batches are also removed so no need to check for new roots. No new + // roots are possible as a result of this subgraph removal. self.roots.remove(&node_id); for transaction in &node.transactions { @@ -86,8 +90,8 @@ impl BatchGraph { // Inform parent that this child no longer exists. // - // The same is not required for children of this batch as we will - // be removing those as well. + // The same is not required for children of this batch as we will be removing those as + // well. for parent in &node.parents { // Parent could already be removed as part of this subgraph removal. if let Some(parent) = self.nodes.get_mut(parent) { @@ -102,7 +106,7 @@ impl BatchGraph { removed } - /// Removes a set of batches from the graph without removing any descendents. + /// Removes a set of batches from the graph without removing any descendants. /// /// This is intended to cull completed batches from stale blocs. pub fn remove_committed(&mut self, batches: BTreeSet) -> Vec { @@ -110,7 +114,7 @@ impl BatchGraph { for batch in batches { let node = self.nodes.remove(&batch).expect("Node must be in graph"); - assert_eq!(node.status, Status::InBlock); + assert_eq!(node.status, Status::Processed); // Remove batch from graph. No need to update parents as they should be removed in this // call as well. @@ -129,32 +133,37 @@ impl BatchGraph { } /// Mark a batch as proven if it exists. - pub fn mark_proven(&mut self, id: BatchJobId) { - // Its possible for inflight batches to have been removed as part - // of another batches failure. + pub fn mark_proven(&mut self, id: BatchJobId, batch: TransactionBatch) { + // Its possible for inflight batches to have been removed as part of another batches + // failure. if let Some(node) = self.nodes.get_mut(&id) { - node.status = Status::Proven; + assert!(node.status == Status::Queued); + node.status = Status::Proven(batch); self.try_make_root(id); } } /// Returns at most `count` __indepedent__ batches which are ready for inclusion in a block. - pub fn select_block(&mut self, count: usize) -> BTreeSet { - let mut batches = BTreeSet::new(); + pub fn select_block(&mut self, count: usize) -> BTreeMap { + let mut batches = BTreeMap::new(); // Track children so we can evaluate them for root afterwards. let mut children = BTreeSet::new(); - for batch in &self.roots { - let mut node = self.nodes.get_mut(batch).expect("Root node must be in graph"); + for batch_id in &self.roots { + let mut node = self.nodes.get_mut(batch_id).expect("Root node must be in graph"); // Filter out batches which have dependencies in our selection so far. - if batches.union(&node.parents).next().is_some() { + if node.parents.iter().any(|parent| batches.contains_key(parent)) { continue; } - batches.insert(*batch); - node.status = Status::Proven; + let Status::Proven(batch) = node.status.clone() else { + unreachable!("Root batch must be in proven state."); + }; + + batches.insert(*batch_id, batch); + node.status = Status::Processed; if batches.len() == count { break; @@ -177,7 +186,7 @@ impl BatchGraph { for parent in node.parents.clone() { let parent = self.nodes.get(&parent).expect("Parent must be in pool"); - if parent.status != Status::InBlock { + if parent.status != Status::Processed { return; } } @@ -193,9 +202,13 @@ struct Node { children: BTreeSet, } -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq)] enum Status { - InFlight, - Proven, - InBlock, + /// The batch is a busy being proven. + Queued, + /// The batch is proven. It may be placed in a block + /// __IFF__ all of its parents are already in a block. + Proven(TransactionBatch), + /// Batch is part of a block. + Processed, } diff --git a/crates/block-producer/src/mempool/inflight_state.rs b/crates/block-producer/src/mempool/inflight_state.rs deleted file mode 100644 index d2df8d2b0..000000000 --- a/crates/block-producer/src/mempool/inflight_state.rs +++ /dev/null @@ -1,314 +0,0 @@ -use std::{ - collections::{BTreeMap, BTreeSet, VecDeque}, - sync::Arc, -}; - -use miden_objects::{ - accounts::AccountId, - notes::Nullifier, - transaction::{ProvenTransaction, TransactionId}, - Digest, -}; - -use crate::{errors::AddTransactionErrorRework, store::TransactionInputs}; - -/// Tracks the inflight state of the mempool. This includes recently committed blocks. -/// -/// Allows appending and reverting transactions as well as marking them -/// as part of a committed block. Committed state can also be pruned once the -/// state is considered past the stale threshold. -#[derive(Default)] -pub struct InflightState { - /// Non-empty inflight account states. - /// - /// Accounts which are [AccountStatus::Empty] are immedietely pruned. - accounts: BTreeMap, - - /// Nullifiers emitted by inflight transactions and recently committed blocks. - nullifiers: BTreeSet, -} - -/// Describes the impact that a set of transactions had on the state. -/// -/// TODO: this is a terrible name. -pub struct StateDiff { - /// The number of transactions that affected each account. - account_transactions: BTreeMap, - - /// The nullifiers that were emitted by the transactions. - nullifiers: BTreeSet, - // TODO: input/output notes -} - -impl StateDiff { - pub fn new(txs: &[Arc]) -> Self { - let mut account_transactions = BTreeMap::::new(); - let mut nullifiers = BTreeSet::new(); - - for tx in txs { - *account_transactions.entry(tx.account_id()).or_default() += 1; - nullifiers.extend(tx.get_nullifiers()); - } - - Self { account_transactions, nullifiers } - } -} - -impl InflightState { - /// Appends the transaction to the inflight state. - /// - /// This operation is atomic i.e. a rejected transaction has no impact of the state. - pub fn add_transaction( - &mut self, - tx: &ProvenTransaction, - inputs: &TransactionInputs, - ) -> Result, AddTransactionErrorRework> { - // Separate verification and state mutation so that a rejected transaction - // does not impact the state (atomicity). - self.verify_transaction(tx, inputs)?; - - let parents = self.insert_transaction(tx); - - Ok(parents) - } - - fn verify_transaction( - &mut self, - tx: &ProvenTransaction, - inputs: &TransactionInputs, - ) -> Result<(), AddTransactionErrorRework> { - // Ensure current account state is correct. - let current = self - .accounts - .get(&tx.account_id()) - .and_then(|account_state| account_state.latest_state()) - .copied() - .or(inputs.account_hash) - .unwrap_or_default(); - let expected = tx.account_update().init_state_hash(); - - if expected != current { - return Err(AddTransactionErrorRework::InvalidAccountState { current, expected }); - } - - // Ensure nullifiers aren't already present. - // TODO: Verifying the inputs nullifiers should be done externally already. - // TODO: The above should cause a change in type for inputs indicating this. - let input_nullifiers = tx.get_nullifiers().collect::>(); - let double_spend = - self.nullifiers.union(&input_nullifiers).copied().collect::>(); - if !double_spend.is_empty() { - return Err(AddTransactionErrorRework::NotesAlreadyConsumed(double_spend)); - } - - // TODO: additional input and output note checks, depends on the transaction type changes. - - Ok(()) - } - - /// Aggregate the transaction into the state, returning its parent transactions. - fn insert_transaction(&mut self, tx: &ProvenTransaction) -> BTreeSet { - let account_parent = self - .accounts - .entry(tx.account_id()) - .or_default() - .insert(tx.account_update().final_state_hash(), tx.id()); - - self.nullifiers.extend(tx.get_nullifiers()); - - // TODO: input and output notes - - account_parent.into_iter().collect() - } - - /// Reverts the given state diff. - /// - /// # Panics - /// - /// Panics if any part of the diff isn't present in the state. Callers should take - /// care to only revert transaction sets who's ancestors are all either committed or reverted. - pub fn revert_transactions(&mut self, diff: StateDiff) { - for (account, count) in diff.account_transactions { - let status = self.accounts.get_mut(&account).expect("Account must exist").revert(count); - - // Prune empty accounts. - if status.is_empty() { - self.accounts.remove(&account); - } - } - - for nullifier in diff.nullifiers { - assert!(self.nullifiers.remove(&nullifier), "Nullifier must exist"); - } - - // TODO: input and output notes - } - - /// Marks the given state diff as committed. - /// - /// These transactions are no longer considered inflight. Callers should take care to only - /// commit transactions who's ancestors are all committed. - /// - /// # Panics - /// - /// Panics if the accounts don't have enough inflight transactions to commit. - pub fn commit_transactions(&mut self, diff: &StateDiff) { - for (account, count) in &diff.account_transactions { - self.accounts.get_mut(account).expect("Account must exist").commit(*count); - } - } - - /// Drops the given state diff from memory. - /// - /// # Panics - /// - /// Panics if the accounts don't have enough inflight transactions to commit. - pub fn prune_committed_state(&mut self, diff: StateDiff) { - for (account, count) in diff.account_transactions { - let status = self - .accounts - .get_mut(&account) - .expect("Account must exist") - .remove_commited(count); - - // Prune empty accounts. - if status.is_empty() { - self.accounts.remove(&account); - } - } - - for nullifier in diff.nullifiers { - self.nullifiers.remove(&nullifier); - } - } -} - -/// Tracks the state of a single account. -#[derive(Default)] -struct AccountState { - /// State progression of this account over all uncommitted inflight transactions. - /// - /// Ordering is chronological from front (oldest) to back (newest). - inflight: VecDeque<(Digest, TransactionId)>, - - /// The latest committed state. - /// - /// Only valid if the committed count is greater than zero. - committed_state: Digest, - - /// The number of committed transactions. - /// - /// If this is zero then the committed state is meaningless. - committed_count: usize, -} - -impl AccountState { - /// Inserts a new transaction and its state, returning the previous inflight transaction, if - /// any. - pub fn insert(&mut self, state: Digest, tx: TransactionId) -> Option { - let previous = self.inflight.back().map(|(_, tx)| tx).copied(); - - self.inflight.push_back((state, tx)); - - previous - } - - /// The latest state of this account. - pub fn latest_state(&self) -> Option<&Digest> { - self.inflight - .back() - .map(|(state, _)| state) - // A count of zero indicates no committed state. - .or((self.committed_count > 1).then_some(&self.committed_state)) - } - - /// Reverts the most recent `n` inflight transactions. - /// - /// # Returns - /// - /// Returns the emptyness state of the account. - /// - /// # Panics - /// - /// Panics if there are less than `n` inflight transactions. - pub fn revert(&mut self, n: usize) -> AccountStatus { - let length = self.inflight.len(); - assert!( - length >= n, "Attempted to revert {n} transactions which is more than the {length} which are inflight.", - ); - - self.inflight.drain(length - n..); - - self.emptyness() - } - - /// Commits the first `n` inflight transactions. - /// - /// # Panics - /// - /// Panics if there are less than `n` inflight transactions. - pub fn commit(&mut self, n: usize) { - if n == 0 { - return; - } - - let length = self.inflight.len(); - assert!( - length >= n, "Attempted to revert {n} transactions which is more than the {length} which are inflight.", - ); - - self.committed_state = self - .inflight - .drain(..n) - .last() - .map(|(state, _)| state) - .expect("Must be Some as n > 0"); - self.committed_count += n; - } - - /// Removes `n` committed transactions. - /// - /// # Returns - /// - /// Returns the emptyness state of the account. - /// - /// # Panics - /// - /// Panics if there are less than `n` committed transactions. - pub fn remove_commited(&mut self, n: usize) -> AccountStatus { - assert!( - n <= self.committed_count, - "Attempted to remove {n} committed transactions, but only {} are present.", - self.committed_count - ); - self.committed_count -= n; - - self.emptyness() - } - - fn emptyness(&self) -> AccountStatus { - if self.inflight.is_empty() && self.committed_count == 0 { - AccountStatus::Empty - } else { - AccountStatus::NonEmpty - } - } -} - -/// Describes the emptyness of an [AccountState]. -/// -/// Is marked as #[must_use] so that callers handle prune empty accounts. -#[must_use] -#[derive(Clone, Copy, PartialEq, Eq)] -enum AccountStatus { - /// [AccountState] contains no state and should be pruned. - Empty, - /// [AccountState] contains state and should be kept. - NonEmpty, -} - -impl AccountStatus { - fn is_empty(&self) -> bool { - *self == Self::Empty - } -} diff --git a/crates/block-producer/src/mempool/inflight_state/account_state.rs b/crates/block-producer/src/mempool/inflight_state/account_state.rs new file mode 100644 index 000000000..d2888d033 --- /dev/null +++ b/crates/block-producer/src/mempool/inflight_state/account_state.rs @@ -0,0 +1,336 @@ +use std::{ + collections::{BTreeMap, BTreeSet, VecDeque}, + sync::Arc, +}; + +use miden_objects::{ + accounts::AccountId, + notes::{NoteId, Nullifier}, + transaction::{OutputNote, OutputNotes, ProvenTransaction, TransactionId}, + Digest, +}; + +use crate::{ + errors::{AddTransactionError, VerifyTxError}, + store::TransactionInputs, +}; + +// IN-FLIGHT ACCOUNT STATE +// ================================================================================================ + +/// Tracks the inflight state of an account. +#[derive(Clone, Default, Debug, PartialEq)] +pub struct InflightAccountState { + /// This account's state transitions due to inflight transactions. + /// + /// Ordering is chronological from front (oldest) to back (newest). + states: VecDeque<(Digest, TransactionId)>, + + /// The number of inflight states that have been committed. + /// + /// This acts as a pivot index for `self.states`, splitting it into two segments. The first + /// contains committed states and the second those that are uncommitted. + committed: usize, +} + +impl InflightAccountState { + /// Appends the new state, returning the previous state's transaction ID __IFF__ it is + /// uncommitted. + pub fn insert(&mut self, state: Digest, tx: TransactionId) -> Option { + let mut parent = self.states.back().map(|(_, tx)| tx).copied(); + + // Only return uncommitted parent ID. + // + // Since this is the latest state's ID, we need at least one uncommitted state. + if self.uncommitted_count() == 0 { + parent.take(); + } + + self.states.push_back((state, tx)); + + parent + } + + /// The latest state of this account. + pub fn current_state(&self) -> Option<&Digest> { + self.states.back().map(|(state, _)| state) + } + + /// Reverts the most recent `n` uncommitted inflight transactions. + /// + /// # Returns + /// + /// Returns the emptiness state of the account. + /// + /// # Panics + /// + /// Panics if there are less than `n` uncommitted inflight transactions. + pub fn revert(&mut self, n: usize) -> AccountStatus { + let uncommitted = self.uncommitted_count(); + assert!( + uncommitted >= n, "Attempted to revert {n} transactions which is more than the {uncommitted} which are uncommitted.", + ); + + self.states.drain(self.states.len() - n..); + + self.emptiness() + } + + /// Commits the next `n` uncommitted inflight transactions. + /// + /// # Panics + /// + /// Panics if there are less than `n` uncommitted inflight transactions. + pub fn commit(&mut self, n: usize) { + let uncommitted = self.uncommitted_count(); + assert!( + uncommitted >= n, "Attempted to revert {n} transactions which is more than the {uncommitted} which are uncommitted." + ); + + self.committed += n; + } + + /// Removes `n` committed transactions. + /// + /// # Returns + /// + /// Returns the emptiness state of the account. + /// + /// # Panics + /// + /// Panics if there are less than `n` committed transactions. + pub fn prune_committed(&mut self, n: usize) -> AccountStatus { + assert!( + self.committed >= n, + "Attempted to prune {n} transactions, which is more than the {} which are committed", + self.committed + ); + + self.committed -= n; + self.states.drain(..n); + + self.emptiness() + } + + /// This is essentially `is_empty` with the additional benefit that [AccountStatus] is marked + /// as `#[must_use]`, forcing callers to handle empty accounts (which should be pruned). + fn emptiness(&self) -> AccountStatus { + if self.states.is_empty() { + AccountStatus::Empty + } else { + AccountStatus::NonEmpty + } + } + + /// The number of uncommitted inflight transactions. + fn uncommitted_count(&self) -> usize { + self.states.len() - self.committed + } +} + +/// Describes the emptiness of an [AccountState]. +/// +/// Is marked as #[must_use] so that callers handle prune empty accounts. +#[must_use] +#[derive(Clone, Copy, PartialEq, Eq)] +pub enum AccountStatus { + /// [AccountState] contains no state and should be pruned. + Empty, + /// [AccountState] contains state and should be kept. + NonEmpty, +} + +impl AccountStatus { + pub fn is_empty(&self) -> bool { + *self == Self::Empty + } +} + +// TESTS +// ================================================================================================ + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_utils::Random; + + #[test] + fn current_state_is_the_most_recently_inserted() { + let mut rng = Random::with_random_seed(); + let mut uut = InflightAccountState::default(); + uut.insert(rng.draw_digest(), rng.draw_tx_id()); + uut.insert(rng.draw_digest(), rng.draw_tx_id()); + uut.insert(rng.draw_digest(), rng.draw_tx_id()); + + let expected = rng.draw_digest(); + uut.insert(expected, rng.draw_tx_id()); + + assert_eq!(uut.current_state(), Some(&expected)); + } + + #[test] + fn parent_is_the_most_recently_inserted() { + let mut rng = Random::with_random_seed(); + let mut uut = InflightAccountState::default(); + uut.insert(rng.draw_digest(), rng.draw_tx_id()); + uut.insert(rng.draw_digest(), rng.draw_tx_id()); + uut.insert(rng.draw_digest(), rng.draw_tx_id()); + + let expected = rng.draw_tx_id(); + uut.insert(rng.draw_digest(), expected); + + let parent = uut.insert(rng.draw_digest(), rng.draw_tx_id()); + + assert_eq!(parent, Some(expected)); + } + + #[test] + fn empty_account_has_no_parent() { + let mut rng = Random::with_random_seed(); + let mut uut = InflightAccountState::default(); + let parent = uut.insert(rng.draw_digest(), rng.draw_tx_id()); + + assert!(parent.is_none()); + } + + #[test] + fn fully_committed_account_has_no_parent() { + let mut rng = Random::with_random_seed(); + let mut uut = InflightAccountState::default(); + uut.insert(rng.draw_digest(), rng.draw_tx_id()); + uut.commit(1); + let parent = uut.insert(rng.draw_digest(), rng.draw_tx_id()); + + assert!(parent.is_none()); + } + + #[test] + fn uncommitted_account_has_a_parent() { + let mut rng = Random::with_random_seed(); + let expected_parent = rng.draw_tx_id(); + + let mut uut = InflightAccountState::default(); + uut.insert(rng.draw_digest(), expected_parent); + + let parent = uut.insert(rng.draw_digest(), rng.draw_tx_id()); + + assert_eq!(parent, Some(expected_parent)); + } + + #[test] + fn partially_committed_account_has_a_parent() { + let mut rng = Random::with_random_seed(); + let expected_parent = rng.draw_tx_id(); + + let mut uut = InflightAccountState::default(); + uut.insert(rng.draw_digest(), rng.draw_tx_id()); + uut.insert(rng.draw_digest(), expected_parent); + uut.commit(1); + + let parent = uut.insert(rng.draw_digest(), rng.draw_tx_id()); + + assert_eq!(parent, Some(expected_parent)); + } + + #[test] + fn reverted_txs_are_nonextant() { + let mut rng = Random::with_random_seed(); + const N: usize = 5; + const REVERT: usize = 2; + + let states = (0..N).map(|_| (rng.draw_digest(), rng.draw_tx_id())).collect::>(); + + let mut uut = InflightAccountState::default(); + for (state, tx) in &states { + uut.insert(*state, *tx); + } + uut.revert(REVERT); + + let mut expected = InflightAccountState::default(); + for (state, tx) in states.iter().rev().skip(REVERT).rev() { + expected.insert(*state, *tx); + } + + assert_eq!(uut, expected); + } + + #[test] + fn pruned_txs_are_nonextant() { + let mut rng = Random::with_random_seed(); + const N: usize = 5; + const PRUNE: usize = 2; + + let states = (0..N).map(|_| (rng.draw_digest(), rng.draw_tx_id())).collect::>(); + + let mut uut = InflightAccountState::default(); + for (state, tx) in &states { + uut.insert(*state, *tx); + } + uut.commit(PRUNE); + uut.prune_committed(PRUNE); + + let mut expected = InflightAccountState::default(); + for (state, tx) in states.iter().skip(PRUNE) { + expected.insert(*state, *tx); + } + + assert_eq!(uut, expected); + } + + #[test] + fn is_empty_after_full_commit_and_prune() { + let mut rng = Random::with_random_seed(); + const N: usize = 5; + let mut uut = InflightAccountState::default(); + for _ in 0..N { + uut.insert(rng.draw_digest(), rng.draw_tx_id()); + } + + uut.commit(N); + uut.prune_committed(N); + + assert_eq!(uut, Default::default()); + } + + #[test] + fn is_empty_after_full_revert() { + let mut rng = Random::with_random_seed(); + const N: usize = 5; + let mut uut = InflightAccountState::default(); + let mut rng = Random::with_random_seed(); + for _ in 0..N { + uut.insert(rng.draw_digest(), rng.draw_tx_id()); + } + + uut.revert(N); + + assert_eq!(uut, Default::default()); + } + + #[test] + #[should_panic] + fn revert_panics_on_out_of_bounds() { + let mut rng = Random::with_random_seed(); + const N: usize = 5; + let mut uut = InflightAccountState::default(); + for _ in 0..N { + uut.insert(rng.draw_digest(), rng.draw_tx_id()); + } + + uut.commit(1); + uut.revert(N); + } + + #[test] + #[should_panic] + fn commit_panics_on_out_of_bounds() { + let mut rng = Random::with_random_seed(); + const N: usize = 5; + let mut uut = InflightAccountState::default(); + for _ in 0..N { + uut.insert(rng.draw_digest(), rng.draw_tx_id()); + } + + uut.commit(N + 1); + } +} diff --git a/crates/block-producer/src/mempool/inflight_state/mod.rs b/crates/block-producer/src/mempool/inflight_state/mod.rs new file mode 100644 index 000000000..673a924f1 --- /dev/null +++ b/crates/block-producer/src/mempool/inflight_state/mod.rs @@ -0,0 +1,653 @@ +use std::{ + collections::{BTreeMap, BTreeSet, VecDeque}, + sync::Arc, +}; + +use miden_objects::{ + accounts::AccountId, + notes::{NoteId, Nullifier}, + transaction::{OutputNote, OutputNotes, ProvenTransaction, TransactionId}, + Digest, +}; + +use crate::{ + domain::transaction::AuthenticatedTransaction, + errors::{AddTransactionError, VerifyTxError}, + store::TransactionInputs, +}; + +mod account_state; + +use account_state::InflightAccountState; + +use super::BlockNumber; + +// IN-FLIGHT STATE +// ================================================================================================ + +/// Tracks the inflight state of the mempool. This includes recently committed blocks. +/// +/// Allows appending and reverting transactions as well as marking them as part of a committed +/// block. Committed state can also be pruned once the state is considered past the stale +/// threshold. +#[derive(Clone, Debug, PartialEq)] +pub struct InflightState { + /// Account states from inflight transactions. + /// + /// Accounts which are [AccountStatus::Empty] are immediately pruned. + accounts: BTreeMap, + + /// Nullifiers produced by the input notes of inflight transactions. + nullifiers: BTreeSet, + + /// Notes created by inflight transactions. + /// + /// Some of these may already be consumed - check the nullifiers. + output_notes: BTreeMap, + + /// Delta's representing the impact of each recently committed blocks on the inflight state. + /// + /// These are used to prune committed state after `num_retained_blocks` have passed. + committed_state: VecDeque, + + /// Amount of recently committed blocks we retain in addition to the inflight state. + /// + /// This provides an overlap between committed and inflight state, giving a grace + /// period for incoming transactions to be verified against both without requiring it + /// to be an atomic action. + num_retained_blocks: usize, + + /// The latest committed block height. + chain_tip: BlockNumber, +} + +/// The aggregated impact of a set of sequential transactions on the [InflightState]. +#[derive(Clone, Default, Debug, PartialEq)] +struct StateDelta { + /// The number of transactions that affected each account. + account_transactions: BTreeMap, + + /// The nullifiers consumed by the transactions. + nullifiers: BTreeSet, + + /// The notes produced by the transactions. + output_notes: BTreeSet, +} + +impl StateDelta { + fn new(txs: &[AuthenticatedTransaction]) -> Self { + let mut account_transactions = BTreeMap::::new(); + let mut nullifiers = BTreeSet::new(); + let mut output_notes = BTreeSet::new(); + + for tx in txs { + *account_transactions.entry(tx.account_id()).or_default() += 1; + nullifiers.extend(tx.nullifiers()); + output_notes.extend(tx.output_notes()); + } + + Self { + account_transactions, + nullifiers, + output_notes, + } + } +} + +impl InflightState { + /// Creates an [InflightState] which will retain committed state for the given + /// amount of blocks before pruning them. + pub fn new(chain_tip: BlockNumber, num_retained_blocks: usize) -> Self { + Self { + num_retained_blocks, + chain_tip, + accounts: Default::default(), + nullifiers: Default::default(), + output_notes: Default::default(), + committed_state: Default::default(), + } + } + + /// Appends the transaction to the inflight state. + /// + /// This operation is atomic i.e. a rejected transaction has no impact of the state. + pub fn add_transaction( + &mut self, + tx: &AuthenticatedTransaction, + ) -> Result, AddTransactionError> { + // Separate verification and state mutation so that a rejected transaction + // does not impact the state (atomicity). + self.verify_transaction(tx)?; + + let parents = self.insert_transaction(tx); + + Ok(parents) + } + + fn oldest_committed_state(&self) -> BlockNumber { + let committed_len: u32 = self + .committed_state + .len() + .try_into() + .expect("We should not be storing many blocks"); + self.chain_tip + .checked_sub(BlockNumber::new(committed_len)) + .expect("Chain height cannot be less than number of committed blocks") + } + + fn verify_transaction(&self, tx: &AuthenticatedTransaction) -> Result<(), AddTransactionError> { + // The mempool retains recently committed blocks, in addition to the state that is currently + // inflight. This overlap with the committed state allows us to verify incoming + // transactions against the current state (committed + inflight). Transactions are + // first authenticated against the committed state prior to being submitted to the + // mempool. The overlap provides a grace period between transaction authentication + // against committed state and verification against inflight state. + // + // Here we just ensure that this authentication point is still within this overlap zone. + // This should only fail if the grace period is too restrictive for the current + // combination of block rate, transaction throughput and database IO. + let stale_limit = self.oldest_committed_state(); + if tx.authentication_height() < stale_limit { + return Err(AddTransactionError::StaleInputs { + input_block: tx.authentication_height(), + stale_limit, + }); + } + + // Ensure current account state is correct. + let current = self + .accounts + .get(&tx.account_id()) + .and_then(|account_state| account_state.current_state()) + .copied() + .or(tx.store_account_state()); + let expected = tx.account_update().init_state_hash(); + + // SAFETY: a new accounts state is set to zero ie default. + if expected != current.unwrap_or_default() { + return Err(VerifyTxError::IncorrectAccountInitialHash { + tx_initial_account_hash: expected, + current_account_hash: current, + } + .into()); + } + + // Ensure nullifiers aren't already present. + // + // We don't need to check the store state here because that was + // already performed as part of authenticated the transaction. + let double_spend = tx + .nullifiers() + .filter(|nullifier| self.nullifiers.contains(nullifier)) + .collect::>(); + if !double_spend.is_empty() { + return Err(VerifyTxError::InputNotesAlreadyConsumed(double_spend).into()); + } + + // Ensure output notes aren't already present. + let duplicates = tx + .output_notes() + .filter(|note| self.output_notes.contains_key(note)) + .collect::>(); + if !duplicates.is_empty() { + return Err(VerifyTxError::OutputNotesAlreadyExist(duplicates).into()); + } + + // Ensure that all unauthenticated notes have an inflight output note to consume. + // + // We don't need to worry about double spending them since we already checked for + // that using the nullifiers. + // + // Note that the authenticated transaction already filters out notes that were + // previously unauthenticated, but were authenticated by the store. + let missing = tx + .unauthenticated_notes() + .filter(|note_id| !self.output_notes.contains_key(note_id)) + .collect::>(); + if !missing.is_empty() { + return Err(VerifyTxError::UnauthenticatedNotesNotFound(missing).into()); + } + + Ok(()) + } + + /// Aggregate the transaction into the state, returning its parent transactions. + fn insert_transaction(&mut self, tx: &AuthenticatedTransaction) -> BTreeSet { + let account_parent = self + .accounts + .entry(tx.account_id()) + .or_default() + .insert(tx.account_update().final_state_hash(), tx.id()); + + self.nullifiers.extend(tx.nullifiers()); + self.output_notes + .extend(tx.output_notes().map(|note_id| (note_id, OutputNoteState::new(tx.id())))); + + // Authenticated input notes (provably) consume notes that are already committed + // on chain. They therefore cannot form part of the inflight dependency chain. + // + // Additionally, we only care about parents which have not been committed yet. + let note_parents = tx + .unauthenticated_notes() + .filter_map(|note_id| self.output_notes.get(¬e_id)) + .filter_map(|note| note.transaction()) + .copied(); + + account_parent.into_iter().chain(note_parents).collect() + } + + /// Reverts the given state diff. + /// + /// # Panics + /// + /// Panics if any part of the diff isn't present in the state. Callers should take + /// care to only revert transaction sets who's ancestors are all either committed or reverted. + pub fn revert_transactions(&mut self, txs: &[AuthenticatedTransaction]) { + let delta = StateDelta::new(txs); + for (account, count) in delta.account_transactions { + let status = self.accounts.get_mut(&account).expect("Account must exist").revert(count); + + // Prune empty accounts. + if status.is_empty() { + self.accounts.remove(&account); + } + } + + for nullifier in delta.nullifiers { + assert!(self.nullifiers.remove(&nullifier), "Nullifier must exist"); + } + + for note in delta.output_notes { + assert!(self.output_notes.remove(¬e).is_some(), "Output note must exist"); + } + } + + /// Marks the given state diff as committed. + /// + /// These transactions are no longer considered inflight. Callers should take care to only + /// commit transactions who's ancestors are all committed. + /// + /// Note that this state is still retained for the configured number of blocks. The oldest + /// retained block is also pruned from the state. + /// + /// # Panics + /// + /// Panics if the accounts don't have enough inflight transactions to commit or if + /// the output notes don't exist. + pub fn commit_block(&mut self, txs: &[AuthenticatedTransaction]) { + let delta = StateDelta::new(txs); + for (account, count) in &delta.account_transactions { + self.accounts.get_mut(account).expect("Account must exist").commit(*count); + } + + for note in &delta.output_notes { + self.output_notes.get_mut(note).expect("Output note must exist").commit(); + } + + self.committed_state.push_back(delta); + + if self.committed_state.len() > self.num_retained_blocks { + let delta = self.committed_state.pop_front().expect("Must be some due to length check"); + self.prune_committed_state(delta); + } + + self.chain_tip.increment(); + } + + /// Removes the delta from inflight state. + /// + /// # Panics + /// + /// Panics if the accounts don't have enough inflight transactions to commit. + fn prune_committed_state(&mut self, diff: StateDelta) { + for (account, count) in diff.account_transactions { + let status = self + .accounts + .get_mut(&account) + .expect("Account must exist") + .prune_committed(count); + + // Prune empty accounts. + if status.is_empty() { + self.accounts.remove(&account); + } + } + + for nullifier in diff.nullifiers { + self.nullifiers.remove(&nullifier); + } + + for output_note in diff.output_notes { + self.output_notes.remove(&output_note); + } + } +} + +/// Describes the state of an inflight output note. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum OutputNoteState { + /// Output note is part of a committed block, and its source transaction should no longer be + /// considered for dependency tracking. + Committed, + /// Output note is still inflight and should be considered for dependency tracking. + Inflight(TransactionId), +} + +impl OutputNoteState { + /// Creates a new inflight output note state. + fn new(tx: TransactionId) -> Self { + Self::Inflight(tx) + } + + /// Commits the output note, removing the source transaction. + fn commit(&mut self) { + *self = Self::Committed; + } + + /// Returns the source transaction ID if the output note is not yet committed. + fn transaction(&self) -> Option<&TransactionId> { + if let Self::Inflight(tx) = self { + Some(tx) + } else { + None + } + } +} + +// TESTS +// ================================================================================================ + +#[cfg(test)] +mod tests { + use miden_air::Felt; + use miden_objects::{accounts::AccountType, testing::account_id::AccountIdBuilder}; + + use super::*; + use crate::test_utils::{ + mock_account_id, mock_proven_tx, + note::{mock_note, mock_output_note}, + MockPrivateAccount, MockProvenTxBuilder, + }; + + #[test] + fn rejects_duplicate_nullifiers() { + let account = mock_account_id(1); + let states = (1u8..=4).map(|x| Digest::from([x, 0, 0, 0])).collect::>(); + + let note_seed = 123; + // We need to make the note available first, in order for it to be consumed at all. + let tx0 = MockProvenTxBuilder::with_account(account, states[0], states[1]) + .output_notes(vec![mock_output_note(note_seed)]) + .build(); + let tx1 = MockProvenTxBuilder::with_account(account, states[1], states[2]) + .unauthenticated_notes(vec![mock_note(note_seed)]) + .build(); + let tx2 = MockProvenTxBuilder::with_account(account, states[2], states[3]) + .unauthenticated_notes(vec![mock_note(note_seed)]) + .build(); + + let mut uut = InflightState::new(BlockNumber::default(), 1); + uut.add_transaction(&AuthenticatedTransaction::from_inner(tx0)).unwrap(); + uut.add_transaction(&AuthenticatedTransaction::from_inner(tx1)).unwrap(); + + let err = uut.add_transaction(&AuthenticatedTransaction::from_inner(tx2)).unwrap_err(); + + assert_eq!( + err, + VerifyTxError::InputNotesAlreadyConsumed(vec![mock_note(note_seed).nullifier()]).into() + ); + } + + #[test] + fn rejects_duplicate_output_notes() { + let account = mock_account_id(1); + let states = (1u8..=3).map(|x| Digest::from([x, 0, 0, 0])).collect::>(); + + let note = mock_output_note(123); + let tx0 = MockProvenTxBuilder::with_account(account, states[0], states[1]) + .output_notes(vec![note.clone()]) + .build(); + let tx1 = MockProvenTxBuilder::with_account(account, states[1], states[2]) + .output_notes(vec![note.clone()]) + .build(); + + let mut uut = InflightState::new(BlockNumber::default(), 1); + uut.add_transaction(&AuthenticatedTransaction::from_inner(tx0)).unwrap(); + + let err = uut.add_transaction(&AuthenticatedTransaction::from_inner(tx1)).unwrap_err(); + + assert_eq!(err, VerifyTxError::OutputNotesAlreadyExist(vec![note.id()]).into()); + } + + #[test] + fn rejects_account_state_mismatch() { + let account = mock_account_id(1); + let states = (1u8..=3).map(|x| Digest::from([x, 0, 0, 0])).collect::>(); + + let tx = MockProvenTxBuilder::with_account(account, states[0], states[1]).build(); + + let mut uut = InflightState::new(BlockNumber::default(), 1); + let err = uut + .add_transaction(&AuthenticatedTransaction::from_inner(tx).with_store_state(states[2])) + .unwrap_err(); + + assert_eq!( + err, + VerifyTxError::IncorrectAccountInitialHash { + tx_initial_account_hash: states[0], + current_account_hash: states[2].into() + } + .into() + ); + } + + #[test] + fn account_state_transitions() { + let account = mock_account_id(1); + let states = (1u8..=3).map(|x| Digest::from([x, 0, 0, 0])).collect::>(); + + let tx0 = MockProvenTxBuilder::with_account(account, states[0], states[1]).build(); + let tx1 = MockProvenTxBuilder::with_account(account, states[1], states[2]).build(); + + let mut uut = InflightState::new(BlockNumber::default(), 1); + uut.add_transaction(&AuthenticatedTransaction::from_inner(tx0)).unwrap(); + uut.add_transaction(&AuthenticatedTransaction::from_inner(tx1).with_empty_store_state()) + .unwrap(); + } + + #[test] + fn new_account_state_defaults_to_zero() { + let account = mock_account_id(1); + + let tx = MockProvenTxBuilder::with_account( + account, + [0u8, 0, 0, 0].into(), + [1u8, 0, 0, 0].into(), + ) + .build(); + + let mut uut = InflightState::new(BlockNumber::default(), 1); + uut.add_transaction(&AuthenticatedTransaction::from_inner(tx).with_empty_store_state()) + .unwrap(); + } + + #[test] + fn inflight_account_state_overrides_input_state() { + let account = mock_account_id(1); + let states = (1u8..=3).map(|x| Digest::from([x, 0, 0, 0])).collect::>(); + + let tx0 = MockProvenTxBuilder::with_account(account, states[0], states[1]).build(); + let tx1 = MockProvenTxBuilder::with_account(account, states[1], states[2]).build(); + + let mut uut = InflightState::new(BlockNumber::default(), 1); + uut.add_transaction(&AuthenticatedTransaction::from_inner(tx0)).unwrap(); + + // Feed in an old state via input. This should be ignored, and the previous tx's final + // state should be used. + uut.add_transaction(&AuthenticatedTransaction::from_inner(tx1).with_store_state(states[0])) + .unwrap(); + } + + #[test] + fn dependency_tracking() { + let account = mock_account_id(1); + let states = (1u8..=3).map(|x| Digest::from([x, 0, 0, 0])).collect::>(); + let note_seed = 123; + + // Parent via account state. + let tx0 = MockProvenTxBuilder::with_account(account, states[0], states[1]).build(); + // Parent via output note. + let tx1 = MockProvenTxBuilder::with_account(mock_account_id(2), states[0], states[1]) + .output_notes(vec![mock_output_note(note_seed)]) + .build(); + + let tx = MockProvenTxBuilder::with_account(account, states[1], states[2]) + .unauthenticated_notes(vec![mock_note(note_seed)]) + .build(); + + let mut uut = InflightState::new(BlockNumber::default(), 1); + uut.add_transaction(&AuthenticatedTransaction::from_inner(tx0.clone())).unwrap(); + uut.add_transaction(&AuthenticatedTransaction::from_inner(tx1.clone())).unwrap(); + + let parents = uut + .add_transaction(&AuthenticatedTransaction::from_inner(tx).with_empty_store_state()) + .unwrap(); + let expected = BTreeSet::from([tx0.id(), tx1.id()]); + + assert_eq!(parents, expected); + } + + #[test] + fn committed_parents_are_not_tracked() { + let account = mock_account_id(1); + let states = (1u8..=3).map(|x| Digest::from([x, 0, 0, 0])).collect::>(); + let note_seed = 123; + + // Parent via account state. + let tx0 = MockProvenTxBuilder::with_account(account, states[0], states[1]).build(); + let tx0 = AuthenticatedTransaction::from_inner(tx0); + // Parent via output note. + let tx1 = MockProvenTxBuilder::with_account(mock_account_id(2), states[0], states[1]) + .output_notes(vec![mock_output_note(note_seed)]) + .build(); + let tx1 = AuthenticatedTransaction::from_inner(tx1); + + let tx = MockProvenTxBuilder::with_account(account, states[1], states[2]) + .unauthenticated_notes(vec![mock_note(note_seed)]) + .build(); + + let mut uut = InflightState::new(BlockNumber::default(), 1); + uut.add_transaction(&tx0.clone()).unwrap(); + uut.add_transaction(&tx1.clone()).unwrap(); + + // Commit the parents, which should remove them from dependency tracking. + uut.commit_block(&[tx0, tx1]); + + let parents = uut + .add_transaction(&AuthenticatedTransaction::from_inner(tx).with_empty_store_state()) + .unwrap(); + + assert!(parents.is_empty()); + } + + #[test] + fn tx_insertions_and_reversions_cancel_out() { + // Reverting txs should be equivalent to them never being inserted. + // + // We test this by reverting some txs and equating it to the remaining set. + // This is a form of proprty test. + let states = (1u8..=5).map(|x| Digest::from([x, 0, 0, 0])).collect::>(); + let txs = vec![ + MockProvenTxBuilder::with_account(mock_account_id(1), states[0], states[1]), + MockProvenTxBuilder::with_account(mock_account_id(1), states[1], states[2]) + .output_notes(vec![mock_output_note(111), mock_output_note(222)]), + MockProvenTxBuilder::with_account(mock_account_id(2), states[0], states[1]) + .unauthenticated_notes(vec![mock_note(222)]), + MockProvenTxBuilder::with_account(mock_account_id(1), states[2], states[3]), + MockProvenTxBuilder::with_account(mock_account_id(2), states[1], states[2]) + .unauthenticated_notes(vec![mock_note(111)]) + .output_notes(vec![mock_output_note(45)]), + ]; + + let txs = txs + .into_iter() + .map(MockProvenTxBuilder::build) + .map(AuthenticatedTransaction::from_inner) + .collect::>(); + + for i in 0..states.len() { + // Insert all txs and then revert the last `i` of them. + // This should match only inserting the first `N-i` of them. + let mut reverted = InflightState::new(BlockNumber::default(), 1); + for (idx, tx) in txs.iter().enumerate() { + reverted.add_transaction(tx).unwrap_or_else(|err| { + panic!("Inserting tx #{idx} in iteration {i} should succeed: {err}") + }); + } + reverted.revert_transactions(&txs[txs.len() - i..]); + + let mut inserted = InflightState::new(BlockNumber::default(), 1); + for (idx, tx) in txs.iter().rev().skip(i).rev().enumerate() { + inserted.add_transaction(tx).unwrap_or_else(|err| { + panic!("Inserting tx #{idx} in iteration {i} should succeed: {err}") + }); + } + + assert_eq!(reverted, inserted, "Iteration {i}"); + } + } + + #[test] + fn pruning_committed_state() { + //! This is a form of property test, where we assert that pruning the first `i` of `N` + //! transactions is equivalent to only inserting the last `N-i` transactions. + let states = (1u8..=5).map(|x| Digest::from([x, 0, 0, 0])).collect::>(); + + // Skipping initial txs means that output notes required for subsequent unauthenticated + // input notes wont' always be present. To work around this, we instead only use + // authenticated input notes. + let txs = vec![ + MockProvenTxBuilder::with_account(mock_account_id(1), states[0], states[1]), + MockProvenTxBuilder::with_account(mock_account_id(1), states[1], states[2]) + .output_notes(vec![mock_output_note(111), mock_output_note(222)]), + MockProvenTxBuilder::with_account(mock_account_id(2), states[0], states[1]) + .nullifiers(vec![mock_note(222).nullifier()]), + MockProvenTxBuilder::with_account(mock_account_id(1), states[2], states[3]), + MockProvenTxBuilder::with_account(mock_account_id(2), states[1], states[2]) + .nullifiers(vec![mock_note(111).nullifier()]) + .output_notes(vec![mock_output_note(45)]), + ]; + + let txs = txs + .into_iter() + .map(MockProvenTxBuilder::build) + .map(AuthenticatedTransaction::from_inner) + .collect::>(); + + for i in 0..states.len() { + // Insert all txs and then commit and prune the first `i` of them. + // + // This should match only inserting the final `N-i` transactions. + // Note: we force all committed state to immedietely be pruned by setting + // it to zero. + let mut committed = InflightState::new(BlockNumber::default(), 0); + for (idx, tx) in txs.iter().enumerate() { + committed.add_transaction(tx).unwrap_or_else(|err| { + panic!("Inserting tx #{idx} in iteration {i} should succeed: {err}") + }); + } + committed.commit_block(&txs[..i]); + + let mut inserted = InflightState::new(BlockNumber::new(1), 0); + for (idx, tx) in txs.iter().skip(i).enumerate() { + // We need to adjust the height since we are effectively at block "1" now. + let tx = tx.clone().with_authentication_height(1); + inserted.add_transaction(&tx).unwrap_or_else(|err| { + panic!("Inserting tx #{idx} in iteration {i} should succeed: {err}") + }); + } + + assert_eq!(committed, inserted, "Iteration {i}"); + } + } +} diff --git a/crates/block-producer/src/mempool/mod.rs b/crates/block-producer/src/mempool/mod.rs index 67081009a..6df7f3479 100644 --- a/crates/block-producer/src/mempool/mod.rs +++ b/crates/block-producer/src/mempool/mod.rs @@ -6,7 +6,7 @@ use std::{ }; use batch_graph::BatchGraph; -use inflight_state::{InflightState, StateDiff}; +use inflight_state::InflightState; use miden_objects::{ accounts::AccountId, notes::{NoteId, Nullifier}, @@ -17,7 +17,9 @@ use miden_tx::{utils::collections::KvMap, TransactionVerifierError}; use transaction_graph::TransactionGraph; use crate::{ - errors::AddTransactionErrorRework, + batch_builder::batch::TransactionBatch, + domain::transaction::AuthenticatedTransaction, + errors::{AddTransactionError, VerifyTxError}, store::{TransactionInputs, TxInputsError}, }; @@ -40,7 +42,7 @@ impl BatchJobId { } } -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] pub struct BlockNumber(u32); impl Display for BlockNumber { @@ -50,6 +52,10 @@ impl Display for BlockNumber { } impl BlockNumber { + pub fn new(x: u32) -> Self { + Self(x) + } + pub fn next(&self) -> Self { let mut ret = *self; ret.increment(); @@ -61,7 +67,7 @@ impl BlockNumber { self.checked_sub(Self(1)) } - pub fn increment(mut self) { + pub fn increment(&mut self) { self.0 += 1; } @@ -70,22 +76,17 @@ impl BlockNumber { } } +// MEMPOOL +// ================================================================================================ + pub struct Mempool { /// The latest inflight state of each account. /// /// Accounts without inflight transactions are not stored. state: InflightState, - /// Note's consumed by inflight transactions. - nullifiers: BTreeSet, - - /// Notes produced by inflight transactions. - /// - /// It is possible for these to already be consumed - check nullifiers. - notes: BTreeMap, - /// Inflight transactions. - transactions: TransactionGraph, + transactions: TransactionGraph, /// Inflight batches. batches: BatchGraph, @@ -93,17 +94,11 @@ pub struct Mempool { /// The next batches ID. next_batch_id: BatchJobId, - /// Blocks which have been committed but are not yet considered stale. - committed_diffs: VecDeque, - /// The current block height of the chain. chain_tip: BlockNumber, block_in_progress: Option>, - /// Number of blocks before transaction input is considered stale. - staleness: BlockNumber, - batch_transaction_limit: usize, block_batch_limit: usize, } @@ -120,23 +115,12 @@ impl Mempool { /// Returns an error if the transaction's initial conditions don't match the current state. pub fn add_transaction( &mut self, - transaction: ProvenTransaction, - inputs: TransactionInputs, - ) -> Result { - // Ensure inputs aren't stale. - if let Some(stale_block) = self.stale_block() { - if inputs.current_block_height <= stale_block.0 { - return Err(AddTransactionErrorRework::StaleInputs { - input_block: BlockNumber(inputs.current_block_height), - stale_limit: stale_block, - }); - } - } - + transaction: AuthenticatedTransaction, + ) -> Result { // Add transaction to inflight state. - let parents = self.state.add_transaction(&transaction, &inputs)?; + let parents = self.state.add_transaction(&transaction)?; - self.transactions.insert(transaction, parents); + self.transactions.insert(transaction.id(), transaction, parents); Ok(self.chain_tip.0) } @@ -146,34 +130,35 @@ impl Mempool { /// Transactions are returned in a valid execution ordering. /// /// Returns `None` if no transactions are available. - pub fn select_batch(&mut self) -> Option<(BatchJobId, Vec)> { + pub fn select_batch(&mut self) -> Option<(BatchJobId, Vec)> { let mut parents = BTreeSet::new(); let mut batch = Vec::with_capacity(self.batch_transaction_limit); + let mut tx_ids = Vec::with_capacity(self.batch_transaction_limit); for _ in 0..self.batch_transaction_limit { // Select transactions according to some strategy here. For now its just arbitrary. - let Some((tx, tx_parents)) = self.transactions.pop_for_batching() else { + let Some((tx, tx_parents)) = self.transactions.pop_for_processing() else { break; }; batch.push(tx); parents.extend(tx_parents); } - // Update the depedency graph to reflect parents at the batch level by removing - // all edges within this batch. + // Update the dependency graph to reflect parents at the batch level by removing all edges + // within this batch. for tx in &batch { - parents.remove(tx); + parents.remove(&tx.id()); } let batch_id = self.next_batch_id; self.next_batch_id.increment(); - self.batches.insert(batch_id, batch.clone(), parents); + self.batches.insert(batch_id, tx_ids, parents); Some((batch_id, batch)) } - /// Drops the failed batch and all of its descendents. + /// Drops the failed batch and all of its descendants. /// /// Transactions are placed back in the queue. pub fn batch_failed(&mut self, batch: BatchJobId) { @@ -194,8 +179,8 @@ impl Mempool { } /// Marks a batch as proven if it exists. - pub fn batch_proved(&mut self, batch_id: BatchJobId) { - self.batches.mark_proven(batch_id); + pub fn batch_proved(&mut self, batch_id: BatchJobId, batch: TransactionBatch) { + self.batches.mark_proven(batch_id, batch); } /// Select batches for the next block. @@ -205,16 +190,16 @@ impl Mempool { /// # Panics /// /// Panics if there is already a block in flight. - pub fn select_block(&mut self) -> (BlockNumber, BTreeSet) { + pub fn select_block(&mut self) -> (BlockNumber, BTreeMap) { assert!(self.block_in_progress.is_none(), "Cannot have two blocks inflight."); let batches = self.batches.select_block(self.block_batch_limit); - self.block_in_progress = Some(batches.clone()); + self.block_in_progress = Some(batches.keys().cloned().collect()); (self.chain_tip.next(), batches) } - /// Notify the pool that the block was succesfully completed. + /// Notify the pool that the block was successfully completed. /// /// # Panics /// @@ -225,19 +210,10 @@ impl Mempool { // Remove committed batches and transactions from graphs. let batches = self.block_in_progress.take().expect("No block in progress to commit"); let transactions = self.batches.remove_committed(batches); - let transactions = self.transactions.remove_committed(&transactions); + let transactions = self.transactions.prune_processed(&transactions); // Inform inflight state about committed data. - let diff = StateDiff::new(&transactions); - self.state.commit_transactions(&diff); - - self.committed_diffs.push_back(diff); - if self.committed_diffs.len() > self.staleness.0 as usize { - // SAFETY: just checked that length is non-zero. - let stale_block = self.committed_diffs.pop_front().unwrap(); - - self.state.prune_committed_state(stale_block); - } + self.state.commit_block(&transactions); self.chain_tip.increment(); } @@ -261,15 +237,6 @@ impl Mempool { let transactions = self.transactions.purge_subgraphs(transactions); // Rollback state. - let impact = StateDiff::new(&transactions); - self.state.revert_transactions(impact); - // TODO: revert nullifiers and notes. - } - - /// The highest block height we consider stale. - /// - /// Returns [None] if the blockchain is so short that all blocks are considered fresh. - fn stale_block(&self) -> Option { - self.chain_tip.checked_sub(self.staleness) + self.state.revert_transactions(&transactions); } } diff --git a/crates/block-producer/src/mempool/transaction_graph.rs b/crates/block-producer/src/mempool/transaction_graph.rs index 2384924b1..1a2d9cd94 100644 --- a/crates/block-producer/src/mempool/transaction_graph.rs +++ b/crates/block-producer/src/mempool/transaction_graph.rs @@ -5,70 +5,120 @@ use std::{ use miden_objects::transaction::{ProvenTransaction, TransactionId}; -use super::BatchJobId; +// TRANSACTION GRAPH +// ================================================================================================ -#[derive(Default, Clone, Debug)] -pub struct TransactionGraph { +/// Tracks the dependency graph and status of transactions. +#[derive(Clone, Debug, PartialEq)] +pub struct TransactionGraph { /// All transactions currently inflight. - nodes: BTreeMap, + nodes: BTreeMap>, - /// Transactions ready for inclusion in a batch. + /// Transactions ready for being processed. /// - /// aka transactions whose parent transactions are already included in batches. + /// aka transactions whose parents are already processed. roots: BTreeSet, } -impl TransactionGraph { - pub fn insert(&mut self, transaction: ProvenTransaction, parents: BTreeSet) { - let id = transaction.id(); - // Inform parent's of their new child. +impl Default for TransactionGraph { + fn default() -> Self { + Self { + nodes: Default::default(), + roots: Default::default(), + } + } +} + +impl TransactionGraph { + /// Inserts a new transaction node, with edges to the given parent nodes. + /// + /// # Panics + /// + /// Panics if: + /// - any of the given parents are not part of the graph, + /// - the transaction is already present + pub fn insert(&mut self, id: TransactionId, data: T, parents: BTreeSet) { + // Inform parents of their new child. for parent in &parents { self.nodes.get_mut(parent).expect("Parent must be in pool").children.insert(id); } - let transaction = Node::new(transaction, parents); - if self.nodes.insert(id, transaction).is_some() { + let node = Node::new(data, parents); + if self.nodes.insert(id, node).is_some() { panic!("Transaction already exists in pool"); } - // This could be optimised by inlining this inside the parent loop. This would prevent the + // This could be optimized by inlining this inside the parent loop. This would prevent the // double iteration over parents, at the cost of some code duplication. self.try_make_root(id); } - pub fn pop_for_batching(&mut self) -> Option<(TransactionId, BTreeSet)> { - let tx_id = self.roots.pop_first()?; - let node = self.nodes.get_mut(&tx_id).expect("Root transaction must be in graph"); - node.status = Status::Processed; + /// Returns the next transaction ready for processing, and its parent edges. + /// + /// Internally this transaction is now marked as processed and is no longer considered inqueue. + pub fn pop_for_processing(&mut self) -> Option<(T, BTreeSet)> { + let tx_id = self.roots.first()?; + + Some(self.process(*tx_id)) + } + + /// Marks the transaction as processed and returns its data and parents. + /// + /// Separated out from the actual strategy of choosing so that we have more + /// fine grained control available for tests. + /// + /// # Panics + /// + /// Panics if the transaction: + /// - does not exist + /// - is already processed + /// - is not ready for processing + fn process(&mut self, id: TransactionId) -> (T, BTreeSet) { + assert!(self.roots.remove(&id), "Process target must form part of roots"); + let node = self.nodes.get_mut(&id).expect("Root transaction must be in graph"); + node.mark_as_processed(); // Work around multiple mutable borrows of self. let parents = node.parents.clone(); let children = node.children.clone(); + let tx = node.data.clone(); for child in children { self.try_make_root(child); } - Some((tx_id, parents)) + (tx, parents) } - /// Marks the given transactions as being back inqueue. + /// Marks the given transactions as being back in queue. + /// + /// # Panics + /// + /// Panics if any of the transactions are + /// - not part of the graph + /// - are already in queue aka not processed pub fn requeue_transactions(&mut self, transactions: BTreeSet) { for tx in &transactions { - self.nodes.get_mut(tx).expect("Node must exist").status = Status::InQueue; + self.nodes.get_mut(tx).expect("Node must exist").mark_as_inqueue(); } // All requeued transactions are potential roots, and current roots may have been // invalidated. - let mut potential_roots = transactions; - potential_roots.extend(&self.roots); - self.roots.clear(); + let mut potential_roots = std::mem::take(&mut self.roots); + potential_roots.extend(transactions); for tx in potential_roots { self.try_make_root(tx); } } - pub fn remove_committed(&mut self, tx_ids: &[TransactionId]) -> Vec> { + /// Prunes processed transactions from the graph. + /// + /// # Panics + /// + /// Panics if any of the given transactions are: + /// - not part of the graph + /// - are in queue aka not processed + pub fn prune_processed(&mut self, tx_ids: &[TransactionId]) -> Vec { let mut transactions = Vec::with_capacity(tx_ids.len()); for transaction in tx_ids { let node = self.nodes.remove(transaction).expect("Node must be in graph"); @@ -90,13 +140,10 @@ impl TransactionGraph { transactions } - /// Removes the transactions and all their descendents from the graph. + /// Removes the transactions and all their descendants from the graph. /// /// Returns all transactions removed. - pub fn purge_subgraphs( - &mut self, - transactions: Vec, - ) -> Vec> { + pub fn purge_subgraphs(&mut self, transactions: Vec) -> Vec { let mut removed = Vec::new(); let mut to_process = transactions; @@ -108,8 +155,9 @@ impl TransactionGraph { continue; }; - // All the child batches are also removed so no need to check - // for new roots. No new roots are possible as a result of this subgraph removal. + // All the children will also be removed so no need to check for new roots. + // + // No new roots are possible as a result of this subgraph removal. self.roots.remove(&node_id); // Inform parent that this child no longer exists. @@ -130,13 +178,20 @@ impl TransactionGraph { removed } + /// Adds the given transaction to the set of roots _IFF_ all of its parents are marked as + /// processed. + /// + /// # Panics + /// + /// Panics if the transaction or any of its parents do not exist. This would constitute an + /// internal bookkeeping failure. fn try_make_root(&mut self, tx_id: TransactionId) { let tx = self.nodes.get_mut(&tx_id).expect("Transaction must be in graph"); for parent in tx.parents.clone() { let parent = self.nodes.get(&parent).expect("Parent must be in pool"); - if parent.status != Status::Processed { + if !parent.is_processed() { return; } } @@ -144,27 +199,398 @@ impl TransactionGraph { } } -#[derive(Clone, Debug)] -struct Node { +#[derive(Clone, Debug, PartialEq)] +struct Node { status: Status, - data: Arc, + data: T, parents: BTreeSet, children: BTreeSet, } -impl Node { - fn new(tx: ProvenTransaction, parents: BTreeSet) -> Self { +impl Node { + /// Creates a new inflight [Node] with no children. + fn new(data: T, parents: BTreeSet) -> Self { Self { - status: Status::InQueue, - data: Arc::new(tx), + status: Status::Queued, + data, parents, children: Default::default(), } } + + /// Marks the node as [Status::Processed]. + /// + /// # Panics + /// + /// Panics if the node is already processed. + fn mark_as_processed(&mut self) { + assert!(!self.is_processed()); + self.status = Status::Processed + } + + /// Marks the node as [Status::Inqueue]. + /// + /// # Panics + /// + /// Panics if the node is already inqueue. + fn mark_as_inqueue(&mut self) { + assert!(!self.is_inqueue()); + self.status = Status::Queued + } + + fn is_processed(&self) -> bool { + self.status == Status::Processed + } + + fn is_inqueue(&self) -> bool { + self.status == Status::Queued + } } #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum Status { - InQueue, + Queued, Processed, } + +// TESTS +// ================================================================================================ + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_utils::Random; + + /// Simplified graph type which uses the transaction ID as the data value. + /// + /// Production usage will have `T: ProvenTransaction` however this is cumbersome + /// to generate. Since this graph doesn't actually care about the data type, we + /// simplify test data generation by just duplicating the ID. + type TestGraph = TransactionGraph; + + /// Test helpers and aliases. + impl TestGraph { + /// Alias to insert a transaction with no parents. + fn insert_with_no_parent(&mut self, id: TransactionId) { + self.insert_with_parents(id, Default::default()); + } + + /// Alias for inserting a transaction with parents. + fn insert_with_parents(&mut self, id: TransactionId, parents: BTreeSet) { + self.insert(id, id, parents); + } + + /// Alias for inserting a transaction with a single parent. + fn insert_with_parent(&mut self, id: TransactionId, parent: TransactionId) { + self.insert_with_parents(id, [parent].into()); + } + + /// Calls `pop_for_processing` until it returns `None`. + /// + /// This should result in a fully processed graph, barring bugs. + /// + /// Panics if the graph is not fully processed. + fn process_all(&mut self) -> Vec { + let mut processed = Vec::new(); + while let Some((id, _)) = self.pop_for_processing() { + processed.push(id); + } + + assert!(self.nodes.values().all(Node::is_processed)); + + processed + } + } + + #[test] + fn pruned_nodes_are_nonextant() { + //! Checks that processed and then pruned nodes behave as if they + //! never existed in the graph. We test this by comparing it to + //! a reference graph created without these ancestor nodes. + let mut rng = Random::with_random_seed(); + + let ancestor_a = rng.draw_tx_id(); + let ancestor_b = rng.draw_tx_id(); + + let child_a = rng.draw_tx_id(); + let child_b = rng.draw_tx_id(); + let child_both = rng.draw_tx_id(); + + let mut uut = TestGraph::default(); + uut.insert_with_no_parent(ancestor_a); + uut.insert_with_no_parent(ancestor_b); + uut.insert_with_parent(child_a, ancestor_a); + uut.insert_with_parent(child_b, ancestor_b); + uut.insert_with_parents(child_both, [ancestor_a, ancestor_b].into()); + + uut.process(ancestor_a); + uut.process(ancestor_b); + uut.prune_processed(&[ancestor_a, ancestor_b]); + + let mut reference = TestGraph::default(); + reference.insert_with_no_parent(child_a); + reference.insert_with_no_parent(child_b); + reference.insert_with_no_parent(child_both); + + assert_eq!(uut, reference); + } + + #[test] + fn inserted_node_is_considered_for_root() { + //! Ensure that a fresh node who's parent is + //! already processed will be considered for processing. + let mut rng = Random::with_random_seed(); + let parent_a = rng.draw_tx_id(); + let parent_b = rng.draw_tx_id(); + let child_a = rng.draw_tx_id(); + let child_b = rng.draw_tx_id(); + + let mut uut = TestGraph::default(); + uut.insert_with_no_parent(parent_a); + uut.insert_with_no_parent(parent_b); + uut.process(parent_a); + + uut.insert_with_parent(child_a, parent_a); + uut.insert_with_parent(child_b, parent_b); + + assert!(uut.roots.contains(&child_a)); + assert!(!uut.roots.contains(&child_b)); + } + + #[test] + fn fifo_order_is_maintained() { + //! This test creates a simple queue graph, expecting that the processed items should + //! be emitted in the same order. + let mut rng = Random::with_random_seed(); + let input = (0..10).map(|_| rng.draw_tx_id()).collect::>(); + + let mut uut = TestGraph::default(); + uut.insert_with_no_parent(input[0]); + for pairs in input.windows(2) { + let (parent, id) = (pairs[0], pairs[1]); + uut.insert_with_parent(id, parent); + } + + let result = uut.process_all(); + assert_eq!(result, input); + } + + #[test] + fn requeuing_resets_graph_state() { + //! Requeuing transactions should cause the internal state to reset + //! to the same state as before these transactions were emitted + //! for processing. + + let mut rng = Random::with_random_seed(); + + let ancestor_a = rng.draw_tx_id(); + let ancestor_b = rng.draw_tx_id(); + let parent_a = rng.draw_tx_id(); + let parent_b = rng.draw_tx_id(); + let child_a = rng.draw_tx_id(); + let child_b = rng.draw_tx_id(); + let child_c = rng.draw_tx_id(); + + let mut uut = TestGraph::default(); + uut.insert_with_no_parent(ancestor_a); + uut.insert_with_no_parent(ancestor_b); + uut.insert_with_parent(parent_a, ancestor_a); + uut.insert_with_parent(parent_b, ancestor_b); + uut.insert_with_parents(child_a, [ancestor_a, parent_a].into()); + uut.insert_with_parents(child_b, [parent_a, parent_b].into()); + uut.insert_with_parent(child_c, parent_b); + + let mut reference = uut.clone(); + + uut.process(ancestor_a); + uut.process(ancestor_b); + uut.process(parent_a); + uut.process(parent_b); + uut.process(child_c); + + // Requeue all except ancestor a. This is a somewhat arbitrary choice. + // The reference graph should therefore only have ancestor a processed. + uut.requeue_transactions([ancestor_b, parent_a, parent_b, child_c].into()); + reference.process(ancestor_a); + + assert_eq!(uut, reference); + } + + #[test] + fn nodes_are_processed_exactly_once() { + let mut rng = Random::with_random_seed(); + + let ancestor_a = rng.draw_tx_id(); + let ancestor_b = rng.draw_tx_id(); + let parent_a = rng.draw_tx_id(); + let parent_b = rng.draw_tx_id(); + let child_a = rng.draw_tx_id(); + let child_b = rng.draw_tx_id(); + let child_c = rng.draw_tx_id(); + + let mut uut = TestGraph::default(); + uut.insert_with_no_parent(ancestor_a); + uut.insert_with_no_parent(ancestor_b); + uut.insert_with_parent(parent_a, ancestor_a); + uut.insert_with_parent(parent_b, ancestor_b); + uut.insert_with_parents(child_a, [ancestor_a, parent_a].into()); + uut.insert_with_parents(child_b, [parent_a, parent_b].into()); + uut.insert_with_parent(child_c, parent_b); + + let mut result = uut.process_all(); + result.sort(); + + let mut expected = + vec![ancestor_a, ancestor_b, parent_a, parent_b, child_a, child_b, child_c]; + expected.sort(); + + assert_eq!(result, expected); + } + + #[test] + fn processed_data_and_parent_tracking() { + let mut rng = Random::with_random_seed(); + + let ancestor_a = rng.draw_tx_id(); + let ancestor_b = rng.draw_tx_id(); + let parent_a = rng.draw_tx_id(); + let parent_b = rng.draw_tx_id(); + let child_a = rng.draw_tx_id(); + let child_b = rng.draw_tx_id(); + let child_c = rng.draw_tx_id(); + + let mut uut = TestGraph::default(); + uut.insert_with_no_parent(ancestor_a); + uut.insert_with_no_parent(ancestor_b); + uut.insert_with_parent(parent_a, ancestor_a); + uut.insert_with_parent(parent_b, ancestor_b); + uut.insert_with_parents(child_a, [ancestor_a, parent_a].into()); + uut.insert_with_parents(child_b, [parent_a, parent_b].into()); + uut.insert_with_parent(child_c, parent_b); + + let result = uut.process(ancestor_a); + assert_eq!(result, (ancestor_a, Default::default())); + + let result = uut.process(ancestor_b); + assert_eq!(result, (ancestor_b, Default::default())); + + let result = uut.process(parent_a); + assert_eq!(result, (parent_a, [ancestor_a].into())); + + let result = uut.process(parent_b); + assert_eq!(result, (parent_b, [ancestor_b].into())); + + let result = uut.process(child_a); + assert_eq!(result, (child_a, [ancestor_a, parent_a].into())); + + let result = uut.process(child_b); + assert_eq!(result, (child_b, [parent_a, parent_b].into())); + + let result = uut.process(child_c); + assert_eq!(result, (child_c, [parent_b].into())); + } + + #[test] + fn purging_subgraph_handles_internal_nodes() { + //! Purging a subgraph should correctly handle nodes already deleted within that subgraph. + //! + //! This is a concern for errors as we are deleting parts of the subgraph while we are + //! iterating through the nodes to purge. This means its likely a node will already + //! have been deleted before processing it as an input. + //! + //! We can somewhat force this to occur by re-ordering the inputs relative to the actual + //! dependency order. + + let mut rng = Random::with_random_seed(); + + let ancestor_a = rng.draw_tx_id(); + let ancestor_b = rng.draw_tx_id(); + let parent_a = rng.draw_tx_id(); + let parent_b = rng.draw_tx_id(); + let child_a = rng.draw_tx_id(); + let child_b = rng.draw_tx_id(); + let child_c = rng.draw_tx_id(); + + let mut uut = TestGraph::default(); + uut.insert_with_no_parent(ancestor_a); + uut.insert_with_no_parent(ancestor_b); + uut.insert_with_parent(parent_a, ancestor_a); + uut.insert_with_parent(parent_b, ancestor_b); + uut.insert_with_parents(child_a, [ancestor_a, parent_a].into()); + uut.insert_with_parents(child_b, [parent_a, parent_b].into()); + uut.insert_with_parent(child_c, parent_b); + + uut.purge_subgraphs(vec![child_b, parent_a]); + + let mut reference = TestGraph::default(); + reference.insert_with_no_parent(ancestor_a); + reference.insert_with_no_parent(ancestor_b); + reference.insert_with_parent(parent_b, ancestor_b); + reference.insert_with_parent(child_c, parent_b); + + assert_eq!(uut, reference); + } + + #[test] + fn purging_removes_all_descendents() { + let mut rng = Random::with_random_seed(); + + let ancestor_a = rng.draw_tx_id(); + let ancestor_b = rng.draw_tx_id(); + let parent_a = rng.draw_tx_id(); + let parent_b = rng.draw_tx_id(); + let child_a = rng.draw_tx_id(); + let child_b = rng.draw_tx_id(); + let child_c = rng.draw_tx_id(); + + let mut uut = TestGraph::default(); + uut.insert_with_no_parent(ancestor_a); + uut.insert_with_no_parent(ancestor_b); + uut.insert_with_parent(parent_a, ancestor_a); + uut.insert_with_parent(parent_b, ancestor_b); + uut.insert_with_parents(child_a, [ancestor_a, parent_a].into()); + uut.insert_with_parents(child_b, [parent_a, parent_b].into()); + uut.insert_with_parent(child_c, parent_b); + + uut.purge_subgraphs(vec![parent_a]); + + let mut reference = TestGraph::default(); + reference.insert_with_no_parent(ancestor_a); + reference.insert_with_no_parent(ancestor_b); + reference.insert_with_parent(parent_b, ancestor_b); + reference.insert_with_parent(child_c, parent_b); + + assert_eq!(uut, reference); + } + + #[test] + #[should_panic] + fn duplicate_insert() { + let mut rng = Random::with_random_seed(); + let mut uut = TestGraph::default(); + + let id = rng.draw_tx_id(); + uut.insert_with_no_parent(id); + uut.insert_with_no_parent(id); + } + + #[test] + #[should_panic] + fn missing_parents_in_insert() { + let mut rng = Random::with_random_seed(); + let mut uut = TestGraph::default(); + + uut.insert_with_parents(rng.draw_tx_id(), [rng.draw_tx_id()].into()); + } + + #[test] + #[should_panic] + fn requeueing_an_already_queued_tx() { + let mut rng = Random::with_random_seed(); + let mut uut = TestGraph::default(); + + let id = rng.draw_tx_id(); + uut.insert_with_no_parent(id); + uut.requeue_transactions([id].into()); + } +} diff --git a/crates/block-producer/src/server/mod.rs b/crates/block-producer/src/server/mod.rs index 56114be7e..a5a8c320a 100644 --- a/crates/block-producer/src/server/mod.rs +++ b/crates/block-producer/src/server/mod.rs @@ -4,9 +4,12 @@ use std::{ sync::Arc, }; -use miden_node_proto::generated::{ - block_producer::api_server, requests::SubmitProvenTransactionRequest, - responses::SubmitProvenTransactionResponse, store::api_client as store_client, +use miden_node_proto::{ + domain::nullifiers, + generated::{ + block_producer::api_server, requests::SubmitProvenTransactionRequest, + responses::SubmitProvenTransactionResponse, store::api_client as store_client, + }, }; use miden_node_utils::{ errors::ApiError, @@ -25,7 +28,8 @@ use crate::{ batch_builder::{DefaultBatchBuilder, DefaultBatchBuilderOptions}, block_builder::DefaultBlockBuilder, config::BlockProducerConfig, - errors::AddTransactionErrorRework, + domain::transaction::AuthenticatedTransaction, + errors::{AddTransactionError, VerifyTxError}, mempool::Mempool, state_view::DefaultStateView, store::{DefaultStore, Store}, @@ -145,19 +149,8 @@ impl api_server::Api for Server { self.submit_proven_transaction(request.into_inner()) .await .map(tonic::Response::new) - .map_err(|err| match err { - AddTransactionErrorRework::InvalidAccountState { .. } - | AddTransactionErrorRework::AuthenticatedNoteNotFound(_) - | AddTransactionErrorRework::UnauthenticatedNoteNotFound(_) - | AddTransactionErrorRework::NotesAlreadyConsumed(_) - | AddTransactionErrorRework::DeserializationError(_) - | AddTransactionErrorRework::ProofVerificationFailed(_) => { - Status::invalid_argument(err.to_string()) - }, - // Internal errors. - AddTransactionErrorRework::StaleInputs { .. } - | AddTransactionErrorRework::TxInputsError(_) => Status::internal("Internal error"), - }) + // This Status::from mapping takes care of hiding internal errors. + .map_err(Into::into) } } @@ -171,11 +164,11 @@ impl Server { async fn submit_proven_transaction( &self, request: SubmitProvenTransactionRequest, - ) -> Result { + ) -> Result { debug!(target: COMPONENT, ?request); let tx = ProvenTransaction::read_from_bytes(&request.transaction) - .map_err(|err| AddTransactionErrorRework::DeserializationError(err.to_string()))?; + .map_err(|err| AddTransactionError::DeserializationError(err.to_string()))?; let tx_id = tx.id(); @@ -192,40 +185,17 @@ impl Server { ); debug!(target: COMPONENT, proof = ?tx.proof()); - let mut inputs = self.store.get_tx_inputs(&tx).await?; - - let mut authenticated_notes = BTreeSet::new(); - let mut unauthenticated_notes = BTreeMap::new(); - - for note in tx.input_notes() { - match note.header() { - Some(header) => { - unauthenticated_notes.insert(header.id(), note.nullifier()); - }, - None => { - authenticated_notes.insert(note.nullifier()); - }, - } - } - - // Authenticated note nullifiers must be present in the store and must be unconsumed. - for nullifier in &authenticated_notes { - let nullifier_state = inputs - .nullifiers - .remove(nullifier) - .ok_or(AddTransactionErrorRework::AuthenticatedNoteNotFound(*nullifier))?; - - if nullifier_state.is_some() { - return Err(AddTransactionErrorRework::NotesAlreadyConsumed([*nullifier].into())); - } - } + let inputs = self.store.get_tx_inputs(&tx).await.map_err(VerifyTxError::from)?; + + // SAFETY: we assume that the rpc component has verified the transaction proof already. + let tx = AuthenticatedTransaction::new(tx, inputs)?; self.mempool .lock() .await .lock() .await - .add_transaction(tx, inputs) + .add_transaction(tx) .map(|block_height| SubmitProvenTransactionResponse { block_height }) } } diff --git a/crates/block-producer/src/test_utils/mod.rs b/crates/block-producer/src/test_utils/mod.rs index 07a7a4b52..3a8d7de41 100644 --- a/crates/block-producer/src/test_utils/mod.rs +++ b/crates/block-producer/src/test_utils/mod.rs @@ -1,7 +1,11 @@ use std::sync::Arc; -use miden_objects::{accounts::AccountId, Digest}; -use tokio::sync::RwLock; +use miden_objects::{ + accounts::AccountId, + crypto::rand::{FeltRng, RpoRandomCoin}, + transaction::TransactionId, + Digest, +}; mod proven_tx; @@ -9,6 +13,7 @@ pub use proven_tx::{mock_proven_tx, MockProvenTxBuilder}; mod store; +use rand::Rng; pub use store::{MockStoreFailure, MockStoreSuccess, MockStoreSuccessBuilder}; mod account; @@ -20,3 +25,31 @@ pub mod block; pub mod batch; pub mod note; + +/// Generates random values for tests. +/// +/// It prints its seed on construction which allows us to reproduce +/// test failures. +pub struct Random(RpoRandomCoin); + +impl Random { + /// Creates a [Random] with a random seed. This seed is logged + /// so that it is known for test failures. + pub fn with_random_seed() -> Self { + let seed: [u32; 4] = rand::random(); + + println!("Random::with_random_seed: {seed:?}"); + + let seed = Digest::from(seed).into(); + + Self(RpoRandomCoin::new(seed)) + } + + pub fn draw_tx_id(&mut self) -> TransactionId { + self.0.draw_word().into() + } + + pub fn draw_digest(&mut self) -> Digest { + self.0.draw_word().into() + } +} diff --git a/crates/block-producer/src/test_utils/store.rs b/crates/block-producer/src/test_utils/store.rs index bf1e89328..f1f65bd21 100644 --- a/crates/block-producer/src/test_utils/store.rs +++ b/crates/block-producer/src/test_utils/store.rs @@ -12,6 +12,7 @@ use miden_objects::{ notes::{NoteId, NoteInclusionProof, Nullifier}, BlockHeader, ACCOUNT_TREE_DEPTH, EMPTY_WORD, ZERO, }; +use tokio::sync::RwLock; use super::*; use crate::{ From 90311b353d8fe0dc64f81004410ba9212bd1af40 Mon Sep 17 00:00:00 2001 From: Mirko <48352201+Mirko-von-Leipzig@users.noreply.github.com> Date: Thu, 24 Oct 2024 07:30:37 +0200 Subject: [PATCH 03/50] feat(block-producer): inject failures and randomise work time (#522) --- crates/block-producer/Cargo.toml | 2 +- .../block-producer/src/batch_builder/batch.rs | 3 +- .../block-producer/src/batch_builder/mod.rs | 50 ++++++++++++++----- .../block-producer/src/block_builder/mod.rs | 40 +++++++++++++-- crates/block-producer/src/errors.rs | 8 ++- 5 files changed, 82 insertions(+), 21 deletions(-) diff --git a/crates/block-producer/Cargo.toml b/crates/block-producer/Cargo.toml index 5355d30fb..ec4eec1b1 100644 --- a/crates/block-producer/Cargo.toml +++ b/crates/block-producer/Cargo.toml @@ -25,6 +25,7 @@ miden-objects = { workspace = true } miden-processor = { workspace = true } miden-stdlib = { workspace = true } miden-tx = { workspace = true } +rand = { version = "0.8.5" } serde = { version = "1.0", features = ["derive"] } thiserror = { workspace = true } tokio = { workspace = true, features = ["rt-multi-thread", "net", "macros", "sync", "time"] } @@ -41,7 +42,6 @@ miden-lib = { workspace = true, features = ["testing"] } miden-node-test-macro = { path = "../test-macro" } miden-objects = { workspace = true, features = ["testing"] } miden-tx = { workspace = true, features = ["testing"] } -rand = { version = "0.8.5" } rand_chacha = { version = "0.3", default-features = false } tokio = { workspace = true, features = ["test-util"] } winterfell = { version = "0.9" } diff --git a/crates/block-producer/src/batch_builder/batch.rs b/crates/block-producer/src/batch_builder/batch.rs index e8a13fb36..6490a95a7 100644 --- a/crates/block-producer/src/batch_builder/batch.rs +++ b/crates/block-producer/src/batch_builder/batch.rs @@ -22,8 +22,7 @@ pub type BatchId = Blake3Digest<32>; // TRANSACTION BATCH // ================================================================================================ -/// A batch of transactions that share a common proof. For any given account, at most 1 transaction -/// in the batch must be addressing that account (issue: #186). +/// A batch of transactions that share a common proof. /// /// Note: Until recursive proofs are available in the Miden VM, we don't include the common proof. #[derive(Debug, Clone, PartialEq, Eq)] diff --git a/crates/block-producer/src/batch_builder/mod.rs b/crates/block-producer/src/batch_builder/mod.rs index 65f693006..1c8e11b05 100644 --- a/crates/block-producer/src/batch_builder/mod.rs +++ b/crates/block-producer/src/batch_builder/mod.rs @@ -2,6 +2,7 @@ use std::{ cmp::min, collections::{BTreeMap, BTreeSet}, num::NonZeroUsize, + ops::Range, sync::Arc, time::Duration, }; @@ -12,6 +13,7 @@ use miden_objects::{ transaction::{OutputNote, TransactionId}, Digest, }; +use rand::Rng; use tokio::{sync::Mutex, task::JoinSet, time}; use tonic::async_trait; use tracing::{debug, info, instrument, Span}; @@ -228,7 +230,12 @@ pub struct BatchProducer { pub workers: NonZeroUsize, pub mempool: Arc>, pub tx_per_batch: usize, - pub simulated_proof_time: Duration, + /// Used to simulate batch proving by sleeping for a random duration selected from this range. + pub simulated_proof_time: Range, + /// Simulated block failure rate as a percentage. + /// + /// Note: this _must_ be sign positive and less than 1.0. + pub failure_rate: f32, } type BatchResult = Result<(BatchJobId, TransactionBatch), (BatchJobId, BuildBatchError)>; @@ -237,13 +244,15 @@ type BatchResult = Result<(BatchJobId, TransactionBatch), (BatchJobId, BuildBatc /// instead of returning None. struct WorkerPool { in_progress: JoinSet, - simulated_proof_time: Duration, + simulated_proof_time: Range, + failure_rate: f32, } impl WorkerPool { - fn new(simulated_proof_time: Duration) -> Self { + fn new(simulated_proof_time: Range, failure_rate: f32) -> Self { Self { simulated_proof_time, + failure_rate, in_progress: JoinSet::new(), } } @@ -263,17 +272,30 @@ impl WorkerPool { fn spawn(&mut self, id: BatchJobId, transactions: Vec) { self.in_progress.spawn({ - let simulated_proof_time = self.simulated_proof_time; + // Select a random work duration from the given proof range. + let simulated_proof_time = + rand::thread_rng().gen_range(self.simulated_proof_time.clone()); + + // Randomly fail batches at the configured rate. + // + // Note: Rng::gen rolls between [0, 1.0) for f32, so this works as expected. + let failed = rand::thread_rng().gen::() < self.failure_rate; + async move { tracing::debug!("Begin proving batch."); let transactions = transactions.into_iter().map(AuthenticatedTransaction::into_raw).collect(); + tokio::time::sleep(simulated_proof_time).await; + if failed { + tracing::debug!("Batch proof failure injected."); + return Err((id, BuildBatchError::InjectedFailure(transactions))); + } + let batch = TransactionBatch::new(transactions, Default::default()) .map_err(|err| (id, err))?; - tokio::time::sleep(simulated_proof_time).await; tracing::debug!("Batch proof completed."); Ok((id, batch)) @@ -283,19 +305,21 @@ impl WorkerPool { } impl BatchProducer { - /// Starts the [BlockProducer], infinitely producing blocks at the configured interval. + /// Starts the [BatchProducer], creating and proving batches at the configured interval. /// - /// Block production is sequential and consists of - /// - /// 1. Pulling the next set of batches from the [Mempool] - /// 2. Compiling these batches into the next block - /// 3. Proving the block (this is not yet implemented) - /// 4. Committing the block to the store + /// A pool of batch-proving workers is spawned, which are fed new batch jobs periodically. + /// A batch is skipped if there are no available workers, or if there are no transactions + /// available to batch. pub async fn run(self) { + assert!( + self.failure_rate < 1.0 && self.failure_rate.is_sign_positive(), + "Failure rate must be a percentage" + ); + let mut interval = tokio::time::interval(self.batch_interval); interval.set_missed_tick_behavior(time::MissedTickBehavior::Delay); - let mut inflight = WorkerPool::new(self.simulated_proof_time); + let mut inflight = WorkerPool::new(self.simulated_proof_time, self.failure_rate); loop { tokio::select! { diff --git a/crates/block-producer/src/block_builder/mod.rs b/crates/block-producer/src/block_builder/mod.rs index 9d9deda00..bf36e4642 100644 --- a/crates/block-producer/src/block_builder/mod.rs +++ b/crates/block-producer/src/block_builder/mod.rs @@ -1,5 +1,6 @@ use std::{ collections::{BTreeMap, BTreeSet}, + ops::Range, sync::Arc, }; @@ -11,7 +12,8 @@ use miden_objects::{ notes::{NoteHeader, Nullifier}, transaction::InputNoteCommitment, }; -use tokio::sync::Mutex; +use rand::Rng; +use tokio::{sync::Mutex, time::Duration}; use tracing::{debug, info, instrument}; use crate::{ @@ -151,12 +153,32 @@ where struct BlockProducer { pub mempool: Arc>, - pub block_interval: tokio::time::Duration, + pub block_interval: Duration, pub block_builder: BB, + /// Used to simulate block proving by sleeping for a random duration selected from this range. + pub simulated_proof_time: Range, + + /// Simulated block failure rate as a percentage. + /// + /// Note: this _must_ be sign positive and less than 1.0. + pub failure_rate: f32, } impl BlockProducer { + /// Starts the [BlockProducer], infinitely producing blocks at the configured interval. + /// + /// Block production is sequential and consists of + /// + /// 1. Pulling the next set of batches from the [Mempool] + /// 2. Compiling these batches into the next block + /// 3. Proving the block (this is simulated using random sleeps) + /// 4. Committing the block to the store pub async fn run(self) { + assert!( + self.failure_rate < 1.0 && self.failure_rate.is_sign_positive(), + "Failure rate must be a percentage" + ); + let mut interval = tokio::time::interval(self.block_interval); interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); @@ -166,9 +188,19 @@ impl BlockProducer { let (block_number, batches) = self.mempool.lock().await.select_block(); let batches = batches.into_values().collect::>(); - let result = self.block_builder.build_block(&batches).await; - let mut mempool = self.mempool.lock().await; + let mut result = self.block_builder.build_block(&batches).await; + let proving_duration = rand::thread_rng().gen_range(self.simulated_proof_time.clone()); + + tokio::time::sleep(proving_duration).await; + // Randomly inject failures at the given rate. + // + // Note: Rng::gen rolls between [0, 1.0) for f32, so this works as expected. + if rand::thread_rng().gen::() < self.failure_rate { + result = Err(BuildBlockError::InjectedFailure); + } + + let mut mempool = self.mempool.lock().await; match result { Ok(_) => mempool.block_committed(block_number), Err(_) => mempool.block_failed(block_number), diff --git a/crates/block-producer/src/errors.rs b/crates/block-producer/src/errors.rs index 4791971a6..a546e5a9e 100644 --- a/crates/block-producer/src/errors.rs +++ b/crates/block-producer/src/errors.rs @@ -149,6 +149,9 @@ pub enum BuildBatchError { error: AccountDeltaError, txs: Vec, }, + + #[error("Nothing actually went wrong, failure was injected on purpose")] + InjectedFailure(Vec), } impl BuildBatchError { @@ -164,6 +167,7 @@ impl BuildBatchError { BuildBatchError::UnauthenticatedNotesNotFound(_, txs) => txs, BuildBatchError::NoteHashesMismatch { txs, .. } => txs, BuildBatchError::AccountUpdateError { txs, .. } => txs, + BuildBatchError::InjectedFailure(txs) => txs, } } } @@ -239,11 +243,13 @@ pub enum BuildBlockError { UnauthenticatedNotesNotFound(Vec), #[error("too many batches in block. Got: {0}, max: {MAX_BATCHES_PER_BLOCK}")] TooManyBatchesInBlock(usize), - #[error("Failed to merge transaction delta into account {account_id}: {error}")] + #[error("failed to merge transaction delta into account {account_id}: {error}")] AccountUpdateError { account_id: AccountId, error: AccountDeltaError, }, + #[error("nothing actually went wrong, failure was injected on purpose")] + InjectedFailure, } // Transaction inputs errors From d784cceb8741b86dd1c92fdfd47c11863b0af6e5 Mon Sep 17 00:00:00 2001 From: Mirko <48352201+Mirko-von-Leipzig@users.noreply.github.com> Date: Fri, 25 Oct 2024 10:58:45 +0200 Subject: [PATCH 04/50] refactor(block-producer): common graph type (#525) --- .../block-producer/src/domain/transaction.rs | 2 +- .../src/mempool/dependency_graph.rs | 830 ++++++++++++++++++ crates/block-producer/src/mempool/mod.rs | 36 +- .../src/mempool/transaction_graph.rs | 650 +++----------- 4 files changed, 957 insertions(+), 561 deletions(-) create mode 100644 crates/block-producer/src/mempool/dependency_graph.rs diff --git a/crates/block-producer/src/domain/transaction.rs b/crates/block-producer/src/domain/transaction.rs index 756af859a..e6d9cdb9e 100644 --- a/crates/block-producer/src/domain/transaction.rs +++ b/crates/block-producer/src/domain/transaction.rs @@ -15,7 +15,7 @@ use crate::{errors::VerifyTxError, mempool::BlockNumber, store::TransactionInput /// previously unauthenticated input notes. /// /// Note that this is of course only valid for the chain height of the authentication. -#[derive(Clone, PartialEq)] +#[derive(Clone, Debug, PartialEq)] pub struct AuthenticatedTransaction { inner: ProvenTransaction, /// The account state provided by the store [inputs](TransactionInputs). diff --git a/crates/block-producer/src/mempool/dependency_graph.rs b/crates/block-producer/src/mempool/dependency_graph.rs new file mode 100644 index 000000000..58ef76fd1 --- /dev/null +++ b/crates/block-producer/src/mempool/dependency_graph.rs @@ -0,0 +1,830 @@ +use std::collections::{BTreeMap, BTreeSet}; + +use miden_tx::utils::collections::KvMap; + +// DEPENDENCY GRAPH +// ================================================================================================ + +/// A dependency graph structure where nodes are inserted, and then made available for processing +/// once all parent nodes have been processed. +/// +/// Forms the basis of our transaction and batch dependency graphs. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DependencyGraph { + /// Each node's data. + vertices: BTreeMap, + + /// Each node's parents. This is redundant with `children`, + /// but we require both for efficient lookups. + parents: BTreeMap>, + + /// Each node's children. This is redundant with `parents`, + /// but we require both for efficient lookups. + children: BTreeMap>, + + /// Nodes that are available to process next. + /// + /// Effectively this is the set of nodes which are + /// unprocessed and whose parent's _are_ all processed. + roots: BTreeSet, + + /// Set of nodes that are already processed. + processed: BTreeSet, +} + +#[derive(thiserror::Error, Debug, Clone, PartialEq, Eq)] +pub enum GraphError { + #[error("Node {0} already exists")] + DuplicateKey(K), + + #[error("Parents not found: {0:?}")] + MissingParents(BTreeSet), + + #[error("Nodes not found: {0:?}")] + UnknownNodes(BTreeSet), + + #[error("Nodes were not yet processed: {0:?}")] + UnprocessedNodes(BTreeSet), + + #[error("Nodes would be left dangling: {0:?}")] + DanglingNodes(BTreeSet), + + #[error("Node {0} is not ready to be processed")] + NotARootNode(K), +} + +/// This cannot be derived without enforcing `Default` bounds on K and V. +impl Default for DependencyGraph { + fn default() -> Self { + Self { + vertices: Default::default(), + parents: Default::default(), + children: Default::default(), + roots: Default::default(), + processed: Default::default(), + } + } +} + +impl DependencyGraph { + /// Inserts a new node into the graph. + /// + /// # Errors + /// + /// Errors if the node already exists, or if any of the parents are not part of the graph. + /// + /// This method is atomic. + pub fn insert(&mut self, key: K, value: V, parents: BTreeSet) -> Result<(), GraphError> { + if self.vertices.contains_key(&key) { + return Err(GraphError::DuplicateKey(key)); + } + + let missing_parents = parents + .iter() + .filter(|parent| !self.vertices.contains_key(parent)) + .cloned() + .collect::>(); + if !missing_parents.is_empty() { + return Err(GraphError::MissingParents(missing_parents)); + } + + // Inform parents of their new child. + for parent in &parents { + self.children.entry(parent.clone()).or_default().insert(key.clone()); + } + self.vertices.insert(key.clone(), value); + self.parents.insert(key.clone(), parents); + self.children.insert(key.clone(), Default::default()); + + self.try_make_root(key); + + Ok(()) + } + + /// Reverts the nodes __and their descendents__, requeueing them for processing. + /// + /// # Errors + /// + /// Returns an error if any of the given nodes: + /// + /// - are not part of the graph, or + /// - were not previously processed + /// + /// This method is atomic. + pub fn revert_subgraphs(&mut self, keys: BTreeSet) -> Result<(), GraphError> { + let missing_nodes = keys + .iter() + .filter(|key| !self.vertices.contains_key(key)) + .cloned() + .collect::>(); + if !missing_nodes.is_empty() { + return Err(GraphError::UnknownNodes(missing_nodes)); + } + let unprocessed = keys.difference(&self.processed).cloned().collect::>(); + if !unprocessed.is_empty() { + return Err(GraphError::UnprocessedNodes(unprocessed)); + } + + let mut reverted = BTreeSet::new(); + let mut to_revert = keys.clone(); + + while let Some(key) = to_revert.pop_first() { + self.processed.remove(&key); + + let unprocessed_children = self + .children + .get(&key) + .map(|children| children.difference(&reverted)) + .into_iter() + .flatten() + .cloned(); + + to_revert.extend(unprocessed_children); + + reverted.insert(key); + } + + // Only the original keys and the current roots need to be considered as roots. + // + // The children of the input keys are disqualified by definition (they're descendents), + // and current roots must be re-evaluated since their parents may have been requeued. + std::mem::take(&mut self.roots) + .into_iter() + .chain(keys) + .for_each(|key| self.try_make_root(key)); + + Ok(()) + } + + /// Removes a set of previously processed nodes from the graph. + /// + /// This is used to bound the size of the graph by removing nodes once they are no longer + /// required. + /// + /// # Errors + /// + /// Errors if + /// - any node is unknown + /// - any node is __not__ processed + /// - any parent node would be left unpruned + /// + /// The last point implies that all parents of the given nodes must either be part of the set, + /// or already been pruned. + /// + /// This method is atomic. + pub fn prune_processed(&mut self, keys: BTreeSet) -> Result, GraphError> { + let missing_nodes = keys + .iter() + .filter(|key| !self.vertices.contains_key(key)) + .cloned() + .collect::>(); + if !missing_nodes.is_empty() { + return Err(GraphError::UnknownNodes(missing_nodes)); + } + + let unprocessed = keys.difference(&self.processed).cloned().collect::>(); + if !unprocessed.is_empty() { + return Err(GraphError::UnprocessedNodes(unprocessed)); + } + + // No parent may be left dangling i.e. all parents must be part of this prune set. + let dangling = keys + .iter() + .flat_map(|key| self.parents.get(key)) + .flatten() + .filter(|parent| !keys.contains(parent)) + .cloned() + .collect::>(); + if !dangling.is_empty() { + return Err(GraphError::DanglingNodes(dangling)); + } + + let mut pruned = Vec::with_capacity(keys.len()); + + for key in keys { + let value = self.vertices.remove(&key).expect("Checked in precondition"); + pruned.push(value); + self.processed.remove(&key); + self.parents.remove(&key); + + let children = self.children.remove(&key).unwrap_or_default(); + + // Remove edges from children to this node. + for child in children { + if let Some(child) = self.parents.get_mut(&child) { + child.remove(&key); + } + } + } + + Ok(pruned) + } + + /// Removes the set of nodes __and all descendents__ from the graph, returning all removed + /// values. + /// + /// # Returns + /// + /// Returns a mapping of each removed key to its value. + /// + /// # Errors + /// + /// Returns an error if any of the given nodes does not exist. + /// + /// This method is atomic. + pub fn purge_subgraphs(&mut self, keys: BTreeSet) -> Result, GraphError> { + let missing_nodes = keys + .iter() + .filter(|key| !self.vertices.contains_key(key)) + .cloned() + .collect::>(); + if !missing_nodes.is_empty() { + return Err(GraphError::UnknownNodes(missing_nodes)); + } + + let mut visited = keys.clone(); + let mut to_remove = keys; + let mut removed = BTreeMap::new(); + + while let Some(key) = to_remove.pop_first() { + let value = self + .vertices + .remove(&key) + .expect("Node was checked in precondition and must therefore exist"); + removed.insert(key.clone(), value); + + self.processed.remove(&key); + self.roots.remove(&key); + + // Children must also be purged. Take care not to visit them twice which is + // possible since children can have multiple purged parents. + let unvisited_children = self.children.remove(&key).unwrap_or_default(); + let unvisited_children = unvisited_children.difference(&visited).cloned(); + to_remove.extend(unvisited_children); + + // Inform parents that this child no longer exists. + let parents = self.parents.remove(&key).unwrap_or_default(); + for parent in parents { + if let Some(parent) = self.children.get_mut(&parent) { + parent.remove(&key); + } + } + } + + Ok(removed) + } + + /// Adds the node to the `roots` list _IFF_ all of its parents are processed. + /// + /// # SAFETY + /// + /// This method assumes the node exists. Caller is responsible for ensuring this is true. + fn try_make_root(&mut self, key: K) { + debug_assert!(self.vertices.contains_key(&key), "Potential root must exist in the graph"); + + let all_parents_processed = self + .parents + .get(&key) + .into_iter() + .flatten() + .all(|parent| self.processed.contains(parent)); + + if all_parents_processed { + self.roots.insert(key); + } + } + + /// Returns the set of nodes that are ready for processing. + /// + /// Nodes can be selected from here and marked as processed using [`Self::process_root`]. + pub fn roots(&self) -> &BTreeSet { + &self.roots + } + + /// Marks a root node as processed, removing it from the roots list. + /// + /// The node's children are [evaluated](Self::try_make_root) as possible roots. + /// + /// # Error + /// + /// Errors if the node is not in the roots list. + /// + /// This method is atomic. + pub fn process_root(&mut self, key: K) -> Result<(), GraphError> { + if !self.roots.remove(&key) { + return Err(GraphError::NotARootNode(key)); + } + + self.processed.insert(key.clone()); + + self.children + .get(&key) + .cloned() + .unwrap_or_default() + .into_iter() + .for_each(|child| self.try_make_root(child)); + + Ok(()) + } + + /// Returns the value of a node. + pub fn get(&self, key: &K) -> Option<&V> { + self.vertices.get(key) + } + + /// Returns the parents of the node, or [None] if the node does not exist. + pub fn parents(&self, key: &K) -> Option<&BTreeSet> { + self.parents.get(key) + } +} + +// TESTS +// ================================================================================================ + +#[cfg(test)] +mod tests { + use super::*; + + // TEST UTILITIES + // ================================================================================================ + + /// Simplified graph variant where a node's key always equals its value. This is done to make + /// generating test values simpler. + type TestGraph = DependencyGraph; + + impl TestGraph { + /// Alias for inserting a node with no parents. + fn insert_root(&mut self, node: u32) -> Result<(), GraphError> { + self.insert_with_parents(node, Default::default()) + } + + /// Alias for inserting a node with a single parent. + fn insert_with_parent(&mut self, node: u32, parent: u32) -> Result<(), GraphError> { + self.insert_with_parents(node, [parent].into()) + } + + /// Alias for inserting a node with multiple parents. + fn insert_with_parents( + &mut self, + node: u32, + parents: BTreeSet, + ) -> Result<(), GraphError> { + self.insert(node, node, parents) + } + + /// Calls process_root until all nodes have been processed. + fn process_all(&mut self) { + while let Some(root) = self.roots().first().cloned() { + /// SAFETY: this is definitely a root since we just took it from there :) + self.process_root(root); + } + } + } + + // INSERT TESTS + // ================================================================================================ + + #[test] + fn inserted_nodes_are_considered_for_root() { + //! Ensure that an inserted node is added to the root list if all parents are already + //! processed. + let parent_a = 1; + let parent_b = 2; + let child_a = 3; + let child_b = 4; + let child_c = 5; + + let mut uut = TestGraph::default(); + uut.insert_root(parent_a).unwrap(); + uut.insert_root(parent_b).unwrap(); + + // Only process one parent so that some children remain unrootable. + uut.process_root(parent_a).unwrap(); + + uut.insert_with_parent(child_a, parent_a).unwrap(); + uut.insert_with_parent(child_b, parent_b).unwrap(); + uut.insert_with_parents(child_c, [parent_a, parent_b].into()).unwrap(); + + // Only child_a should be added (in addition to the parents), since the other children + // are dependent on parent_b which is incomplete. + let expected_roots = [parent_b, child_a].into(); + + assert_eq!(uut.roots, expected_roots); + } + + #[test] + fn insert_with_known_parents_succeeds() { + let parent_a = 10; + let parent_b = 20; + let grandfather = 123; + let uncle = 222; + + let mut uut = TestGraph::default(); + uut.insert_root(grandfather).unwrap(); + uut.insert_root(parent_a).unwrap(); + uut.insert_with_parent(parent_b, grandfather).unwrap(); + uut.insert_with_parent(uncle, grandfather).unwrap(); + uut.insert_with_parents(1, [parent_a, parent_b].into()).unwrap(); + } + + #[test] + fn insert_duplicate_is_rejected() { + //! Ensure that inserting a duplicate node + //! - results in an error, and + //! - does not mutate the state (atomicity) + const KEY: u32 = 123; + let mut uut = TestGraph::default(); + uut.insert_root(KEY).unwrap(); + + let err = uut.insert_root(KEY).unwrap_err(); + let expected = GraphError::DuplicateKey(KEY); + assert_eq!(err, expected); + + let mut atomic_reference = TestGraph::default(); + atomic_reference.insert_root(KEY); + assert_eq!(uut, atomic_reference); + } + + #[test] + fn insert_with_all_parents_missing_is_rejected() { + //! Ensure that inserting a node with unknown parents + //! - results in an error, and + //! - does not mutate the state (atomicity) + const MISSING: [u32; 4] = [1, 2, 3, 4]; + let mut uut = TestGraph::default(); + + let err = uut.insert_with_parents(0xABC, MISSING.into()).unwrap_err(); + let expected = GraphError::MissingParents(MISSING.into()); + assert_eq!(err, expected); + + let atomic_reference = TestGraph::default(); + assert_eq!(uut, atomic_reference); + } + + #[test] + fn insert_with_some_parents_missing_is_rejected() { + //! Ensure that inserting a node with unknown parents + //! - results in an error, and + //! - does not mutate the state (atomicity) + const MISSING: u32 = 123; + let mut uut = TestGraph::default(); + + uut.insert_root(1).unwrap(); + uut.insert_root(2).unwrap(); + uut.insert_root(3).unwrap(); + + let atomic_reference = uut.clone(); + + let err = uut.insert_with_parents(0xABC, [1, 2, 3, MISSING].into()).unwrap_err(); + let expected = GraphError::MissingParents([MISSING].into()); + assert_eq!(err, expected); + assert_eq!(uut, atomic_reference); + } + + // REVERT TESTS + // ================================================================================================ + + #[test] + fn reverting_unprocessed_nodes_is_rejected() { + let mut uut = TestGraph::default(); + uut.insert_root(1).unwrap(); + uut.insert_root(2).unwrap(); + uut.insert_root(3).unwrap(); + uut.process_root(1).unwrap(); + + let err = uut.revert_subgraphs([1, 2, 3].into()).unwrap_err(); + let expected = GraphError::UnprocessedNodes([2, 3].into()); + + assert_eq!(err, expected); + } + + #[test] + fn reverting_unknown_nodes_is_rejected() { + let err = TestGraph::default().revert_subgraphs([1].into()).unwrap_err(); + let expected = GraphError::UnknownNodes([1].into()); + assert_eq!(err, expected); + } + + #[test] + fn reverting_resets_the_entire_subgraph() { + //! Reverting should reset the state to before any of the nodes where processed. + let grandparent = 1; + let parent_a = 2; + let parent_b = 3; + let child_a = 4; + let child_b = 5; + let child_c = 6; + + let disjoint = 7; + + let mut uut = TestGraph::default(); + uut.insert_root(grandparent).unwrap(); + uut.insert_root(disjoint).unwrap(); + uut.insert_with_parent(parent_a, grandparent).unwrap(); + uut.insert_with_parent(parent_b, grandparent).unwrap(); + uut.insert_with_parent(child_a, parent_a).unwrap(); + uut.insert_with_parent(child_b, parent_b).unwrap(); + uut.insert_with_parents(child_c, [parent_a, parent_b].into()).unwrap(); + + uut.process_root(disjoint).unwrap(); + + let reference = uut.clone(); + + uut.process_all(); + uut.revert_subgraphs([grandparent].into()).unwrap(); + + assert_eq!(uut, reference); + } + + #[test] + fn reverting_reevaluates_roots() { + //! Node reverting from processed to unprocessed should cause the root nodes to be + //! re-evaluated. Only nodes with all parents processed should remain in the set. + let disjoint_parent = 1; + let disjoint_child = 2; + + let parent_a = 3; + let parent_b = 4; + let child_a = 5; + let child_b = 6; + + let partially_disjoin_child = 7; + + let mut uut = TestGraph::default(); + // This pair of nodes should not be impacted by the reverted subgraph. + uut.insert_root(disjoint_parent).unwrap(); + uut.insert_with_parent(disjoint_child, disjoint_parent).unwrap(); + + uut.insert_root(parent_a).unwrap(); + uut.insert_root(parent_b).unwrap(); + uut.insert_with_parent(child_a, parent_a); + uut.insert_with_parent(child_b, parent_b); + uut.insert_with_parents(partially_disjoin_child, [disjoint_parent, parent_a].into()); + + // Since we are reverting the other parents, we expect the roots to match the current state. + uut.process_root(disjoint_parent).unwrap(); + let reference = uut.roots().clone(); + + uut.process_root(parent_a).unwrap(); + uut.process_root(parent_b).unwrap(); + uut.revert_subgraphs([parent_a, parent_b].into()).unwrap(); + + assert_eq!(uut.roots(), &reference); + } + + // PRUNING TESTS + // ================================================================================================ + + #[test] + fn pruned_nodes_are_nonextant() { + //! Checks that processed and then pruned nodes behave as if they never existed in the + //! graph. We test this by comparing it to a reference graph created without these ancestor + //! nodes. + let ancestor_a = 1; + let ancestor_b = 2; + + let child_a = 3; + let child_b = 4; + let child_both = 5; + + let mut uut = TestGraph::default(); + uut.insert_root(ancestor_a).unwrap(); + uut.insert_root(ancestor_b).unwrap(); + uut.insert_with_parent(child_a, ancestor_a).unwrap(); + uut.insert_with_parent(child_b, ancestor_b).unwrap(); + uut.insert_with_parents(child_both, [ancestor_a, ancestor_b].into()).unwrap(); + + uut.process_root(ancestor_a).unwrap(); + uut.process_root(ancestor_b).unwrap(); + uut.prune_processed([ancestor_a, ancestor_b].into()).unwrap(); + + let mut reference = TestGraph::default(); + reference.insert_root(child_a).unwrap(); + reference.insert_root(child_b).unwrap(); + reference.insert_root(child_both).unwrap(); + + assert_eq!(uut, reference); + } + + #[test] + fn pruning_unknown_nodes_is_rejected() { + let err = TestGraph::default().prune_processed([1].into()).unwrap_err(); + let expected = GraphError::UnknownNodes([1].into()); + assert_eq!(err, expected); + } + + #[test] + fn pruning_unprocessed_nodes_is_rejected() { + let mut uut = TestGraph::default(); + uut.insert_root(1).unwrap(); + + let err = uut.prune_processed([1].into()).unwrap_err(); + let expected = GraphError::UnprocessedNodes([1].into()); + assert_eq!(err, expected); + } + + #[test] + fn pruning_cannot_leave_parents_dangling() { + //! Pruning processed nodes must always prune all parent nodes as well. No parent node may + //! be left behind. + let dangling = 1; + let pruned = 2; + let mut uut = TestGraph::default(); + uut.insert_root(dangling).unwrap(); + uut.insert_with_parent(pruned, dangling).unwrap(); + uut.process_all(); + + let err = uut.prune_processed([pruned].into()).unwrap_err(); + let expected = GraphError::DanglingNodes([dangling].into()); + assert_eq!(err, expected); + } + + // PURGING TESTS + // ================================================================================================ + + #[test] + fn purging_subgraph_handles_internal_nodes() { + //! Purging a subgraph should correctly handle nodes already deleted within that subgraph. + //! + //! This is a concern for errors as we are deleting parts of the subgraph while we are + //! iterating through the nodes to purge. This means its likely a node will already have + //! been deleted before processing it as an input. + //! + //! We can force this to occur by re-ordering the inputs relative to the actual dependency + //! order. This means this test is a bit weaker because it relies on implementation details. + + let ancestor_a = 1; + let ancestor_b = 2; + let parent_a = 3; + let parent_b = 4; + let child_a = 5; + // This should be purged prior to parent_a. Relies on the fact that we are iterating over a + // btree which is ordered by value. + let child_b = 0; + let child_c = 6; + + let mut uut = TestGraph::default(); + uut.insert_root(ancestor_a).unwrap(); + uut.insert_root(ancestor_b).unwrap(); + uut.insert_with_parent(parent_a, ancestor_a).unwrap(); + uut.insert_with_parent(parent_b, ancestor_b).unwrap(); + uut.insert_with_parents(child_a, [ancestor_a, parent_a].into()).unwrap(); + uut.insert_with_parents(child_b, [parent_a, parent_b].into()).unwrap(); + uut.insert_with_parent(child_c, parent_b).unwrap(); + + uut.purge_subgraphs([child_b, parent_a].into()); + + let mut reference = TestGraph::default(); + reference.insert_root(ancestor_a).unwrap(); + reference.insert_root(ancestor_b).unwrap(); + reference.insert_with_parent(parent_b, ancestor_b).unwrap(); + reference.insert_with_parent(child_c, parent_b).unwrap(); + + assert_eq!(uut, reference); + } + + #[test] + fn purging_removes_all_descendents() { + let ancestor_a = 1; + let ancestor_b = 2; + let parent_a = 3; + let parent_b = 4; + let child_a = 5; + let child_b = 6; + let child_c = 7; + + let mut uut = TestGraph::default(); + uut.insert_root(ancestor_a).unwrap(); + uut.insert_root(ancestor_b).unwrap(); + uut.insert_with_parent(parent_a, ancestor_a).unwrap(); + uut.insert_with_parent(parent_b, ancestor_b).unwrap(); + uut.insert_with_parents(child_a, [ancestor_a, parent_a].into()).unwrap(); + uut.insert_with_parents(child_b, [parent_a, parent_b].into()).unwrap(); + uut.insert_with_parent(child_c, parent_b).unwrap(); + + uut.purge_subgraphs([parent_a].into()).unwrap(); + + let mut reference = TestGraph::default(); + reference.insert_root(ancestor_a).unwrap(); + reference.insert_root(ancestor_b).unwrap(); + reference.insert_with_parent(parent_b, ancestor_b).unwrap(); + reference.insert_with_parent(child_c, parent_b).unwrap(); + + assert_eq!(uut, reference); + } + + // PROCESSING TESTS + // ================================================================================================ + + #[test] + fn process_root_evaluates_children_as_roots() { + let parent_a = 1; + let parent_b = 2; + let child_a = 3; + let child_b = 4; + let child_c = 5; + + let mut uut = TestGraph::default(); + uut.insert_root(parent_a).unwrap(); + uut.insert_root(parent_b).unwrap(); + uut.insert_with_parent(child_a, parent_a).unwrap(); + uut.insert_with_parent(child_b, parent_b).unwrap(); + uut.insert_with_parents(child_c, [parent_a, parent_b].into()).unwrap(); + + // This should promote only child_a to root, in addition to the remaining parent_b root. + uut.process_root(parent_a).unwrap(); + assert_eq!(uut.roots(), &[parent_b, child_a].into()); + } + + #[test] + fn process_root_rejects_non_root_node() { + let mut uut = TestGraph::default(); + uut.insert_root(1).unwrap(); + uut.insert_with_parent(2, 1).unwrap(); + + let err = uut.process_root(2).unwrap_err(); + let expected = GraphError::NotARootNode(2); + assert_eq!(err, expected); + } + + #[test] + fn process_root_cannot_reprocess_same_node() { + let mut uut = TestGraph::default(); + uut.insert_root(1).unwrap(); + uut.process_root(1).unwrap(); + + let err = uut.process_root(1).unwrap_err(); + let expected = GraphError::NotARootNode(1); + assert_eq!(err, expected); + } + + #[test] + fn processing_a_queue_graph() { + //! Creates a queue graph and ensures that nodes processed in FIFO order. + let nodes = (0..10).collect::>(); + + let mut uut = TestGraph::default(); + uut.insert_root(nodes[0]); + for pairs in nodes.windows(2) { + let (parent, id) = (pairs[0], pairs[1]); + uut.insert_with_parent(id, parent); + } + + let mut ordered_roots = Vec::::new(); + for node in &nodes { + let current_roots = uut.roots().clone(); + ordered_roots.extend(¤t_roots); + + for root in current_roots { + uut.process_root(root).unwrap(); + } + } + + assert_eq!(ordered_roots, nodes); + } + + #[test] + fn processing_and_root_tracking() { + //! Creates a somewhat arbitrarily connected graph and ensures that roots are tracked as + //! expected as the they are processed. + let ancestor_a = 1; + let ancestor_b = 2; + let parent_a = 3; + let parent_b = 4; + let child_a = 5; + let child_b = 6; + let child_c = 7; + + let mut uut = TestGraph::default(); + uut.insert_root(ancestor_a).unwrap(); + uut.insert_root(ancestor_b).unwrap(); + uut.insert_with_parent(parent_a, ancestor_a).unwrap(); + uut.insert_with_parent(parent_b, ancestor_b).unwrap(); + uut.insert_with_parents(child_a, [ancestor_a, parent_a].into()).unwrap(); + uut.insert_with_parents(child_b, [parent_a, parent_b].into()).unwrap(); + uut.insert_with_parent(child_c, parent_b).unwrap(); + + assert_eq!(uut.roots(), &[ancestor_a, ancestor_b].into()); + + uut.process_root(ancestor_a).unwrap(); + assert_eq!(uut.roots(), &[ancestor_b, parent_a].into()); + + uut.process_root(ancestor_b).unwrap(); + assert_eq!(uut.roots(), &[parent_a, parent_b].into()); + + uut.process_root(parent_a).unwrap(); + assert_eq!(uut.roots(), &[parent_b, child_a].into()); + + uut.process_root(parent_b).unwrap(); + assert_eq!(uut.roots(), &[child_a, child_b, child_c].into()); + + uut.process_root(child_a).unwrap(); + assert_eq!(uut.roots(), &[child_b, child_c].into()); + + uut.process_root(child_b).unwrap(); + assert_eq!(uut.roots(), &[child_c].into()); + + uut.process_root(child_c).unwrap(); + assert!(uut.roots().is_empty()); + } +} diff --git a/crates/block-producer/src/mempool/mod.rs b/crates/block-producer/src/mempool/mod.rs index 6df7f3479..3f64325c7 100644 --- a/crates/block-producer/src/mempool/mod.rs +++ b/crates/block-producer/src/mempool/mod.rs @@ -24,6 +24,7 @@ use crate::{ }; mod batch_graph; +mod dependency_graph; mod inflight_state; mod transaction_graph; @@ -86,7 +87,7 @@ pub struct Mempool { state: InflightState, /// Inflight transactions. - transactions: TransactionGraph, + transactions: TransactionGraph, /// Inflight batches. batches: BatchGraph, @@ -120,7 +121,7 @@ impl Mempool { // Add transaction to inflight state. let parents = self.state.add_transaction(&transaction)?; - self.transactions.insert(transaction.id(), transaction, parents); + self.transactions.insert(transaction, parents); Ok(self.chain_tip.0) } @@ -131,24 +132,11 @@ impl Mempool { /// /// Returns `None` if no transactions are available. pub fn select_batch(&mut self) -> Option<(BatchJobId, Vec)> { - let mut parents = BTreeSet::new(); - let mut batch = Vec::with_capacity(self.batch_transaction_limit); - let mut tx_ids = Vec::with_capacity(self.batch_transaction_limit); - - for _ in 0..self.batch_transaction_limit { - // Select transactions according to some strategy here. For now its just arbitrary. - let Some((tx, tx_parents)) = self.transactions.pop_for_processing() else { - break; - }; - batch.push(tx); - parents.extend(tx_parents); - } - - // Update the dependency graph to reflect parents at the batch level by removing all edges - // within this batch. - for tx in &batch { - parents.remove(&tx.id()); + let (batch, parents) = self.transactions.select_batch(self.batch_transaction_limit); + if batch.is_empty() { + return None; } + let tx_ids = batch.iter().map(AuthenticatedTransaction::id).collect(); let batch_id = self.next_batch_id; self.next_batch_id.increment(); @@ -210,7 +198,10 @@ impl Mempool { // Remove committed batches and transactions from graphs. let batches = self.block_in_progress.take().expect("No block in progress to commit"); let transactions = self.batches.remove_committed(batches); - let transactions = self.transactions.prune_processed(&transactions); + let transactions = self + .transactions + .commit_transactions(&transactions) + .expect("Transaction graph malformed"); // Inform inflight state about committed data. self.state.commit_block(&transactions); @@ -234,7 +225,10 @@ impl Mempool { let batches = purged.keys().collect::>(); let transactions = purged.into_values().flatten().collect(); - let transactions = self.transactions.purge_subgraphs(transactions); + let transactions = self + .transactions + .purge_subgraphs(transactions) + .expect("Transaction graph is malformed"); // Rollback state. self.state.revert_transactions(&transactions); diff --git a/crates/block-producer/src/mempool/transaction_graph.rs b/crates/block-producer/src/mempool/transaction_graph.rs index 1a2d9cd94..9e3f4d61b 100644 --- a/crates/block-producer/src/mempool/transaction_graph.rs +++ b/crates/block-producer/src/mempool/transaction_graph.rs @@ -5,592 +5,164 @@ use std::{ use miden_objects::transaction::{ProvenTransaction, TransactionId}; +use super::dependency_graph::{DependencyGraph, GraphError}; +use crate::domain::transaction::AuthenticatedTransaction; + // TRANSACTION GRAPH // ================================================================================================ /// Tracks the dependency graph and status of transactions. -#[derive(Clone, Debug, PartialEq)] -pub struct TransactionGraph { - /// All transactions currently inflight. - nodes: BTreeMap>, - - /// Transactions ready for being processed. - /// - /// aka transactions whose parents are already processed. - roots: BTreeSet, +/// +/// It handles insertion of transactions, locking them inqueue until they are ready to be processed. +/// A transaction is considered eligible for batch selection once all of its parents have also been +/// selected. Essentially this graph ensures that transaction dependency ordering is adhered to. +/// +/// Transactions from failed batches may be [re-queued](Self::requeue_transactions) for batch +/// selection. Successful batches will eventually form part of a committed block at which point the +/// transaction data may be safely [pruned](Self::prune_committed). +/// +/// Transactions may also be outright [purged](Self::purge_subgraphs) from the graph. This is useful +/// for transactions which may have become invalid due to external considerations e.g. expired +/// transactions. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct TransactionGraph { + inner: DependencyGraph, } -impl Default for TransactionGraph { - fn default() -> Self { - Self { - nodes: Default::default(), - roots: Default::default(), - } - } -} - -impl TransactionGraph { +impl TransactionGraph { /// Inserts a new transaction node, with edges to the given parent nodes. /// - /// # Panics - /// - /// Panics if: - /// - any of the given parents are not part of the graph, - /// - the transaction is already present - pub fn insert(&mut self, id: TransactionId, data: T, parents: BTreeSet) { - // Inform parents of their new child. - for parent in &parents { - self.nodes.get_mut(parent).expect("Parent must be in pool").children.insert(id); - } - - let node = Node::new(data, parents); - if self.nodes.insert(id, node).is_some() { - panic!("Transaction already exists in pool"); - } - - // This could be optimized by inlining this inside the parent loop. This would prevent the - // double iteration over parents, at the cost of some code duplication. - self.try_make_root(id); - } - - /// Returns the next transaction ready for processing, and its parent edges. + /// # Errors /// - /// Internally this transaction is now marked as processed and is no longer considered inqueue. - pub fn pop_for_processing(&mut self) -> Option<(T, BTreeSet)> { - let tx_id = self.roots.first()?; - - Some(self.process(*tx_id)) + /// Follows the error conditions of [DependencyGraph::insert]. + pub fn insert( + &mut self, + transaction: AuthenticatedTransaction, + parents: BTreeSet, + ) -> Result<(), GraphError> { + self.inner.insert(transaction.id(), transaction, parents) } - /// Marks the transaction as processed and returns its data and parents. + /// Selects a set of up-to count transactions for the next batch, as well as their parents. /// - /// Separated out from the actual strategy of choosing so that we have more - /// fine grained control available for tests. + /// Internally these transactions are considered processed and cannot be emitted in future + /// batches. /// - /// # Panics + /// Note: this may emit empty batches. /// - /// Panics if the transaction: - /// - does not exist - /// - is already processed - /// - is not ready for processing - fn process(&mut self, id: TransactionId) -> (T, BTreeSet) { - assert!(self.roots.remove(&id), "Process target must form part of roots"); - let node = self.nodes.get_mut(&id).expect("Root transaction must be in graph"); - node.mark_as_processed(); + /// See also: + /// - [Self::requeue_transactions] + /// - [Self::prune_committed] + pub fn select_batch( + &mut self, + count: usize, + ) -> (Vec, BTreeSet) { + // This strategy just selects arbitrary roots for now. This is valid but not very + // interesting or efficient. + let mut batch = Vec::with_capacity(count); + let mut parents = BTreeSet::new(); + + for _ in 0..count { + let Some(root) = self.inner.roots().first().cloned() else { + break; + }; - // Work around multiple mutable borrows of self. - let parents = node.parents.clone(); - let children = node.children.clone(); - let tx = node.data.clone(); + // SAFETY: we retieved a root node, and therefore this node must exist. + self.inner.process_root(root).unwrap(); + let tx = self.inner.get(&root).unwrap(); + let tx_parents = self.inner.parents(&root).unwrap(); - for child in children { - self.try_make_root(child); + batch.push(tx.clone()); + parents.extend(tx_parents); } - (tx, parents) + (batch, parents) } /// Marks the given transactions as being back in queue. /// - /// # Panics + /// # Errors /// - /// Panics if any of the transactions are - /// - not part of the graph - /// - are already in queue aka not processed - pub fn requeue_transactions(&mut self, transactions: BTreeSet) { - for tx in &transactions { - self.nodes.get_mut(tx).expect("Node must exist").mark_as_inqueue(); - } - - // All requeued transactions are potential roots, and current roots may have been - // invalidated. - let mut potential_roots = std::mem::take(&mut self.roots); - potential_roots.extend(transactions); - for tx in potential_roots { - self.try_make_root(tx); - } + /// Follows the error conditions of [DependencyGraph::requeue]. + pub fn requeue_transactions( + &mut self, + transactions: BTreeSet, + ) -> Result<(), GraphError> { + self.inner.revert_subgraphs(transactions) } - /// Prunes processed transactions from the graph. + /// Removes the provided transactions from the graph. /// - /// # Panics + /// # Errors /// - /// Panics if any of the given transactions are: - /// - not part of the graph - /// - are in queue aka not processed - pub fn prune_processed(&mut self, tx_ids: &[TransactionId]) -> Vec { - let mut transactions = Vec::with_capacity(tx_ids.len()); - for transaction in tx_ids { - let node = self.nodes.remove(transaction).expect("Node must be in graph"); - assert_eq!(node.status, Status::Processed); - - transactions.push(node.data); - - // Remove node from graph. No need to update parents as they should be removed in this - // call as well. - for child in node.children { - // Its possible for the child to part of this same set of batches and therefore - // already removed. - if let Some(child) = self.nodes.get_mut(&child) { - child.parents.remove(transaction); - } - } - } - - transactions + /// Follows the error conditions of [DependencyGraph::prune_processed]. + pub fn commit_transactions( + &mut self, + tx_ids: &[TransactionId], + ) -> Result, GraphError> { + // TODO: revisit this api. + let tx_ids = tx_ids.iter().cloned().collect(); + self.inner.prune_processed(tx_ids) } /// Removes the transactions and all their descendants from the graph. /// - /// Returns all transactions removed. - pub fn purge_subgraphs(&mut self, transactions: Vec) -> Vec { - let mut removed = Vec::new(); - - let mut to_process = transactions; - - while let Some(node_id) = to_process.pop() { - // Its possible for a node to already have been removed as part of this subgraph - // removal. - let Some(node) = self.nodes.remove(&node_id) else { - continue; - }; - - // All the children will also be removed so no need to check for new roots. - // - // No new roots are possible as a result of this subgraph removal. - self.roots.remove(&node_id); - - // Inform parent that this child no longer exists. - // - // The same is not required for children of this batch as we will - // be removing those as well. - for parent in &node.parents { - // Parent could already be removed as part of this subgraph removal. - if let Some(parent) = self.nodes.get_mut(parent) { - parent.children.remove(&node_id); - } - } - - to_process.extend(node.children); - removed.push(node.data); - } - - removed - } - - /// Adds the given transaction to the set of roots _IFF_ all of its parents are marked as - /// processed. + /// Returns the removed transactions. /// - /// # Panics + /// # Errors /// - /// Panics if the transaction or any of its parents do not exist. This would constitute an - /// internal bookkeeping failure. - fn try_make_root(&mut self, tx_id: TransactionId) { - let tx = self.nodes.get_mut(&tx_id).expect("Transaction must be in graph"); - - for parent in tx.parents.clone() { - let parent = self.nodes.get(&parent).expect("Parent must be in pool"); - - if !parent.is_processed() { - return; - } - } - self.roots.insert(tx_id); + /// Follows the error conditions of [DependencyGraph::purge_subgraphs]. + pub fn purge_subgraphs( + &mut self, + transactions: Vec, + ) -> Result, GraphError> { + // TODO: revisit this api. + let transactions = transactions.into_iter().collect(); + self.inner.purge_subgraphs(transactions).map(|kv| kv.into_values().collect()) } } -#[derive(Clone, Debug, PartialEq)] -struct Node { - status: Status, - data: T, - parents: BTreeSet, - children: BTreeSet, -} - -impl Node { - /// Creates a new inflight [Node] with no children. - fn new(data: T, parents: BTreeSet) -> Self { - Self { - status: Status::Queued, - data, - parents, - children: Default::default(), - } - } - - /// Marks the node as [Status::Processed]. - /// - /// # Panics - /// - /// Panics if the node is already processed. - fn mark_as_processed(&mut self) { - assert!(!self.is_processed()); - self.status = Status::Processed - } - - /// Marks the node as [Status::Inqueue]. - /// - /// # Panics - /// - /// Panics if the node is already inqueue. - fn mark_as_inqueue(&mut self) { - assert!(!self.is_inqueue()); - self.status = Status::Queued - } - - fn is_processed(&self) -> bool { - self.status == Status::Processed - } - - fn is_inqueue(&self) -> bool { - self.status == Status::Queued - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum Status { - Queued, - Processed, -} - -// TESTS -// ================================================================================================ - #[cfg(test)] mod tests { use super::*; - use crate::test_utils::Random; - - /// Simplified graph type which uses the transaction ID as the data value. - /// - /// Production usage will have `T: ProvenTransaction` however this is cumbersome - /// to generate. Since this graph doesn't actually care about the data type, we - /// simplify test data generation by just duplicating the ID. - type TestGraph = TransactionGraph; - - /// Test helpers and aliases. - impl TestGraph { - /// Alias to insert a transaction with no parents. - fn insert_with_no_parent(&mut self, id: TransactionId) { - self.insert_with_parents(id, Default::default()); - } - - /// Alias for inserting a transaction with parents. - fn insert_with_parents(&mut self, id: TransactionId, parents: BTreeSet) { - self.insert(id, id, parents); - } - - /// Alias for inserting a transaction with a single parent. - fn insert_with_parent(&mut self, id: TransactionId, parent: TransactionId) { - self.insert_with_parents(id, [parent].into()); - } - - /// Calls `pop_for_processing` until it returns `None`. - /// - /// This should result in a fully processed graph, barring bugs. - /// - /// Panics if the graph is not fully processed. - fn process_all(&mut self) -> Vec { - let mut processed = Vec::new(); - while let Some((id, _)) = self.pop_for_processing() { - processed.push(id); - } - - assert!(self.nodes.values().all(Node::is_processed)); - - processed - } - } - - #[test] - fn pruned_nodes_are_nonextant() { - //! Checks that processed and then pruned nodes behave as if they - //! never existed in the graph. We test this by comparing it to - //! a reference graph created without these ancestor nodes. - let mut rng = Random::with_random_seed(); - - let ancestor_a = rng.draw_tx_id(); - let ancestor_b = rng.draw_tx_id(); - - let child_a = rng.draw_tx_id(); - let child_b = rng.draw_tx_id(); - let child_both = rng.draw_tx_id(); - - let mut uut = TestGraph::default(); - uut.insert_with_no_parent(ancestor_a); - uut.insert_with_no_parent(ancestor_b); - uut.insert_with_parent(child_a, ancestor_a); - uut.insert_with_parent(child_b, ancestor_b); - uut.insert_with_parents(child_both, [ancestor_a, ancestor_b].into()); - - uut.process(ancestor_a); - uut.process(ancestor_b); - uut.prune_processed(&[ancestor_a, ancestor_b]); - - let mut reference = TestGraph::default(); - reference.insert_with_no_parent(child_a); - reference.insert_with_no_parent(child_b); - reference.insert_with_no_parent(child_both); - - assert_eq!(uut, reference); - } - - #[test] - fn inserted_node_is_considered_for_root() { - //! Ensure that a fresh node who's parent is - //! already processed will be considered for processing. - let mut rng = Random::with_random_seed(); - let parent_a = rng.draw_tx_id(); - let parent_b = rng.draw_tx_id(); - let child_a = rng.draw_tx_id(); - let child_b = rng.draw_tx_id(); + use crate::test_utils::mock_proven_tx; - let mut uut = TestGraph::default(); - uut.insert_with_no_parent(parent_a); - uut.insert_with_no_parent(parent_b); - uut.process(parent_a); - - uut.insert_with_parent(child_a, parent_a); - uut.insert_with_parent(child_b, parent_b); - - assert!(uut.roots.contains(&child_a)); - assert!(!uut.roots.contains(&child_b)); - } + // BATCH SELECTION TESTS + // ================================================================================================ #[test] - fn fifo_order_is_maintained() { - //! This test creates a simple queue graph, expecting that the processed items should - //! be emitted in the same order. - let mut rng = Random::with_random_seed(); - let input = (0..10).map(|_| rng.draw_tx_id()).collect::>(); - - let mut uut = TestGraph::default(); - uut.insert_with_no_parent(input[0]); - for pairs in input.windows(2) { - let (parent, id) = (pairs[0], pairs[1]); - uut.insert_with_parent(id, parent); + fn select_batch_respects_limit() { + // These transactions are independent and just used to ensure we have more available + // transactions than we want in the batch. + let txs = (0..10) + .map(|i| mock_proven_tx(i, vec![], vec![])) + .map(AuthenticatedTransaction::from_inner); + + let mut uut = TransactionGraph::default(); + for tx in txs { + uut.insert(tx, [].into()).unwrap(); } - let result = uut.process_all(); - assert_eq!(result, input); - } - - #[test] - fn requeuing_resets_graph_state() { - //! Requeuing transactions should cause the internal state to reset - //! to the same state as before these transactions were emitted - //! for processing. - - let mut rng = Random::with_random_seed(); - - let ancestor_a = rng.draw_tx_id(); - let ancestor_b = rng.draw_tx_id(); - let parent_a = rng.draw_tx_id(); - let parent_b = rng.draw_tx_id(); - let child_a = rng.draw_tx_id(); - let child_b = rng.draw_tx_id(); - let child_c = rng.draw_tx_id(); - - let mut uut = TestGraph::default(); - uut.insert_with_no_parent(ancestor_a); - uut.insert_with_no_parent(ancestor_b); - uut.insert_with_parent(parent_a, ancestor_a); - uut.insert_with_parent(parent_b, ancestor_b); - uut.insert_with_parents(child_a, [ancestor_a, parent_a].into()); - uut.insert_with_parents(child_b, [parent_a, parent_b].into()); - uut.insert_with_parent(child_c, parent_b); - - let mut reference = uut.clone(); - - uut.process(ancestor_a); - uut.process(ancestor_b); - uut.process(parent_a); - uut.process(parent_b); - uut.process(child_c); - - // Requeue all except ancestor a. This is a somewhat arbitrary choice. - // The reference graph should therefore only have ancestor a processed. - uut.requeue_transactions([ancestor_b, parent_a, parent_b, child_c].into()); - reference.process(ancestor_a); - - assert_eq!(uut, reference); - } + let (batch, parents) = uut.select_batch(0); + assert!(batch.is_empty()); + assert!(parents.is_empty()); - #[test] - fn nodes_are_processed_exactly_once() { - let mut rng = Random::with_random_seed(); - - let ancestor_a = rng.draw_tx_id(); - let ancestor_b = rng.draw_tx_id(); - let parent_a = rng.draw_tx_id(); - let parent_b = rng.draw_tx_id(); - let child_a = rng.draw_tx_id(); - let child_b = rng.draw_tx_id(); - let child_c = rng.draw_tx_id(); - - let mut uut = TestGraph::default(); - uut.insert_with_no_parent(ancestor_a); - uut.insert_with_no_parent(ancestor_b); - uut.insert_with_parent(parent_a, ancestor_a); - uut.insert_with_parent(parent_b, ancestor_b); - uut.insert_with_parents(child_a, [ancestor_a, parent_a].into()); - uut.insert_with_parents(child_b, [parent_a, parent_b].into()); - uut.insert_with_parent(child_c, parent_b); - - let mut result = uut.process_all(); - result.sort(); - - let mut expected = - vec![ancestor_a, ancestor_b, parent_a, parent_b, child_a, child_b, child_c]; - expected.sort(); - - assert_eq!(result, expected); - } - - #[test] - fn processed_data_and_parent_tracking() { - let mut rng = Random::with_random_seed(); + let (batch, parents) = uut.select_batch(3); + assert_eq!(batch.len(), 3); + assert!(parents.is_empty()); - let ancestor_a = rng.draw_tx_id(); - let ancestor_b = rng.draw_tx_id(); - let parent_a = rng.draw_tx_id(); - let parent_b = rng.draw_tx_id(); - let child_a = rng.draw_tx_id(); - let child_b = rng.draw_tx_id(); - let child_c = rng.draw_tx_id(); + let (batch, parents) = uut.select_batch(4); + assert_eq!(batch.len(), 4); + assert!(parents.is_empty()); - let mut uut = TestGraph::default(); - uut.insert_with_no_parent(ancestor_a); - uut.insert_with_no_parent(ancestor_b); - uut.insert_with_parent(parent_a, ancestor_a); - uut.insert_with_parent(parent_b, ancestor_b); - uut.insert_with_parents(child_a, [ancestor_a, parent_a].into()); - uut.insert_with_parents(child_b, [parent_a, parent_b].into()); - uut.insert_with_parent(child_c, parent_b); - - let result = uut.process(ancestor_a); - assert_eq!(result, (ancestor_a, Default::default())); - - let result = uut.process(ancestor_b); - assert_eq!(result, (ancestor_b, Default::default())); - - let result = uut.process(parent_a); - assert_eq!(result, (parent_a, [ancestor_a].into())); - - let result = uut.process(parent_b); - assert_eq!(result, (parent_b, [ancestor_b].into())); - - let result = uut.process(child_a); - assert_eq!(result, (child_a, [ancestor_a, parent_a].into())); - - let result = uut.process(child_b); - assert_eq!(result, (child_b, [parent_a, parent_b].into())); - - let result = uut.process(child_c); - assert_eq!(result, (child_c, [parent_b].into())); - } - - #[test] - fn purging_subgraph_handles_internal_nodes() { - //! Purging a subgraph should correctly handle nodes already deleted within that subgraph. - //! - //! This is a concern for errors as we are deleting parts of the subgraph while we are - //! iterating through the nodes to purge. This means its likely a node will already - //! have been deleted before processing it as an input. - //! - //! We can somewhat force this to occur by re-ordering the inputs relative to the actual - //! dependency order. - - let mut rng = Random::with_random_seed(); - - let ancestor_a = rng.draw_tx_id(); - let ancestor_b = rng.draw_tx_id(); - let parent_a = rng.draw_tx_id(); - let parent_b = rng.draw_tx_id(); - let child_a = rng.draw_tx_id(); - let child_b = rng.draw_tx_id(); - let child_c = rng.draw_tx_id(); - - let mut uut = TestGraph::default(); - uut.insert_with_no_parent(ancestor_a); - uut.insert_with_no_parent(ancestor_b); - uut.insert_with_parent(parent_a, ancestor_a); - uut.insert_with_parent(parent_b, ancestor_b); - uut.insert_with_parents(child_a, [ancestor_a, parent_a].into()); - uut.insert_with_parents(child_b, [parent_a, parent_b].into()); - uut.insert_with_parent(child_c, parent_b); - - uut.purge_subgraphs(vec![child_b, parent_a]); - - let mut reference = TestGraph::default(); - reference.insert_with_no_parent(ancestor_a); - reference.insert_with_no_parent(ancestor_b); - reference.insert_with_parent(parent_b, ancestor_b); - reference.insert_with_parent(child_c, parent_b); - - assert_eq!(uut, reference); - } - - #[test] - fn purging_removes_all_descendents() { - let mut rng = Random::with_random_seed(); - - let ancestor_a = rng.draw_tx_id(); - let ancestor_b = rng.draw_tx_id(); - let parent_a = rng.draw_tx_id(); - let parent_b = rng.draw_tx_id(); - let child_a = rng.draw_tx_id(); - let child_b = rng.draw_tx_id(); - let child_c = rng.draw_tx_id(); - - let mut uut = TestGraph::default(); - uut.insert_with_no_parent(ancestor_a); - uut.insert_with_no_parent(ancestor_b); - uut.insert_with_parent(parent_a, ancestor_a); - uut.insert_with_parent(parent_b, ancestor_b); - uut.insert_with_parents(child_a, [ancestor_a, parent_a].into()); - uut.insert_with_parents(child_b, [parent_a, parent_b].into()); - uut.insert_with_parent(child_c, parent_b); - - uut.purge_subgraphs(vec![parent_a]); - - let mut reference = TestGraph::default(); - reference.insert_with_no_parent(ancestor_a); - reference.insert_with_no_parent(ancestor_b); - reference.insert_with_parent(parent_b, ancestor_b); - reference.insert_with_parent(child_c, parent_b); - - assert_eq!(uut, reference); - } - - #[test] - #[should_panic] - fn duplicate_insert() { - let mut rng = Random::with_random_seed(); - let mut uut = TestGraph::default(); - - let id = rng.draw_tx_id(); - uut.insert_with_no_parent(id); - uut.insert_with_no_parent(id); - } - - #[test] - #[should_panic] - fn missing_parents_in_insert() { - let mut rng = Random::with_random_seed(); - let mut uut = TestGraph::default(); - - uut.insert_with_parents(rng.draw_tx_id(), [rng.draw_tx_id()].into()); - } - - #[test] - #[should_panic] - fn requeueing_an_already_queued_tx() { - let mut rng = Random::with_random_seed(); - let mut uut = TestGraph::default(); + // We expect this to be partially filled. + let (batch, parents) = uut.select_batch(4); + assert_eq!(batch.len(), 3); + assert!(parents.is_empty()); - let id = rng.draw_tx_id(); - uut.insert_with_no_parent(id); - uut.requeue_transactions([id].into()); + // And thereafter empty. + let (batch, parents) = uut.select_batch(100); + assert!(batch.is_empty()); + assert!(parents.is_empty()); } } From fd020921b3d4f4406ac6bb5c1fa9bb9d3a4a5486 Mon Sep 17 00:00:00 2001 From: Mirko <48352201+Mirko-von-Leipzig@users.noreply.github.com> Date: Tue, 29 Oct 2024 09:35:51 +0200 Subject: [PATCH 05/50] feat(block-producer): arc transaction data (#530) This makes `AuthenticatedTransaction` much cheaper to clone and move around. --- .../block-producer/src/batch_builder/mod.rs | 10 ++++-- .../block-producer/src/domain/transaction.rs | 32 ++++++++++------- .../src/mempool/dependency_graph.rs | 34 +++++++++---------- 3 files changed, 45 insertions(+), 31 deletions(-) diff --git a/crates/block-producer/src/batch_builder/mod.rs b/crates/block-producer/src/batch_builder/mod.rs index 1c8e11b05..59b45949b 100644 --- a/crates/block-producer/src/batch_builder/mod.rs +++ b/crates/block-producer/src/batch_builder/mod.rs @@ -9,6 +9,7 @@ use std::{ use miden_objects::{ accounts::AccountId, + assembly::SourceManager, notes::NoteId, transaction::{OutputNote, TransactionId}, Digest, @@ -284,8 +285,13 @@ impl WorkerPool { async move { tracing::debug!("Begin proving batch."); - let transactions = - transactions.into_iter().map(AuthenticatedTransaction::into_raw).collect(); + // TODO: This is a deep clone which can be avoided by change batch building to using + // refs or arcs. + let transactions = transactions + .iter() + .map(AuthenticatedTransaction::raw_proven_transaction) + .cloned() + .collect(); tokio::time::sleep(simulated_proof_time).await; if failed { diff --git a/crates/block-producer/src/domain/transaction.rs b/crates/block-producer/src/domain/transaction.rs index e6d9cdb9e..e040fe52b 100644 --- a/crates/block-producer/src/domain/transaction.rs +++ b/crates/block-producer/src/domain/transaction.rs @@ -1,4 +1,4 @@ -use std::collections::BTreeSet; +use std::{collections::BTreeSet, sync::Arc}; use miden_objects::{ accounts::AccountId, @@ -14,10 +14,12 @@ use crate::{errors::VerifyTxError, mempool::BlockNumber, store::TransactionInput /// Authentication ensures that all nullifiers are unspent, and additionally authenticates some /// previously unauthenticated input notes. /// +/// This struct is cheap to clone as it uses an Arc for the heavy data. +/// /// Note that this is of course only valid for the chain height of the authentication. #[derive(Clone, Debug, PartialEq)] pub struct AuthenticatedTransaction { - inner: ProvenTransaction, + inner: Arc, /// The account state provided by the store [inputs](TransactionInputs). /// /// This does not necessarily have to match the transaction's initial state @@ -62,7 +64,7 @@ impl AuthenticatedTransaction { .collect(); Ok(AuthenticatedTransaction { - inner: tx, + inner: Arc::new(tx), notes_authenticated_by_store: authenticated_notes, authentication_height: BlockNumber::new(inputs.current_block_height), store_account_state: inputs.account_hash, @@ -107,8 +109,8 @@ impl AuthenticatedTransaction { .filter(|note_id| !self.notes_authenticated_by_store.contains(note_id)) } - pub fn into_raw(self) -> ProvenTransaction { - self.inner + pub fn raw_proven_transaction(&self) -> &ProvenTransaction { + &self.inner } } @@ -117,18 +119,24 @@ impl AuthenticatedTransaction { //! Builder methods intended for easier test setup. /// Short-hand for `Self::new` where the input's are setup to match the transaction's initial - /// account state. + /// account state. This covers the account's initial state and nullifiers being set to unspent. pub fn from_inner(inner: ProvenTransaction) -> Self { let store_account_state = match inner.account_update().init_state_hash() { zero if zero == Digest::default() => None, non_zero => Some(non_zero), }; - Self { - inner, - store_account_state, - notes_authenticated_by_store: Default::default(), - authentication_height: Default::default(), - } + let inputs = TransactionInputs { + account_id: inner.account_id(), + account_hash: store_account_state, + nullifiers: inner.get_nullifiers().map(|nullifier| (nullifier, None)).collect(), + missing_unauthenticated_notes: inner + .get_unauthenticated_notes() + .map(|header| header.id()) + .collect(), + current_block_height: Default::default(), + }; + // SAFETY: nullifiers were set to None aka are definitely unspent. + Self::new(inner, inputs).unwrap() } /// Overrides the authentication height with the given value. diff --git a/crates/block-producer/src/mempool/dependency_graph.rs b/crates/block-producer/src/mempool/dependency_graph.rs index 58ef76fd1..09582980d 100644 --- a/crates/block-producer/src/mempool/dependency_graph.rs +++ b/crates/block-producer/src/mempool/dependency_graph.rs @@ -66,7 +66,7 @@ impl Default for DependencyGraph { } } -impl DependencyGraph { +impl DependencyGraph { /// Inserts a new node into the graph. /// /// # Errors @@ -82,7 +82,7 @@ impl DependencyGraph { let missing_parents = parents .iter() .filter(|parent| !self.vertices.contains_key(parent)) - .cloned() + .copied() .collect::>(); if !missing_parents.is_empty() { return Err(GraphError::MissingParents(missing_parents)); @@ -90,11 +90,11 @@ impl DependencyGraph { // Inform parents of their new child. for parent in &parents { - self.children.entry(parent.clone()).or_default().insert(key.clone()); + self.children.entry(*parent).or_default().insert(key); } - self.vertices.insert(key.clone(), value); - self.parents.insert(key.clone(), parents); - self.children.insert(key.clone(), Default::default()); + self.vertices.insert(key, value); + self.parents.insert(key, parents); + self.children.insert(key, Default::default()); self.try_make_root(key); @@ -115,12 +115,12 @@ impl DependencyGraph { let missing_nodes = keys .iter() .filter(|key| !self.vertices.contains_key(key)) - .cloned() + .copied() .collect::>(); if !missing_nodes.is_empty() { return Err(GraphError::UnknownNodes(missing_nodes)); } - let unprocessed = keys.difference(&self.processed).cloned().collect::>(); + let unprocessed = keys.difference(&self.processed).copied().collect::>(); if !unprocessed.is_empty() { return Err(GraphError::UnprocessedNodes(unprocessed)); } @@ -137,7 +137,7 @@ impl DependencyGraph { .map(|children| children.difference(&reverted)) .into_iter() .flatten() - .cloned(); + .copied(); to_revert.extend(unprocessed_children); @@ -176,13 +176,13 @@ impl DependencyGraph { let missing_nodes = keys .iter() .filter(|key| !self.vertices.contains_key(key)) - .cloned() + .copied() .collect::>(); if !missing_nodes.is_empty() { return Err(GraphError::UnknownNodes(missing_nodes)); } - let unprocessed = keys.difference(&self.processed).cloned().collect::>(); + let unprocessed = keys.difference(&self.processed).copied().collect::>(); if !unprocessed.is_empty() { return Err(GraphError::UnprocessedNodes(unprocessed)); } @@ -193,7 +193,7 @@ impl DependencyGraph { .flat_map(|key| self.parents.get(key)) .flatten() .filter(|parent| !keys.contains(parent)) - .cloned() + .copied() .collect::>(); if !dangling.is_empty() { return Err(GraphError::DanglingNodes(dangling)); @@ -236,7 +236,7 @@ impl DependencyGraph { let missing_nodes = keys .iter() .filter(|key| !self.vertices.contains_key(key)) - .cloned() + .copied() .collect::>(); if !missing_nodes.is_empty() { return Err(GraphError::UnknownNodes(missing_nodes)); @@ -251,7 +251,7 @@ impl DependencyGraph { .vertices .remove(&key) .expect("Node was checked in precondition and must therefore exist"); - removed.insert(key.clone(), value); + removed.insert(key, value); self.processed.remove(&key); self.roots.remove(&key); @@ -259,7 +259,7 @@ impl DependencyGraph { // Children must also be purged. Take care not to visit them twice which is // possible since children can have multiple purged parents. let unvisited_children = self.children.remove(&key).unwrap_or_default(); - let unvisited_children = unvisited_children.difference(&visited).cloned(); + let unvisited_children = unvisited_children.difference(&visited); to_remove.extend(unvisited_children); // Inform parents that this child no longer exists. @@ -315,7 +315,7 @@ impl DependencyGraph { return Err(GraphError::NotARootNode(key)); } - self.processed.insert(key.clone()); + self.processed.insert(key); self.children .get(&key) @@ -374,7 +374,7 @@ mod tests { /// Calls process_root until all nodes have been processed. fn process_all(&mut self) { - while let Some(root) = self.roots().first().cloned() { + while let Some(root) = self.roots().first().copied() { /// SAFETY: this is definitely a root since we just took it from there :) self.process_root(root); } From ad20a303d0a282aec709d9b2253c59cb32d53123 Mon Sep 17 00:00:00 2001 From: Mirko <48352201+Mirko-von-Leipzig@users.noreply.github.com> Date: Wed, 30 Oct 2024 11:32:27 +0200 Subject: [PATCH 06/50] feat(block-producer): refactor batch graph to use dependency graph (#533) `BatchGraph` now uses `DependencyGraph ◄────┘ +/// └───────────┘ +/// ``` #[derive(Default, Clone)] pub struct BatchGraph { - nodes: BTreeMap, - roots: BTreeSet, + /// Tracks the interdependencies between batches. + inner: DependencyGraph, - /// Allows for reverse lookup of transaction -> batch. + /// Maps each transaction to its batch, allowing for reverse lookups. + /// + /// Incoming batches are defined entirely in terms of transactions, including parent edges. + /// This let's us transform these parent transactions into the relevant parent batches. transactions: BTreeMap, + + /// Maps each batch to its transaction set. + /// + /// Required because the dependency graph is defined in terms of batches. This let's us + /// translate between batches and their transactions when required. + batches: BTreeMap>, +} + +#[derive(thiserror::Error, Debug, Clone, PartialEq, Eq)] +pub enum BatchInsertError { + #[error("Transactions are already in the graph: {0:?}")] + DuplicateTransactions(BTreeSet), + #[error("Unknown parent transaction {0}")] + UnknownParentTransaction(TransactionId), + #[error(transparent)] + GraphError(#[from] GraphError), } impl BatchGraph { + /// Inserts a new batch into the graph. + /// + /// # Errors + /// + /// Returns an error if: + /// - the batch ID is already in use + /// - any transactions are already in the graph + /// - any parent transactions are _not_ in the graph pub fn insert( &mut self, id: BatchJobId, transactions: Vec, parents: BTreeSet, - ) { - // Reverse lookup parent transaction batches. - let parents = parents - .into_iter() - .map(|tx| self.transactions.get(&tx).expect("Parent transaction must be in a batch")) + ) -> Result<(), BatchInsertError> { + let duplicates = transactions + .iter() + .filter(|tx| self.transactions.contains_key(tx)) .copied() - .collect(); - - // Inform parents of their new child. - for parent in &parents { - self.nodes - .get_mut(parent) - .expect("Parent batch must be present") - .children - .insert(id); + .collect::>(); + if !duplicates.is_empty() { + return Err(BatchInsertError::DuplicateTransactions(duplicates)); } - // Insert transactions for reverse lookup. - for tx in &transactions { - self.transactions.insert(*tx, id); + // Reverse lookup parent transaction batches. + let parent_batches = parents + .into_iter() + .map(|tx| { + self.transactions + .get(&tx) + .copied() + .ok_or(BatchInsertError::UnknownParentTransaction(tx)) + }) + .collect::>()?; + + self.inner.insert_pending(id, parent_batches)?; + + for tx in transactions.iter().copied() { + self.transactions.insert(tx, id); } + self.batches.insert(id, transactions); - // Insert the new node into the graph. - let batch = Node { - status: Status::Queued, - transactions, - parents, - children: Default::default(), - }; - self.nodes.insert(id, batch); - - // New node might be a root. - // - // This could be optimized by inlining this inside the parent loop. This would prevent the - // double iteration over parents, at the cost of some code duplication. - self.try_make_root(id); + Ok(()) } - /// Removes the batches and all their descendants from the graph. + /// Removes the batches and their descendants from the graph. + /// + /// # Returns + /// + /// Returns all removes batches and their transactions. /// - /// Returns all removed batches and their transactions. - pub fn purge_subgraphs( + /// # Errors + /// + /// Returns an error if any of the batches are not currently in the graph. + pub fn remove_batches( &mut self, - batches: BTreeSet, - ) -> BTreeMap> { - let mut removed = BTreeMap::new(); - - let mut to_process = batches; - - while let Some(node_id) = to_process.pop_first() { - // Its possible for a node to already have been removed as part of this subgraph - // removal. - let Some(node) = self.nodes.remove(&node_id) else { - continue; - }; + batch_ids: BTreeSet, + ) -> Result>, GraphError> { + // This returns all descendent batches as well. + let batch_ids = self.inner.purge_subgraphs(batch_ids)?; + + // SAFETY: These batches must all have been inserted since they are emitted from the inner + // dependency graph, and therefore must all be in the batches mapping. + let batches = batch_ids + .into_iter() + .map(|batch_id| (batch_id, self.batches.remove(&batch_id).unwrap())) + .collect::>(); - // All the child batches are also removed so no need to check for new roots. No new - // roots are possible as a result of this subgraph removal. - self.roots.remove(&node_id); - - for transaction in &node.transactions { - self.transactions.remove(transaction); - } - - // Inform parent that this child no longer exists. - // - // The same is not required for children of this batch as we will be removing those as - // well. - for parent in &node.parents { - // Parent could already be removed as part of this subgraph removal. - if let Some(parent) = self.nodes.get_mut(parent) { - parent.children.remove(&node_id); - } - } - - to_process.extend(node.children); - removed.insert(node_id, node.transactions); + for tx in batches.values().flatten() { + self.transactions.remove(tx); } - removed + Ok(batches) } - /// Removes a set of batches from the graph without removing any descendants. + /// Removes the set of committed batches from the graph. + /// + /// The batches _must_ have been previously selected for inclusion in a block using + /// [`select_block`](Self::select_block). This is intended for limiting the size of the graph by + /// culling committed data. + /// + /// # Returns /// - /// This is intended to cull completed batches from stale blocs. - pub fn remove_committed(&mut self, batches: BTreeSet) -> Vec { + /// Returns the transactions of the pruned batches. + /// + /// # Errors + /// + /// Returns an error if + /// - any batch was not previously selected for inclusion in a block + /// - any batch is unknown + /// - any parent batch would be left dangling in the graph + /// + /// The last point implies that batches should be removed in block order. + pub fn prune_committed( + &mut self, + batch_ids: BTreeSet, + ) -> Result, GraphError> { + self.inner.prune_processed(batch_ids.clone())?; let mut transactions = Vec::new(); - for batch in batches { - let node = self.nodes.remove(&batch).expect("Node must be in graph"); - assert_eq!(node.status, Status::Processed); - - // Remove batch from graph. No need to update parents as they should be removed in this - // call as well. - for child in node.children { - // Its possible for the child to part of this same set of batches and therefore - // already removed. - if let Some(child) = self.nodes.get_mut(&child) { - child.parents.remove(&batch); - } - } - - transactions.extend_from_slice(&node.transactions); + for batch_id in &batch_ids { + transactions.extend(self.batches.remove(batch_id).into_iter().flatten()); + } + + for tx in &transactions { + self.transactions.remove(tx); } - transactions + Ok(transactions) } - /// Mark a batch as proven if it exists. - pub fn mark_proven(&mut self, id: BatchJobId, batch: TransactionBatch) { - // Its possible for inflight batches to have been removed as part of another batches - // failure. - if let Some(node) = self.nodes.get_mut(&id) { - assert!(node.status == Status::Queued); - node.status = Status::Proven(batch); - self.try_make_root(id); - } + /// Submits a proof for the given batch, promoting it from pending to ready for inclusion in a + /// block once all its parents have themselves been included. + /// + /// # Errors + /// + /// Returns an error if the batch is not in the graph or if it was already previously proven. + pub fn submit_proof( + &mut self, + id: BatchJobId, + batch: TransactionBatch, + ) -> Result<(), GraphError> { + self.inner.promote_pending(id, batch) } - /// Returns at most `count` __indepedent__ batches which are ready for inclusion in a block. + /// Returns at most `count` batches which are ready for inclusion in a block. pub fn select_block(&mut self, count: usize) -> BTreeMap { let mut batches = BTreeMap::new(); - // Track children so we can evaluate them for root afterwards. - let mut children = BTreeSet::new(); + for _ in 0..count { + // This strategy just selects arbitrary roots for now. This is valid but not very + // interesting or efficient. + let Some(batch_id) = self.inner.roots().first().copied() else { + break; + }; - for batch_id in &self.roots { - let mut node = self.nodes.get_mut(batch_id).expect("Root node must be in graph"); + // SAFETY: This is definitely a root since we just selected it from the set of roots. + self.inner.process_root(batch_id).unwrap(); + // SAFETY: Since it was a root batch, it must definitely have a processed batch + // associated with it. + let batch = self.inner.get(&batch_id).unwrap(); - // Filter out batches which have dependencies in our selection so far. - if node.parents.iter().any(|parent| batches.contains_key(parent)) { - continue; - } + batches.insert(batch_id, batch.clone()); + } - let Status::Proven(batch) = node.status.clone() else { - unreachable!("Root batch must be in proven state."); - }; + batches + } +} - batches.insert(*batch_id, batch); - node.status = Status::Processed; +#[cfg(test)] +mod tests { + use super::*; + use crate::test_utils::Random; - if batches.len() == count { - break; - } - } + // INSERT TESTS + // ================================================================================================ - // Performing this outside the main loop has two benefits: - // 1. We avoid checking the same child multiple times. - // 2. We avoid evaluating children for selection (they're dependents). - for child in children { - self.try_make_root(child); - } + #[test] + fn insert_rejects_duplicate_batch_ids() { + let id = BatchJobId::new(1); + let mut uut = BatchGraph::default(); - batches + uut.insert(id, Default::default(), Default::default()).unwrap(); + let err = uut.insert(id, Default::default(), Default::default()).unwrap_err(); + let expected = BatchInsertError::GraphError(GraphError::DuplicateKey(id)); + + assert_eq!(err, expected); } - fn try_make_root(&mut self, id: BatchJobId) { - let node = self.nodes.get_mut(&id).expect("Node must be in graph"); + #[test] + fn insert_rejects_duplicate_transactions() { + let mut rng = Random::with_random_seed(); + let tx_dup = rng.draw_tx_id(); + let tx_non_dup = rng.draw_tx_id(); - for parent in node.parents.clone() { - let parent = self.nodes.get(&parent).expect("Parent must be in pool"); + let mut uut = BatchGraph::default(); - if parent.status != Status::Processed { - return; - } - } - self.roots.insert(id); + uut.insert(BatchJobId::new(1), vec![tx_dup], Default::default()).unwrap(); + let err = uut + .insert(BatchJobId::new(2), vec![tx_dup, tx_non_dup], Default::default()) + .unwrap_err(); + let expected = BatchInsertError::DuplicateTransactions([tx_dup].into()); + + assert_eq!(err, expected); } -} -#[derive(Clone, Debug)] -struct Node { - status: Status, - transactions: Vec, - parents: BTreeSet, - children: BTreeSet, -} + #[test] + fn insert_rejects_missing_parents() { + let mut rng = Random::with_random_seed(); + let tx = rng.draw_tx_id(); + let missing = rng.draw_tx_id(); + + let mut uut = BatchGraph::default(); + + let err = uut.insert(BatchJobId::new(2), vec![tx], [missing].into()).unwrap_err(); + let expected = BatchInsertError::UnknownParentTransaction(missing); -#[derive(Debug, Clone, PartialEq, Eq)] -enum Status { - /// The batch is a busy being proven. - Queued, - /// The batch is proven. It may be placed in a block - /// __IFF__ all of its parents are already in a block. - Proven(TransactionBatch), - /// Batch is part of a block. - Processed, + assert_eq!(err, expected); + } + + // PURGE_SUBGRAPHS TESTS + // ================================================================================================ + + #[test] + fn purge_subgraphs_returns_all_purged_transaction_sets() { + //! Ensure that purge_subgraphs returns both parent and child batches when the parent is + //! pruned. Further ensure that a disjoint batch is not pruned. + let mut rng = Random::with_random_seed(); + let parent_batch_txs = (0..5).map(|_| rng.draw_tx_id()).collect::>(); + let child_batch_txs = (0..5).map(|_| rng.draw_tx_id()).collect::>(); + let disjoint_batch_txs = (0..5).map(|_| rng.draw_tx_id()).collect(); + + let parent_batch_id = BatchJobId::new(0); + let child_batch_id = BatchJobId::new(1); + let disjoint_batch_id = BatchJobId::new(2); + + let mut uut = BatchGraph::default(); + uut.insert(parent_batch_id, parent_batch_txs.clone(), Default::default()) + .unwrap(); + uut.insert(child_batch_id, child_batch_txs.clone(), [parent_batch_txs[0]].into()) + .unwrap(); + uut.insert(disjoint_batch_id, disjoint_batch_txs, Default::default()).unwrap(); + + let result = uut.remove_batches([parent_batch_id].into()).unwrap(); + let expected = + [(parent_batch_id, parent_batch_txs), (child_batch_id, child_batch_txs)].into(); + + assert_eq!(result, expected); + } } diff --git a/crates/block-producer/src/mempool/dependency_graph.rs b/crates/block-producer/src/mempool/dependency_graph.rs index 09582980d..258f6d7cb 100644 --- a/crates/block-producer/src/mempool/dependency_graph.rs +++ b/crates/block-producer/src/mempool/dependency_graph.rs @@ -1,4 +1,7 @@ -use std::collections::{BTreeMap, BTreeSet}; +use std::{ + collections::{BTreeMap, BTreeSet}, + fmt::Display, +}; use miden_tx::utils::collections::KvMap; @@ -9,8 +12,36 @@ use miden_tx::utils::collections::KvMap; /// once all parent nodes have been processed. /// /// Forms the basis of our transaction and batch dependency graphs. +/// +/// # Node lifecycle +/// ``` +/// │ +/// │ +/// insert_pending│ +/// ┌─────▼─────┐ +/// │ pending │────┐ +/// └─────┬─────┘ │ +/// │ │ +/// promote_pending│ │ +/// ┌─────▼─────┐ │ +/// ┌──────────► in queue │────│ +/// │ └─────┬─────┘ │ +/// revert_processed│ │ │ +/// │ process_root│ │ +/// │ ┌─────▼─────┐ │ +/// └──────────┼ processed │────│ +/// └─────┬─────┘ │ +/// │ │ +/// prune_processed│ │purge_subgraphs +/// ┌─────▼─────┐ │ +/// │ ◄────┘ +/// └───────────┘ +/// ``` #[derive(Debug, Clone, PartialEq, Eq)] pub struct DependencyGraph { + /// Node's who's data is still pending. + pending: BTreeSet, + /// Each node's data. vertices: BTreeMap, @@ -49,8 +80,11 @@ pub enum GraphError { #[error("Nodes would be left dangling: {0:?}")] DanglingNodes(BTreeSet), - #[error("Node {0} is not ready to be processed")] - NotARootNode(K), + #[error("Node {0} is not a root node")] + InvalidRootNode(K), + + #[error("Node {0} is not a pending node")] + InvalidPendingNode(K), } /// This cannot be derived without enforcing `Default` bounds on K and V. @@ -58,6 +92,7 @@ impl Default for DependencyGraph { fn default() -> Self { Self { vertices: Default::default(), + pending: Default::default(), parents: Default::default(), children: Default::default(), roots: Default::default(), @@ -66,22 +101,22 @@ impl Default for DependencyGraph { } } -impl DependencyGraph { - /// Inserts a new node into the graph. +impl DependencyGraph { + /// Inserts a new pending node into the graph. /// /// # Errors /// /// Errors if the node already exists, or if any of the parents are not part of the graph. /// /// This method is atomic. - pub fn insert(&mut self, key: K, value: V, parents: BTreeSet) -> Result<(), GraphError> { - if self.vertices.contains_key(&key) { + pub fn insert_pending(&mut self, key: K, parents: BTreeSet) -> Result<(), GraphError> { + if self.contains(&key) { return Err(GraphError::DuplicateKey(key)); } let missing_parents = parents .iter() - .filter(|parent| !self.vertices.contains_key(parent)) + .filter(|parent| !self.contains(parent)) .copied() .collect::>(); if !missing_parents.is_empty() { @@ -92,10 +127,27 @@ impl DependencyGraph { for parent in &parents { self.children.entry(*parent).or_default().insert(key); } - self.vertices.insert(key, value); + self.pending.insert(key); self.parents.insert(key, parents); self.children.insert(key, Default::default()); + Ok(()) + } + + /// Promotes a pending node, associating it with the provided value and allowing it to be + /// considered for processing. + /// + /// # Errors + /// + /// Errors if the given node is not pending. + /// + /// This method is atomic. + pub fn promote_pending(&mut self, key: K, value: V) -> Result<(), GraphError> { + if !self.pending.remove(&key) { + return Err(GraphError::InvalidPendingNode(key)); + } + + self.vertices.insert(key, value); self.try_make_root(key); Ok(()) @@ -103,6 +155,8 @@ impl DependencyGraph { /// Reverts the nodes __and their descendents__, requeueing them for processing. /// + /// Descendents which are pending remain unchanged. + /// /// # Errors /// /// Returns an error if any of the given nodes: @@ -137,6 +191,8 @@ impl DependencyGraph { .map(|children| children.difference(&reverted)) .into_iter() .flatten() + // We should not revert children which are pending. + .filter(|child| self.vertices.contains_key(child)) .copied(); to_revert.extend(unprocessed_children); @@ -173,11 +229,8 @@ impl DependencyGraph { /// /// This method is atomic. pub fn prune_processed(&mut self, keys: BTreeSet) -> Result, GraphError> { - let missing_nodes = keys - .iter() - .filter(|key| !self.vertices.contains_key(key)) - .copied() - .collect::>(); + let missing_nodes = + keys.iter().filter(|key| !self.contains(key)).copied().collect::>(); if !missing_nodes.is_empty() { return Err(GraphError::UnknownNodes(missing_nodes)); } @@ -221,37 +274,32 @@ impl DependencyGraph { } /// Removes the set of nodes __and all descendents__ from the graph, returning all removed - /// values. + /// nodes. This __includes__ pending nodes. /// /// # Returns /// - /// Returns a mapping of each removed key to its value. + /// All nodes removed. /// /// # Errors /// /// Returns an error if any of the given nodes does not exist. /// /// This method is atomic. - pub fn purge_subgraphs(&mut self, keys: BTreeSet) -> Result, GraphError> { - let missing_nodes = keys - .iter() - .filter(|key| !self.vertices.contains_key(key)) - .copied() - .collect::>(); + pub fn purge_subgraphs(&mut self, keys: BTreeSet) -> Result, GraphError> { + let missing_nodes = + keys.iter().filter(|key| !self.contains(key)).copied().collect::>(); if !missing_nodes.is_empty() { return Err(GraphError::UnknownNodes(missing_nodes)); } let mut visited = keys.clone(); let mut to_remove = keys; - let mut removed = BTreeMap::new(); + let mut removed = BTreeSet::new(); while let Some(key) = to_remove.pop_first() { - let value = self - .vertices - .remove(&key) - .expect("Node was checked in precondition and must therefore exist"); - removed.insert(key, value); + self.vertices.remove(&key); + self.pending.remove(&key); + removed.insert(key); self.processed.remove(&key); self.roots.remove(&key); @@ -280,7 +328,17 @@ impl DependencyGraph { /// /// This method assumes the node exists. Caller is responsible for ensuring this is true. fn try_make_root(&mut self, key: K) { - debug_assert!(self.vertices.contains_key(&key), "Potential root must exist in the graph"); + if self.pending.contains(&key) { + return; + } + debug_assert!( + self.vertices.contains_key(&key), + "Potential root {key} must exist in the graph" + ); + debug_assert!( + !self.processed.contains(&key), + "Potential root {key} cannot already be processed" + ); let all_parents_processed = self .parents @@ -312,7 +370,7 @@ impl DependencyGraph { /// This method is atomic. pub fn process_root(&mut self, key: K) -> Result<(), GraphError> { if !self.roots.remove(&key) { - return Err(GraphError::NotARootNode(key)); + return Err(GraphError::InvalidRootNode(key)); } self.processed.insert(key); @@ -336,6 +394,11 @@ impl DependencyGraph { pub fn parents(&self, key: &K) -> Option<&BTreeSet> { self.parents.get(key) } + + /// Returns true if the node exists, in either the pending or non-pending sets. + fn contains(&self, key: &K) -> bool { + self.pending.contains(key) || self.vertices.contains_key(key) + } } // TESTS @@ -354,7 +417,7 @@ mod tests { impl TestGraph { /// Alias for inserting a node with no parents. - fn insert_root(&mut self, node: u32) -> Result<(), GraphError> { + fn insert_with_no_parents(&mut self, node: u32) -> Result<(), GraphError> { self.insert_with_parents(node, Default::default()) } @@ -369,7 +432,21 @@ mod tests { node: u32, parents: BTreeSet, ) -> Result<(), GraphError> { - self.insert(node, node, parents) + self.insert_pending(node, parents) + } + + /// Alias for promoting nodes with the same value as the key. + fn promote(&mut self, nodes: impl IntoIterator) -> Result<(), GraphError> { + for node in nodes { + self.promote_pending(node, node)?; + } + Ok(()) + } + + /// Promotes all nodes in the pending list with value=key. + fn promote_all(&mut self) { + /// SAFETY: these are definitely pending nodes. + self.promote(self.pending.clone()).unwrap(); } /// Calls process_root until all nodes have been processed. @@ -381,12 +458,12 @@ mod tests { } } - // INSERT TESTS + // PROMOTE TESTS // ================================================================================================ #[test] - fn inserted_nodes_are_considered_for_root() { - //! Ensure that an inserted node is added to the root list if all parents are already + fn promoted_nodes_are_considered_for_root() { + //! Ensure that a promoted node is added to the root list if all parents are already //! processed. let parent_a = 1; let parent_b = 2; @@ -395,8 +472,9 @@ mod tests { let child_c = 5; let mut uut = TestGraph::default(); - uut.insert_root(parent_a).unwrap(); - uut.insert_root(parent_b).unwrap(); + uut.insert_with_no_parents(parent_a).unwrap(); + uut.insert_with_no_parents(parent_b).unwrap(); + uut.promote_all(); // Only process one parent so that some children remain unrootable. uut.process_root(parent_a).unwrap(); @@ -405,6 +483,8 @@ mod tests { uut.insert_with_parent(child_b, parent_b).unwrap(); uut.insert_with_parents(child_c, [parent_a, parent_b].into()).unwrap(); + uut.promote_all(); + // Only child_a should be added (in addition to the parents), since the other children // are dependent on parent_b which is incomplete. let expected_roots = [parent_b, child_a].into(); @@ -412,6 +492,69 @@ mod tests { assert_eq!(uut.roots, expected_roots); } + #[test] + fn pending_nodes_are_not_considered_for_root() { + //! Ensure that an unpromoted node is _not_ added to the root list even if all parents are + //! already processed. + let parent_a = 1; + let parent_b = 2; + let child_a = 3; + let child_b = 4; + let child_c = 5; + + let mut uut = TestGraph::default(); + uut.insert_with_no_parents(parent_a).unwrap(); + uut.insert_with_no_parents(parent_b).unwrap(); + uut.promote_all(); + uut.process_all(); + + uut.insert_with_parent(child_a, parent_a).unwrap(); + uut.insert_with_parent(child_b, parent_b).unwrap(); + uut.insert_with_parents(child_c, [parent_a, parent_b].into()).unwrap(); + + uut.promote([child_b]).unwrap(); + + // Only child b is valid as it was promoted. + let expected = [child_b].into(); + + assert_eq!(uut.roots, expected); + } + + #[test] + fn promoted_nodes_are_moved() { + let mut uut = TestGraph::default(); + uut.insert_with_no_parents(123).unwrap(); + + assert!(uut.pending.contains(&123)); + assert!(!uut.vertices.contains_key(&123)); + + uut.promote_pending(123, 123); + + assert!(!uut.pending.contains(&123)); + assert!(uut.vertices.contains_key(&123)); + } + + #[test] + fn promote_rejects_already_promoted_nodes() { + let mut uut = TestGraph::default(); + uut.insert_with_no_parents(123).unwrap(); + uut.promote_all(); + + let err = uut.promote_pending(123, 123).unwrap_err(); + let expected = GraphError::InvalidPendingNode(123); + assert_eq!(err, expected); + } + + #[test] + fn promote_rejects_unknown_nodes() { + let err = TestGraph::default().promote_pending(123, 123).unwrap_err(); + let expected = GraphError::InvalidPendingNode(123); + assert_eq!(err, expected); + } + + // INSERT TESTS + // ================================================================================================ + #[test] fn insert_with_known_parents_succeeds() { let parent_a = 10; @@ -420,8 +563,8 @@ mod tests { let uncle = 222; let mut uut = TestGraph::default(); - uut.insert_root(grandfather).unwrap(); - uut.insert_root(parent_a).unwrap(); + uut.insert_with_no_parents(grandfather).unwrap(); + uut.insert_with_no_parents(parent_a).unwrap(); uut.insert_with_parent(parent_b, grandfather).unwrap(); uut.insert_with_parent(uncle, grandfather).unwrap(); uut.insert_with_parents(1, [parent_a, parent_b].into()).unwrap(); @@ -434,14 +577,14 @@ mod tests { //! - does not mutate the state (atomicity) const KEY: u32 = 123; let mut uut = TestGraph::default(); - uut.insert_root(KEY).unwrap(); + uut.insert_with_no_parents(KEY).unwrap(); - let err = uut.insert_root(KEY).unwrap_err(); + let err = uut.insert_with_no_parents(KEY).unwrap_err(); let expected = GraphError::DuplicateKey(KEY); assert_eq!(err, expected); let mut atomic_reference = TestGraph::default(); - atomic_reference.insert_root(KEY); + atomic_reference.insert_with_no_parents(KEY); assert_eq!(uut, atomic_reference); } @@ -469,9 +612,9 @@ mod tests { const MISSING: u32 = 123; let mut uut = TestGraph::default(); - uut.insert_root(1).unwrap(); - uut.insert_root(2).unwrap(); - uut.insert_root(3).unwrap(); + uut.insert_with_no_parents(1).unwrap(); + uut.insert_with_no_parents(2).unwrap(); + uut.insert_with_no_parents(3).unwrap(); let atomic_reference = uut.clone(); @@ -487,9 +630,10 @@ mod tests { #[test] fn reverting_unprocessed_nodes_is_rejected() { let mut uut = TestGraph::default(); - uut.insert_root(1).unwrap(); - uut.insert_root(2).unwrap(); - uut.insert_root(3).unwrap(); + uut.insert_with_no_parents(1).unwrap(); + uut.insert_with_no_parents(2).unwrap(); + uut.insert_with_no_parents(3).unwrap(); + uut.promote_all(); uut.process_root(1).unwrap(); let err = uut.revert_subgraphs([1, 2, 3].into()).unwrap_err(); @@ -518,14 +662,16 @@ mod tests { let disjoint = 7; let mut uut = TestGraph::default(); - uut.insert_root(grandparent).unwrap(); - uut.insert_root(disjoint).unwrap(); + uut.insert_with_no_parents(grandparent).unwrap(); + uut.insert_with_no_parents(disjoint).unwrap(); uut.insert_with_parent(parent_a, grandparent).unwrap(); uut.insert_with_parent(parent_b, grandparent).unwrap(); uut.insert_with_parent(child_a, parent_a).unwrap(); uut.insert_with_parent(child_b, parent_b).unwrap(); uut.insert_with_parents(child_c, [parent_a, parent_b].into()).unwrap(); + uut.promote([disjoint, grandparent, parent_a, parent_b, child_a, child_c]) + .unwrap(); uut.process_root(disjoint).unwrap(); let reference = uut.clone(); @@ -552,16 +698,17 @@ mod tests { let mut uut = TestGraph::default(); // This pair of nodes should not be impacted by the reverted subgraph. - uut.insert_root(disjoint_parent).unwrap(); + uut.insert_with_no_parents(disjoint_parent).unwrap(); uut.insert_with_parent(disjoint_child, disjoint_parent).unwrap(); - uut.insert_root(parent_a).unwrap(); - uut.insert_root(parent_b).unwrap(); + uut.insert_with_no_parents(parent_a).unwrap(); + uut.insert_with_no_parents(parent_b).unwrap(); uut.insert_with_parent(child_a, parent_a); uut.insert_with_parent(child_b, parent_b); uut.insert_with_parents(partially_disjoin_child, [disjoint_parent, parent_a].into()); // Since we are reverting the other parents, we expect the roots to match the current state. + uut.promote_all(); uut.process_root(disjoint_parent).unwrap(); let reference = uut.roots().clone(); @@ -588,20 +735,22 @@ mod tests { let child_both = 5; let mut uut = TestGraph::default(); - uut.insert_root(ancestor_a).unwrap(); - uut.insert_root(ancestor_b).unwrap(); + uut.insert_with_no_parents(ancestor_a).unwrap(); + uut.insert_with_no_parents(ancestor_b).unwrap(); uut.insert_with_parent(child_a, ancestor_a).unwrap(); uut.insert_with_parent(child_b, ancestor_b).unwrap(); uut.insert_with_parents(child_both, [ancestor_a, ancestor_b].into()).unwrap(); + uut.promote_all(); uut.process_root(ancestor_a).unwrap(); uut.process_root(ancestor_b).unwrap(); uut.prune_processed([ancestor_a, ancestor_b].into()).unwrap(); let mut reference = TestGraph::default(); - reference.insert_root(child_a).unwrap(); - reference.insert_root(child_b).unwrap(); - reference.insert_root(child_both).unwrap(); + reference.insert_with_no_parents(child_a).unwrap(); + reference.insert_with_no_parents(child_b).unwrap(); + reference.insert_with_no_parents(child_both).unwrap(); + reference.promote_all(); assert_eq!(uut, reference); } @@ -616,7 +765,8 @@ mod tests { #[test] fn pruning_unprocessed_nodes_is_rejected() { let mut uut = TestGraph::default(); - uut.insert_root(1).unwrap(); + uut.insert_with_no_parents(1).unwrap(); + uut.promote_all(); let err = uut.prune_processed([1].into()).unwrap_err(); let expected = GraphError::UnprocessedNodes([1].into()); @@ -630,8 +780,9 @@ mod tests { let dangling = 1; let pruned = 2; let mut uut = TestGraph::default(); - uut.insert_root(dangling).unwrap(); + uut.insert_with_no_parents(dangling).unwrap(); uut.insert_with_parent(pruned, dangling).unwrap(); + uut.promote_all(); uut.process_all(); let err = uut.prune_processed([pruned].into()).unwrap_err(); @@ -664,8 +815,8 @@ mod tests { let child_c = 6; let mut uut = TestGraph::default(); - uut.insert_root(ancestor_a).unwrap(); - uut.insert_root(ancestor_b).unwrap(); + uut.insert_with_no_parents(ancestor_a).unwrap(); + uut.insert_with_no_parents(ancestor_b).unwrap(); uut.insert_with_parent(parent_a, ancestor_a).unwrap(); uut.insert_with_parent(parent_b, ancestor_b).unwrap(); uut.insert_with_parents(child_a, [ancestor_a, parent_a].into()).unwrap(); @@ -675,8 +826,8 @@ mod tests { uut.purge_subgraphs([child_b, parent_a].into()); let mut reference = TestGraph::default(); - reference.insert_root(ancestor_a).unwrap(); - reference.insert_root(ancestor_b).unwrap(); + reference.insert_with_no_parents(ancestor_a).unwrap(); + reference.insert_with_no_parents(ancestor_b).unwrap(); reference.insert_with_parent(parent_b, ancestor_b).unwrap(); reference.insert_with_parent(child_c, parent_b).unwrap(); @@ -694,8 +845,8 @@ mod tests { let child_c = 7; let mut uut = TestGraph::default(); - uut.insert_root(ancestor_a).unwrap(); - uut.insert_root(ancestor_b).unwrap(); + uut.insert_with_no_parents(ancestor_a).unwrap(); + uut.insert_with_no_parents(ancestor_b).unwrap(); uut.insert_with_parent(parent_a, ancestor_a).unwrap(); uut.insert_with_parent(parent_b, ancestor_b).unwrap(); uut.insert_with_parents(child_a, [ancestor_a, parent_a].into()).unwrap(); @@ -705,8 +856,8 @@ mod tests { uut.purge_subgraphs([parent_a].into()).unwrap(); let mut reference = TestGraph::default(); - reference.insert_root(ancestor_a).unwrap(); - reference.insert_root(ancestor_b).unwrap(); + reference.insert_with_no_parents(ancestor_a).unwrap(); + reference.insert_with_no_parents(ancestor_b).unwrap(); reference.insert_with_parent(parent_b, ancestor_b).unwrap(); reference.insert_with_parent(child_c, parent_b).unwrap(); @@ -725,13 +876,13 @@ mod tests { let child_c = 5; let mut uut = TestGraph::default(); - uut.insert_root(parent_a).unwrap(); - uut.insert_root(parent_b).unwrap(); + uut.insert_with_no_parents(parent_a).unwrap(); + uut.insert_with_no_parents(parent_b).unwrap(); uut.insert_with_parent(child_a, parent_a).unwrap(); uut.insert_with_parent(child_b, parent_b).unwrap(); uut.insert_with_parents(child_c, [parent_a, parent_b].into()).unwrap(); + uut.promote_all(); - // This should promote only child_a to root, in addition to the remaining parent_b root. uut.process_root(parent_a).unwrap(); assert_eq!(uut.roots(), &[parent_b, child_a].into()); } @@ -739,22 +890,24 @@ mod tests { #[test] fn process_root_rejects_non_root_node() { let mut uut = TestGraph::default(); - uut.insert_root(1).unwrap(); + uut.insert_with_no_parents(1).unwrap(); uut.insert_with_parent(2, 1).unwrap(); + uut.promote_all(); let err = uut.process_root(2).unwrap_err(); - let expected = GraphError::NotARootNode(2); + let expected = GraphError::InvalidRootNode(2); assert_eq!(err, expected); } #[test] fn process_root_cannot_reprocess_same_node() { let mut uut = TestGraph::default(); - uut.insert_root(1).unwrap(); + uut.insert_with_no_parents(1).unwrap(); + uut.promote_all(); uut.process_root(1).unwrap(); let err = uut.process_root(1).unwrap_err(); - let expected = GraphError::NotARootNode(1); + let expected = GraphError::InvalidRootNode(1); assert_eq!(err, expected); } @@ -764,11 +917,12 @@ mod tests { let nodes = (0..10).collect::>(); let mut uut = TestGraph::default(); - uut.insert_root(nodes[0]); + uut.insert_with_no_parents(nodes[0]); for pairs in nodes.windows(2) { let (parent, id) = (pairs[0], pairs[1]); uut.insert_with_parent(id, parent); } + uut.promote_all(); let mut ordered_roots = Vec::::new(); for node in &nodes { @@ -796,13 +950,14 @@ mod tests { let child_c = 7; let mut uut = TestGraph::default(); - uut.insert_root(ancestor_a).unwrap(); - uut.insert_root(ancestor_b).unwrap(); + uut.insert_with_no_parents(ancestor_a).unwrap(); + uut.insert_with_no_parents(ancestor_b).unwrap(); uut.insert_with_parent(parent_a, ancestor_a).unwrap(); uut.insert_with_parent(parent_b, ancestor_b).unwrap(); uut.insert_with_parents(child_a, [ancestor_a, parent_a].into()).unwrap(); uut.insert_with_parents(child_b, [parent_a, parent_b].into()).unwrap(); uut.insert_with_parent(child_c, parent_b).unwrap(); + uut.promote_all(); assert_eq!(uut.roots(), &[ancestor_a, ancestor_b].into()); diff --git a/crates/block-producer/src/mempool/mod.rs b/crates/block-producer/src/mempool/mod.rs index 3f64325c7..5e542372e 100644 --- a/crates/block-producer/src/mempool/mod.rs +++ b/crates/block-producer/src/mempool/mod.rs @@ -38,9 +38,14 @@ impl Display for BatchJobId { } impl BatchJobId { - pub fn increment(mut self) { + pub fn increment(&mut self) { self.0 += 1; } + + #[cfg(test)] + pub fn new(value: u64) -> Self { + Self(value) + } } #[derive(Debug, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] @@ -150,7 +155,8 @@ impl Mempool { /// /// Transactions are placed back in the queue. pub fn batch_failed(&mut self, batch: BatchJobId) { - let removed_batches = self.batches.purge_subgraphs([batch].into()); + let removed_batches = + self.batches.remove_batches([batch].into()).expect("Batch was not present"); // Its possible to receive failures for batches which were already removed // as part of a prior failure. Early exit to prevent logging these no-ops. @@ -168,7 +174,7 @@ impl Mempool { /// Marks a batch as proven if it exists. pub fn batch_proved(&mut self, batch_id: BatchJobId, batch: TransactionBatch) { - self.batches.mark_proven(batch_id, batch); + self.batches.submit_proof(batch_id, batch); } /// Select batches for the next block. @@ -197,7 +203,7 @@ impl Mempool { // Remove committed batches and transactions from graphs. let batches = self.block_in_progress.take().expect("No block in progress to commit"); - let transactions = self.batches.remove_committed(batches); + let transactions = self.batches.prune_committed(batches).expect("Batches failed to commit"); let transactions = self .transactions .commit_transactions(&transactions) @@ -221,16 +227,21 @@ impl Mempool { let batches = self.block_in_progress.take().expect("No block in progress to be failed"); // Remove all transactions from the graphs. - let purged = self.batches.purge_subgraphs(batches); + let purged = self.batches.remove_batches(batches).expect("Bad graph"); let batches = purged.keys().collect::>(); let transactions = purged.into_values().flatten().collect(); let transactions = self .transactions - .purge_subgraphs(transactions) + .remove_transactions(transactions) .expect("Transaction graph is malformed"); // Rollback state. + let transactions = transactions + .into_iter() + // FIXME + .map(|tx_id| todo!("Inflight state should remember diffs")) + .collect::>(); self.state.revert_transactions(&transactions); } } diff --git a/crates/block-producer/src/mempool/transaction_graph.rs b/crates/block-producer/src/mempool/transaction_graph.rs index 9e3f4d61b..37fafbd4b 100644 --- a/crates/block-producer/src/mempool/transaction_graph.rs +++ b/crates/block-producer/src/mempool/transaction_graph.rs @@ -21,9 +21,28 @@ use crate::domain::transaction::AuthenticatedTransaction; /// selection. Successful batches will eventually form part of a committed block at which point the /// transaction data may be safely [pruned](Self::prune_committed). /// -/// Transactions may also be outright [purged](Self::purge_subgraphs) from the graph. This is useful -/// for transactions which may have become invalid due to external considerations e.g. expired -/// transactions. +/// Transactions may also be outright [purged](Self::remove_transactions) from the graph. This is +/// useful for transactions which may have become invalid due to external considerations e.g. +/// expired transactions. +/// +/// # Transaction lifecycle: +/// ``` +/// │ +/// insert│ +/// ┌─────▼─────┐ +/// ┌─────────► ┼────┐ +/// │ └─────┬─────┘ │ +/// │ │ │ +/// requeue_transactions│ select_batch│ │ +/// │ ┌─────▼─────┐ │ +/// └─────────┼ in batch ┼────┤ +/// └─────┬─────┘ │ +/// │ │ +/// commit_transactions│ │remove_transactions +/// ┌─────▼─────┐ │ +/// │ ◄────┘ +/// └───────────┘ +/// ``` #[derive(Clone, Debug, Default, PartialEq)] pub struct TransactionGraph { inner: DependencyGraph, @@ -40,7 +59,8 @@ impl TransactionGraph { transaction: AuthenticatedTransaction, parents: BTreeSet, ) -> Result<(), GraphError> { - self.inner.insert(transaction.id(), transaction, parents) + self.inner.insert_pending(transaction.id(), parents)?; + self.inner.promote_pending(transaction.id(), transaction) } /// Selects a set of up-to count transactions for the next batch, as well as their parents. @@ -67,8 +87,10 @@ impl TransactionGraph { break; }; - // SAFETY: we retieved a root node, and therefore this node must exist. + // SAFETY: This is definitely a root since we just selected it from the set of roots. self.inner.process_root(root).unwrap(); + // SAFETY: Since it was a root batch, it must definitely have a processed batch + // associated with it. let tx = self.inner.get(&root).unwrap(); let tx_parents = self.inner.parents(&root).unwrap(); @@ -112,13 +134,13 @@ impl TransactionGraph { /// # Errors /// /// Follows the error conditions of [DependencyGraph::purge_subgraphs]. - pub fn purge_subgraphs( + pub fn remove_transactions( &mut self, transactions: Vec, - ) -> Result, GraphError> { + ) -> Result, GraphError> { // TODO: revisit this api. let transactions = transactions.into_iter().collect(); - self.inner.purge_subgraphs(transactions).map(|kv| kv.into_values().collect()) + self.inner.purge_subgraphs(transactions) } } From 225b6f436ad8d76d79d3d1954c68019f7cf4e23a Mon Sep 17 00:00:00 2001 From: Mirko <48352201+Mirko-von-Leipzig@users.noreply.github.com> Date: Sat, 2 Nov 2024 11:22:26 +0200 Subject: [PATCH 07/50] feat(block-producer): inflight state custody of transaction data (#534) --- .../src/mempool/inflight_state/mod.rs | 160 +++++++++--------- crates/block-producer/src/mempool/mod.rs | 12 +- .../src/mempool/transaction_graph.rs | 5 +- 3 files changed, 86 insertions(+), 91 deletions(-) diff --git a/crates/block-producer/src/mempool/inflight_state/mod.rs b/crates/block-producer/src/mempool/inflight_state/mod.rs index 673a924f1..76b8f8f84 100644 --- a/crates/block-producer/src/mempool/inflight_state/mod.rs +++ b/crates/block-producer/src/mempool/inflight_state/mod.rs @@ -1,5 +1,5 @@ use std::{ - collections::{BTreeMap, BTreeSet, VecDeque}, + collections::{btree_map::Entry, BTreeMap, BTreeSet, VecDeque}, sync::Arc, }; @@ -45,10 +45,13 @@ pub struct InflightState { /// Some of these may already be consumed - check the nullifiers. output_notes: BTreeMap, - /// Delta's representing the impact of each recently committed blocks on the inflight state. + /// Inflight transaction deltas. /// - /// These are used to prune committed state after `num_retained_blocks` have passed. - committed_state: VecDeque, + /// This _excludes_ deltas in committed blocks. + transaction_deltas: BTreeMap, + + /// Committed block deltas. + committed_blocks: VecDeque>, /// Amount of recently committed blocks we retain in addition to the inflight state. /// @@ -61,35 +64,23 @@ pub struct InflightState { chain_tip: BlockNumber, } -/// The aggregated impact of a set of sequential transactions on the [InflightState]. -#[derive(Clone, Default, Debug, PartialEq)] -struct StateDelta { - /// The number of transactions that affected each account. - account_transactions: BTreeMap, - - /// The nullifiers consumed by the transactions. +/// A summary of a transaction's impact on the state. +#[derive(Clone, Debug, PartialEq)] +struct Delta { + /// The account this transaction updated. + account: AccountId, + /// The nullifiers produced by this transaction. nullifiers: BTreeSet, - - /// The notes produced by the transactions. + /// The output notes created by this transaction. output_notes: BTreeSet, } -impl StateDelta { - fn new(txs: &[AuthenticatedTransaction]) -> Self { - let mut account_transactions = BTreeMap::::new(); - let mut nullifiers = BTreeSet::new(); - let mut output_notes = BTreeSet::new(); - - for tx in txs { - *account_transactions.entry(tx.account_id()).or_default() += 1; - nullifiers.extend(tx.nullifiers()); - output_notes.extend(tx.output_notes()); - } - +impl Delta { + fn new(tx: &AuthenticatedTransaction) -> Self { Self { - account_transactions, - nullifiers, - output_notes, + account: tx.account_id(), + nullifiers: tx.nullifiers().collect(), + output_notes: tx.output_notes().collect(), } } } @@ -104,7 +95,8 @@ impl InflightState { accounts: Default::default(), nullifiers: Default::default(), output_notes: Default::default(), - committed_state: Default::default(), + transaction_deltas: Default::default(), + committed_blocks: Default::default(), } } @@ -126,7 +118,7 @@ impl InflightState { fn oldest_committed_state(&self) -> BlockNumber { let committed_len: u32 = self - .committed_state + .committed_blocks .len() .try_into() .expect("We should not be storing many blocks"); @@ -213,6 +205,7 @@ impl InflightState { /// Aggregate the transaction into the state, returning its parent transactions. fn insert_transaction(&mut self, tx: &AuthenticatedTransaction) -> BTreeSet { + self.transaction_deltas.insert(tx.id(), Delta::new(tx)); let account_parent = self .accounts .entry(tx.account_id()) @@ -236,29 +229,30 @@ impl InflightState { account_parent.into_iter().chain(note_parents).collect() } - /// Reverts the given state diff. + /// Reverts the given set of _uncommitted_ transactions. /// /// # Panics /// - /// Panics if any part of the diff isn't present in the state. Callers should take - /// care to only revert transaction sets who's ancestors are all either committed or reverted. - pub fn revert_transactions(&mut self, txs: &[AuthenticatedTransaction]) { - let delta = StateDelta::new(txs); - for (account, count) in delta.account_transactions { - let status = self.accounts.get_mut(&account).expect("Account must exist").revert(count); + /// Panics if any transactions is not part of the uncommitted state. Callers should take care to + /// only revert transaction sets who's ancestors are all either committed or reverted. + pub fn revert_transactions(&mut self, txs: BTreeSet) { + for tx in txs { + let delta = self.transaction_deltas.remove(&tx).expect("Transaction delta must exist"); + // SAFETY: Since the delta exists, so must the account. + let account_status = self.accounts.get_mut(&delta.account).unwrap().revert(1); // Prune empty accounts. - if status.is_empty() { - self.accounts.remove(&account); + if account_status.is_empty() { + self.accounts.remove(&delta.account); } - } - for nullifier in delta.nullifiers { - assert!(self.nullifiers.remove(&nullifier), "Nullifier must exist"); - } + for nullifier in delta.nullifiers { + assert!(self.nullifiers.remove(&nullifier), "Nullifier must exist"); + } - for note in delta.output_notes { - assert!(self.output_notes.remove(¬e).is_some(), "Output note must exist"); + for note in delta.output_notes { + assert!(self.output_notes.remove(¬e).is_some(), "Output note must exist"); + } } } @@ -272,53 +266,57 @@ impl InflightState { /// /// # Panics /// - /// Panics if the accounts don't have enough inflight transactions to commit or if - /// the output notes don't exist. - pub fn commit_block(&mut self, txs: &[AuthenticatedTransaction]) { - let delta = StateDelta::new(txs); - for (account, count) in &delta.account_transactions { - self.accounts.get_mut(account).expect("Account must exist").commit(*count); - } + /// Panics if any transactions is not part of the uncommitted state. + pub fn commit_block(&mut self, txs: impl IntoIterator) { + let mut block_deltas = BTreeMap::new(); + for tx in txs.into_iter() { + let delta = self.transaction_deltas.remove(&tx).expect("Transaction delta must exist"); - for note in &delta.output_notes { - self.output_notes.get_mut(note).expect("Output note must exist").commit(); - } + // SAFETY: Since the delta exists, so must the account. + self.accounts.get_mut(&delta.account).unwrap().commit(1); - self.committed_state.push_back(delta); + for note in &delta.output_notes { + self.output_notes.get_mut(note).expect("Output note must exist").commit(); + } - if self.committed_state.len() > self.num_retained_blocks { - let delta = self.committed_state.pop_front().expect("Must be some due to length check"); - self.prune_committed_state(delta); + block_deltas.insert(tx, delta); } + self.committed_blocks.push_back(block_deltas); + self.prune_block(); self.chain_tip.increment(); } - /// Removes the delta from inflight state. + /// Prunes the state from the oldest committed block _IFF_ there are more than the number we + /// wish to retain. /// - /// # Panics - /// - /// Panics if the accounts don't have enough inflight transactions to commit. - fn prune_committed_state(&mut self, diff: StateDelta) { - for (account, count) in diff.account_transactions { - let status = self - .accounts - .get_mut(&account) - .expect("Account must exist") - .prune_committed(count); + /// This is used to bound the size of the inflight state. + fn prune_block(&mut self) { + // Keep the required number of committed blocks. + // + // This would occur on startup until we have accumulated enough blocks. + if self.committed_blocks.len() <= self.num_retained_blocks { + return; + } + // SAFETY: The length check above guarantees that we have at least one committed block. + let block = self.committed_blocks.pop_front().unwrap(); + + for (tx_id, delta) in block { + // SAFETY: Since the delta exists, so must the account. + let status = self.accounts.get_mut(&delta.account).unwrap().prune_committed(1); // Prune empty accounts. if status.is_empty() { - self.accounts.remove(&account); + self.accounts.remove(&delta.account); } - } - for nullifier in diff.nullifiers { - self.nullifiers.remove(&nullifier); - } + for nullifier in delta.nullifiers { + self.nullifiers.remove(&nullifier); + } - for output_note in diff.output_notes { - self.output_notes.remove(&output_note); + for output_note in delta.output_notes { + self.output_notes.remove(&output_note); + } } } } @@ -541,7 +539,7 @@ mod tests { uut.add_transaction(&tx1.clone()).unwrap(); // Commit the parents, which should remove them from dependency tracking. - uut.commit_block(&[tx0, tx1]); + uut.commit_block([tx0.id(), tx1.id()]); let parents = uut .add_transaction(&AuthenticatedTransaction::from_inner(tx).with_empty_store_state()) @@ -584,7 +582,9 @@ mod tests { panic!("Inserting tx #{idx} in iteration {i} should succeed: {err}") }); } - reverted.revert_transactions(&txs[txs.len() - i..]); + reverted.revert_transactions( + txs.iter().rev().take(i).rev().map(AuthenticatedTransaction::id).collect(), + ); let mut inserted = InflightState::new(BlockNumber::default(), 1); for (idx, tx) in txs.iter().rev().skip(i).rev().enumerate() { @@ -636,7 +636,7 @@ mod tests { panic!("Inserting tx #{idx} in iteration {i} should succeed: {err}") }); } - committed.commit_block(&txs[..i]); + committed.commit_block(txs.iter().take(i).map(AuthenticatedTransaction::id)); let mut inserted = InflightState::new(BlockNumber::new(1), 0); for (idx, tx) in txs.iter().skip(i).enumerate() { diff --git a/crates/block-producer/src/mempool/mod.rs b/crates/block-producer/src/mempool/mod.rs index 5e542372e..71743c074 100644 --- a/crates/block-producer/src/mempool/mod.rs +++ b/crates/block-producer/src/mempool/mod.rs @@ -204,13 +204,12 @@ impl Mempool { // Remove committed batches and transactions from graphs. let batches = self.block_in_progress.take().expect("No block in progress to commit"); let transactions = self.batches.prune_committed(batches).expect("Batches failed to commit"); - let transactions = self - .transactions + self.transactions .commit_transactions(&transactions) .expect("Transaction graph malformed"); // Inform inflight state about committed data. - self.state.commit_block(&transactions); + self.state.commit_block(transactions); self.chain_tip.increment(); } @@ -237,11 +236,6 @@ impl Mempool { .expect("Transaction graph is malformed"); // Rollback state. - let transactions = transactions - .into_iter() - // FIXME - .map(|tx_id| todo!("Inflight state should remember diffs")) - .collect::>(); - self.state.revert_transactions(&transactions); + self.state.revert_transactions(transactions); } } diff --git a/crates/block-producer/src/mempool/transaction_graph.rs b/crates/block-producer/src/mempool/transaction_graph.rs index 37fafbd4b..f11c968f3 100644 --- a/crates/block-producer/src/mempool/transaction_graph.rs +++ b/crates/block-producer/src/mempool/transaction_graph.rs @@ -121,10 +121,11 @@ impl TransactionGraph { pub fn commit_transactions( &mut self, tx_ids: &[TransactionId], - ) -> Result, GraphError> { + ) -> Result<(), GraphError> { // TODO: revisit this api. let tx_ids = tx_ids.iter().cloned().collect(); - self.inner.prune_processed(tx_ids) + self.inner.prune_processed(tx_ids)?; + Ok(()) } /// Removes the transactions and all their descendants from the graph. From 49846e3ea60bf56222057883618d1cb6b0dabb77 Mon Sep 17 00:00:00 2001 From: Bobbin Threadbare Date: Sat, 2 Nov 2024 10:30:23 -0700 Subject: [PATCH 08/50] chore: address comments --- CHANGELOG.md | 8 ++++---- Cargo.lock | 51 +++++++++++++++++++++++++++------------------------ Makefile | 2 +- 3 files changed, 32 insertions(+), 29 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ed2ecf837..7257a5da3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,21 +2,21 @@ ## v0.6.0 (TBD) -- Added `AccountCode` as part of `GetAccountProofs` endpoint response (#521). -- [BREAKING] Added `kernel_root` to block header's protobuf message definitions (#496). -- [BREAKING] Renamed `off-chain` and `on-chain` to `private` and `public` respectively for the account storage modes (#489). - Optimized state synchronizations by removing unnecessary fetching and parsing of note details (#462). - [BREAKING] Changed `GetAccountDetailsResponse` field to `details` (#481). - Improve `--version` by adding build metadata (#495). +- [BREAKING] Added `kernel_root` to block header's protobuf message definitions (#496). +- [BREAKING] Renamed `off-chain` and `on-chain` to `private` and `public` respectively for the account storage modes (#489). - [BREAKING] Introduced additional limits for note/account number (#503). - [BREAKING] Removed support for basic wallets in genesis creation (#510). - Added `GetAccountProofs` endpoint (#506). - Migrated faucet from actix-web to axum (#511). - Changed the `BlockWitness` to pass the inputs to the VM using only advice provider (#516). +- [BREAKING] Changed faucet storage type in the genesis to public. Using faucet from the genesis for faucet web app. Added support for faucet restarting without blockchain restarting (#517). - [BREAKING] Improved store API errors (return "not found" instead of "internal error" status if requested account(s) not found) (#518). +- Added `AccountCode` as part of `GetAccountProofs` endpoint response (#521). - [BREAKING] Migrated to v0.11 version of Miden VM (#528). - Reduce cloning in the store's `apply_block` (#532). -- [BREAKING] Changed faucet storage type in the genesis to public. Using faucet from the genesis for faucet web app. Added support for faucet restarting without blockchain restarting (#517). ## 0.5.1 (2024-09-12) diff --git a/Cargo.lock b/Cargo.lock index 7456d865a..4de7bc004 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -70,9 +70,9 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.9" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8365de52b16c035ff4fcafe0092ba9390540e3e352870ac09933bebcaa2c8c56" +checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" [[package]] name = "anstyle-parse" @@ -104,9 +104,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.91" +version = "1.0.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c042108f3ed77fd83760a5fd79b53be043192bb3b9dba91d8c574c0ada7850c8" +checksum = "74f37166d7d48a0284b99dd824694c26119c700b53bf0d1540cdb147dbdaaf13" [[package]] name = "arrayref" @@ -395,9 +395,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.1.31" +version = "1.1.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2e7962b54006dcfcc61cb72735f4d89bb97061dd6a7ed882ec6b8ee53714c6f" +checksum = "67b9470d453346108f93a59222a9a1a5724db32d0a4727b7ab7ace4b4d822dc9" dependencies = [ "jobserver", "libc", @@ -912,8 +912,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" dependencies = [ "cfg-if", + "js-sys", "libc", "wasi", + "wasm-bindgen", ] [[package]] @@ -1284,9 +1286,9 @@ dependencies = [ [[package]] name = "libm" -version = "0.2.10" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a00419de735aac21d53b0de5ce2c03bd3627277cf471300f27ebc89f7d828047" +checksum = "8355be11b20d696c8f18f6cc018c4e372165b1fa8126cef092399c9951984ffa" [[package]] name = "libredox" @@ -1402,7 +1404,7 @@ checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" [[package]] name = "miden-air" version = "0.10.5" -source = "git+https://github.com/0xPolygonMiden/miden-vm.git?branch=next#9f9cc63b709d8b7c83841d8e421701dcd33792bb" +source = "git+https://github.com/0xPolygonMiden/miden-vm.git?branch=next#f1c0553859c42c56f28d1d6aaf66d3e2f9907bd1" dependencies = [ "miden-core", "miden-thiserror", @@ -1413,7 +1415,7 @@ dependencies = [ [[package]] name = "miden-assembly" version = "0.10.5" -source = "git+https://github.com/0xPolygonMiden/miden-vm.git?branch=next#9f9cc63b709d8b7c83841d8e421701dcd33792bb" +source = "git+https://github.com/0xPolygonMiden/miden-vm.git?branch=next#f1c0553859c42c56f28d1d6aaf66d3e2f9907bd1" dependencies = [ "aho-corasick", "lalrpop", @@ -1430,7 +1432,7 @@ dependencies = [ [[package]] name = "miden-core" version = "0.10.5" -source = "git+https://github.com/0xPolygonMiden/miden-vm.git?branch=next#9f9cc63b709d8b7c83841d8e421701dcd33792bb" +source = "git+https://github.com/0xPolygonMiden/miden-vm.git?branch=next#f1c0553859c42c56f28d1d6aaf66d3e2f9907bd1" dependencies = [ "lock_api", "loom", @@ -1506,7 +1508,7 @@ dependencies = [ [[package]] name = "miden-lib" version = "0.6.0" -source = "git+https://github.com/0xPolygonMiden/miden-base.git?branch=next#b624d0810b1b3813f106aa38b796b9d62a39265d" +source = "git+https://github.com/0xPolygonMiden/miden-base.git?branch=next#f621c437bd5eb67d7b4a82d98a3eec8d238bb6cb" dependencies = [ "miden-assembly", "miden-objects", @@ -1703,8 +1705,9 @@ dependencies = [ [[package]] name = "miden-objects" version = "0.6.0" -source = "git+https://github.com/0xPolygonMiden/miden-base.git?branch=next#b624d0810b1b3813f106aa38b796b9d62a39265d" +source = "git+https://github.com/0xPolygonMiden/miden-base.git?branch=next#f621c437bd5eb67d7b4a82d98a3eec8d238bb6cb" dependencies = [ + "getrandom", "miden-assembly", "miden-core", "miden-crypto", @@ -1717,7 +1720,7 @@ dependencies = [ [[package]] name = "miden-processor" version = "0.10.6" -source = "git+https://github.com/0xPolygonMiden/miden-vm.git?branch=next#9f9cc63b709d8b7c83841d8e421701dcd33792bb" +source = "git+https://github.com/0xPolygonMiden/miden-vm.git?branch=next#f1c0553859c42c56f28d1d6aaf66d3e2f9907bd1" dependencies = [ "miden-air", "miden-core", @@ -1728,7 +1731,7 @@ dependencies = [ [[package]] name = "miden-prover" version = "0.10.5" -source = "git+https://github.com/0xPolygonMiden/miden-vm.git?branch=next#9f9cc63b709d8b7c83841d8e421701dcd33792bb" +source = "git+https://github.com/0xPolygonMiden/miden-vm.git?branch=next#f1c0553859c42c56f28d1d6aaf66d3e2f9907bd1" dependencies = [ "miden-air", "miden-processor", @@ -1743,7 +1746,7 @@ version = "0.6.0" [[package]] name = "miden-stdlib" version = "0.10.5" -source = "git+https://github.com/0xPolygonMiden/miden-vm.git?branch=next#9f9cc63b709d8b7c83841d8e421701dcd33792bb" +source = "git+https://github.com/0xPolygonMiden/miden-vm.git?branch=next#f1c0553859c42c56f28d1d6aaf66d3e2f9907bd1" dependencies = [ "miden-assembly", ] @@ -1771,7 +1774,7 @@ dependencies = [ [[package]] name = "miden-tx" version = "0.6.0" -source = "git+https://github.com/0xPolygonMiden/miden-base.git?branch=next#b624d0810b1b3813f106aa38b796b9d62a39265d" +source = "git+https://github.com/0xPolygonMiden/miden-base.git?branch=next#f621c437bd5eb67d7b4a82d98a3eec8d238bb6cb" dependencies = [ "async-trait", "miden-assembly", @@ -1790,7 +1793,7 @@ dependencies = [ [[package]] name = "miden-verifier" version = "0.10.5" -source = "git+https://github.com/0xPolygonMiden/miden-vm.git?branch=next#9f9cc63b709d8b7c83841d8e421701dcd33792bb" +source = "git+https://github.com/0xPolygonMiden/miden-vm.git?branch=next#f1c0553859c42c56f28d1d6aaf66d3e2f9907bd1" dependencies = [ "miden-air", "miden-core", @@ -2804,9 +2807,9 @@ checksum = "b7401a30af6cb5818bb64852270bb722533397edcfc7344954a38f420819ece2" [[package]] name = "syn" -version = "2.0.85" +version = "2.0.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5023162dfcd14ef8f32034d8bcd4cc5ddc61ef7a247c024a33e24e1f24d21b56" +checksum = "25aa4ce346d03a6dcd68dd8b4010bcb74e54e62c90c573f394c46eae99aba32d" dependencies = [ "proc-macro2", "quote", @@ -2887,18 +2890,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.65" +version = "1.0.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d11abd9594d9b38965ef50805c5e469ca9cc6f197f883f717e0269a3057b3d5" +checksum = "5d171f59dbaa811dbbb1aee1e73db92ec2b122911a48e1390dfe327a821ddede" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.65" +version = "1.0.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae71770322cbd277e69d762a16c444af02aa0575ac0d174f0b9562d3b37f8602" +checksum = "b08be0f17bd307950653ce45db00cd31200d82b624b36e181337d9c7d92765b5" dependencies = [ "proc-macro2", "quote", diff --git a/Makefile b/Makefile index b10ad72a9..afa1a2da9 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,7 @@ BUILD_PROTO=BUILD_PROTO=1 .PHONY: clippy clippy: ## Runs Clippy with configs - cargo clippy --locked --workspace --all-targets --all-features -- -D warnings --allow clippy::arc_with_non_send_sync + cargo clippy --locked --workspace --all-targets --all-features -- -D warnings .PHONY: fix From 2ceec74ebf8e5a3bee92171b57c6fdece0f25bb6 Mon Sep 17 00:00:00 2001 From: Mirko <48352201+Mirko-von-Leipzig@users.noreply.github.com> Date: Tue, 5 Nov 2024 09:29:45 +0200 Subject: [PATCH 09/50] feat(block-producer): promote redesign (#541) Replace FIFO with the mempool block-producer rework. This still has known bugs and issues; notably the client's integration tests sometimes hang. --- .../block-producer/src/batch_builder/batch.rs | 4 +- .../block-producer/src/batch_builder/mod.rs | 365 +++++------------- .../block-producer/src/block_builder/mod.rs | 168 ++++---- .../src/block_builder/prover/block_witness.rs | 3 +- .../src/block_builder/prover/tests.rs | 2 +- crates/block-producer/src/errors.rs | 41 +- crates/block-producer/src/lib.rs | 23 +- .../block-producer/src/mempool/batch_graph.rs | 1 - .../src/mempool/dependency_graph.rs | 29 +- .../mempool/inflight_state/account_state.rs | 30 +- .../src/mempool/inflight_state/mod.rs | 18 +- crates/block-producer/src/mempool/mod.rs | 70 +++- .../src/mempool/transaction_graph.rs | 7 +- crates/block-producer/src/server/api.rs | 77 ---- crates/block-producer/src/server/mod.rs | 218 ++++++----- .../src/state_view/account_state.rs | 123 ------ crates/block-producer/src/state_view/mod.rs | 276 ------------- .../src/state_view/tests/apply_block.rs | 170 -------- .../src/state_view/tests/mod.rs | 42 -- .../src/state_view/tests/verify_tx.rs | 306 --------------- crates/block-producer/src/store/mod.rs | 59 +-- crates/block-producer/src/test_utils/batch.rs | 2 +- crates/block-producer/src/test_utils/block.rs | 2 +- crates/block-producer/src/test_utils/mod.rs | 1 - crates/block-producer/src/test_utils/store.rs | 43 +-- crates/block-producer/src/txqueue/mod.rs | 174 --------- .../block-producer/src/txqueue/tests/mod.rs | 225 ----------- 27 files changed, 409 insertions(+), 2070 deletions(-) delete mode 100644 crates/block-producer/src/server/api.rs delete mode 100644 crates/block-producer/src/state_view/account_state.rs delete mode 100644 crates/block-producer/src/state_view/mod.rs delete mode 100644 crates/block-producer/src/state_view/tests/apply_block.rs delete mode 100644 crates/block-producer/src/state_view/tests/mod.rs delete mode 100644 crates/block-producer/src/state_view/tests/verify_tx.rs delete mode 100644 crates/block-producer/src/txqueue/mod.rs delete mode 100644 crates/block-producer/src/txqueue/tests/mod.rs diff --git a/crates/block-producer/src/batch_builder/batch.rs b/crates/block-producer/src/batch_builder/batch.rs index 6490a95a7..be0c28cb8 100644 --- a/crates/block-producer/src/batch_builder/batch.rs +++ b/crates/block-producer/src/batch_builder/batch.rs @@ -9,13 +9,13 @@ use miden_objects::{ batches::BatchNoteTree, crypto::hash::blake::{Blake3Digest, Blake3_256}, notes::{NoteHeader, NoteId, Nullifier}, - transaction::{InputNoteCommitment, OutputNote, TransactionId}, + transaction::{InputNoteCommitment, OutputNote, ProvenTransaction, TransactionId}, AccountDeltaError, Digest, MAX_ACCOUNTS_PER_BATCH, MAX_INPUT_NOTES_PER_BATCH, MAX_OUTPUT_NOTES_PER_BATCH, }; use tracing::instrument; -use crate::{errors::BuildBatchError, ProvenTransaction}; +use crate::errors::BuildBatchError; pub type BatchId = Blake3Digest<32>; diff --git a/crates/block-producer/src/batch_builder/mod.rs b/crates/block-producer/src/batch_builder/mod.rs index 59b45949b..830c68c3b 100644 --- a/crates/block-producer/src/batch_builder/mod.rs +++ b/crates/block-producer/src/batch_builder/mod.rs @@ -1,243 +1,111 @@ -use std::{ - cmp::min, - collections::{BTreeMap, BTreeSet}, - num::NonZeroUsize, - ops::Range, - sync::Arc, - time::Duration, -}; +use std::{num::NonZeroUsize, ops::Range, time::Duration}; -use miden_objects::{ - accounts::AccountId, - assembly::SourceManager, - notes::NoteId, - transaction::{OutputNote, TransactionId}, - Digest, -}; use rand::Rng; -use tokio::{sync::Mutex, task::JoinSet, time}; -use tonic::async_trait; +use tokio::{task::JoinSet, time}; use tracing::{debug, info, instrument, Span}; use crate::{ - block_builder::BlockBuilder, domain::transaction::AuthenticatedTransaction, - mempool::{BatchJobId, Mempool}, - ProvenTransaction, SharedRwVec, COMPONENT, + mempool::{BatchJobId, SharedMempool}, + COMPONENT, SERVER_BUILD_BATCH_FREQUENCY, }; -#[cfg(test)] -mod tests; +// FIXME: fix the batch builder tests. +// #[cfg(test)] +// mod tests; pub mod batch; pub use batch::TransactionBatch; use miden_node_utils::formatting::{format_array, format_blake3_digest}; -use crate::{errors::BuildBatchError, store::Store}; +use crate::errors::BuildBatchError; // BATCH BUILDER // ================================================================================================ -/// Abstraction over batch proving of transactions. -/// -/// Transactions are aggregated into batches prior to being added to blocks. This trait abstracts -/// over this responsibility. The trait's goal is to be implementation agnostic, allowing for -/// multiple implementations, e.g.: -/// -/// - in-process cpu based prover -/// - out-of-process gpu based prover -/// - distributed prover on another machine -#[async_trait] -pub trait BatchBuilder: Send + Sync + 'static { - /// Start proving of a new batch. - async fn build_batch(&self, txs: Vec) -> Result<(), BuildBatchError>; -} - -// DEFAULT BATCH BUILDER -// ================================================================================================ - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct DefaultBatchBuilderOptions { - /// The frequency at which blocks are created - pub block_frequency: Duration, - - /// Maximum number of batches in any given block - pub max_batches_per_block: usize, -} - -pub struct DefaultBatchBuilder { - store: Arc, - - block_builder: Arc, - - options: DefaultBatchBuilderOptions, - - /// Batches ready to be included in a block - ready_batches: SharedRwVec, +pub struct BatchBuilder { + pub batch_interval: Duration, + pub workers: NonZeroUsize, + /// Used to simulate batch proving by sleeping for a random duration selected from this range. + pub simulated_proof_time: Range, + /// Simulated block failure rate as a percentage. + /// + /// Note: this _must_ be sign positive and less than 1.0. + pub failure_rate: f32, } -// FIXME: remove the allow when the upstream clippy issue is fixed: -// https://github.com/rust-lang/rust-clippy/issues/12281 -#[allow(clippy::blocks_in_conditions)] -impl DefaultBatchBuilder -where - S: Store, - BB: BlockBuilder, -{ - // CONSTRUCTOR - // -------------------------------------------------------------------------------------------- - /// Returns an new [BatchBuilder] instantiated with the provided [BlockBuilder] and the - /// specified options. - pub fn new(store: Arc, block_builder: Arc, options: DefaultBatchBuilderOptions) -> Self { +impl Default for BatchBuilder { + fn default() -> Self { Self { - store, - block_builder, - options, - ready_batches: Default::default(), + batch_interval: SERVER_BUILD_BATCH_FREQUENCY, + // SAFETY: 2 is non-zero so this always succeeds. + workers: 2.try_into().unwrap(), + // Note: The range cannot be empty. + simulated_proof_time: Duration::ZERO..Duration::from_millis(1), + failure_rate: 0.0, } } +} - // BATCH BUILDER STARTER - // -------------------------------------------------------------------------------------------- - pub async fn run(self: Arc) { - let mut interval = time::interval(self.options.block_frequency); - - info!(target: COMPONENT, period_ms = interval.period().as_millis(), "Batch builder started"); - - loop { - interval.tick().await; - self.try_build_block().await; - } - } - - // HELPER METHODS - // -------------------------------------------------------------------------------------------- - - /// Note that we call `build_block()` regardless of whether the `ready_batches` queue is empty. - /// A call to an empty `build_block()` indicates that an empty block should be created. - #[instrument(target = "miden-block-producer", skip_all)] - async fn try_build_block(&self) { - let mut batches_in_block: Vec = { - let mut locked_ready_batches = self.ready_batches.write().await; - - let num_batches_in_block = - min(self.options.max_batches_per_block, locked_ready_batches.len()); - - locked_ready_batches.drain(..num_batches_in_block).collect() - }; - - match self.block_builder.build_block(&batches_in_block).await { - Ok(_) => { - // block successfully built, do nothing - }, - Err(_) => { - // Block building failed; add back the batches at the end of the queue - self.ready_batches.write().await.append(&mut batches_in_block); - }, - } - } - - /// Returns a list of IDs for unauthenticated notes which are not output notes of any ready - /// transaction batch or the candidate batch itself. - async fn find_dangling_notes(&self, txs: &[ProvenTransaction]) -> Vec { - // TODO: We can optimize this by looking at the notes created in the previous batches - - // build a set of output notes from all ready batches and the candidate batch - let mut all_output_notes: BTreeSet = txs - .iter() - .flat_map(|tx| tx.output_notes().iter().map(OutputNote::id)) - .chain( - self.ready_batches - .read() - .await - .iter() - .flat_map(|batch| batch.output_notes().iter().map(OutputNote::id)), - ) - .collect(); +impl BatchBuilder { + /// Starts the [BatchProducer], creating and proving batches at the configured interval. + /// + /// A pool of batch-proving workers is spawned, which are fed new batch jobs periodically. + /// A batch is skipped if there are no available workers, or if there are no transactions + /// available to batch. + pub async fn run(self, mempool: SharedMempool) { + assert!( + self.failure_rate < 1.0 && self.failure_rate.is_sign_positive(), + "Failure rate must be a percentage" + ); - // from the list of unauthenticated notes in the candidate batch, filter out any note - // which is also an output note either in any of the ready batches or in the candidate - // batch itself - txs.iter() - .flat_map(|tx| tx.get_unauthenticated_notes().map(|note| note.id())) - .filter(|note_id| !all_output_notes.remove(note_id)) - .collect() - } -} + let mut interval = tokio::time::interval(self.batch_interval); + interval.set_missed_tick_behavior(time::MissedTickBehavior::Delay); -// FIXME: remove the allow when the upstream clippy issue is fixed: -// https://github.com/rust-lang/rust-clippy/issues/12281 -#[allow(clippy::blocks_in_conditions)] -#[async_trait] -impl BatchBuilder for DefaultBatchBuilder -where - S: Store, - BB: BlockBuilder, -{ - #[instrument(target = "miden-block-producer", skip_all, err, fields(batch_id))] - async fn build_batch(&self, txs: Vec) -> Result<(), BuildBatchError> { - let num_txs = txs.len(); + let mut inflight = WorkerPool::new(self.simulated_proof_time, self.failure_rate); - info!(target: COMPONENT, num_txs, "Building a transaction batch"); - debug!(target: COMPONENT, txs = %format_array(txs.iter().map(|tx| tx.id().to_hex()))); + loop { + tokio::select! { + _ = interval.tick() => { + if inflight.len() >= self.workers.get() { + tracing::info!("All batch workers occupied."); + continue; + } - // make sure that all unauthenticated notes in the transactions of the proposed batch - // have been either created in any of the ready batches (or the batch itself) or are - // already in the store - // - // TODO: this can be optimized by first computing dangling notes of the batch itself, - // and only then checking against the other ready batches - let dangling_notes = self.find_dangling_notes(&txs).await; - let found_unauthenticated_notes = match dangling_notes.is_empty() { - true => Default::default(), - false => { - let stored_notes = - match self.store.get_note_authentication_info(dangling_notes.iter()).await { - Ok(stored_notes) => stored_notes, - Err(err) => return Err(BuildBatchError::NotePathsError(err, txs)), + // Transactions available? + let Some((batch_id, transactions)) = + mempool.lock().await.select_batch() + else { + tracing::info!("No transactions available for batch."); + continue; }; - let missing_notes: Vec<_> = dangling_notes - .into_iter() - .filter(|note_id| !stored_notes.contains_note(note_id)) - .collect(); - if !missing_notes.is_empty() { - return Err(BuildBatchError::UnauthenticatedNotesNotFound(missing_notes, txs)); + inflight.spawn(batch_id, transactions); + }, + result = inflight.join_next() => { + let mut mempool = mempool.lock().await; + match result { + Err(err) => { + tracing::warn!(%err, "Batch job panic'd.") + // TODO: somehow embed the batch ID into the join error, though this doesn't seem possible? + // mempool.batch_failed(batch_id); + }, + Ok(Err((batch_id, err))) => { + tracing::warn!(%batch_id, %err, "Batch job failed."); + mempool.batch_failed(batch_id); + }, + Ok(Ok((batch_id, batch))) => { + mempool.batch_proved(batch_id, batch); + } + } } - - stored_notes - }, - }; - - let batch = TransactionBatch::new(txs, found_unauthenticated_notes)?; - - info!(target: COMPONENT, "Transaction batch built"); - Span::current().record("batch_id", format_blake3_digest(batch.id())); - - let num_batches = { - let mut write_guard = self.ready_batches.write().await; - write_guard.push(batch); - write_guard.len() - }; - - info!(target: COMPONENT, num_batches, "Transaction batch added to the batch queue"); - - Ok(()) + } + } } } -pub struct BatchProducer { - pub batch_interval: Duration, - pub workers: NonZeroUsize, - pub mempool: Arc>, - pub tx_per_batch: usize, - /// Used to simulate batch proving by sleeping for a random duration selected from this range. - pub simulated_proof_time: Range, - /// Simulated block failure rate as a percentage. - /// - /// Note: this _must_ be sign positive and less than 1.0. - pub failure_rate: f32, -} +// BATCH WORKER +// ================================================================================================ type BatchResult = Result<(BatchJobId, TransactionBatch), (BatchJobId, BuildBatchError)>; @@ -285,84 +153,43 @@ impl WorkerPool { async move { tracing::debug!("Begin proving batch."); - // TODO: This is a deep clone which can be avoided by change batch building to using - // refs or arcs. - let transactions = transactions - .iter() - .map(AuthenticatedTransaction::raw_proven_transaction) - .cloned() - .collect(); + let batch = Self::build_batch(transactions).map_err(|err| (id, err))?; tokio::time::sleep(simulated_proof_time).await; if failed { tracing::debug!("Batch proof failure injected."); - return Err((id, BuildBatchError::InjectedFailure(transactions))); + return Err((id, BuildBatchError::InjectedFailure)); } - let batch = TransactionBatch::new(transactions, Default::default()) - .map_err(|err| (id, err))?; - tracing::debug!("Batch proof completed."); Ok((id, batch)) } }); } -} -impl BatchProducer { - /// Starts the [BatchProducer], creating and proving batches at the configured interval. - /// - /// A pool of batch-proving workers is spawned, which are fed new batch jobs periodically. - /// A batch is skipped if there are no available workers, or if there are no transactions - /// available to batch. - pub async fn run(self) { - assert!( - self.failure_rate < 1.0 && self.failure_rate.is_sign_positive(), - "Failure rate must be a percentage" - ); - - let mut interval = tokio::time::interval(self.batch_interval); - interval.set_missed_tick_behavior(time::MissedTickBehavior::Delay); + #[instrument(target = "miden-block-producer", skip_all, err, fields(batch_id))] + fn build_batch( + txs: Vec, + ) -> Result { + let num_txs = txs.len(); - let mut inflight = WorkerPool::new(self.simulated_proof_time, self.failure_rate); + info!(target: COMPONENT, num_txs, "Building a transaction batch"); + debug!(target: COMPONENT, txs = %format_array(txs.iter().map(|tx| tx.id().to_hex()))); - loop { - tokio::select! { - _ = interval.tick() => { - if inflight.len() >= self.workers.get() { - tracing::info!("All batch workers occupied."); - continue; - } + // TODO: This is a deep clone which can be avoided by change batch building to using + // refs or arcs. + let txs = txs + .iter() + .map(AuthenticatedTransaction::raw_proven_transaction) + .cloned() + .collect(); + // TODO: Found unauthenticated notes are no longer required.. potentially? + let batch = TransactionBatch::new(txs, Default::default())?; - // Transactions available? - let Some((batch_id, transactions)) = - self.mempool.lock().await.select_batch() - else { - tracing::info!("No transactions available for batch."); - continue; - }; + Span::current().record("batch_id", format_blake3_digest(batch.id())); + info!(target: COMPONENT, "Transaction batch built"); - inflight.spawn(batch_id, transactions); - }, - result = inflight.join_next() => { - let mut mempool = self.mempool.lock().await; - match result { - Err(err) => { - tracing::warn!(%err, "Batch job panic'd.") - // TODO: somehow embed the batch ID into the join error, though this doesn't seem possible? - // mempool.batch_failed(batch_id); - }, - Ok(Err((batch_id, err))) => { - tracing::warn!(%batch_id, %err, "Batch job failed."); - mempool.batch_failed(batch_id); - }, - Ok(Ok((batch_id, batch))) => { - mempool.batch_proved(batch_id, batch); - } - } - } - } - } + Ok(batch) } } diff --git a/crates/block-producer/src/block_builder/mod.rs b/crates/block-producer/src/block_builder/mod.rs index bf36e4642..f67501cbe 100644 --- a/crates/block-producer/src/block_builder/mod.rs +++ b/crates/block-producer/src/block_builder/mod.rs @@ -1,10 +1,5 @@ -use std::{ - collections::{BTreeMap, BTreeSet}, - ops::Range, - sync::Arc, -}; +use std::{collections::BTreeSet, ops::Range}; -use async_trait::async_trait; use miden_node_utils::formatting::{format_array, format_blake3_digest}; use miden_objects::{ accounts::AccountId, @@ -13,67 +8,96 @@ use miden_objects::{ transaction::InputNoteCommitment, }; use rand::Rng; -use tokio::{sync::Mutex, time::Duration}; +use tokio::time::Duration; use tracing::{debug, info, instrument}; use crate::{ batch_builder::batch::TransactionBatch, errors::BuildBlockError, - mempool::{BatchJobId, Mempool}, + mempool::SharedMempool, store::{ApplyBlock, DefaultStore, Store}, - COMPONENT, + COMPONENT, SERVER_BLOCK_FREQUENCY, }; pub(crate) mod prover; use self::prover::{block_witness::BlockWitness, BlockProver}; -#[cfg(test)] -mod tests; +// FIXME: reimplement the tests. +// #[cfg(test)] +// mod tests; // BLOCK BUILDER // ================================================================================================= -#[async_trait] -pub trait BlockBuilder: Send + Sync + 'static { - /// Receive batches to be included in a block. An empty vector indicates that no batches were - /// ready, and that an empty block should be created. +pub struct BlockBuilder { + pub block_interval: Duration, + /// Used to simulate block proving by sleeping for a random duration selected from this range. + pub simulated_proof_time: Range, + + /// Simulated block failure rate as a percentage. /// - /// The `BlockBuilder` relies on `build_block()` to be called as a precondition to creating a - /// block. In other words, if `build_block()` is never called, then no blocks are produced. - async fn build_block(&self, batches: &[TransactionBatch]) -> Result<(), BuildBlockError>; -} + /// Note: this _must_ be sign positive and less than 1.0. + pub failure_rate: f32, -#[derive(Debug)] -pub struct DefaultBlockBuilder { - store: Arc, - state_view: Arc, - block_kernel: BlockProver, + pub store: DefaultStore, + pub block_kernel: BlockProver, } -impl DefaultBlockBuilder -where - S: Store, - A: ApplyBlock, -{ - pub fn new(store: Arc, state_view: Arc) -> Self { +impl BlockBuilder { + pub fn new(store: DefaultStore) -> Self { Self { - store, - state_view, + block_interval: SERVER_BLOCK_FREQUENCY, + // Note: The range cannot be empty. + simulated_proof_time: Duration::ZERO..Duration::from_millis(1), + failure_rate: 0.0, block_kernel: BlockProver::new(), + store, + } + } + /// Starts the [BlockBuilder], infinitely producing blocks at the configured interval. + /// + /// Block production is sequential and consists of + /// + /// 1. Pulling the next set of batches from the [Mempool] + /// 2. Compiling these batches into the next block + /// 3. Proving the block (this is simulated using random sleeps) + /// 4. Committing the block to the store + pub async fn run(self, mempool: SharedMempool) { + assert!( + self.failure_rate < 1.0 && self.failure_rate.is_sign_positive(), + "Failure rate must be a percentage" + ); + + let mut interval = tokio::time::interval(self.block_interval); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + + loop { + interval.tick().await; + + let (block_number, batches) = mempool.lock().await.select_block(); + let batches = batches.into_values().collect::>(); + + let mut result = self.build_block(&batches).await; + let proving_duration = rand::thread_rng().gen_range(self.simulated_proof_time.clone()); + + tokio::time::sleep(proving_duration).await; + + // Randomly inject failures at the given rate. + // + // Note: Rng::gen rolls between [0, 1.0) for f32, so this works as expected. + if rand::thread_rng().gen::() < self.failure_rate { + result = Err(BuildBlockError::InjectedFailure); + } + + let mut mempool = mempool.lock().await; + match result { + Ok(_) => mempool.block_committed(block_number), + Err(_) => mempool.block_failed(block_number), + } } } -} -// FIXME: remove the allow when the upstream clippy issue is fixed: -// https://github.com/rust-lang/rust-clippy/issues/12281 -#[allow(clippy::blocks_in_conditions)] -#[async_trait] -impl BlockBuilder for DefaultBlockBuilder -where - S: Store, - A: ApplyBlock, -{ #[instrument(target = "miden-block-producer", skip_all, err)] async fn build_block(&self, batches: &[TransactionBatch]) -> Result<(), BuildBlockError> { info!( @@ -143,68 +167,10 @@ where info!(target: COMPONENT, block_num, %block_hash, "block built"); debug!(target: COMPONENT, ?block); - self.state_view.apply_block(&block).await?; + self.store.apply_block(&block).await?; info!(target: COMPONENT, block_num, %block_hash, "block committed"); Ok(()) } } - -struct BlockProducer { - pub mempool: Arc>, - pub block_interval: Duration, - pub block_builder: BB, - /// Used to simulate block proving by sleeping for a random duration selected from this range. - pub simulated_proof_time: Range, - - /// Simulated block failure rate as a percentage. - /// - /// Note: this _must_ be sign positive and less than 1.0. - pub failure_rate: f32, -} - -impl BlockProducer { - /// Starts the [BlockProducer], infinitely producing blocks at the configured interval. - /// - /// Block production is sequential and consists of - /// - /// 1. Pulling the next set of batches from the [Mempool] - /// 2. Compiling these batches into the next block - /// 3. Proving the block (this is simulated using random sleeps) - /// 4. Committing the block to the store - pub async fn run(self) { - assert!( - self.failure_rate < 1.0 && self.failure_rate.is_sign_positive(), - "Failure rate must be a percentage" - ); - - let mut interval = tokio::time::interval(self.block_interval); - interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); - - loop { - interval.tick().await; - - let (block_number, batches) = self.mempool.lock().await.select_block(); - let batches = batches.into_values().collect::>(); - - let mut result = self.block_builder.build_block(&batches).await; - let proving_duration = rand::thread_rng().gen_range(self.simulated_proof_time.clone()); - - tokio::time::sleep(proving_duration).await; - - // Randomly inject failures at the given rate. - // - // Note: Rng::gen rolls between [0, 1.0) for f32, so this works as expected. - if rand::thread_rng().gen::() < self.failure_rate { - result = Err(BuildBlockError::InjectedFailure); - } - - let mut mempool = self.mempool.lock().await; - match result { - Ok(_) => mempool.block_committed(block_number), - Err(_) => mempool.block_failed(block_number), - } - } - } -} diff --git a/crates/block-producer/src/block_builder/prover/block_witness.rs b/crates/block-producer/src/block_builder/prover/block_witness.rs index 0b4f06567..aad05ce51 100644 --- a/crates/block-producer/src/block_builder/prover/block_witness.rs +++ b/crates/block-producer/src/block_builder/prover/block_witness.rs @@ -11,10 +11,9 @@ use miden_objects::{ }; use crate::{ - batch_builder::batch::AccountUpdate, + batch_builder::{batch::AccountUpdate, TransactionBatch}, block::BlockInputs, errors::{BlockProverError, BuildBlockError}, - TransactionBatch, }; // BLOCK WITNESS diff --git a/crates/block-producer/src/block_builder/prover/tests.rs b/crates/block-producer/src/block_builder/prover/tests.rs index 3d776ffc6..a28f86f9f 100644 --- a/crates/block-producer/src/block_builder/prover/tests.rs +++ b/crates/block-producer/src/block_builder/prover/tests.rs @@ -20,13 +20,13 @@ use miden_objects::{ use self::block_witness::AccountUpdateWitness; use super::*; use crate::{ + batch_builder::TransactionBatch, block::{AccountWitness, BlockInputs}, store::Store, test_utils::{ block::{build_actual_block_header, build_expected_block_header, MockBlockBuilder}, MockProvenTxBuilder, MockStoreSuccessBuilder, }, - TransactionBatch, }; // BLOCK WITNESS TESTS diff --git a/crates/block-producer/src/errors.rs b/crates/block-producer/src/errors.rs index a546e5a9e..493116f4c 100644 --- a/crates/block-producer/src/errors.rs +++ b/crates/block-producer/src/errors.rs @@ -120,21 +120,12 @@ pub enum BuildBatchError { )] TooManyAccountsInBatch(Vec), - #[error("Failed to create notes SMT: {0}")] - NotesSmtError(MerkleError, Vec), - - #[error("Failed to get note paths: {0}")] - NotePathsError(NotePathsError, Vec), - #[error("Duplicated unauthenticated transaction input note ID in the batch: {0}")] DuplicateUnauthenticatedNote(NoteId, Vec), #[error("Duplicated transaction output note ID in the batch: {0}")] DuplicateOutputNote(NoteId, Vec), - #[error("Unauthenticated transaction notes not found in the store: {0:?}")] - UnauthenticatedNotesNotFound(Vec, Vec), - #[error("Note hashes mismatch for note {id}: (input: {input_hash}, output: {output_hash})")] NoteHashesMismatch { id: NoteId, @@ -151,25 +142,7 @@ pub enum BuildBatchError { }, #[error("Nothing actually went wrong, failure was injected on purpose")] - InjectedFailure(Vec), -} - -impl BuildBatchError { - pub fn into_transactions(self) -> Vec { - match self { - BuildBatchError::TooManyInputNotes(_, txs) => txs, - BuildBatchError::TooManyNotesCreated(_, txs) => txs, - BuildBatchError::TooManyAccountsInBatch(txs) => txs, - BuildBatchError::NotesSmtError(_, txs) => txs, - BuildBatchError::NotePathsError(_, txs) => txs, - BuildBatchError::DuplicateUnauthenticatedNote(_, txs) => txs, - BuildBatchError::DuplicateOutputNote(_, txs) => txs, - BuildBatchError::UnauthenticatedNotesNotFound(_, txs) => txs, - BuildBatchError::NoteHashesMismatch { txs, .. } => txs, - BuildBatchError::AccountUpdateError { txs, .. } => txs, - BuildBatchError::InjectedFailure(txs) => txs, - } - } + InjectedFailure, } // Block prover errors @@ -199,18 +172,6 @@ pub enum BlockInputsError { GrpcClientError(String), } -// Note paths errors -// ================================================================================================= - -#[allow(clippy::enum_variant_names)] -#[derive(Debug, PartialEq, Eq, Error)] -pub enum NotePathsError { - #[error("failed to parse protobuf message: {0}")] - ConversionError(#[from] ConversionError), - #[error("gRPC client failed with error: {0}")] - GrpcClientError(String), -} - // Block applying errors // ================================================================================================= diff --git a/crates/block-producer/src/lib.rs b/crates/block-producer/src/lib.rs index 421718ce9..961e1b5e7 100644 --- a/crates/block-producer/src/lib.rs +++ b/crates/block-producer/src/lib.rs @@ -1,11 +1,4 @@ -// TODO: remove once block-producer rework is complete -#![allow(unused)] - -use std::{sync::Arc, time::Duration}; - -use batch_builder::batch::TransactionBatch; -use miden_objects::transaction::ProvenTransaction; -use tokio::sync::RwLock; +use std::time::Duration; #[cfg(test)] pub mod test_utils; @@ -15,20 +8,12 @@ mod block_builder; mod domain; mod errors; mod mempool; -mod state_view; mod store; -mod txqueue; pub mod block; pub mod config; pub mod server; -// TYPE ALIASES -// ================================================================================================= - -/// A vector that can be shared across threads -pub(crate) type SharedRwVec = Arc>>; - // CONSTANTS // ================================================================================================= @@ -46,3 +31,9 @@ const SERVER_BUILD_BATCH_FREQUENCY: Duration = Duration::from_secs(2); /// Maximum number of batches per block const SERVER_MAX_BATCHES_PER_BLOCK: usize = 4; + +/// The number of blocks of committed state that the mempool retains. +/// +/// This determines the grace period incoming transactions have between fetching their input from +/// the store and verification in the mempool. +const SERVER_MEMPOOL_STATE_RETENTION: usize = 5; diff --git a/crates/block-producer/src/mempool/batch_graph.rs b/crates/block-producer/src/mempool/batch_graph.rs index 9ce5370d0..b83af277d 100644 --- a/crates/block-producer/src/mempool/batch_graph.rs +++ b/crates/block-producer/src/mempool/batch_graph.rs @@ -1,7 +1,6 @@ use std::collections::{BTreeMap, BTreeSet}; use miden_objects::transaction::TransactionId; -use miden_tx::utils::collections::KvMap; use super::{ dependency_graph::{DependencyGraph, GraphError}, diff --git a/crates/block-producer/src/mempool/dependency_graph.rs b/crates/block-producer/src/mempool/dependency_graph.rs index 258f6d7cb..37b247590 100644 --- a/crates/block-producer/src/mempool/dependency_graph.rs +++ b/crates/block-producer/src/mempool/dependency_graph.rs @@ -3,8 +3,6 @@ use std::{ fmt::Display, }; -use miden_tx::utils::collections::KvMap; - // DEPENDENCY GRAPH // ================================================================================================ @@ -292,7 +290,7 @@ impl DependencyGraph return Err(GraphError::UnknownNodes(missing_nodes)); } - let mut visited = keys.clone(); + let visited = keys.clone(); let mut to_remove = keys; let mut removed = BTreeSet::new(); @@ -445,15 +443,15 @@ mod tests { /// Promotes all nodes in the pending list with value=key. fn promote_all(&mut self) { - /// SAFETY: these are definitely pending nodes. + // SAFETY: these are definitely pending nodes. self.promote(self.pending.clone()).unwrap(); } /// Calls process_root until all nodes have been processed. fn process_all(&mut self) { while let Some(root) = self.roots().first().copied() { - /// SAFETY: this is definitely a root since we just took it from there :) - self.process_root(root); + // SAFETY: this is definitely a root since we just took it from there :) + self.process_root(root).unwrap(); } } } @@ -528,7 +526,7 @@ mod tests { assert!(uut.pending.contains(&123)); assert!(!uut.vertices.contains_key(&123)); - uut.promote_pending(123, 123); + uut.promote_pending(123, 123).unwrap(); assert!(!uut.pending.contains(&123)); assert!(uut.vertices.contains_key(&123)); @@ -584,7 +582,7 @@ mod tests { assert_eq!(err, expected); let mut atomic_reference = TestGraph::default(); - atomic_reference.insert_with_no_parents(KEY); + atomic_reference.insert_with_no_parents(KEY).unwrap(); assert_eq!(uut, atomic_reference); } @@ -703,9 +701,10 @@ mod tests { uut.insert_with_no_parents(parent_a).unwrap(); uut.insert_with_no_parents(parent_b).unwrap(); - uut.insert_with_parent(child_a, parent_a); - uut.insert_with_parent(child_b, parent_b); - uut.insert_with_parents(partially_disjoin_child, [disjoint_parent, parent_a].into()); + uut.insert_with_parent(child_a, parent_a).unwrap(); + uut.insert_with_parent(child_b, parent_b).unwrap(); + uut.insert_with_parents(partially_disjoin_child, [disjoint_parent, parent_a].into()) + .unwrap(); // Since we are reverting the other parents, we expect the roots to match the current state. uut.promote_all(); @@ -823,7 +822,7 @@ mod tests { uut.insert_with_parents(child_b, [parent_a, parent_b].into()).unwrap(); uut.insert_with_parent(child_c, parent_b).unwrap(); - uut.purge_subgraphs([child_b, parent_a].into()); + uut.purge_subgraphs([child_b, parent_a].into()).unwrap(); let mut reference = TestGraph::default(); reference.insert_with_no_parents(ancestor_a).unwrap(); @@ -917,15 +916,15 @@ mod tests { let nodes = (0..10).collect::>(); let mut uut = TestGraph::default(); - uut.insert_with_no_parents(nodes[0]); + uut.insert_with_no_parents(nodes[0]).unwrap(); for pairs in nodes.windows(2) { let (parent, id) = (pairs[0], pairs[1]); - uut.insert_with_parent(id, parent); + uut.insert_with_parent(id, parent).unwrap(); } uut.promote_all(); let mut ordered_roots = Vec::::new(); - for node in &nodes { + for _ in &nodes { let current_roots = uut.roots().clone(); ordered_roots.extend(¤t_roots); diff --git a/crates/block-producer/src/mempool/inflight_state/account_state.rs b/crates/block-producer/src/mempool/inflight_state/account_state.rs index d2888d033..f929ab114 100644 --- a/crates/block-producer/src/mempool/inflight_state/account_state.rs +++ b/crates/block-producer/src/mempool/inflight_state/account_state.rs @@ -1,19 +1,6 @@ -use std::{ - collections::{BTreeMap, BTreeSet, VecDeque}, - sync::Arc, -}; - -use miden_objects::{ - accounts::AccountId, - notes::{NoteId, Nullifier}, - transaction::{OutputNote, OutputNotes, ProvenTransaction, TransactionId}, - Digest, -}; - -use crate::{ - errors::{AddTransactionError, VerifyTxError}, - store::TransactionInputs, -}; +use std::collections::VecDeque; + +use miden_objects::{transaction::TransactionId, Digest}; // IN-FLIGHT ACCOUNT STATE // ================================================================================================ @@ -244,7 +231,7 @@ mod tests { for (state, tx) in &states { uut.insert(*state, *tx); } - uut.revert(REVERT); + let _ = uut.revert(REVERT); let mut expected = InflightAccountState::default(); for (state, tx) in states.iter().rev().skip(REVERT).rev() { @@ -267,7 +254,7 @@ mod tests { uut.insert(*state, *tx); } uut.commit(PRUNE); - uut.prune_committed(PRUNE); + let _ = uut.prune_committed(PRUNE); let mut expected = InflightAccountState::default(); for (state, tx) in states.iter().skip(PRUNE) { @@ -287,14 +274,13 @@ mod tests { } uut.commit(N); - uut.prune_committed(N); + let _ = uut.prune_committed(N); assert_eq!(uut, Default::default()); } #[test] fn is_empty_after_full_revert() { - let mut rng = Random::with_random_seed(); const N: usize = 5; let mut uut = InflightAccountState::default(); let mut rng = Random::with_random_seed(); @@ -302,7 +288,7 @@ mod tests { uut.insert(rng.draw_digest(), rng.draw_tx_id()); } - uut.revert(N); + let _ = uut.revert(N); assert_eq!(uut, Default::default()); } @@ -318,7 +304,7 @@ mod tests { } uut.commit(1); - uut.revert(N); + let _ = uut.revert(N); } #[test] diff --git a/crates/block-producer/src/mempool/inflight_state/mod.rs b/crates/block-producer/src/mempool/inflight_state/mod.rs index 76b8f8f84..233613c26 100644 --- a/crates/block-producer/src/mempool/inflight_state/mod.rs +++ b/crates/block-producer/src/mempool/inflight_state/mod.rs @@ -1,19 +1,14 @@ -use std::{ - collections::{btree_map::Entry, BTreeMap, BTreeSet, VecDeque}, - sync::Arc, -}; +use std::collections::{BTreeMap, BTreeSet, VecDeque}; use miden_objects::{ accounts::AccountId, notes::{NoteId, Nullifier}, - transaction::{OutputNote, OutputNotes, ProvenTransaction, TransactionId}, - Digest, + transaction::TransactionId, }; use crate::{ domain::transaction::AuthenticatedTransaction, errors::{AddTransactionError, VerifyTxError}, - store::TransactionInputs, }; mod account_state; @@ -301,7 +296,7 @@ impl InflightState { // SAFETY: The length check above guarantees that we have at least one committed block. let block = self.committed_blocks.pop_front().unwrap(); - for (tx_id, delta) in block { + for (_, delta) in block { // SAFETY: Since the delta exists, so must the account. let status = self.accounts.get_mut(&delta.account).unwrap().prune_committed(1); @@ -357,14 +352,13 @@ impl OutputNoteState { #[cfg(test)] mod tests { - use miden_air::Felt; - use miden_objects::{accounts::AccountType, testing::account_id::AccountIdBuilder}; + use miden_objects::Digest; use super::*; use crate::test_utils::{ - mock_account_id, mock_proven_tx, + mock_account_id, note::{mock_note, mock_output_note}, - MockPrivateAccount, MockProvenTxBuilder, + MockProvenTxBuilder, }; #[test] diff --git a/crates/block-producer/src/mempool/mod.rs b/crates/block-producer/src/mempool/mod.rs index 71743c074..bbe13767c 100644 --- a/crates/block-producer/src/mempool/mod.rs +++ b/crates/block-producer/src/mempool/mod.rs @@ -1,26 +1,17 @@ use std::{ - collections::{btree_map::Entry, BTreeMap, BTreeSet, VecDeque}, + collections::{BTreeMap, BTreeSet}, fmt::Display, - ops::Sub, sync::Arc, }; use batch_graph::BatchGraph; use inflight_state::InflightState; -use miden_objects::{ - accounts::AccountId, - notes::{NoteId, Nullifier}, - transaction::{ProvenTransaction, TransactionId}, - Digest, -}; -use miden_tx::{utils::collections::KvMap, TransactionVerifierError}; +use tokio::sync::Mutex; use transaction_graph::TransactionGraph; use crate::{ - batch_builder::batch::TransactionBatch, - domain::transaction::AuthenticatedTransaction, - errors::{AddTransactionError, VerifyTxError}, - store::{TransactionInputs, TxInputsError}, + batch_builder::batch::TransactionBatch, domain::transaction::AuthenticatedTransaction, + errors::AddTransactionError, }; mod batch_graph; @@ -28,7 +19,7 @@ mod dependency_graph; mod inflight_state; mod transaction_graph; -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] pub struct BatchJobId(u64); impl Display for BatchJobId { @@ -85,6 +76,29 @@ impl BlockNumber { // MEMPOOL // ================================================================================================ +#[derive(Clone)] +pub struct SharedMempool(Arc>); + +impl SharedMempool { + pub fn new( + chain_tip: BlockNumber, + batch_limit: usize, + block_limit: usize, + state_retention: usize, + ) -> Self { + Self(Arc::new(Mutex::new(Mempool::new( + chain_tip, + batch_limit, + block_limit, + state_retention, + )))) + } + + pub async fn lock(&self) -> tokio::sync::MutexGuard { + self.0.lock().await + } +} + pub struct Mempool { /// The latest inflight state of each account. /// @@ -110,6 +124,25 @@ pub struct Mempool { } impl Mempool { + /// Creates a new [Mempool] with the provided configuration. + fn new( + chain_tip: BlockNumber, + batch_limit: usize, + block_limit: usize, + state_retention: usize, + ) -> Self { + Self { + chain_tip, + batch_transaction_limit: batch_limit, + block_batch_limit: block_limit, + state: InflightState::new(chain_tip, state_retention), + block_in_progress: Default::default(), + transactions: Default::default(), + batches: Default::default(), + next_batch_id: Default::default(), + } + } + /// Adds a transaction to the mempool. /// /// # Returns @@ -126,7 +159,7 @@ impl Mempool { // Add transaction to inflight state. let parents = self.state.add_transaction(&transaction)?; - self.transactions.insert(transaction, parents); + self.transactions.insert(transaction, parents).expect("Malformed graph"); Ok(self.chain_tip.0) } @@ -146,7 +179,7 @@ impl Mempool { let batch_id = self.next_batch_id; self.next_batch_id.increment(); - self.batches.insert(batch_id, tx_ids, parents); + self.batches.insert(batch_id, tx_ids, parents).expect("Malformed graph"); Some((batch_id, batch)) } @@ -167,14 +200,14 @@ impl Mempool { let batches = removed_batches.keys().copied().collect::>(); let transactions = removed_batches.into_values().flatten().collect(); - self.transactions.requeue_transactions(transactions); + self.transactions.requeue_transactions(transactions).expect("Malformed graph"); tracing::warn!(%batch, descendents=?batches, "Batch failed, dropping all inflight descendent batches, impacted transactions are back in queue."); } /// Marks a batch as proven if it exists. pub fn batch_proved(&mut self, batch_id: BatchJobId, batch: TransactionBatch) { - self.batches.submit_proof(batch_id, batch); + self.batches.submit_proof(batch_id, batch).expect("Malformed graph"); } /// Select batches for the next block. @@ -227,7 +260,6 @@ impl Mempool { // Remove all transactions from the graphs. let purged = self.batches.remove_batches(batches).expect("Bad graph"); - let batches = purged.keys().collect::>(); let transactions = purged.into_values().flatten().collect(); let transactions = self diff --git a/crates/block-producer/src/mempool/transaction_graph.rs b/crates/block-producer/src/mempool/transaction_graph.rs index f11c968f3..089566269 100644 --- a/crates/block-producer/src/mempool/transaction_graph.rs +++ b/crates/block-producer/src/mempool/transaction_graph.rs @@ -1,9 +1,6 @@ -use std::{ - collections::{BTreeMap, BTreeSet}, - sync::Arc, -}; +use std::collections::BTreeSet; -use miden_objects::transaction::{ProvenTransaction, TransactionId}; +use miden_objects::transaction::TransactionId; use super::dependency_graph::{DependencyGraph, GraphError}; use crate::domain::transaction::AuthenticatedTransaction; diff --git a/crates/block-producer/src/server/api.rs b/crates/block-producer/src/server/api.rs deleted file mode 100644 index 8a917242e..000000000 --- a/crates/block-producer/src/server/api.rs +++ /dev/null @@ -1,77 +0,0 @@ -use std::sync::Arc; - -use miden_node_proto::generated::{ - block_producer::api_server, requests::SubmitProvenTransactionRequest, - responses::SubmitProvenTransactionResponse, -}; -use miden_node_utils::formatting::{format_input_notes, format_output_notes}; -use miden_objects::{transaction::ProvenTransaction, utils::serde::Deserializable}; -use tonic::Status; -use tracing::{debug, info, instrument}; - -use crate::{ - batch_builder::BatchBuilder, - txqueue::{TransactionQueue, TransactionValidator}, - COMPONENT, -}; - -// BLOCK PRODUCER -// ================================================================================================ - -pub struct BlockProducerApi { - queue: Arc>, -} - -impl BlockProducerApi { - pub fn new(queue: Arc>) -> Self { - Self { queue } - } -} - -// FIXME: remove the allow when the upstream clippy issue is fixed: -// https://github.com/rust-lang/rust-clippy/issues/12281 -#[allow(clippy::blocks_in_conditions)] -#[tonic::async_trait] -impl api_server::Api for BlockProducerApi -where - TV: TransactionValidator, - BB: BatchBuilder, -{ - #[instrument( - target = "miden-block-producer", - name = "block_producer:submit_proven_transaction", - skip_all, - err - )] - async fn submit_proven_transaction( - &self, - request: tonic::Request, - ) -> Result, Status> { - let request = request.into_inner(); - debug!(target: COMPONENT, ?request); - - let tx = ProvenTransaction::read_from_bytes(&request.transaction) - .map_err(|_| Status::invalid_argument("Invalid transaction"))?; - - info!( - target: COMPONENT, - tx_id = %tx.id().to_hex(), - account_id = %tx.account_id().to_hex(), - initial_account_hash = %tx.account_update().init_state_hash(), - final_account_hash = %tx.account_update().final_state_hash(), - input_notes = %format_input_notes(tx.input_notes()), - output_notes = %format_output_notes(tx.output_notes()), - block_ref = %tx.block_ref(), - "Deserialized transaction" - ); - debug!(target: COMPONENT, proof = ?tx.proof()); - - let block_height = self - .queue - .add_transaction(tx) - .await - .map_err(|err| Status::invalid_argument(format!("{:?}", err)))?; - - Ok(tonic::Response::new(SubmitProvenTransactionResponse { block_height })) - } -} diff --git a/crates/block-producer/src/server/mod.rs b/crates/block-producer/src/server/mod.rs index a5a8c320a..ff4c57364 100644 --- a/crates/block-producer/src/server/mod.rs +++ b/crates/block-producer/src/server/mod.rs @@ -1,53 +1,30 @@ -use std::{ - collections::{BTreeMap, BTreeSet}, - net::ToSocketAddrs, - sync::Arc, -}; +use std::net::ToSocketAddrs; -use miden_node_proto::{ - domain::nullifiers, - generated::{ - block_producer::api_server, requests::SubmitProvenTransactionRequest, - responses::SubmitProvenTransactionResponse, store::api_client as store_client, - }, +use miden_node_proto::generated::{ + block_producer::api_server, requests::SubmitProvenTransactionRequest, + responses::SubmitProvenTransactionResponse, store::api_client as store_client, }; use miden_node_utils::{ errors::ApiError, formatting::{format_input_notes, format_output_notes}, }; -use miden_objects::{ - transaction::ProvenTransaction, utils::serde::Deserializable, MIN_PROOF_SECURITY_LEVEL, -}; -use miden_tx::TransactionVerifier; +use miden_objects::{transaction::ProvenTransaction, utils::serde::Deserializable}; use tokio::{net::TcpListener, sync::Mutex}; use tokio_stream::wrappers::TcpListenerStream; use tonic::Status; use tracing::{debug, info, instrument}; use crate::{ - batch_builder::{DefaultBatchBuilder, DefaultBatchBuilderOptions}, - block_builder::DefaultBlockBuilder, + batch_builder::BatchBuilder, + block_builder::BlockBuilder, config::BlockProducerConfig, domain::transaction::AuthenticatedTransaction, errors::{AddTransactionError, VerifyTxError}, - mempool::Mempool, - state_view::DefaultStateView, + mempool::{BlockNumber, SharedMempool}, store::{DefaultStore, Store}, - txqueue::{TransactionQueue, TransactionQueueOptions}, - COMPONENT, SERVER_BATCH_SIZE, SERVER_BLOCK_FREQUENCY, SERVER_BUILD_BATCH_FREQUENCY, - SERVER_MAX_BATCHES_PER_BLOCK, + COMPONENT, SERVER_BATCH_SIZE, SERVER_MAX_BATCHES_PER_BLOCK, SERVER_MEMPOOL_STATE_RETENTION, }; -pub mod api; - -type Api = api::BlockProducerApi< - DefaultBatchBuilder< - DefaultStore, - DefaultBlockBuilder>, - >, - DefaultStateView, ->; - /// Represents an initialized block-producer component where the RPC connection is open, /// but not yet actively responding to requests. /// @@ -55,8 +32,14 @@ type Api = api::BlockProducerApi< /// components to the store without resorting to sleeps or other mechanisms to spawn dependent /// components. pub struct BlockProducer { - api_service: api_server::ApiServer, - listener: TcpListener, + batch_builder: BatchBuilder, + block_builder: BlockBuilder, + batch_limit: usize, + block_limit: usize, + state_retention: usize, + rpc_listener: TcpListener, + store: DefaultStore, + chain_tip: BlockNumber, } impl BlockProducer { @@ -66,82 +49,132 @@ impl BlockProducer { pub async fn init(config: BlockProducerConfig) -> Result { info!(target: COMPONENT, %config, "Initializing server"); - let store = Arc::new(DefaultStore::new( + // TODO: Does this actually need an arc to be properly shared? + let store = DefaultStore::new( store_client::ApiClient::connect(config.store_url.to_string()) .await .map_err(|err| ApiError::DatabaseConnectionFailed(err.to_string()))?, - )); - let state_view = - Arc::new(DefaultStateView::new(Arc::clone(&store), config.verify_tx_proofs)); - - let block_builder = DefaultBlockBuilder::new(Arc::clone(&store), Arc::clone(&state_view)); - let batch_builder_options = DefaultBatchBuilderOptions { - block_frequency: SERVER_BLOCK_FREQUENCY, - max_batches_per_block: SERVER_MAX_BATCHES_PER_BLOCK, - }; - let batch_builder = Arc::new(DefaultBatchBuilder::new( - Arc::clone(&store), - Arc::new(block_builder), - batch_builder_options, - )); - - let transaction_queue_options = TransactionQueueOptions { - build_batch_frequency: SERVER_BUILD_BATCH_FREQUENCY, - batch_size: SERVER_BATCH_SIZE, - }; - let queue = Arc::new(TransactionQueue::new( - state_view, - Arc::clone(&batch_builder), - transaction_queue_options, - )); - - let api_service = - api_server::ApiServer::new(api::BlockProducerApi::new(Arc::clone(&queue))); + ); - tokio::spawn(async move { queue.run().await }); - tokio::spawn(async move { batch_builder.run().await }); + let latest_header = store + .latest_header() + .await + .map_err(|err| ApiError::DatabaseConnectionFailed(err.to_string()))?; + let chain_tip = BlockNumber::new(latest_header.block_num()); - let addr = config + let rpc_listener = config .endpoint .to_socket_addrs() .map_err(ApiError::EndpointToSocketFailed)? .next() - .ok_or_else(|| ApiError::AddressResolutionFailed(config.endpoint.to_string()))?; - - let listener = TcpListener::bind(addr).await?; + .ok_or_else(|| ApiError::AddressResolutionFailed(config.endpoint.to_string())) + .map(TcpListener::bind)? + .await?; info!(target: COMPONENT, "Server initialized"); - Ok(Self { api_service, listener }) + Ok(Self { + batch_builder: Default::default(), + block_builder: BlockBuilder::new(store.clone()), + batch_limit: SERVER_BATCH_SIZE, + block_limit: SERVER_MAX_BATCHES_PER_BLOCK, + state_retention: SERVER_MEMPOOL_STATE_RETENTION, + store, + rpc_listener, + chain_tip, + }) } - /// Serves the block-producers's RPC API. - /// - /// Note: this blocks until the server dies. pub async fn serve(self) -> Result<(), ApiError> { - tonic::transport::Server::builder() - .add_service(self.api_service) - .serve_with_incoming(TcpListenerStream::new(self.listener)) - .await - .map_err(ApiError::ApiServeFailed) + let Self { + batch_builder, + block_builder, + batch_limit, + block_limit, + state_retention, + rpc_listener, + store, + chain_tip, + } = self; + + let mempool = SharedMempool::new(chain_tip, batch_limit, block_limit, state_retention); + + // Spawn rpc server and batch and block provers. + // + // These communicate indirectly via a shared mempool. + // + // These should run forever, so we combine them into a joinset so that if + // any complete or fail, we can shutdown the rest (somewhat) gracefully. + let mut tasks = tokio::task::JoinSet::new(); + + // TODO: improve the error situationship. + let batch_builder_id = tasks + .spawn({ + let mempool = mempool.clone(); + async { batch_builder.run(mempool).await } + }) + .id(); + let block_builder_id = tasks + .spawn({ + let mempool = mempool.clone(); + async { block_builder.run(mempool).await } + }) + .id(); + let rpc_id = tasks + .spawn(async move { + BlockProducerRpcServer::new(mempool, store) + .serve(rpc_listener) + .await + .expect("Really the rest should throw errors instead of panic'ing.") + }) + .id(); + + // Wait for any task to end. They should run forever, so this is an unexpected result. + + // SAFETY: The JoinSet is definitely not empty. + let task_result = tasks.join_next_with_id().await.unwrap(); + let task_id = match &task_result { + Ok((id, _)) => *id, + Err(err) => err.id(), + }; + + let task_name = match task_id { + id if id == batch_builder_id => "batch-builder", + id if id == block_builder_id => "block-builder", + id if id == rpc_id => "rpc", + _ => { + tracing::warn!("An unknown task ID was detected in the block-producer."); + "unknown" + }, + }; + + tracing::error!( + task = task_name, + result = ?task_result, + "Block-producer task ended unexpectedly, aborting" + ); + + tasks.abort_all(); + + Ok(()) } } -pub struct Server { - /// This outer mutex enforces that the incoming transactions won't crowd out more important - /// mempool locks. +/// Serves the block producer's RPC [api](api_server::Api). +struct BlockProducerRpcServer { + /// The mutex effectively rate limits incoming transactions into the mempool by forcing them + /// through a queue. /// - /// The inner mutex will be abstracted away once we are happy with the api. - mempool: Mutex>>, + /// This gives mempool users such as the batch and block builders equal footing with __all__ + /// incoming transactions combined. Without this incoming transactions would greatly restrict + /// the block-producers usage of the mempool. + mempool: Mutex, store: DefaultStore, } -// FIXME: remove the allow when the upstream clippy issue is fixed: -// https://github.com/rust-lang/rust-clippy/issues/12281 -#[allow(clippy::blocks_in_conditions)] #[tonic::async_trait] -impl api_server::Api for Server { +impl api_server::Api for BlockProducerRpcServer { async fn submit_proven_transaction( &self, request: tonic::Request, @@ -154,7 +187,18 @@ impl api_server::Api for Server { } } -impl Server { +impl BlockProducerRpcServer { + pub fn new(mempool: SharedMempool, store: DefaultStore) -> Self { + Self { mempool: Mutex::new(mempool), store } + } + + async fn serve(self, listener: TcpListener) -> Result<(), tonic::transport::Error> { + tonic::transport::Server::builder() + .add_service(api_server::ApiServer::new(self)) + .serve_with_incoming(TcpListenerStream::new(listener)) + .await + } + #[instrument( target = "miden-block-producer", name = "block_producer:submit_proven_transaction", diff --git a/crates/block-producer/src/state_view/account_state.rs b/crates/block-producer/src/state_view/account_state.rs deleted file mode 100644 index b3c538cc9..000000000 --- a/crates/block-producer/src/state_view/account_state.rs +++ /dev/null @@ -1,123 +0,0 @@ -use std::collections::{BTreeMap, VecDeque}; - -use miden_objects::{accounts::AccountId, Digest}; - -use crate::errors::VerifyTxError; - -/// Tracks the list of inflight account updates. -/// -/// New transactions can be registered with [Self::verify_and_add]. States that are no longer -/// considered inflight (e.g. due to being applied) may be removed using [Self::remove]. -/// -/// Both functions perform safety checks to ensure the states match what we expect. -#[derive(Debug, Default)] -pub struct InflightAccountStates(BTreeMap>); - -impl InflightAccountStates { - /// Verifies that the provided initial state matches the latest inflight account state (if any). - pub fn verify_update(&self, id: AccountId, init_state: Digest) -> Result<(), VerifyTxError> { - if let Some(latest) = self.get(id) { - if latest != &init_state { - return Err(VerifyTxError::IncorrectAccountInitialHash { - tx_initial_account_hash: init_state, - current_account_hash: Some(*latest), - }); - } - } - - Ok(()) - } - - /// [Verifies](Self::verify_update) the update and appends it to the list of inflight account - /// updates. - pub fn verify_and_add( - &mut self, - id: AccountId, - init_state: Digest, - final_state: Digest, - ) -> Result<(), VerifyTxError> { - let states = self.0.entry(id).or_default(); - - // Ensure the latest state matches the new inital state. - if let Some(latest) = states.back() { - if latest != &init_state { - return Err(VerifyTxError::IncorrectAccountInitialHash { - tx_initial_account_hash: init_state, - current_account_hash: Some(*latest), - }); - } - } - - states.push_back(final_state); - - Ok(()) - } - - /// Remove state transitions from earliest up until a state that matches the given - /// final state. Returns an error if no match was found. - /// - /// In other words, if an account has state transitions `a->b->c->d` then calling `remove(b)` - /// would leave behind `c->d`. - pub fn remove(&mut self, id: AccountId, final_state: Digest) -> Result<(), ()> { - let states = self.0.get_mut(&id).ok_or(())?; - let Some(idx) = states.iter().position(|x| x == &final_state) else { - return Err(()); - }; - - states.drain(..=idx); - // Prevent infinite growth by removing entries which have no - // inflight state changes. - if states.is_empty() { - self.0.remove(&id); - } - - Ok(()) - } - - /// The latest value of the given account. - pub fn get(&self, id: AccountId) -> Option<&Digest> { - self.0.get(&id).and_then(|states| states.back()) - } - - /// Number of accounts with inflight transactions. - #[cfg(test)] - pub fn contains(&self, id: AccountId) -> bool { - self.0.contains_key(&id) - } - - /// Number of accounts with inflight transactions. - #[cfg(test)] - pub fn len(&self) -> usize { - self.0.len() - } -} - -#[cfg(test)] -mod tests { - use miden_air::Felt; - use miden_objects::accounts::AccountId; - - use super::*; - - #[test] - fn account_states_must_chain() { - let account: AccountId = AccountId::new_unchecked(Felt::new(10)); - const ONE: Digest = Digest::new([Felt::new(1), Felt::new(1), Felt::new(1), Felt::new(1)]); - const TWO: Digest = Digest::new([Felt::new(2), Felt::new(2), Felt::new(2), Felt::new(2)]); - const THREE: Digest = Digest::new([Felt::new(3), Felt::new(3), Felt::new(3), Felt::new(3)]); - let mut uut = InflightAccountStates::default(); - - assert!(uut.verify_and_add(account, Digest::default(), ONE).is_ok()); - assert!(uut.verify_and_add(account, ONE, TWO).is_ok()); - assert!(uut.verify_and_add(account, TWO, THREE).is_ok()); - assert!(uut.verify_and_add(account, TWO, ONE).is_err()); - - assert!(uut.remove(account, TWO).is_ok()); - // Repeat removal should fail since this is no longer present. - assert!(uut.remove(account, TWO).is_err()); - assert!(uut.remove(account, THREE).is_ok()); - - // Check that cleanup is performed. - assert!(uut.0.is_empty()); - } -} diff --git a/crates/block-producer/src/state_view/mod.rs b/crates/block-producer/src/state_view/mod.rs deleted file mode 100644 index f0bcf1b93..000000000 --- a/crates/block-producer/src/state_view/mod.rs +++ /dev/null @@ -1,276 +0,0 @@ -use std::{collections::BTreeSet, sync::Arc}; - -use async_trait::async_trait; -use miden_node_utils::formatting::format_array; -use miden_objects::{ - block::Block, - notes::{NoteId, Nullifier}, - transaction::OutputNote, - Digest, MIN_PROOF_SECURITY_LEVEL, -}; -use miden_tx::TransactionVerifier; -use tokio::sync::RwLock; -use tracing::{debug, instrument}; - -use self::account_state::InflightAccountStates; -use crate::{ - errors::VerifyTxError, - store::{ApplyBlock, ApplyBlockError, Store, TransactionInputs}, - txqueue::TransactionValidator, - ProvenTransaction, COMPONENT, -}; - -mod account_state; -#[cfg(test)] -mod tests; - -pub struct DefaultStateView { - store: Arc, - - /// Enables or disables the verification of transaction proofs in `verify_tx` - verify_tx_proofs: bool, - - /// The account states modified by transactions currently in the block production pipeline. - accounts_in_flight: Arc>, - - /// The nullifiers of notes consumed by transactions currently in the block production - /// pipeline. - nullifiers_in_flight: Arc>>, - - /// The output notes of transactions currently in the block production pipeline. - notes_in_flight: Arc>>, -} - -impl DefaultStateView -where - S: Store, -{ - pub fn new(store: Arc, verify_tx_proofs: bool) -> Self { - Self { - store, - verify_tx_proofs, - accounts_in_flight: Default::default(), - nullifiers_in_flight: Default::default(), - notes_in_flight: Default::default(), - } - } -} - -// FIXME: remove the allow when the upstream clippy issue is fixed: -// https://github.com/rust-lang/rust-clippy/issues/12281 -#[allow(clippy::blocks_in_conditions)] -#[async_trait] -impl TransactionValidator for DefaultStateView -where - S: Store, -{ - #[instrument(skip_all, err)] - async fn verify_tx(&self, candidate_tx: &ProvenTransaction) -> Result { - if self.verify_tx_proofs { - // Make sure that the transaction proof is valid and meets the required security level - let tx_verifier = TransactionVerifier::new(MIN_PROOF_SECURITY_LEVEL); - tx_verifier - .verify(candidate_tx.clone()) - .map_err(|_| VerifyTxError::InvalidTransactionProof(candidate_tx.id()))?; - } - - // Soft-check if `tx` violates in-flight requirements. - // - // This is a "soft" check, because we'll need to redo it at the end. We do this soft check - // to quickly reject clearly infracting transactions before hitting the store (slow). - // - // At this stage we don't provide missing notes, they will be available on the second check - // after getting the transaction inputs. - ensure_in_flight_constraints( - candidate_tx, - &*self.accounts_in_flight.read().await, - &*self.nullifiers_in_flight.read().await, - &*self.notes_in_flight.read().await, - &[], - )?; - - // Fetch the transaction inputs from the store, and check tx input constraints; this will - // identify a set of unauthenticated input notes which are not in the store yet; we'll use - // this set to verify that these notes are currently in flight (i.e., they are output notes - // of one of the inflight transactions) - let mut tx_inputs = self.store.get_tx_inputs(candidate_tx).await?; - - let current_block_height = tx_inputs.current_block_height; - - // The latest inflight account state takes precedence since this is the current block being - // constructed. - if let Some(inflight) = self.accounts_in_flight.read().await.get(candidate_tx.account_id()) - { - tx_inputs.account_hash = Some(*inflight); - } - let missing_notes = ensure_tx_inputs_constraints(candidate_tx, tx_inputs)?; - - // Re-check in-flight transaction constraints, and if verification passes, register - // transaction - // - // Note: We need to re-check these constraints because we dropped the locks since we last - // checked - { - let mut locked_accounts_in_flight = self.accounts_in_flight.write().await; - let mut locked_nullifiers_in_flight = self.nullifiers_in_flight.write().await; - let mut locked_notes_in_flight = self.notes_in_flight.write().await; - - ensure_in_flight_constraints( - candidate_tx, - &locked_accounts_in_flight, - &locked_nullifiers_in_flight, - &locked_notes_in_flight, - &missing_notes, - )?; - - locked_accounts_in_flight.verify_and_add( - candidate_tx.account_id(), - candidate_tx.account_update().init_state_hash(), - candidate_tx.account_update().final_state_hash(), - )?; - locked_nullifiers_in_flight.extend(&mut candidate_tx.get_nullifiers()); - locked_notes_in_flight.extend(candidate_tx.output_notes().iter().map(OutputNote::id)); - } - - Ok(current_block_height) - } -} - -// FIXME: remove the allow when the upstream clippy issue is fixed: -// https://github.com/rust-lang/rust-clippy/issues/12281 -#[allow(clippy::blocks_in_conditions)] -#[async_trait] -impl ApplyBlock for DefaultStateView -where - S: Store, -{ - #[instrument(target = "miden-block-producer", skip_all, err)] - async fn apply_block(&self, block: &Block) -> Result<(), ApplyBlockError> { - self.store.apply_block(block).await?; - - let mut locked_accounts_in_flight = self.accounts_in_flight.write().await; - let mut locked_nullifiers_in_flight = self.nullifiers_in_flight.write().await; - let mut locked_notes_in_flight = self.notes_in_flight.write().await; - - // Remove account ids of transactions in block - for update in block.updated_accounts() { - locked_accounts_in_flight - .remove(update.account_id(), update.new_state_hash()) - .expect("Account update should be valid"); - } - - // Remove new nullifiers of transactions in block - for nullifier in block.nullifiers() { - let was_in_flight = locked_nullifiers_in_flight.remove(nullifier); - debug_assert!(was_in_flight); - } - - // Remove new notes of transactions in block - for (_, note) in block.notes() { - let was_in_flight = locked_notes_in_flight.remove(¬e.id()); - debug_assert!(was_in_flight); - } - - Ok(()) - } -} - -// HELPERS -// ------------------------------------------------------------------------------------------------- - -/// Ensures the constraints related to in-flight transactions: -/// - the candidate tx's initial state matches the latest inflight state (if any) -/// - no note's nullifier in candidate tx's consumed notes is already contained in -/// `already_consumed_nullifiers` -/// - all notes in `tx_notes_not_in_store` are currently in flight -/// -/// The account state is not verified as it is performed by [InflightAccountStates]. -#[instrument(target = "miden-block-producer", skip_all, err)] -fn ensure_in_flight_constraints( - candidate_tx: &ProvenTransaction, - accounts_in_flight: &InflightAccountStates, - already_consumed_nullifiers: &BTreeSet, - notes_in_flight: &BTreeSet, - tx_notes_not_in_store: &[NoteId], -) -> Result<(), VerifyTxError> { - debug!(target: COMPONENT, already_consumed_nullifiers = %format_array(already_consumed_nullifiers)); - - accounts_in_flight.verify_update( - candidate_tx.account_id(), - candidate_tx.account_update().init_state_hash(), - )?; - - // Check no consumed notes were already consumed - let infracting_nullifiers: Vec = { - candidate_tx - .get_nullifiers() - .filter(|nullifier| already_consumed_nullifiers.contains(nullifier)) - .collect() - }; - - if !infracting_nullifiers.is_empty() { - return Err(VerifyTxError::InputNotesAlreadyConsumed(infracting_nullifiers)); - } - - // Check all notes not found in store are in in-flight notes, return list of missing notes - let missing_notes: Vec = tx_notes_not_in_store - .iter() - .filter(|note_id| !notes_in_flight.contains(note_id)) - .copied() - .collect(); - if !missing_notes.is_empty() { - return Err(VerifyTxError::UnauthenticatedNotesNotFound(missing_notes)); - } - - Ok(()) -} - -/// Ensures the constraints related to transaction inputs: -/// - the candidate transaction's initial account state hash must be the same as the one in the -/// Store or empty for new accounts -/// - input notes must not be already consumed -/// -/// Returns a list of unauthenticated input notes that were not found in the store. -#[instrument(target = "miden-block-producer", skip_all, err)] -fn ensure_tx_inputs_constraints( - candidate_tx: &ProvenTransaction, - tx_inputs: TransactionInputs, -) -> Result, VerifyTxError> { - debug!(target: COMPONENT, %tx_inputs); - - match tx_inputs.account_hash { - // if the account is present in the Store, make sure that the account state hash - // from the received transaction is the same as the one from the Store - Some(store_account_hash) => { - if candidate_tx.account_update().init_state_hash() != store_account_hash { - return Err(VerifyTxError::IncorrectAccountInitialHash { - tx_initial_account_hash: candidate_tx.account_update().init_state_hash(), - current_account_hash: Some(store_account_hash), - }); - } - }, - // if the account is not present in the Store, it must be a new account - None => { - // if the initial account hash is not equal to `Digest::default()` it - // signifies that the account is not new but is also not recorded in the Store - if candidate_tx.account_update().init_state_hash() != Digest::default() { - return Err(VerifyTxError::IncorrectAccountInitialHash { - tx_initial_account_hash: candidate_tx.account_update().init_state_hash(), - current_account_hash: None, - }); - } - }, - } - - let infracting_nullifiers: Vec = tx_inputs - .nullifiers - .into_iter() - .filter_map(|(nullifier_in_tx, block_num)| block_num.is_some().then_some(nullifier_in_tx)) - .collect(); - - if !infracting_nullifiers.is_empty() { - return Err(VerifyTxError::InputNotesAlreadyConsumed(infracting_nullifiers)); - } - - Ok(tx_inputs.missing_unauthenticated_notes) -} diff --git a/crates/block-producer/src/state_view/tests/apply_block.rs b/crates/block-producer/src/state_view/tests/apply_block.rs deleted file mode 100644 index 9465ec951..000000000 --- a/crates/block-producer/src/state_view/tests/apply_block.rs +++ /dev/null @@ -1,170 +0,0 @@ -//! Requirements for `apply_block()`: -//! -//! AB1: the internal store's `apply_block` is called once -//! AB2: All accounts modified by transactions in the block are removed from the internal state -//! AB3: All consumed notes by some transaction in the block are still not consumable after -//! `apply_block` - -use std::iter; - -use miden_objects::{accounts::delta::AccountUpdateDetails, block::BlockAccountUpdate}; - -use super::*; -use crate::test_utils::{block::MockBlockBuilder, MockStoreSuccessBuilder}; - -/// Tests requirement AB1 -#[tokio::test] -#[miden_node_test_macro::enable_logging] -async fn test_apply_block_ab1() { - let account: MockPrivateAccount<3> = MockPrivateAccount::from(0); - - let store = Arc::new( - MockStoreSuccessBuilder::from_accounts(iter::once((account.id, account.states[0]))).build(), - ); - - let tx = - MockProvenTxBuilder::with_account(account.id, account.states[0], account.states[1]).build(); - - let state_view = DefaultStateView::new(store.clone(), false); - - // Verify transaction so it can be tracked in state view - let verify_tx_res = state_view.verify_tx(&tx).await; - assert!(verify_tx_res.is_ok()); - - let block = MockBlockBuilder::new(&store) - .await - .account_updates( - std::iter::once(account) - .map(|mock_account| { - BlockAccountUpdate::new( - mock_account.id, - mock_account.states[1], - AccountUpdateDetails::Private, - vec![tx.id()], - ) - }) - .collect(), - ) - .build(); - - let apply_block_res = state_view.apply_block(&block).await; - assert!(apply_block_res.is_ok()); - - assert_eq!(*store.num_apply_block_called.read().await, 1); -} - -/// Tests requirement AB2 -#[tokio::test] -#[miden_node_test_macro::enable_logging] -async fn test_apply_block_ab2() { - let (txs, accounts): (Vec<_>, Vec<_>) = get_txs_and_accounts(0, 3).unzip(); - - let store = Arc::new( - MockStoreSuccessBuilder::from_accounts( - accounts - .clone() - .into_iter() - .map(|mock_account| (mock_account.id, mock_account.states[0])), - ) - .build(), - ); - - let state_view = DefaultStateView::new(store.clone(), false); - - // Verify transactions so it can be tracked in state view - for tx in txs { - let verify_tx_res = state_view.verify_tx(&tx).await; - assert_eq!(verify_tx_res, Ok(0)); - } - - // All except the first account will go into the block. - let accounts_in_block: Vec = accounts.iter().skip(1).cloned().collect(); - - let block = MockBlockBuilder::new(&store) - .await - .account_updates( - accounts_in_block - .into_iter() - .map(|mock_account| { - BlockAccountUpdate::new( - mock_account.id, - mock_account.states[1], - AccountUpdateDetails::Private, - vec![], - ) - }) - .collect(), - ) - .build(); - - let apply_block_res = state_view.apply_block(&block).await; - assert!(apply_block_res.is_ok()); - - let accounts_still_in_flight = state_view.accounts_in_flight.read().await; - - // Only the first account should still be in flight - assert_eq!(accounts_still_in_flight.len(), 1); - assert!(accounts_still_in_flight.contains(accounts[0].id)); -} - -/// Tests requirement AB3 -#[tokio::test] -#[miden_node_test_macro::enable_logging] -async fn test_apply_block_ab3() { - let (txs, accounts): (Vec<_>, Vec<_>) = get_txs_and_accounts(0, 3).unzip(); - - let store = Arc::new( - MockStoreSuccessBuilder::from_accounts( - accounts - .clone() - .into_iter() - .map(|mock_account| (mock_account.id, mock_account.states[0])), - ) - .build(), - ); - - let state_view = DefaultStateView::new(store.clone(), false); - - // Verify transactions so it can be tracked in state view - for tx in txs.clone() { - let verify_tx_res = state_view.verify_tx(&tx).await; - assert_eq!(verify_tx_res, Ok(0)); - } - - let block = MockBlockBuilder::new(&store) - .await - .account_updates( - accounts - .clone() - .into_iter() - .map(|mock_account| { - BlockAccountUpdate::new( - mock_account.id, - mock_account.states[1], - AccountUpdateDetails::Private, - vec![], - ) - }) - .collect(), - ) - .build(); - - let apply_block_res = state_view.apply_block(&block).await; - assert!(apply_block_res.is_ok()); - - // Craft a new transaction which tries to consume the same note that was consumed in the - // first tx - let tx_new = MockProvenTxBuilder::with_account( - accounts[0].id, - accounts[0].states[1], - accounts[0].states[2], - ) - .nullifiers(txs[0].get_nullifiers().collect()) - .build(); - - let verify_tx_res = state_view.verify_tx(&tx_new).await; - assert_eq!( - verify_tx_res, - Err(VerifyTxError::InputNotesAlreadyConsumed(txs[0].get_nullifiers().collect())) - ); -} diff --git a/crates/block-producer/src/state_view/tests/mod.rs b/crates/block-producer/src/state_view/tests/mod.rs deleted file mode 100644 index 4fbf87c60..000000000 --- a/crates/block-producer/src/state_view/tests/mod.rs +++ /dev/null @@ -1,42 +0,0 @@ -use miden_objects::{Hasher, EMPTY_WORD, ZERO}; - -use super::*; -use crate::test_utils::{MockPrivateAccount, MockProvenTxBuilder}; - -mod apply_block; -mod verify_tx; - -// HELPERS -// ------------------------------------------------------------------------------------------------- - -pub fn nullifier_by_index(index: u32) -> Nullifier { - Nullifier::new( - Hasher::hash(&index.to_be_bytes()), - Hasher::hash( - &[index.to_be_bytes(), index.to_be_bytes()] - .into_iter() - .flatten() - .collect::>(), - ), - EMPTY_WORD.into(), - [ZERO, ZERO, ZERO, index.into()], - ) -} - -/// Returns `num` transactions, and the corresponding account they modify. -/// The transactions each consume a single different note -pub fn get_txs_and_accounts( - starting_account_index: u32, - num: u32, -) -> impl Iterator { - (0..num).map(move |index| { - let account = MockPrivateAccount::from(starting_account_index + index); - let nullifier_starting_index = (starting_account_index + index) as u64; - let tx = - MockProvenTxBuilder::with_account(account.id, account.states[0], account.states[1]) - .nullifiers_range(nullifier_starting_index..(nullifier_starting_index + 1)) - .build(); - - (tx, account) - }) -} diff --git a/crates/block-producer/src/state_view/tests/verify_tx.rs b/crates/block-producer/src/state_view/tests/verify_tx.rs deleted file mode 100644 index c3f576193..000000000 --- a/crates/block-producer/src/state_view/tests/verify_tx.rs +++ /dev/null @@ -1,306 +0,0 @@ -//! `verify_tx(tx)` requirements: -//! -//! Store-related requirements -//! VT1: `tx.initial_account_hash` must match the account hash in store -//! VT2: If store doesn't contain account, `verify_tx` should check that it is a new account ( TODO -//! ) and succeed VT3: If `tx` consumes an already-consumed note in the store, `verify_tx` must fail -//! -//! in-flight related requirements -//! VT4: In each block, at most 1 transaction is allowed to modify any given account -//! VT5: `verify_tx(tx)` must fail if a previous transaction, not yet in the block, consumed a note -//! that `tx` is also consuming - -use std::iter; - -use miden_objects::notes::Note; -use tokio::task::JoinSet; - -use super::*; -use crate::test_utils::{block::MockBlockBuilder, note::mock_note, MockStoreSuccessBuilder}; - -/// Tests the happy path where 3 transactions who modify different accounts and consume different -/// notes all verify successfully -#[tokio::test] -#[miden_node_test_macro::enable_logging] -async fn test_verify_tx_happy_path() { - let (txs, accounts): (Vec, Vec) = - get_txs_and_accounts(0, 3).unzip(); - - let store = Arc::new( - MockStoreSuccessBuilder::from_accounts( - accounts - .into_iter() - .map(|mock_account| (mock_account.id, mock_account.states[0])), - ) - .build(), - ); - - let state_view = DefaultStateView::new(store, false); - - for tx in txs { - state_view.verify_tx(&tx).await.unwrap(); - } -} - -/// Tests the happy path where 3 transactions who modify different accounts and consume different -/// notes all verify successfully. -/// -/// In this test, all calls to `verify_tx()` are concurrent -#[tokio::test] -#[miden_node_test_macro::enable_logging] -async fn test_verify_tx_happy_path_concurrent() { - let (txs, accounts): (Vec, Vec) = - get_txs_and_accounts(0, 3).unzip(); - - let store = Arc::new( - MockStoreSuccessBuilder::from_accounts( - accounts - .into_iter() - .map(|mock_account| (mock_account.id, mock_account.states[0])), - ) - .build(), - ); - - let state_view = Arc::new(DefaultStateView::new(store, false)); - - let mut set = JoinSet::new(); - - for tx in txs { - let state_view = state_view.clone(); - set.spawn(async move { state_view.verify_tx(&tx).await }); - } - - while let Some(res) = set.join_next().await { - res.unwrap().unwrap(); - } -} - -/// Verifies requirement VT1 -#[tokio::test] -#[miden_node_test_macro::enable_logging] -async fn test_verify_tx_vt1() { - let account = MockPrivateAccount::<3>::from(1); - - let store = Arc::new( - MockStoreSuccessBuilder::from_accounts(iter::once((account.id, account.states[0]))).build(), - ); - - // The transaction's initial account hash uses `account.states[1]`, where the store expects - // `account.states[0]` - let tx = MockProvenTxBuilder::with_account(account.id, account.states[1], account.states[2]) - .nullifiers_range(0..1) - .build(); - - let state_view = DefaultStateView::new(store, false); - - let verify_tx_result = state_view.verify_tx(&tx).await; - - assert_eq!( - verify_tx_result, - Err(VerifyTxError::IncorrectAccountInitialHash { - tx_initial_account_hash: account.states[1], - current_account_hash: Some(account.states[0]), - }) - ); -} - -/// Verifies requirement VT2 -#[tokio::test] -#[miden_node_test_macro::enable_logging] -async fn test_verify_tx_vt2() { - let account_not_in_store: MockPrivateAccount<3> = MockPrivateAccount::from(0); - - // Notice: account is not added to the store - let store = Arc::new(MockStoreSuccessBuilder::from_batches(iter::empty()).build()); - - let tx = MockProvenTxBuilder::with_account( - account_not_in_store.id, - account_not_in_store.states[0], - account_not_in_store.states[1], - ) - .nullifiers_range(0..1) - .build(); - - let state_view = DefaultStateView::new(store, false); - - let verify_tx_result = state_view.verify_tx(&tx).await; - - assert!(verify_tx_result.is_ok()); -} - -/// Verifies requirement VT3 -#[tokio::test] -#[miden_node_test_macro::enable_logging] -async fn test_verify_tx_vt3() { - let account: MockPrivateAccount<3> = MockPrivateAccount::from(1); - - let nullifier_in_store = nullifier_by_index(0); - - // Notice: `consumed_note_in_store` is added to the store - let store = Arc::new( - MockStoreSuccessBuilder::from_accounts(iter::once((account.id, account.states[0]))) - .initial_nullifiers(BTreeSet::from_iter(iter::once(nullifier_in_store.inner()))) - .initial_block_num(1) - .build(), - ); - - let tx = MockProvenTxBuilder::with_account(account.id, account.states[0], account.states[1]) - .nullifiers(vec![nullifier_in_store]) - .build(); - - let state_view = DefaultStateView::new(store, false); - - let verify_tx_result = state_view.verify_tx(&tx).await; - - assert_eq!( - verify_tx_result, - Err(VerifyTxError::InputNotesAlreadyConsumed(vec![nullifier_in_store])) - ); -} - -/// Verifies requirement VT4 -#[tokio::test] -#[miden_node_test_macro::enable_logging] -async fn test_verify_tx_vt4() { - let account: MockPrivateAccount<3> = MockPrivateAccount::from(1); - - let store = Arc::new( - MockStoreSuccessBuilder::from_accounts(iter::once((account.id, account.states[0]))).build(), - ); - - let tx1 = - MockProvenTxBuilder::with_account(account.id, account.states[0], account.states[1]).build(); - - // Notice: tx2 follows tx1, using the same account and with an initial state matching the final - // state of the first. We expect both to pass. - let tx2 = - MockProvenTxBuilder::with_account(account.id, account.states[1], account.states[2]).build(); - - let state_view = DefaultStateView::new(store, false); - - let verify_tx1_result = state_view.verify_tx(&tx1).await; - assert!(verify_tx1_result.is_ok()); - - let verify_tx2_result = state_view.verify_tx(&tx2).await; - assert!(verify_tx2_result.is_ok()); -} - -/// Verifies requirement VT5 -#[tokio::test] -#[miden_node_test_macro::enable_logging] -async fn test_verify_tx_vt5() { - let account_1: MockPrivateAccount<3> = MockPrivateAccount::from(1); - let account_2: MockPrivateAccount<3> = MockPrivateAccount::from(2); - let nullifier_in_both_txs = nullifier_by_index(0); - - // Notice: `consumed_note_in_both_txs` is NOT in the store - let store = Arc::new( - MockStoreSuccessBuilder::from_accounts( - [account_1, account_2] - .into_iter() - .map(|account| (account.id, account.states[0])), - ) - .build(), - ); - - let tx1 = - MockProvenTxBuilder::with_account(account_1.id, account_1.states[0], account_1.states[1]) - .nullifiers(vec![nullifier_in_both_txs]) - .build(); - - // Notice: tx2 modifies the same account as tx1, even though from a different initial state, - // which is currently disallowed - let tx2 = - MockProvenTxBuilder::with_account(account_2.id, account_2.states[1], account_2.states[2]) - .nullifiers(vec![nullifier_in_both_txs]) - .build(); - - let state_view = DefaultStateView::new(store, false); - - let verify_tx1_result = state_view.verify_tx(&tx1).await; - assert!(verify_tx1_result.is_ok()); - - let verify_tx2_result = state_view.verify_tx(&tx2).await; - assert_eq!( - verify_tx2_result, - Err(VerifyTxError::InputNotesAlreadyConsumed(vec![nullifier_in_both_txs])) - ); -} - -/// Tests that `verify_tx()` succeeds when the unauthenticated input note found in the in-flight -/// notes -#[tokio::test] -#[miden_node_test_macro::enable_logging] -async fn test_verify_tx_dangling_note_found_in_inflight_notes() { - let account_1: MockPrivateAccount<3> = MockPrivateAccount::from(1); - let account_2: MockPrivateAccount<3> = MockPrivateAccount::from(2); - let store = Arc::new( - MockStoreSuccessBuilder::from_accounts( - [account_1, account_2] - .into_iter() - .map(|account| (account.id, account.states[0])), - ) - .build(), - ); - let state_view = DefaultStateView::new(Arc::clone(&store), false); - - let dangling_notes = vec![mock_note(1)]; - let output_notes = dangling_notes.iter().cloned().map(OutputNote::Full).collect(); - - let tx1 = MockProvenTxBuilder::with_account_index(1).output_notes(output_notes).build(); - - let verify_tx1_result = state_view.verify_tx(&tx1).await; - assert_eq!(verify_tx1_result, Ok(0)); - - let tx2 = MockProvenTxBuilder::with_account_index(2) - .unauthenticated_notes(dangling_notes.clone()) - .build(); - - let verify_tx2_result = state_view.verify_tx(&tx2).await; - assert_eq!( - verify_tx2_result, - Ok(0), - "Dangling unauthenticated notes must be found in the in-flight notes after previous tx verification" - ); -} - -/// Tests that `verify_tx()` fails when the unauthenticated input note not found not in the -/// in-flight notes nor in the store -#[tokio::test] -#[miden_node_test_macro::enable_logging] -async fn test_verify_tx_stored_unauthenticated_notes() { - let account_1: MockPrivateAccount<3> = MockPrivateAccount::from(1); - let store = Arc::new( - MockStoreSuccessBuilder::from_accounts( - [account_1].into_iter().map(|account| (account.id, account.states[0])), - ) - .build(), - ); - let dangling_notes = vec![mock_note(1)]; - let tx1 = MockProvenTxBuilder::with_account_index(1) - .unauthenticated_notes(dangling_notes.clone()) - .build(); - - let state_view = DefaultStateView::new(Arc::clone(&store), false); - - let verify_tx1_result = state_view.verify_tx(&tx1).await; - assert_eq!( - verify_tx1_result, - Err(VerifyTxError::UnauthenticatedNotesNotFound( - dangling_notes.iter().map(Note::id).collect() - )), - "Dangling unauthenticated notes must not be found in the store by this moment" - ); - - let output_notes = dangling_notes.into_iter().map(OutputNote::Full).collect(); - let block = MockBlockBuilder::new(&store).await.created_notes(vec![output_notes]).build(); - - store.apply_block(&block).await.unwrap(); - - let verify_tx1_result = state_view.verify_tx(&tx1).await; - assert_eq!( - verify_tx1_result, - Ok(0), - "Dangling unauthenticated notes must be found in the store after block applying" - ); -} diff --git a/crates/block-producer/src/store/mod.rs b/crates/block-producer/src/store/mod.rs index 8c06005a1..476c05a07 100644 --- a/crates/block-producer/src/store/mod.rs +++ b/crates/block-producer/src/store/mod.rs @@ -7,14 +7,10 @@ use std::{ use async_trait::async_trait; use itertools::Itertools; use miden_node_proto::{ - domain::notes::NoteAuthenticationInfo, errors::{ConversionError, MissingFieldHelper}, generated::{ digest, - requests::{ - ApplyBlockRequest, GetBlockInputsRequest, GetNoteAuthenticationInfoRequest, - GetTransactionInputsRequest, - }, + requests::{ApplyBlockRequest, GetBlockInputsRequest, GetTransactionInputsRequest}, responses::{GetTransactionInputsResponse, NullifierTransactionInputRecord}, store::api_client as store_client, }, @@ -25,15 +21,16 @@ use miden_objects::{ accounts::AccountId, block::Block, notes::{NoteId, Nullifier}, + transaction::ProvenTransaction, utils::Serializable, - Digest, + BlockHeader, Digest, }; use miden_processor::crypto::RpoDigest; use tonic::transport::Channel; use tracing::{debug, info, instrument}; pub use crate::errors::{ApplyBlockError, BlockInputsError, TxInputsError}; -use crate::{block::BlockInputs, errors::NotePathsError, ProvenTransaction, COMPONENT}; +use crate::{block::BlockInputs, COMPONENT}; // STORE TRAIT // ================================================================================================ @@ -53,15 +50,6 @@ pub trait Store: ApplyBlock { produced_nullifiers: impl Iterator + Send, notes: impl Iterator + Send, ) -> Result; - - /// Returns note authentication information for the set of specified notes. - /// - /// If authentication info for a note does not exist in the store, the note is omitted - /// from the returned set of notes. - async fn get_note_authentication_info( - &self, - notes: impl Iterator + Send, - ) -> Result; } #[async_trait] @@ -154,6 +142,7 @@ impl TryFrom for TransactionInputs { // DEFAULT STORE IMPLEMENTATION // ================================================================================================ +#[derive(Clone)] pub struct DefaultStore { store: store_client::ApiClient, } @@ -163,6 +152,20 @@ impl DefaultStore { pub fn new(store: store_client::ApiClient) -> Self { Self { store } } + + /// Returns the latest block's header from the store. + pub async fn latest_header(&self) -> Result { + // TODO: fixup the errors types. + let response = self + .store + .clone() + .get_block_header_by_number(tonic::Request::new(Default::default())) + .await + .map_err(|err| err.to_string())? + .into_inner(); + + BlockHeader::try_from(response.block_header.unwrap()).map_err(|err| err.to_string()) + } } // FIXME: remove the allow when the upstream clippy issue is fixed: @@ -255,28 +258,4 @@ impl Store for DefaultStore { Ok(store_response.try_into()?) } - - async fn get_note_authentication_info( - &self, - notes: impl Iterator + Send, - ) -> Result { - let request = tonic::Request::new(GetNoteAuthenticationInfoRequest { - note_ids: notes.map(digest::Digest::from).collect(), - }); - - let store_response = self - .store - .clone() - .get_note_authentication_info(request) - .await - .map_err(|err| NotePathsError::GrpcClientError(err.message().to_string()))? - .into_inner(); - - let note_authentication_info = store_response - .proofs - .ok_or(GetTransactionInputsResponse::missing_field("proofs"))? - .try_into()?; - - Ok(note_authentication_info) - } } diff --git a/crates/block-producer/src/test_utils/batch.rs b/crates/block-producer/src/test_utils/batch.rs index 889fc8829..2717a3a79 100644 --- a/crates/block-producer/src/test_utils/batch.rs +++ b/crates/block-producer/src/test_utils/batch.rs @@ -1,4 +1,4 @@ -use crate::{test_utils::MockProvenTxBuilder, TransactionBatch}; +use crate::{batch_builder::TransactionBatch, test_utils::MockProvenTxBuilder}; pub trait TransactionBatchConstructor { /// Returns a `TransactionBatch` with `notes_per_tx.len()` transactions, where the i'th diff --git a/crates/block-producer/src/test_utils/block.rs b/crates/block-producer/src/test_utils/block.rs index 71d15f7e6..35abbe6a0 100644 --- a/crates/block-producer/src/test_utils/block.rs +++ b/crates/block-producer/src/test_utils/block.rs @@ -10,10 +10,10 @@ use miden_objects::{ use super::MockStoreSuccess; use crate::{ + batch_builder::TransactionBatch, block::BlockInputs, block_builder::prover::{block_witness::BlockWitness, BlockProver}, store::Store, - TransactionBatch, }; /// Constructs the block we expect to be built given the store state, and a set of transaction diff --git a/crates/block-producer/src/test_utils/mod.rs b/crates/block-producer/src/test_utils/mod.rs index 3a8d7de41..e2385e747 100644 --- a/crates/block-producer/src/test_utils/mod.rs +++ b/crates/block-producer/src/test_utils/mod.rs @@ -13,7 +13,6 @@ pub use proven_tx::{mock_proven_tx, MockProvenTxBuilder}; mod store; -use rand::Rng; pub use store::{MockStoreFailure, MockStoreSuccess, MockStoreSuccessBuilder}; mod account; diff --git a/crates/block-producer/src/test_utils/store.rs b/crates/block-producer/src/test_utils/store.rs index 3df5ca2b0..c04ff3923 100644 --- a/crates/block-producer/src/test_utils/store.rs +++ b/crates/block-producer/src/test_utils/store.rs @@ -10,6 +10,7 @@ use miden_objects::{ block::{Block, NoteBatch}, crypto::merkle::{Mmr, SimpleSmt, Smt, ValuePath}, notes::{NoteId, NoteInclusionProof, Nullifier}, + transaction::ProvenTransaction, BlockHeader, ACCOUNT_TREE_DEPTH, EMPTY_WORD, ZERO, }; use tokio::sync::RwLock; @@ -18,14 +19,12 @@ use super::*; use crate::{ batch_builder::TransactionBatch, block::{AccountWitness, BlockInputs}, - errors::NotePathsError, store::{ ApplyBlock, ApplyBlockError, BlockInputsError, Store, TransactionInputs, TxInputsError, }, test_utils::block::{ block_output_notes, flatten_output_notes, note_created_smt_from_note_batches, }, - ProvenTransaction, }; /// Builds a [`MockStoreSuccess`] @@ -352,39 +351,6 @@ impl Store for MockStoreSuccess { found_unauthenticated_notes, }) } - - async fn get_note_authentication_info( - &self, - notes: impl Iterator + Send, - ) -> Result { - let locked_notes = self.notes.read().await; - let locked_headers = self.block_headers.read().await; - let locked_chain_mmr = self.chain_mmr.read().await; - - let note_proofs = notes - .filter_map(|id| locked_notes.get(id).map(|proof| (*id, proof.clone()))) - .collect::>(); - - let latest_header = - *locked_headers.iter().max_by_key(|(block_num, _)| *block_num).unwrap().1; - let chain_length = latest_header.block_num(); - - let block_proofs = note_proofs - .values() - .map(|note_proof| { - let block_num = note_proof.location().block_num(); - let block_header = *locked_headers.get(&block_num).unwrap(); - let mmr_path = locked_chain_mmr - .open_at(block_num as usize, latest_header.block_num() as usize) - .unwrap() - .merkle_path; - - BlockInclusionProof { block_header, mmr_path, chain_length } - }) - .collect(); - - Ok(NoteAuthenticationInfo { block_proofs, note_proofs }) - } } #[derive(Default)] @@ -414,11 +380,4 @@ impl Store for MockStoreFailure { ) -> Result { Err(BlockInputsError::GrpcClientError(String::new())) } - - async fn get_note_authentication_info( - &self, - _notes: impl Iterator + Send, - ) -> Result { - Err(NotePathsError::GrpcClientError(String::new())) - } } diff --git a/crates/block-producer/src/txqueue/mod.rs b/crates/block-producer/src/txqueue/mod.rs deleted file mode 100644 index 5f5f724c5..000000000 --- a/crates/block-producer/src/txqueue/mod.rs +++ /dev/null @@ -1,174 +0,0 @@ -use std::{sync::Arc, time::Duration}; - -use async_trait::async_trait; -use miden_objects::MAX_OUTPUT_NOTES_PER_BATCH; -use tokio::{sync::RwLock, time}; -use tracing::{debug, info, info_span, instrument, Instrument}; - -use crate::{ - batch_builder::BatchBuilder, - errors::{AddTransactionError, VerifyTxError}, - ProvenTransaction, SharedRwVec, COMPONENT, -}; - -#[cfg(test)] -mod tests; - -// TRANSACTION VALIDATOR -// ================================================================================================ - -/// Implementations are responsible to track in-flight transactions and verify that new transactions -/// added to the queue are not conflicting. -/// -/// See [crate::store::ApplyBlock], that trait's `apply_block` is called when a block is sealed, and -/// it can determine when transactions are no longer in-flight. -#[async_trait] -pub trait TransactionValidator: Send + Sync + 'static { - /// Method to receive a `tx` for processing and return the current block height. - /// - /// This method should: - /// - Verify the transaction is valid, against the current's rollup state, and also against - /// in-flight transactions. - /// - Track the necessary state of the transaction until it is committed to the `store`, to - /// perform the check above. - async fn verify_tx(&self, tx: &ProvenTransaction) -> Result; -} - -// TRANSACTION QUEUE -// ================================================================================================ - -pub struct TransactionQueueOptions { - /// The frequency at which we try to build batches from transactions in the queue - pub build_batch_frequency: Duration, - - /// The size of a batch - pub batch_size: usize, -} - -pub struct TransactionQueue { - ready_queue: SharedRwVec, - tx_validator: Arc, - batch_builder: Arc, - options: TransactionQueueOptions, -} - -impl TransactionQueue -where - TV: TransactionValidator, - BB: BatchBuilder, -{ - pub fn new( - tx_validator: Arc, - batch_builder: Arc, - options: TransactionQueueOptions, - ) -> Self { - Self { - ready_queue: Arc::new(RwLock::new(Vec::new())), - tx_validator, - batch_builder, - options, - } - } - - pub async fn run(self: Arc) { - let mut interval = time::interval(self.options.build_batch_frequency); - - info!(target: COMPONENT, period_ms = interval.period().as_millis(), "Transaction queue started"); - - loop { - interval.tick().await; - self.try_build_batches().await; - } - } - - /// Divides the queue in groups to be batched; those that failed are appended back on the queue - #[instrument(target = "miden-block-producer", skip_all)] - async fn try_build_batches(&self) { - let mut txs: Vec = { - let mut locked_ready_queue = self.ready_queue.write().await; - - // If there are no transactions in the queue, this call is a no-op. The [BatchBuilder] - // will produce empty blocks if necessary. - if locked_ready_queue.is_empty() { - debug!(target: COMPONENT, "Transaction queue empty"); - return; - } - - locked_ready_queue.drain(..).rev().collect() - }; - - while !txs.is_empty() { - let mut batch = Vec::with_capacity(self.options.batch_size); - let mut output_notes_in_batch = 0; - - while let Some(tx) = txs.pop() { - output_notes_in_batch += tx.output_notes().num_notes(); - - debug_assert!( - tx.output_notes().num_notes() <= MAX_OUTPUT_NOTES_PER_BATCH, - "Sanity check, the number of output notes of a single transaction must never be larger than the batch maximum", - ); - - if output_notes_in_batch > MAX_OUTPUT_NOTES_PER_BATCH - || batch.len() == self.options.batch_size - { - // Batch would be too big in number of notes or transactions. Push the tx back - // to the list of available transactions and forward the current batch. - txs.push(tx); - break; - } - - // The tx fits in the current batch - batch.push(tx) - } - - let ready_queue = self.ready_queue.clone(); - let batch_builder = self.batch_builder.clone(); - - tokio::spawn( - async move { - match batch_builder.build_batch(batch).await { - Ok(_) => { - // batch was successfully built, do nothing - }, - Err(e) => { - // batch building failed, add txs back to the beginning of the queue - let mut locked_ready_queue = ready_queue.write().await; - e.into_transactions() - .into_iter() - .enumerate() - .for_each(|(i, tx)| locked_ready_queue.insert(i, tx)); - }, - } - } - .instrument(info_span!(target: COMPONENT, "batch_builder")), - ); - } - } - - /// Queues `tx` to be added in a batch and subsequently into a block and returns the current - /// block height. - /// - /// This method will validate the `tx` and ensure it is valid w.r.t. the rollup state, and the - /// current in-flight transactions. - #[instrument(target = "miden-block-producer", skip_all, err)] - pub async fn add_transaction(&self, tx: ProvenTransaction) -> Result { - info!(target: COMPONENT, tx_id = %tx.id().to_hex(), account_id = %tx.account_id().to_hex()); - - let block_height = self - .tx_validator - .verify_tx(&tx) - .await - .map_err(AddTransactionError::VerificationFailed)?; - - let queue_len = { - let mut queue_write_guard = self.ready_queue.write().await; - queue_write_guard.push(tx); - queue_write_guard.len() - }; - - info!(target: COMPONENT, queue_len, "Transaction added to tx queue"); - - Ok(block_height) - } -} diff --git a/crates/block-producer/src/txqueue/tests/mod.rs b/crates/block-producer/src/txqueue/tests/mod.rs deleted file mode 100644 index 79ae32b21..000000000 --- a/crates/block-producer/src/txqueue/tests/mod.rs +++ /dev/null @@ -1,225 +0,0 @@ -use tokio::sync::mpsc::{self, error::TryRecvError}; - -use super::*; -use crate::{errors::BuildBatchError, test_utils::MockProvenTxBuilder, TransactionBatch}; - -// STRUCTS -// ================================================================================================ - -/// All transactions verify successfully -struct TransactionValidatorSuccess; - -#[async_trait] -impl TransactionValidator for TransactionValidatorSuccess { - async fn verify_tx(&self, _tx: &ProvenTransaction) -> Result { - Ok(0) - } -} - -/// All transactions fail to verify -struct TransactionValidatorFailure; - -#[async_trait] -impl TransactionValidator for TransactionValidatorFailure { - async fn verify_tx(&self, tx: &ProvenTransaction) -> Result { - Err(VerifyTxError::InvalidTransactionProof(tx.id())) - } -} - -/// Records all batches built in `ready_batches` -struct BatchBuilderSuccess { - ready_batches: mpsc::UnboundedSender, -} - -impl BatchBuilderSuccess { - fn new(ready_batches: mpsc::UnboundedSender) -> Self { - Self { ready_batches } - } -} - -#[async_trait] -impl BatchBuilder for BatchBuilderSuccess { - async fn build_batch(&self, txs: Vec) -> Result<(), BuildBatchError> { - let batch = TransactionBatch::new(txs, Default::default()) - .expect("Tx batch building should have succeeded"); - self.ready_batches - .send(batch) - .expect("Sending to channel should have succeeded"); - - Ok(()) - } -} - -/// Always fails to build batch -#[derive(Default)] -struct BatchBuilderFailure; - -#[async_trait] -impl BatchBuilder for BatchBuilderFailure { - async fn build_batch(&self, txs: Vec) -> Result<(), BuildBatchError> { - Err(BuildBatchError::TooManyNotesCreated(0, txs)) - } -} - -// TESTS -// ================================================================================================ - -/// Tests that when the internal "build batch timer" hits, all transactions in the queue are sent to -/// be built in some batch -#[tokio::test(start_paused = true)] -#[miden_node_test_macro::enable_logging] -async fn test_build_batch_success() { - let build_batch_frequency = Duration::from_millis(5); - let batch_size = 3; - let (sender, mut receiver) = mpsc::unbounded_channel::(); - - let tx_queue = Arc::new(TransactionQueue::new( - Arc::new(TransactionValidatorSuccess), - Arc::new(BatchBuilderSuccess::new(sender)), - TransactionQueueOptions { build_batch_frequency, batch_size }, - )); - - // Starts the transaction queue task. - tokio::spawn(tx_queue.clone().run()); - - // the queue start empty - assert_eq!(Err(TryRecvError::Empty), receiver.try_recv()); - - // if no transactions have been added to the queue in the batch build interval, the queue does - // nothing - tokio::time::advance(build_batch_frequency).await; - assert_eq!(Err(TryRecvError::Empty), receiver.try_recv(), "queue starts empty"); - - // if there is a single transaction in the queue when it is time to build a batch, the batch is - // created with that single transaction - let tx = MockProvenTxBuilder::with_account_index(0).build(); - tx_queue - .add_transaction(tx.clone()) - .await - .expect("Transaction queue is running"); - - tokio::time::advance(build_batch_frequency).await; - let batch = receiver.try_recv().expect("Queue not empty"); - assert_eq!( - Err(TryRecvError::Empty), - receiver.try_recv(), - "A single transaction produces a single batch" - ); - let expected = - TransactionBatch::new(vec![tx.clone()], Default::default()).expect("Valid transactions"); - assert_eq!(expected, batch, "The batch should have the one transaction added to the queue"); - - // a batch will include up to `batch_size` transactions - let mut txs = Vec::new(); - for i in 0..batch_size { - let tx = MockProvenTxBuilder::with_account_index(i as u32).build(); - tx_queue - .add_transaction(tx.clone()) - .await - .expect("Transaction queue is running"); - txs.push(tx); - } - tokio::time::advance(build_batch_frequency).await; - let batch = receiver.try_recv().expect("Queue not empty"); - assert_eq!( - Err(TryRecvError::Empty), - receiver.try_recv(), - "{batch_size} transactions create a single batch" - ); - let expected = TransactionBatch::new(txs, Default::default()).expect("Valid transactions"); - assert_eq!(expected, batch, "The batch should the transactions to fill a batch"); - - // the transaction queue eagerly produces batches - let mut txs = Vec::new(); - for i in 0..(2 * batch_size + 1) { - let tx = MockProvenTxBuilder::with_account_index(i as u32).build(); - tx_queue - .add_transaction(tx.clone()) - .await - .expect("Transaction queue is running"); - txs.push(tx.clone()) - } - for expected_batch in txs.chunks(batch_size).map(|txs| txs.to_vec()) { - tokio::time::advance(build_batch_frequency).await; - let batch = receiver.try_recv().expect("Queue not empty"); - let expected = - TransactionBatch::new(expected_batch, Default::default()).expect("Valid transactions"); - assert_eq!(expected, batch, "The batch should the transactions to fill a batch"); - } - - // ensure all transactions have been consumed - tokio::time::advance(build_batch_frequency * 2).await; - assert_eq!( - Err(TryRecvError::Empty), - receiver.try_recv(), - "If there are no transactions, no batches are produced" - ); -} - -/// Tests that when transactions fail to verify, they are not added to the queue -#[tokio::test(start_paused = true)] -#[miden_node_test_macro::enable_logging] -async fn test_tx_verify_failure() { - let build_batch_frequency = Duration::from_millis(5); - let batch_size = 3; - - let (sender, mut receiver) = mpsc::unbounded_channel::(); - let batch_builder = Arc::new(BatchBuilderSuccess::new(sender)); - - let tx_queue = Arc::new(TransactionQueue::new( - Arc::new(TransactionValidatorFailure), - batch_builder.clone(), - TransactionQueueOptions { build_batch_frequency, batch_size }, - )); - - // Start the queue - tokio::spawn(tx_queue.clone().run()); - - // Add a bunch of transactions that will all fail tx verification - for i in 0..(3 * batch_size as u32) { - let r = tx_queue - .add_transaction(MockProvenTxBuilder::with_account_index(i).build()) - .await; - - assert!(matches!(r, Err(AddTransactionError::VerificationFailed(_)))); - assert_eq!( - Err(TryRecvError::Empty), - receiver.try_recv(), - "If there are no transactions, no batches are produced" - ); - } -} - -/// Tests that when batch building fails, transactions are added back to the ready queue -#[tokio::test] -#[miden_node_test_macro::enable_logging] -async fn test_build_batch_failure() { - let build_batch_frequency = Duration::from_millis(30); - let batch_size = 3; - - let batch_builder = Arc::new(BatchBuilderFailure); - - let tx_queue = TransactionQueue::new( - Arc::new(TransactionValidatorSuccess), - batch_builder.clone(), - TransactionQueueOptions { build_batch_frequency, batch_size }, - ); - - let internal_ready_queue = tx_queue.ready_queue.clone(); - - // Add enough transactions so that we have 1 batch - for i in 0..batch_size { - tx_queue - .add_transaction(MockProvenTxBuilder::with_account_index(i as u32).build()) - .await - .unwrap(); - } - - // Start the queue - tokio::spawn(Arc::new(tx_queue).run()); - - // Wait for tx queue to fail once to build the batch - time::sleep(Duration::from_millis(45)).await; - - assert_eq!(internal_ready_queue.read().await.len(), 3); -} From 16c3087a5e0b6cfa718b334996622eb2b43f17b2 Mon Sep 17 00:00:00 2001 From: polydez <155382956+polydez@users.noreply.github.com> Date: Fri, 8 Nov 2024 14:01:00 +0500 Subject: [PATCH 10/50] refactor: simplify shared mempool (#548) --- crates/block-producer/src/mempool/mod.rs | 31 ++++-------------------- crates/block-producer/src/server/mod.rs | 4 +-- 2 files changed, 7 insertions(+), 28 deletions(-) diff --git a/crates/block-producer/src/mempool/mod.rs b/crates/block-producer/src/mempool/mod.rs index bbe13767c..e793ef1df 100644 --- a/crates/block-producer/src/mempool/mod.rs +++ b/crates/block-producer/src/mempool/mod.rs @@ -76,28 +76,7 @@ impl BlockNumber { // MEMPOOL // ================================================================================================ -#[derive(Clone)] -pub struct SharedMempool(Arc>); - -impl SharedMempool { - pub fn new( - chain_tip: BlockNumber, - batch_limit: usize, - block_limit: usize, - state_retention: usize, - ) -> Self { - Self(Arc::new(Mutex::new(Mempool::new( - chain_tip, - batch_limit, - block_limit, - state_retention, - )))) - } - - pub async fn lock(&self) -> tokio::sync::MutexGuard { - self.0.lock().await - } -} +pub type SharedMempool = Arc>; pub struct Mempool { /// The latest inflight state of each account. @@ -125,13 +104,13 @@ pub struct Mempool { impl Mempool { /// Creates a new [Mempool] with the provided configuration. - fn new( + pub fn new( chain_tip: BlockNumber, batch_limit: usize, block_limit: usize, state_retention: usize, - ) -> Self { - Self { + ) -> SharedMempool { + Arc::new(Mutex::new(Self { chain_tip, batch_transaction_limit: batch_limit, block_batch_limit: block_limit, @@ -140,7 +119,7 @@ impl Mempool { transactions: Default::default(), batches: Default::default(), next_batch_id: Default::default(), - } + })) } /// Adds a transaction to the mempool. diff --git a/crates/block-producer/src/server/mod.rs b/crates/block-producer/src/server/mod.rs index ff4c57364..e1f3bf0b7 100644 --- a/crates/block-producer/src/server/mod.rs +++ b/crates/block-producer/src/server/mod.rs @@ -20,7 +20,7 @@ use crate::{ config::BlockProducerConfig, domain::transaction::AuthenticatedTransaction, errors::{AddTransactionError, VerifyTxError}, - mempool::{BlockNumber, SharedMempool}, + mempool::{BlockNumber, Mempool, SharedMempool}, store::{DefaultStore, Store}, COMPONENT, SERVER_BATCH_SIZE, SERVER_MAX_BATCHES_PER_BLOCK, SERVER_MEMPOOL_STATE_RETENTION, }; @@ -97,7 +97,7 @@ impl BlockProducer { chain_tip, } = self; - let mempool = SharedMempool::new(chain_tip, batch_limit, block_limit, state_retention); + let mempool = Mempool::new(chain_tip, batch_limit, block_limit, state_retention); // Spawn rpc server and batch and block provers. // From 8329a776202bd4ff85e1121d0fae834af9a28e13 Mon Sep 17 00:00:00 2001 From: Varun Doshi <61531351+varun-doshi@users.noreply.github.com> Date: Thu, 28 Nov 2024 17:18:39 +0530 Subject: [PATCH 11/50] chore: update deadpool-sqlite (#555) --- Cargo.lock | 16 ++++++++-------- crates/store/Cargo.toml | 4 ++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6dff0370d..184a24b62 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -619,9 +619,9 @@ dependencies = [ [[package]] name = "deadpool-sqlite" -version = "0.8.1" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f9cc6210316f8b7ced394e2a5d2833ce7097fb28afb5881299c61bc18e8e0e9" +checksum = "656f14fc1ab819c65f332045ea7cb38841bbe551f3b2bc7e3abefb559af4155c" dependencies = [ "deadpool", "deadpool-sync", @@ -1302,9 +1302,9 @@ dependencies = [ [[package]] name = "libsqlite3-sys" -version = "0.28.0" +version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c10584274047cb335c23d3e61bcef8e323adae7c5c8c760540f73610177fc3f" +checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149" dependencies = [ "bindgen", "cc", @@ -2490,9 +2490,9 @@ checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "rusqlite" -version = "0.31.0" +version = "0.32.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b838eba278d213a8beaf485bd313fd580ca4505a00d5871caeb1457c55322cae" +checksum = "7753b721174eb8ff87a9a0e799e2d7bc3749323e773db92e0984debb00019d6e" dependencies = [ "bitflags", "fallible-iterator", @@ -2504,9 +2504,9 @@ dependencies = [ [[package]] name = "rusqlite_migration" -version = "1.2.0" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55709bc01054c69e2f1cefdc886642b5e6376a8db3c86f761be0c423eebf178b" +checksum = "923b42e802f7dc20a0a6b5e097ba7c83fe4289da07e49156fecf6af08aa9cd1c" dependencies = [ "log", "rusqlite", diff --git a/crates/store/Cargo.toml b/crates/store/Cargo.toml index bb9a63103..1337fcd86 100644 --- a/crates/store/Cargo.toml +++ b/crates/store/Cargo.toml @@ -12,7 +12,7 @@ homepage.workspace = true repository.workspace = true [dependencies] -deadpool-sqlite = { version = "0.8", features = ["rt_tokio_1"] } +deadpool-sqlite = { version = "0.9.0", features = ["rt_tokio_1"] } directories = { version = "5.0" } figment = { version = "0.10", features = ["toml", "env"] } hex = { version = "0.4" } @@ -21,7 +21,7 @@ miden-node-proto = { workspace = true } miden-node-utils = { workspace = true } miden-objects = { workspace = true } prost = { workspace = true } -rusqlite = { version = "0.31", features = ["array", "buildtime_bindgen", "bundled"] } +rusqlite = { version = "0.32.1", features = ["array", "buildtime_bindgen", "bundled"] } rusqlite_migration = { version = "1.0" } serde = { version = "1.0", features = ["derive"] } thiserror = { workspace = true } From 312d06a8f35fe952106d24c537646df350762e4f Mon Sep 17 00:00:00 2001 From: Mirko <48352201+Mirko-von-Leipzig@users.noreply.github.com> Date: Fri, 29 Nov 2024 16:22:17 +0200 Subject: [PATCH 12/50] feat(block-producer): improve mempool config (#543) --- .../block-producer/src/batch_builder/batch.rs | 46 ++----- .../src/block_builder/prover/block_witness.rs | 5 +- .../block-producer/src/domain/transaction.rs | 8 ++ crates/block-producer/src/errors.rs | 38 +----- crates/block-producer/src/lib.rs | 12 +- .../block-producer/src/mempool/batch_graph.rs | 34 ++--- crates/block-producer/src/mempool/mod.rs | 118 ++++++++++++++++-- .../src/mempool/transaction_graph.rs | 46 ++++--- crates/block-producer/src/server/mod.rs | 18 +-- 9 files changed, 203 insertions(+), 122 deletions(-) diff --git a/crates/block-producer/src/batch_builder/batch.rs b/crates/block-producer/src/batch_builder/batch.rs index be0c28cb8..225112d00 100644 --- a/crates/block-producer/src/batch_builder/batch.rs +++ b/crates/block-producer/src/batch_builder/batch.rs @@ -10,8 +10,7 @@ use miden_objects::{ crypto::hash::blake::{Blake3Digest, Blake3_256}, notes::{NoteHeader, NoteId, Nullifier}, transaction::{InputNoteCommitment, OutputNote, ProvenTransaction, TransactionId}, - AccountDeltaError, Digest, MAX_ACCOUNTS_PER_BATCH, MAX_INPUT_NOTES_PER_BATCH, - MAX_OUTPUT_NOTES_PER_BATCH, + AccountDeltaError, Digest, }; use tracing::instrument; @@ -76,13 +75,11 @@ impl TransactionBatch { /// for transforming unauthenticated notes into authenticated notes. /// /// # Errors + /// /// Returns an error if: - /// - The number of output notes across all transactions exceeds 4096. /// - There are duplicated output notes or unauthenticated notes found across all transactions /// in the batch. /// - Hashes for corresponding input notes and output notes don't match. - /// - /// TODO: enforce limit on the number of created nullifiers. #[instrument(target = "miden-block-producer", name = "new_batch", skip_all, err)] pub fn new( txs: Vec, @@ -101,11 +98,7 @@ impl TransactionBatch { vacant.insert(AccountUpdate::new(tx)); }, Entry::Occupied(occupied) => occupied.into_mut().merge_tx(tx).map_err(|error| { - BuildBatchError::AccountUpdateError { - account_id: tx.account_id(), - error, - txs: txs.clone(), - } + BuildBatchError::AccountUpdateError { account_id: tx.account_id(), error } })?, }; @@ -113,15 +106,11 @@ impl TransactionBatch { for note in tx.get_unauthenticated_notes() { let id = note.id(); if !unauthenticated_input_notes.insert(id) { - return Err(BuildBatchError::DuplicateUnauthenticatedNote(id, txs.clone())); + return Err(BuildBatchError::DuplicateUnauthenticatedNote(id)); } } } - if updated_accounts.len() > MAX_ACCOUNTS_PER_BATCH { - return Err(BuildBatchError::TooManyAccountsInBatch(txs)); - } - // Populate batch produced nullifiers and match output notes with corresponding // unauthenticated input notes in the same batch, which are removed from the unauthenticated // input notes set. @@ -136,7 +125,7 @@ impl TransactionBatch { // Header is presented only for unauthenticated input notes. let input_note = match input_note.header() { Some(input_note_header) => { - if output_notes.remove_note(input_note_header, &txs)? { + if output_notes.remove_note(input_note_header)? { continue; } @@ -154,16 +143,8 @@ impl TransactionBatch { input_notes.push(input_note) } - if input_notes.len() > MAX_INPUT_NOTES_PER_BATCH { - return Err(BuildBatchError::TooManyInputNotes(input_notes.len(), txs)); - } - let output_notes = output_notes.into_notes(); - if output_notes.len() > MAX_OUTPUT_NOTES_PER_BATCH { - return Err(BuildBatchError::TooManyNotesCreated(output_notes.len(), txs)); - } - // Build the output notes SMT. let output_notes_smt = BatchNoteTree::with_contiguous_leaves( output_notes.iter().map(|note| (note.id(), note.metadata())), @@ -249,7 +230,7 @@ impl OutputNoteTracker { for tx in txs { for note in tx.output_notes().iter() { if output_note_index.insert(note.id(), output_notes.len()).is_some() { - return Err(BuildBatchError::DuplicateOutputNote(note.id(), txs.to_vec())); + return Err(BuildBatchError::DuplicateOutputNote(note.id())); } output_notes.push(Some(note.clone())); } @@ -258,11 +239,7 @@ impl OutputNoteTracker { Ok(Self { output_notes, output_note_index }) } - pub fn remove_note( - &mut self, - input_note_header: &NoteHeader, - txs: &[ProvenTransaction], - ) -> Result { + pub fn remove_note(&mut self, input_note_header: &NoteHeader) -> Result { let id = input_note_header.id(); if let Some(note_index) = self.output_note_index.remove(&id) { if let Some(output_note) = mem::take(&mut self.output_notes[note_index]) { @@ -273,7 +250,6 @@ impl OutputNoteTracker { id, input_hash, output_hash, - txs: txs.to_vec(), }); } @@ -322,7 +298,7 @@ mod tests { )); match OutputNoteTracker::new(&txs) { - Err(BuildBatchError::DuplicateOutputNote(note_id, _)) => { + Err(BuildBatchError::DuplicateOutputNote(note_id)) => { assert_eq!(note_id, duplicate_output_note.id()) }, res => panic!("Unexpected result: {res:?}"), @@ -336,8 +312,8 @@ mod tests { let note_to_remove = mock_note(4); - assert!(tracker.remove_note(note_to_remove.header(), &txs).unwrap()); - assert!(!tracker.remove_note(note_to_remove.header(), &txs).unwrap()); + assert!(tracker.remove_note(note_to_remove.header()).unwrap()); + assert!(!tracker.remove_note(note_to_remove.header()).unwrap()); // Check that output notes are in the expected order and consumed note was removed assert_eq!( @@ -358,7 +334,7 @@ mod tests { let duplicate_note = mock_note(5); txs.push(mock_proven_tx(4, vec![duplicate_note.clone()], vec![mock_output_note(9)])); match TransactionBatch::new(txs, Default::default()) { - Err(BuildBatchError::DuplicateUnauthenticatedNote(note_id, _)) => { + Err(BuildBatchError::DuplicateUnauthenticatedNote(note_id)) => { assert_eq!(note_id, duplicate_note.id()) }, res => panic!("Unexpected result: {res:?}"), diff --git a/crates/block-producer/src/block_builder/prover/block_witness.rs b/crates/block-producer/src/block_builder/prover/block_witness.rs index aad05ce51..af837339c 100644 --- a/crates/block-producer/src/block_builder/prover/block_witness.rs +++ b/crates/block-producer/src/block_builder/prover/block_witness.rs @@ -7,7 +7,7 @@ use miden_objects::{ notes::Nullifier, transaction::TransactionId, vm::{AdviceInputs, StackInputs}, - BlockHeader, Digest, Felt, BLOCK_NOTE_TREE_DEPTH, MAX_BATCHES_PER_BLOCK, ZERO, + BlockHeader, Digest, Felt, BLOCK_NOTE_TREE_DEPTH, ZERO, }; use crate::{ @@ -35,9 +35,6 @@ impl BlockWitness { mut block_inputs: BlockInputs, batches: &[TransactionBatch], ) -> Result<(Self, Vec), BuildBlockError> { - if batches.len() > MAX_BATCHES_PER_BLOCK { - return Err(BuildBlockError::TooManyBatchesInBlock(batches.len())); - } Self::validate_nullifiers(&block_inputs, batches)?; let batch_created_notes_roots = batches diff --git a/crates/block-producer/src/domain/transaction.rs b/crates/block-producer/src/domain/transaction.rs index e040fe52b..738fccf36 100644 --- a/crates/block-producer/src/domain/transaction.rs +++ b/crates/block-producer/src/domain/transaction.rs @@ -99,6 +99,14 @@ impl AuthenticatedTransaction { self.inner.output_notes().iter().map(|note| note.id()) } + pub fn output_note_count(&self) -> usize { + self.inner.output_notes().num_notes() + } + + pub fn input_note_count(&self) -> usize { + self.inner.input_notes().num_notes() + } + /// Notes which were unauthenticate in the transaction __and__ which were /// not authenticated by the store inputs. pub fn unauthenticated_notes(&self) -> impl Iterator + '_ { diff --git a/crates/block-producer/src/errors.rs b/crates/block-producer/src/errors.rs index 493116f4c..46a50be6e 100644 --- a/crates/block-producer/src/errors.rs +++ b/crates/block-producer/src/errors.rs @@ -4,9 +4,8 @@ use miden_objects::{ accounts::AccountId, crypto::merkle::{MerkleError, MmrError}, notes::{NoteId, Nullifier}, - transaction::{ProvenTransaction, TransactionId}, - AccountDeltaError, Digest, TransactionInputError, MAX_ACCOUNTS_PER_BATCH, - MAX_BATCHES_PER_BLOCK, MAX_INPUT_NOTES_PER_BATCH, MAX_OUTPUT_NOTES_PER_BATCH, + transaction::TransactionId, + AccountDeltaError, Digest, TransactionInputError, }; use miden_processor::ExecutionError; use thiserror::Error; @@ -94,51 +93,26 @@ impl From for tonic::Status { // Batch building errors // ================================================================================================= -/// Error that may happen while building a transaction batch. -/// -/// These errors are returned from the batch builder to the transaction queue, instead of -/// dropping the transactions, they are included into the error values, so that the transaction -/// queue can re-queue them. +/// Error encountered while building a batch. #[derive(Debug, PartialEq, Eq, Error)] pub enum BuildBatchError { - #[error( - "Too many input notes in the batch. Got: {0}, limit: {}", - MAX_INPUT_NOTES_PER_BATCH - )] - TooManyInputNotes(usize, Vec), - - #[error( - "Too many notes created in the batch. Got: {0}, limit: {}", - MAX_OUTPUT_NOTES_PER_BATCH - )] - TooManyNotesCreated(usize, Vec), - - #[error( - "Too many account updates in the batch. Got: {}, limit: {}", - .0.len(), - MAX_ACCOUNTS_PER_BATCH - )] - TooManyAccountsInBatch(Vec), - #[error("Duplicated unauthenticated transaction input note ID in the batch: {0}")] - DuplicateUnauthenticatedNote(NoteId, Vec), + DuplicateUnauthenticatedNote(NoteId), #[error("Duplicated transaction output note ID in the batch: {0}")] - DuplicateOutputNote(NoteId, Vec), + DuplicateOutputNote(NoteId), #[error("Note hashes mismatch for note {id}: (input: {input_hash}, output: {output_hash})")] NoteHashesMismatch { id: NoteId, input_hash: Digest, output_hash: Digest, - txs: Vec, }, #[error("Failed to merge transaction delta into account {account_id}: {error}")] AccountUpdateError { account_id: AccountId, error: AccountDeltaError, - txs: Vec, }, #[error("Nothing actually went wrong, failure was injected on purpose")] @@ -202,8 +176,6 @@ pub enum BuildBlockError { InconsistentNullifiers(Vec), #[error("unauthenticated transaction notes not found in the store or in outputs of other transactions in the block: {0:?}")] UnauthenticatedNotesNotFound(Vec), - #[error("too many batches in block. Got: {0}, max: {MAX_BATCHES_PER_BLOCK}")] - TooManyBatchesInBlock(usize), #[error("failed to merge transaction delta into account {account_id}: {error}")] AccountUpdateError { account_id: AccountId, diff --git a/crates/block-producer/src/lib.rs b/crates/block-producer/src/lib.rs index 961e1b5e7..cf82a3570 100644 --- a/crates/block-producer/src/lib.rs +++ b/crates/block-producer/src/lib.rs @@ -21,7 +21,7 @@ pub mod server; pub const COMPONENT: &str = "miden-block-producer"; /// The number of transactions per batch -const SERVER_BATCH_SIZE: usize = 2; +const SERVER_MAX_TXS_PER_BATCH: usize = 2; /// The frequency at which blocks are produced const SERVER_BLOCK_FREQUENCY: Duration = Duration::from_secs(10); @@ -37,3 +37,13 @@ const SERVER_MAX_BATCHES_PER_BLOCK: usize = 4; /// This determines the grace period incoming transactions have between fetching their input from /// the store and verification in the mempool. const SERVER_MEMPOOL_STATE_RETENTION: usize = 5; + +const _: () = assert!( + SERVER_MAX_BATCHES_PER_BLOCK <= miden_objects::MAX_BATCHES_PER_BLOCK, + "Server constraint cannot exceed the protocol's constraint" +); + +const _: () = assert!( + SERVER_MAX_TXS_PER_BATCH <= miden_objects::MAX_ACCOUNTS_PER_BATCH, + "Server constraint cannot exceed the protocol's constraint" +); diff --git a/crates/block-producer/src/mempool/batch_graph.rs b/crates/block-producer/src/mempool/batch_graph.rs index b83af277d..1eebd71e8 100644 --- a/crates/block-producer/src/mempool/batch_graph.rs +++ b/crates/block-producer/src/mempool/batch_graph.rs @@ -4,7 +4,7 @@ use miden_objects::transaction::TransactionId; use super::{ dependency_graph::{DependencyGraph, GraphError}, - BatchJobId, + BatchJobId, BlockBudget, BudgetStatus, }; use crate::batch_builder::batch::TransactionBatch; @@ -143,7 +143,9 @@ impl BatchGraph { // dependency graph, and therefore must all be in the batches mapping. let batches = batch_ids .into_iter() - .map(|batch_id| (batch_id, self.batches.remove(&batch_id).unwrap())) + .map(|batch_id| { + (batch_id, self.batches.remove(&batch_id).expect("batch should be removed")) + }) .collect::>(); for tx in batches.values().flatten() { @@ -203,24 +205,28 @@ impl BatchGraph { self.inner.promote_pending(id, batch) } - /// Returns at most `count` batches which are ready for inclusion in a block. - pub fn select_block(&mut self, count: usize) -> BTreeMap { + /// Selects the next set of batches ready for inclusion in a block while adhering to the given + /// budget. + pub fn select_block( + &mut self, + mut budget: BlockBudget, + ) -> BTreeMap { let mut batches = BTreeMap::new(); - for _ in 0..count { - // This strategy just selects arbitrary roots for now. This is valid but not very - // interesting or efficient. - let Some(batch_id) = self.inner.roots().first().copied() else { + while let Some(batch_id) = self.inner.roots().first().copied() { + // SAFETY: Since it was a root batch, it must definitely have a processed batch + // associated with it. + let batch = self.inner.get(&batch_id).expect("root should be in graph").clone(); + + // Adhere to block's budget. + if budget.check_then_subtract(&batch) == BudgetStatus::Exceeded { break; - }; + } // SAFETY: This is definitely a root since we just selected it from the set of roots. - self.inner.process_root(batch_id).unwrap(); - // SAFETY: Since it was a root batch, it must definitely have a processed batch - // associated with it. - let batch = self.inner.get(&batch_id).unwrap(); + self.inner.process_root(batch_id).expect("root should be processed"); - batches.insert(batch_id, batch.clone()); + batches.insert(batch_id, batch); } batches diff --git a/crates/block-producer/src/mempool/mod.rs b/crates/block-producer/src/mempool/mod.rs index e793ef1df..5a59f2aca 100644 --- a/crates/block-producer/src/mempool/mod.rs +++ b/crates/block-producer/src/mempool/mod.rs @@ -6,12 +6,15 @@ use std::{ use batch_graph::BatchGraph; use inflight_state::InflightState; +use miden_objects::{ + MAX_ACCOUNTS_PER_BATCH, MAX_INPUT_NOTES_PER_BATCH, MAX_OUTPUT_NOTES_PER_BATCH, +}; use tokio::sync::Mutex; use transaction_graph::TransactionGraph; use crate::{ batch_builder::batch::TransactionBatch, domain::transaction::AuthenticatedTransaction, - errors::AddTransactionError, + errors::AddTransactionError, SERVER_MAX_BATCHES_PER_BLOCK, SERVER_MAX_TXS_PER_BATCH, }; mod batch_graph; @@ -73,6 +76,102 @@ impl BlockNumber { } } +// MEMPOOL BUDGET +// ================================================================================================ + +/// Limits placed on a batch's contents. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub struct BatchBudget { + /// Maximum number of transactions allowed in a batch. + transactions: usize, + /// Maximum number of input notes allowed. + input_notes: usize, + /// Maximum number of output notes allowed. + output_notes: usize, + /// Maximum number of updated accounts. + accounts: usize, +} + +/// Limits placed on a blocks's contents. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub struct BlockBudget { + /// Maximum number of batches allowed in a block. + batches: usize, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum BudgetStatus { + /// The operation remained within the budget. + WithinScope, + /// The operation exceeded the budget. + Exceeded, +} + +impl Default for BatchBudget { + fn default() -> Self { + Self { + transactions: SERVER_MAX_TXS_PER_BATCH, + input_notes: MAX_INPUT_NOTES_PER_BATCH, + output_notes: MAX_OUTPUT_NOTES_PER_BATCH, + accounts: MAX_ACCOUNTS_PER_BATCH, + } + } +} + +impl Default for BlockBudget { + fn default() -> Self { + Self { batches: SERVER_MAX_BATCHES_PER_BLOCK } + } +} + +impl BatchBudget { + /// Attempts to consume the transaction's resources from the budget. + /// + /// Returns [BudgetStatus::Exceeded] if the transaction would exceed the remaining budget, + /// otherwise returns [BudgetStatus::Ok] and subtracts the resources from the budger. + #[must_use] + fn check_then_subtract(&mut self, tx: &AuthenticatedTransaction) -> BudgetStatus { + // This type assertion reminds us to update the account check if we ever support multiple + // account updates per tx. + let _: miden_objects::accounts::AccountId = tx.account_update().account_id(); + const ACCOUNT_UPDATES_PER_TX: usize = 1; + + let output_notes = tx.output_note_count(); + let input_notes = tx.input_note_count(); + + if self.transactions == 0 + || self.accounts < ACCOUNT_UPDATES_PER_TX + || self.input_notes < input_notes + || self.output_notes < output_notes + { + return BudgetStatus::Exceeded; + } + + self.transactions -= 1; + self.accounts -= ACCOUNT_UPDATES_PER_TX; + self.input_notes -= input_notes; + self.output_notes -= output_notes; + + BudgetStatus::WithinScope + } +} + +impl BlockBudget { + /// Attempts to consume the batch's resources from the budget. + /// + /// Returns [BudgetStatus::Exceeded] if the batch would exceed the remaining budget, + /// otherwise returns [BudgetStatus::Ok]. + #[must_use] + fn check_then_subtract(&mut self, _batch: &TransactionBatch) -> BudgetStatus { + if self.batches == 0 { + BudgetStatus::Exceeded + } else { + self.batches -= 1; + BudgetStatus::WithinScope + } + } +} + // MEMPOOL // ================================================================================================ @@ -96,24 +195,25 @@ pub struct Mempool { /// The current block height of the chain. chain_tip: BlockNumber, + /// The current inflight block, if any. block_in_progress: Option>, - batch_transaction_limit: usize, - block_batch_limit: usize, + batch_budget: BatchBudget, + block_budget: BlockBudget, } impl Mempool { /// Creates a new [Mempool] with the provided configuration. pub fn new( chain_tip: BlockNumber, - batch_limit: usize, - block_limit: usize, + batch_budget: BatchBudget, + block_budget: BlockBudget, state_retention: usize, ) -> SharedMempool { Arc::new(Mutex::new(Self { chain_tip, - batch_transaction_limit: batch_limit, - block_batch_limit: block_limit, + batch_budget, + block_budget, state: InflightState::new(chain_tip, state_retention), block_in_progress: Default::default(), transactions: Default::default(), @@ -149,7 +249,7 @@ impl Mempool { /// /// Returns `None` if no transactions are available. pub fn select_batch(&mut self) -> Option<(BatchJobId, Vec)> { - let (batch, parents) = self.transactions.select_batch(self.batch_transaction_limit); + let (batch, parents) = self.transactions.select_batch(self.batch_budget); if batch.is_empty() { return None; } @@ -199,7 +299,7 @@ impl Mempool { pub fn select_block(&mut self) -> (BlockNumber, BTreeMap) { assert!(self.block_in_progress.is_none(), "Cannot have two blocks inflight."); - let batches = self.batches.select_block(self.block_batch_limit); + let batches = self.batches.select_block(self.block_budget); self.block_in_progress = Some(batches.keys().cloned().collect()); (self.chain_tip.next(), batches) diff --git a/crates/block-producer/src/mempool/transaction_graph.rs b/crates/block-producer/src/mempool/transaction_graph.rs index 089566269..abefdbce2 100644 --- a/crates/block-producer/src/mempool/transaction_graph.rs +++ b/crates/block-producer/src/mempool/transaction_graph.rs @@ -2,7 +2,10 @@ use std::collections::BTreeSet; use miden_objects::transaction::TransactionId; -use super::dependency_graph::{DependencyGraph, GraphError}; +use super::{ + dependency_graph::{DependencyGraph, GraphError}, + BatchBudget, BudgetStatus, +}; use crate::domain::transaction::AuthenticatedTransaction; // TRANSACTION GRAPH @@ -60,7 +63,9 @@ impl TransactionGraph { self.inner.promote_pending(transaction.id(), transaction) } - /// Selects a set of up-to count transactions for the next batch, as well as their parents. + /// Selects a set transactions for the next batch such that they adhere to the given budget. + /// + /// Also returns the transactions' parents. /// /// Internally these transactions are considered processed and cannot be emitted in future /// batches. @@ -72,26 +77,28 @@ impl TransactionGraph { /// - [Self::prune_committed] pub fn select_batch( &mut self, - count: usize, + mut budget: BatchBudget, ) -> (Vec, BTreeSet) { // This strategy just selects arbitrary roots for now. This is valid but not very // interesting or efficient. - let mut batch = Vec::with_capacity(count); + let mut batch = Vec::with_capacity(budget.transactions); let mut parents = BTreeSet::new(); - for _ in 0..count { - let Some(root) = self.inner.roots().first().cloned() else { + while let Some(root) = self.inner.roots().first().cloned() { + // SAFETY: Since it was a root batch, it must definitely have a processed batch + // associated with it. + let tx = self.inner.get(&root).unwrap().clone(); + + // Adhere to batch budget. + if budget.check_then_subtract(&tx) == BudgetStatus::Exceeded { break; - }; + } // SAFETY: This is definitely a root since we just selected it from the set of roots. self.inner.process_root(root).unwrap(); - // SAFETY: Since it was a root batch, it must definitely have a processed batch - // associated with it. - let tx = self.inner.get(&root).unwrap(); let tx_parents = self.inner.parents(&root).unwrap(); - batch.push(tx.clone()); + batch.push(tx); parents.extend(tx_parents); } @@ -151,7 +158,7 @@ mod tests { // ================================================================================================ #[test] - fn select_batch_respects_limit() { + fn select_batch_respects_transaction_limit() { // These transactions are independent and just used to ensure we have more available // transactions than we want in the batch. let txs = (0..10) @@ -163,25 +170,30 @@ mod tests { uut.insert(tx, [].into()).unwrap(); } - let (batch, parents) = uut.select_batch(0); + let (batch, parents) = + uut.select_batch(BatchBudget { transactions: 0, ..Default::default() }); assert!(batch.is_empty()); assert!(parents.is_empty()); - let (batch, parents) = uut.select_batch(3); + let (batch, parents) = + uut.select_batch(BatchBudget { transactions: 3, ..Default::default() }); assert_eq!(batch.len(), 3); assert!(parents.is_empty()); - let (batch, parents) = uut.select_batch(4); + let (batch, parents) = + uut.select_batch(BatchBudget { transactions: 4, ..Default::default() }); assert_eq!(batch.len(), 4); assert!(parents.is_empty()); // We expect this to be partially filled. - let (batch, parents) = uut.select_batch(4); + let (batch, parents) = + uut.select_batch(BatchBudget { transactions: 4, ..Default::default() }); assert_eq!(batch.len(), 3); assert!(parents.is_empty()); // And thereafter empty. - let (batch, parents) = uut.select_batch(100); + let (batch, parents) = + uut.select_batch(BatchBudget { transactions: 100, ..Default::default() }); assert!(batch.is_empty()); assert!(parents.is_empty()); } diff --git a/crates/block-producer/src/server/mod.rs b/crates/block-producer/src/server/mod.rs index e1f3bf0b7..ba9abaea5 100644 --- a/crates/block-producer/src/server/mod.rs +++ b/crates/block-producer/src/server/mod.rs @@ -20,9 +20,9 @@ use crate::{ config::BlockProducerConfig, domain::transaction::AuthenticatedTransaction, errors::{AddTransactionError, VerifyTxError}, - mempool::{BlockNumber, Mempool, SharedMempool}, + mempool::{BatchBudget, BlockBudget, BlockNumber, Mempool, SharedMempool}, store::{DefaultStore, Store}, - COMPONENT, SERVER_BATCH_SIZE, SERVER_MAX_BATCHES_PER_BLOCK, SERVER_MEMPOOL_STATE_RETENTION, + COMPONENT, SERVER_MEMPOOL_STATE_RETENTION, }; /// Represents an initialized block-producer component where the RPC connection is open, @@ -34,8 +34,8 @@ use crate::{ pub struct BlockProducer { batch_builder: BatchBuilder, block_builder: BlockBuilder, - batch_limit: usize, - block_limit: usize, + batch_budget: BatchBudget, + block_budget: BlockBudget, state_retention: usize, rpc_listener: TcpListener, store: DefaultStore, @@ -76,8 +76,8 @@ impl BlockProducer { Ok(Self { batch_builder: Default::default(), block_builder: BlockBuilder::new(store.clone()), - batch_limit: SERVER_BATCH_SIZE, - block_limit: SERVER_MAX_BATCHES_PER_BLOCK, + batch_budget: Default::default(), + block_budget: Default::default(), state_retention: SERVER_MEMPOOL_STATE_RETENTION, store, rpc_listener, @@ -89,15 +89,15 @@ impl BlockProducer { let Self { batch_builder, block_builder, - batch_limit, - block_limit, + batch_budget, + block_budget, state_retention, rpc_listener, store, chain_tip, } = self; - let mempool = Mempool::new(chain_tip, batch_limit, block_limit, state_retention); + let mempool = Mempool::new(chain_tip, batch_budget, block_budget, state_retention); // Spawn rpc server and batch and block provers. // From 39828bff73772907d2c1b6adf9c269cc05303e9b Mon Sep 17 00:00:00 2001 From: Mirko <48352201+Mirko-von-Leipzig@users.noreply.github.com> Date: Fri, 29 Nov 2024 16:52:10 +0200 Subject: [PATCH 13/50] fix(mempool): allow internal batch dependencies (#549) Notably also merges `next` in. --- CHANGELOG.md | 16 +- Cargo.lock | 146 ++++++++--------- Cargo.toml | 12 +- bin/node/Cargo.toml | 2 +- bin/node/src/commands/start.rs | 43 +++-- crates/block-producer/Cargo.toml | 2 +- .../block-producer/src/batch_builder/mod.rs | 2 +- .../block-producer/src/block_builder/mod.rs | 2 +- crates/block-producer/src/errors.rs | 15 ++ .../block-producer/src/mempool/batch_graph.rs | 35 ++++- .../src/mempool/dependency_graph.rs | 2 +- .../mempool/inflight_state/account_state.rs | 8 +- .../src/mempool/inflight_state/mod.rs | 2 +- crates/block-producer/src/mempool/mod.rs | 18 ++- .../src/mempool/transaction_graph.rs | 10 +- crates/block-producer/src/server/mod.rs | 44 +++--- crates/block-producer/src/store/mod.rs | 6 - crates/rpc/src/server/api.rs | 3 - crates/store/src/db/mod.rs | 4 +- crates/store/src/db/sql.rs | 4 +- crates/store/src/db/tests.rs | 50 +++--- crates/store/src/errors.rs | 147 +++++++++++------- crates/store/src/server/api.rs | 6 +- crates/store/src/state.rs | 47 +++--- 24 files changed, 363 insertions(+), 263 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7257a5da3..ed4c12a1b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,22 +1,28 @@ # Changelog -## v0.6.0 (TBD) +## v0.6.0 (2024-11-05) +### Enhancements + +- Added `GetAccountProofs` endpoint (#506). + +### Changes + +- [BREAKING] Added `kernel_root` to block header's protobuf message definitions (#496). +- [BREAKING] Renamed `off-chain` and `on-chain` to `private` and `public` respectively for the account storage modes (#489). - Optimized state synchronizations by removing unnecessary fetching and parsing of note details (#462). - [BREAKING] Changed `GetAccountDetailsResponse` field to `details` (#481). - Improve `--version` by adding build metadata (#495). -- [BREAKING] Added `kernel_root` to block header's protobuf message definitions (#496). -- [BREAKING] Renamed `off-chain` and `on-chain` to `private` and `public` respectively for the account storage modes (#489). - [BREAKING] Introduced additional limits for note/account number (#503). - [BREAKING] Removed support for basic wallets in genesis creation (#510). -- Added `GetAccountProofs` endpoint (#506). - Migrated faucet from actix-web to axum (#511). - Changed the `BlockWitness` to pass the inputs to the VM using only advice provider (#516). -- [BREAKING] Changed faucet storage type in the genesis to public. Using faucet from the genesis for faucet web app. Added support for faucet restarting without blockchain restarting (#517). - [BREAKING] Improved store API errors (return "not found" instead of "internal error" status if requested account(s) not found) (#518). - Added `AccountCode` as part of `GetAccountProofs` endpoint response (#521). - [BREAKING] Migrated to v0.11 version of Miden VM (#528). - Reduce cloning in the store's `apply_block` (#532). +- [BREAKING] Changed faucet storage type in the genesis to public. Using faucet from the genesis for faucet web app. Added support for faucet restarting without blockchain restarting (#517). +- [BREAKING] Improved `ApplyBlockError` in the store (#535). ## 0.5.1 (2024-09-12) diff --git a/Cargo.lock b/Cargo.lock index 4de7bc004..e5468123c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -55,9 +55,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.6.17" +version = "0.6.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23a1e53f0f5d86382dafe1cf314783b2044280f406e7e1506368220ad11b1338" +checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" dependencies = [ "anstyle", "anstyle-parse", @@ -395,9 +395,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.1.34" +version = "1.1.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67b9470d453346108f93a59222a9a1a5724db32d0a4727b7ab7ace4b4d822dc9" +checksum = "0f57c4b4da2a9d619dd035f27316d7a426305b75be93d09e92f2b9229c34feaf" dependencies = [ "jobserver", "libc", @@ -966,9 +966,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.15.0" +version = "0.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e087f84d4f86bf4b218b927129862374b72199ae7d8657835f1e89000eea4fb" +checksum = "3a9bfc1af68b1726ea47d3d5109de126281def866b33970e10fbab11b5dafab3" [[package]] name = "hashlink" @@ -1066,9 +1066,9 @@ dependencies = [ [[package]] name = "hyper-timeout" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3203a961e5c83b6f5498933e78b6b263e208c197b63e9c6c53cc82ffd3f63793" +checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" dependencies = [ "hyper", "hyper-util", @@ -1148,7 +1148,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" dependencies = [ "equivalent", - "hashbrown 0.15.0", + "hashbrown 0.15.1", ] [[package]] @@ -1403,8 +1403,9 @@ checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" [[package]] name = "miden-air" -version = "0.10.5" -source = "git+https://github.com/0xPolygonMiden/miden-vm.git?branch=next#f1c0553859c42c56f28d1d6aaf66d3e2f9907bd1" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea00ea451d1547978817a37c1904ff34f4a24de2c23f9b77b282a0bc570ac3f1" dependencies = [ "miden-core", "miden-thiserror", @@ -1414,8 +1415,9 @@ dependencies = [ [[package]] name = "miden-assembly" -version = "0.10.5" -source = "git+https://github.com/0xPolygonMiden/miden-vm.git?branch=next#f1c0553859c42c56f28d1d6aaf66d3e2f9907bd1" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58925d5ab3d625b8e828267a64dd555b1fe37cd5a1d89d09224920d3255de5a9" dependencies = [ "aho-corasick", "lalrpop", @@ -1426,13 +1428,14 @@ dependencies = [ "rustc_version 0.4.1", "smallvec", "tracing", - "unicode-width", + "unicode-width 0.2.0", ] [[package]] name = "miden-core" -version = "0.10.5" -source = "git+https://github.com/0xPolygonMiden/miden-vm.git?branch=next#f1c0553859c42c56f28d1d6aaf66d3e2f9907bd1" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ba2f0ffd0362c91c0eedba391a3c17a8047389e15020ec95b0d7e840375bf03" dependencies = [ "lock_api", "loom", @@ -1450,9 +1453,9 @@ dependencies = [ [[package]] name = "miden-crypto" -version = "0.11.0" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e0ca714c8242f329b9ea6f1a5bf0e93f1490f348f982e3a606d91b884254308" +checksum = "f50a68deed96cde1f51eb623f75828e320f699e0d798f11592f8958ba8b512c3" dependencies = [ "blake3", "cc", @@ -1502,13 +1505,14 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7e392e0a8c34b32671012b439de35fa8987bf14f0f8aac279b97f8b8cc6e263b" dependencies = [ - "unicode-width", + "unicode-width 0.1.14", ] [[package]] name = "miden-lib" version = "0.6.0" -source = "git+https://github.com/0xPolygonMiden/miden-base.git?branch=next#f621c437bd5eb67d7b4a82d98a3eec8d238bb6cb" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "816404c10b0799f12d3b53b3a9baa9af99fa340fe1a579e0919bba57718fa97a" dependencies = [ "miden-assembly", "miden-objects", @@ -1544,7 +1548,7 @@ dependencies = [ "terminal_size", "textwrap", "trybuild", - "unicode-width", + "unicode-width 0.1.14", ] [[package]] @@ -1705,7 +1709,8 @@ dependencies = [ [[package]] name = "miden-objects" version = "0.6.0" -source = "git+https://github.com/0xPolygonMiden/miden-base.git?branch=next#f621c437bd5eb67d7b4a82d98a3eec8d238bb6cb" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a6f54dde939928e438488b36651485a0e80057025b7e30343d4340a524a1651" dependencies = [ "getrandom", "miden-assembly", @@ -1719,8 +1724,9 @@ dependencies = [ [[package]] name = "miden-processor" -version = "0.10.6" -source = "git+https://github.com/0xPolygonMiden/miden-vm.git?branch=next#f1c0553859c42c56f28d1d6aaf66d3e2f9907bd1" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e1a756be679bfe2bdb209b9f4f12b4b59e8cac7c9f884ad90535b32bbd75923" dependencies = [ "miden-air", "miden-core", @@ -1730,12 +1736,14 @@ dependencies = [ [[package]] name = "miden-prover" -version = "0.10.5" -source = "git+https://github.com/0xPolygonMiden/miden-vm.git?branch=next#f1c0553859c42c56f28d1d6aaf66d3e2f9907bd1" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b6c2fc7ed9831493e1c18fb9d15a4b780ec624e9192f494ad3900c19bfe5b17" dependencies = [ "miden-air", "miden-processor", "tracing", + "winter-maybe-async", "winter-prover", ] @@ -1745,8 +1753,9 @@ version = "0.6.0" [[package]] name = "miden-stdlib" -version = "0.10.5" -source = "git+https://github.com/0xPolygonMiden/miden-vm.git?branch=next#f1c0553859c42c56f28d1d6aaf66d3e2f9907bd1" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09948fda011d044273d1bbc56669da410ee3167f96bf1ad9885c946cfbed5757" dependencies = [ "miden-assembly", ] @@ -1774,7 +1783,8 @@ dependencies = [ [[package]] name = "miden-tx" version = "0.6.0" -source = "git+https://github.com/0xPolygonMiden/miden-base.git?branch=next#f621c437bd5eb67d7b4a82d98a3eec8d238bb6cb" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d433744a3f02233ec53b151bfe88b7a008b4eae5dfd41bb38a9889eb67efaf7" dependencies = [ "async-trait", "miden-assembly", @@ -1787,13 +1797,14 @@ dependencies = [ "rand_chacha", "regex", "walkdir", - "winter-maybe-async 0.10.1", + "winter-maybe-async", ] [[package]] name = "miden-verifier" -version = "0.10.5" -source = "git+https://github.com/0xPolygonMiden/miden-vm.git?branch=next#f1c0553859c42c56f28d1d6aaf66d3e2f9907bd1" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "636b2eab9dc0a9fdd6d30d9ece3cf276b012e9ecbccd934d4c0befa07c84cfd0" dependencies = [ "miden-air", "miden-core", @@ -1818,7 +1829,7 @@ dependencies = [ "terminal_size", "textwrap", "thiserror", - "unicode-width", + "unicode-width 0.1.14", ] [[package]] @@ -2534,9 +2545,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.38" +version = "0.38.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa260229e6538e52293eeb577aabd09945a09d6d9cc0fc550ed7529056c2e32a" +checksum = "375116bee2be9ed569afe2154ea6a99dfdffd257f533f187498c2a8f5feaf4ee" dependencies = [ "bitflags", "errno", @@ -2885,23 +2896,23 @@ checksum = "23d434d3f8967a09480fb04132ebe0a3e088c173e6d0ee7897abbdf4eab0f8b9" dependencies = [ "smawk", "unicode-linebreak", - "unicode-width", + "unicode-width 0.1.14", ] [[package]] name = "thiserror" -version = "1.0.66" +version = "1.0.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d171f59dbaa811dbbb1aee1e73db92ec2b122911a48e1390dfe327a821ddede" +checksum = "02dd99dc800bbb97186339685293e1cc5d9df1f8fae2d0aecd9ff1c77efea892" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.66" +version = "1.0.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b08be0f17bd307950653ce45db00cd31200d82b624b36e181337d9c7d92765b5" +checksum = "a7c61ec9a6f64d2793d8a45faba21efbe3ced62a886d44c36a009b2b519b4c7e" dependencies = [ "proc-macro2", "quote", @@ -3344,6 +3355,12 @@ version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" +[[package]] +name = "unicode-width" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd" + [[package]] name = "unicode-xid" version = "0.2.6" @@ -3786,9 +3803,9 @@ dependencies = [ [[package]] name = "winter-air" -version = "0.9.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b72f12b88ebb060b52c0e9aece9bb64a9fc38daf7ba689dd5ce63271b456c883" +checksum = "29bec0b06b741543f43e3a6677b95b200d4cad2daab76e6721e14345345bfd0e" dependencies = [ "libm", "winter-crypto", @@ -3799,9 +3816,9 @@ dependencies = [ [[package]] name = "winter-crypto" -version = "0.9.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00fbb724d2d9fbfd3aa16ea27f5e461d4fe1d74b0c9e0ed1bf79e9e2a955f4d5" +checksum = "163da45f1d4d65cac361b8df4835a6daa95b3399154e16eb0305c178c6f6c1f4" dependencies = [ "blake3", "sha3", @@ -3811,9 +3828,9 @@ dependencies = [ [[package]] name = "winter-fri" -version = "0.9.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ab6077cf4c23c0411f591f4ba29378e27f26acb8cef3c51cadd93daaf6080b3" +checksum = "3b7b394670d68979a4cc21a37a95ef8ef350cf84be9256c53effe3052df50d26" dependencies = [ "winter-crypto", "winter-math", @@ -3822,24 +3839,13 @@ dependencies = [ [[package]] name = "winter-math" -version = "0.9.3" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b0e685b3b872d82e58a86519294a814b7bc7a4d3cd2c93570a7d80c0c5a1aba" +checksum = "5a8ba832121679e79b004b0003018c85873956d742a39c348c247f680fe15e00" dependencies = [ "winter-utils", ] -[[package]] -name = "winter-maybe-async" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ce0f4161cdde50de809b3869c1cb083a09e92e949428ea28f04c0d64045875c" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "winter-maybe-async" version = "0.10.1" @@ -3852,24 +3858,24 @@ dependencies = [ [[package]] name = "winter-prover" -version = "0.9.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f17e3dbae97050f58e01ed4f12906e247841575a0518632e052941a1c37468df" +checksum = "2f55f0153d26691caaf969066a13a824bcf3c98719d71b0f569bf8dc40a06fb9" dependencies = [ "tracing", "winter-air", "winter-crypto", "winter-fri", "winter-math", - "winter-maybe-async 0.9.0", + "winter-maybe-async", "winter-utils", ] [[package]] name = "winter-rand-utils" -version = "0.9.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2b827c901ab0c316d89812858ff451d60855c0a5c7ae734b098c62a28624181" +checksum = "4a7616d11fcc26552dada45c803a884ac97c253218835b83a2c63e1c2a988639" dependencies = [ "rand", "winter-utils", @@ -3877,18 +3883,18 @@ dependencies = [ [[package]] name = "winter-utils" -version = "0.9.3" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "961e81e9388877a25db1c034ba38253de2055f569633ae6a665d857a0556391b" +checksum = "76b116c8ade0172506f8bda32dc674cf6b230adc8516e5138a0173ae69158a4f" dependencies = [ "rayon", ] [[package]] name = "winter-verifier" -version = "0.9.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "324002ade90f21e85599d51a232a80781efc8cb46f511f8bc89f9c5a4eb9cb65" +checksum = "2ae1648768f96f5e6321a48a5bff5cc3101d2e51b23a6a095c6c9c9e133ecb61" dependencies = [ "winter-air", "winter-crypto", @@ -3899,9 +3905,9 @@ dependencies = [ [[package]] name = "winterfell" -version = "0.9.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01151ac5fe2d783950743e8a110e0a2f26994f888b4cbe848699142cb3ea1e5b" +checksum = "39c8336dc6a035698780b8cc624f875e479bd6bf6e1846670f3ef4485c125882" dependencies = [ "winter-air", "winter-prover", diff --git a/Cargo.toml b/Cargo.toml index 9e0d53b99..ce5eb685e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,8 +25,8 @@ exclude = [".github/"] readme = "README.md" [workspace.dependencies] -miden-air = { git = "https://github.com/0xPolygonMiden/miden-vm.git", branch = "next" } -miden-lib = { git = "https://github.com/0xPolygonMiden/miden-base.git", branch = "next" } +miden-air = { version = "0.11" } +miden-lib = { version = "0.6" } miden-node-block-producer = { path = "crates/block-producer", version = "0.6" } miden-node-faucet = { path = "bin/faucet", version = "0.6" } miden-node-proto = { path = "crates/proto", version = "0.6" } @@ -35,10 +35,10 @@ miden-node-rpc-proto = { path = "crates/rpc-proto", version = "0.6" } miden-node-store = { path = "crates/store", version = "0.6" } miden-node-test-macro = { path = "crates/test-macro" } miden-node-utils = { path = "crates/utils", version = "0.6" } -miden-objects = { git = "https://github.com/0xPolygonMiden/miden-base.git", branch = "next" } -miden-processor = { git = "https://github.com/0xPolygonMiden/miden-vm.git", branch = "next" } -miden-stdlib = { git = "https://github.com/0xPolygonMiden/miden-vm.git", branch = "next", default-features = false } -miden-tx = { git = "https://github.com/0xPolygonMiden/miden-base.git", branch = "next" } +miden-objects = { version = "0.6"} +miden-processor = { version = "0.11" } +miden-stdlib = { version = "0.11", default-features = false } +miden-tx = { version = "0.6"} prost = { version = "0.13" } rand = { version = "0.8" } thiserror = { version = "1.0" } diff --git a/bin/node/Cargo.toml b/bin/node/Cargo.toml index b855cbe64..d2d9b5d91 100644 --- a/bin/node/Cargo.toml +++ b/bin/node/Cargo.toml @@ -14,7 +14,7 @@ repository.workspace = true [features] # Makes `make-genesis` subcommand run faster. Is only suitable for testing. # INFO: Make sure that all your components have matching features for them to function. -testing = ["miden-lib/testing"] +testing = ["miden-lib/testing", "miden-objects/testing"] tracing-forest = ["miden-node-block-producer/tracing-forest"] [dependencies] diff --git a/bin/node/src/commands/start.rs b/bin/node/src/commands/start.rs index acbd356ac..f1e46fbf9 100644 --- a/bin/node/src/commands/start.rs +++ b/bin/node/src/commands/start.rs @@ -1,3 +1,5 @@ +use std::collections::HashMap; + use anyhow::{Context, Result}; use miden_node_block_producer::server::BlockProducer; use miden_node_rpc::server::Rpc; @@ -16,22 +18,41 @@ pub async fn start_node(config: NodeConfig) -> Result<()> { // Start store. The store endpoint is available after loading completes. let store = Store::init(store).await.context("Loading store")?; - join_set.spawn(async move { store.serve().await.context("Serving store") }); + let store_id = join_set.spawn(async move { store.serve().await.context("Serving store") }).id(); // Start block-producer. The block-producer's endpoint is available after loading completes. let block_producer = BlockProducer::init(block_producer).await.context("Loading block-producer")?; - join_set.spawn(async move { block_producer.serve().await.context("Serving block-producer") }); + let block_producer_id = join_set + .spawn(async move { block_producer.serve().await.context("Serving block-producer") }) + .id(); // Start RPC component. let rpc = Rpc::init(rpc).await.context("Loading RPC")?; - join_set.spawn(async move { rpc.serve().await.context("Serving RPC") }); - - // block on all tasks - while let Some(res) = join_set.join_next().await { - // For now, if one of the components fails, crash the node - res??; - } - - Ok(()) + let rpc_id = join_set.spawn(async move { rpc.serve().await.context("Serving RPC") }).id(); + + // Lookup table so we can identify the failed component. + let component_ids = HashMap::from([ + (store_id, "store"), + (block_producer_id, "block-producer"), + (rpc_id, "rpc"), + ]); + + // SAFETY: The joinset is definitely not empty. + let component_result = join_set.join_next_with_id().await.unwrap(); + + // We expect components to run indefinitely, so we treat any return as fatal. + // + // Map all outcomes to an error, and provide component context. + let (id, err) = match component_result { + Ok((id, Ok(_))) => (id, Err(anyhow::anyhow!("Component completed unexpectedly"))), + Ok((id, Err(err))) => (id, Err(err)), + Err(join_err) => (join_err.id(), Err(join_err).context("Joining component task")), + }; + let component = component_ids.get(&id).unwrap_or(&"unknown"); + + // We could abort and gracefully shutdown the other components, but since we're crashing the + // node there is no point. + + err.context(format!("Component {component} failed")) } diff --git a/crates/block-producer/Cargo.toml b/crates/block-producer/Cargo.toml index ec4eec1b1..f6d987b2e 100644 --- a/crates/block-producer/Cargo.toml +++ b/crates/block-producer/Cargo.toml @@ -44,4 +44,4 @@ miden-objects = { workspace = true, features = ["testing"] } miden-tx = { workspace = true, features = ["testing"] } rand_chacha = { version = "0.3", default-features = false } tokio = { workspace = true, features = ["test-util"] } -winterfell = { version = "0.9" } +winterfell = { version = "0.10" } diff --git a/crates/block-producer/src/batch_builder/mod.rs b/crates/block-producer/src/batch_builder/mod.rs index 830c68c3b..30070b58e 100644 --- a/crates/block-producer/src/batch_builder/mod.rs +++ b/crates/block-producer/src/batch_builder/mod.rs @@ -48,7 +48,7 @@ impl Default for BatchBuilder { } impl BatchBuilder { - /// Starts the [BatchProducer], creating and proving batches at the configured interval. + /// Starts the [BatchBuilder], creating and proving batches at the configured interval. /// /// A pool of batch-proving workers is spawned, which are fed new batch jobs periodically. /// A batch is skipped if there are no available workers, or if there are no transactions diff --git a/crates/block-producer/src/block_builder/mod.rs b/crates/block-producer/src/block_builder/mod.rs index f67501cbe..a8c017c7f 100644 --- a/crates/block-producer/src/block_builder/mod.rs +++ b/crates/block-producer/src/block_builder/mod.rs @@ -59,7 +59,7 @@ impl BlockBuilder { /// /// Block production is sequential and consists of /// - /// 1. Pulling the next set of batches from the [Mempool] + /// 1. Pulling the next set of batches from the mempool /// 2. Compiling these batches into the next block /// 3. Proving the block (this is simulated using random sleeps) /// 4. Committing the block to the store diff --git a/crates/block-producer/src/errors.rs b/crates/block-producer/src/errors.rs index 46a50be6e..9ff3a76bb 100644 --- a/crates/block-producer/src/errors.rs +++ b/crates/block-producer/src/errors.rs @@ -9,9 +9,24 @@ use miden_objects::{ }; use miden_processor::ExecutionError; use thiserror::Error; +use tokio::task::JoinError; use crate::mempool::BlockNumber; +// Block-producer errors +// ================================================================================================= + +#[derive(Debug, Error)] +pub enum BlockProducerError { + /// A block-producer task completed although it should have ran indefinitely. + #[error("task {task} completed unexpectedly")] + TaskFailedSuccesfully { task: &'static str }, + + /// A block-producer task panic'd. + #[error("error joining {task} task")] + JoinError { task: &'static str, source: JoinError }, +} + // Transaction verification errors // ================================================================================================= diff --git a/crates/block-producer/src/mempool/batch_graph.rs b/crates/block-producer/src/mempool/batch_graph.rs index 1eebd71e8..dea63d992 100644 --- a/crates/block-producer/src/mempool/batch_graph.rs +++ b/crates/block-producer/src/mempool/batch_graph.rs @@ -24,11 +24,11 @@ use crate::batch_builder::batch::TransactionBatch; /// Committed batches (i.e. included in blocks) may be [pruned](Self::prune_committed) from the /// graph to bound the graph's size. /// -/// Batches may also be outright [purged](Self::purge_subgraphs) from the graph. This is useful for +/// Batches may also be outright [purged](Self::remove_batches) from the graph. This is useful for /// batches which may have become invalid due to external considerations e.g. expired transactions. /// /// # Batch lifecycle -/// ``` +/// ```text /// │ /// insert│ /// ┌─────▼─────┐ @@ -50,7 +50,7 @@ use crate::batch_builder::batch::TransactionBatch; /// │ ◄────┘ /// └───────────┘ /// ``` -#[derive(Default, Clone)] +#[derive(Default, Debug, Clone)] pub struct BatchGraph { /// Tracks the interdependencies between batches. inner: DependencyGraph, @@ -81,6 +81,10 @@ pub enum BatchInsertError { impl BatchGraph { /// Inserts a new batch into the graph. /// + /// Parents are the transactions on which the given transactions have a direct dependency. This + /// includes transactions within the same batch i.e. a transaction and parent transaction may + /// both be in this batch. + /// /// # Errors /// /// Returns an error if: @@ -91,7 +95,7 @@ impl BatchGraph { &mut self, id: BatchJobId, transactions: Vec, - parents: BTreeSet, + mut parents: BTreeSet, ) -> Result<(), BatchInsertError> { let duplicates = transactions .iter() @@ -102,7 +106,11 @@ impl BatchGraph { return Err(BatchInsertError::DuplicateTransactions(duplicates)); } - // Reverse lookup parent transaction batches. + // Reverse lookup parent batch IDs. Take care to allow for parent transactions within this + // batch i.e. internal dependencies. + transactions.iter().for_each(|tx| { + parents.remove(tx); + }); let parent_batches = parents .into_iter() .map(|tx| { @@ -233,7 +241,7 @@ impl BatchGraph { } } -#[cfg(test)] +#[cfg(any(test, doctest))] mod tests { use super::*; use crate::test_utils::Random; @@ -284,13 +292,24 @@ mod tests { assert_eq!(err, expected); } + #[test] + fn insert_with_internal_parent_succeeds() { + // Ensure that a batch with internal dependencies can be inserted. + let mut rng = Random::with_random_seed(); + let parent = rng.draw_tx_id(); + let child = rng.draw_tx_id(); + + let mut uut = BatchGraph::default(); + uut.insert(BatchJobId::new(2), vec![parent, child], [parent].into()).unwrap(); + } + // PURGE_SUBGRAPHS TESTS // ================================================================================================ #[test] fn purge_subgraphs_returns_all_purged_transaction_sets() { - //! Ensure that purge_subgraphs returns both parent and child batches when the parent is - //! pruned. Further ensure that a disjoint batch is not pruned. + // Ensure that purge_subgraphs returns both parent and child batches when the parent is + // pruned. Further ensure that a disjoint batch is not pruned. let mut rng = Random::with_random_seed(); let parent_batch_txs = (0..5).map(|_| rng.draw_tx_id()).collect::>(); let child_batch_txs = (0..5).map(|_| rng.draw_tx_id()).collect::>(); diff --git a/crates/block-producer/src/mempool/dependency_graph.rs b/crates/block-producer/src/mempool/dependency_graph.rs index 37b247590..9814d0059 100644 --- a/crates/block-producer/src/mempool/dependency_graph.rs +++ b/crates/block-producer/src/mempool/dependency_graph.rs @@ -12,7 +12,7 @@ use std::{ /// Forms the basis of our transaction and batch dependency graphs. /// /// # Node lifecycle -/// ``` +/// ```text /// │ /// │ /// insert_pending│ diff --git a/crates/block-producer/src/mempool/inflight_state/account_state.rs b/crates/block-producer/src/mempool/inflight_state/account_state.rs index f929ab114..dc79a339a 100644 --- a/crates/block-producer/src/mempool/inflight_state/account_state.rs +++ b/crates/block-producer/src/mempool/inflight_state/account_state.rs @@ -115,15 +115,15 @@ impl InflightAccountState { } } -/// Describes the emptiness of an [AccountState]. +/// Describes the emptiness of an [InflightAccountState]. /// -/// Is marked as #[must_use] so that callers handle prune empty accounts. +/// Is marked as `#[must_use]` so that callers handle prune empty accounts. #[must_use] #[derive(Clone, Copy, PartialEq, Eq)] pub enum AccountStatus { - /// [AccountState] contains no state and should be pruned. + /// [InflightAccountState] contains no state and should be pruned. Empty, - /// [AccountState] contains state and should be kept. + /// [InflightAccountState] contains state and should be kept. NonEmpty, } diff --git a/crates/block-producer/src/mempool/inflight_state/mod.rs b/crates/block-producer/src/mempool/inflight_state/mod.rs index 233613c26..e26c1b73e 100644 --- a/crates/block-producer/src/mempool/inflight_state/mod.rs +++ b/crates/block-producer/src/mempool/inflight_state/mod.rs @@ -29,7 +29,7 @@ use super::BlockNumber; pub struct InflightState { /// Account states from inflight transactions. /// - /// Accounts which are [AccountStatus::Empty] are immediately pruned. + /// Accounts which are empty are immediately pruned. accounts: BTreeMap, /// Nullifiers produced by the input notes of inflight transactions. diff --git a/crates/block-producer/src/mempool/mod.rs b/crates/block-producer/src/mempool/mod.rs index 5a59f2aca..c56b72208 100644 --- a/crates/block-producer/src/mempool/mod.rs +++ b/crates/block-producer/src/mempool/mod.rs @@ -238,7 +238,9 @@ impl Mempool { // Add transaction to inflight state. let parents = self.state.add_transaction(&transaction)?; - self.transactions.insert(transaction, parents).expect("Malformed graph"); + self.transactions + .insert(transaction, parents) + .expect("Transaction should insert after passing inflight state"); Ok(self.chain_tip.0) } @@ -258,7 +260,9 @@ impl Mempool { let batch_id = self.next_batch_id; self.next_batch_id.increment(); - self.batches.insert(batch_id, tx_ids, parents).expect("Malformed graph"); + self.batches + .insert(batch_id, tx_ids, parents) + .expect("Selected batch should insert"); Some((batch_id, batch)) } @@ -279,14 +283,16 @@ impl Mempool { let batches = removed_batches.keys().copied().collect::>(); let transactions = removed_batches.into_values().flatten().collect(); - self.transactions.requeue_transactions(transactions).expect("Malformed graph"); + self.transactions + .requeue_transactions(transactions) + .expect("Transaction should requeue"); tracing::warn!(%batch, descendents=?batches, "Batch failed, dropping all inflight descendent batches, impacted transactions are back in queue."); } /// Marks a batch as proven if it exists. pub fn batch_proved(&mut self, batch_id: BatchJobId, batch: TransactionBatch) { - self.batches.submit_proof(batch_id, batch).expect("Malformed graph"); + self.batches.submit_proof(batch_id, batch).expect("Batch proof should submit"); } /// Select batches for the next block. @@ -338,13 +344,13 @@ impl Mempool { let batches = self.block_in_progress.take().expect("No block in progress to be failed"); // Remove all transactions from the graphs. - let purged = self.batches.remove_batches(batches).expect("Bad graph"); + let purged = self.batches.remove_batches(batches).expect("Batch should be removed"); let transactions = purged.into_values().flatten().collect(); let transactions = self .transactions .remove_transactions(transactions) - .expect("Transaction graph is malformed"); + .expect("Failed transactions should be removed"); // Rollback state. self.state.revert_transactions(transactions); diff --git a/crates/block-producer/src/mempool/transaction_graph.rs b/crates/block-producer/src/mempool/transaction_graph.rs index abefdbce2..ced50cdd1 100644 --- a/crates/block-producer/src/mempool/transaction_graph.rs +++ b/crates/block-producer/src/mempool/transaction_graph.rs @@ -19,14 +19,14 @@ use crate::domain::transaction::AuthenticatedTransaction; /// /// Transactions from failed batches may be [re-queued](Self::requeue_transactions) for batch /// selection. Successful batches will eventually form part of a committed block at which point the -/// transaction data may be safely [pruned](Self::prune_committed). +/// transaction data may be safely [pruned](Self::commit_transactions). /// /// Transactions may also be outright [purged](Self::remove_transactions) from the graph. This is /// useful for transactions which may have become invalid due to external considerations e.g. /// expired transactions. /// /// # Transaction lifecycle: -/// ``` +/// ```text /// │ /// insert│ /// ┌─────▼─────┐ @@ -53,7 +53,7 @@ impl TransactionGraph { /// /// # Errors /// - /// Follows the error conditions of [DependencyGraph::insert]. + /// Follows the error conditions of [DependencyGraph::insert_pending]. pub fn insert( &mut self, transaction: AuthenticatedTransaction, @@ -74,7 +74,7 @@ impl TransactionGraph { /// /// See also: /// - [Self::requeue_transactions] - /// - [Self::prune_committed] + /// - [Self::commit_transactions] pub fn select_batch( &mut self, mut budget: BatchBudget, @@ -109,7 +109,7 @@ impl TransactionGraph { /// /// # Errors /// - /// Follows the error conditions of [DependencyGraph::requeue]. + /// Follows the error conditions of [DependencyGraph::revert_subgraphs]. pub fn requeue_transactions( &mut self, transactions: BTreeSet, diff --git a/crates/block-producer/src/server/mod.rs b/crates/block-producer/src/server/mod.rs index ba9abaea5..dbd32ed30 100644 --- a/crates/block-producer/src/server/mod.rs +++ b/crates/block-producer/src/server/mod.rs @@ -1,4 +1,4 @@ -use std::net::ToSocketAddrs; +use std::{collections::HashMap, net::ToSocketAddrs}; use miden_node_proto::generated::{ block_producer::api_server, requests::SubmitProvenTransactionRequest, @@ -19,7 +19,7 @@ use crate::{ block_builder::BlockBuilder, config::BlockProducerConfig, domain::transaction::AuthenticatedTransaction, - errors::{AddTransactionError, VerifyTxError}, + errors::{AddTransactionError, BlockProducerError, VerifyTxError}, mempool::{BatchBudget, BlockBudget, BlockNumber, Mempool, SharedMempool}, store::{DefaultStore, Store}, COMPONENT, SERVER_MEMPOOL_STATE_RETENTION, @@ -49,7 +49,6 @@ impl BlockProducer { pub async fn init(config: BlockProducerConfig) -> Result { info!(target: COMPONENT, %config, "Initializing server"); - // TODO: Does this actually need an arc to be properly shared? let store = DefaultStore::new( store_client::ApiClient::connect(config.store_url.to_string()) .await @@ -85,7 +84,7 @@ impl BlockProducer { }) } - pub async fn serve(self) -> Result<(), ApiError> { + pub async fn serve(self) -> Result<(), BlockProducerError> { let Self { batch_builder, block_builder, @@ -125,38 +124,33 @@ impl BlockProducer { BlockProducerRpcServer::new(mempool, store) .serve(rpc_listener) .await - .expect("Really the rest should throw errors instead of panic'ing.") + .expect("block-producer failed") }) .id(); - // Wait for any task to end. They should run forever, so this is an unexpected result. + let task_ids = HashMap::from([ + (batch_builder_id, "batch-builder"), + (block_builder_id, "block-builder"), + (rpc_id, "rpc"), + ]); + // Wait for any task to end. They should run indefinitely, so this is an unexpected result. + // // SAFETY: The JoinSet is definitely not empty. let task_result = tasks.join_next_with_id().await.unwrap(); + let task_id = match &task_result { - Ok((id, _)) => *id, + Ok((id, ())) => *id, Err(err) => err.id(), }; + let task = task_ids.get(&task_id).unwrap_or(&"unknown"); - let task_name = match task_id { - id if id == batch_builder_id => "batch-builder", - id if id == block_builder_id => "block-builder", - id if id == rpc_id => "rpc", - _ => { - tracing::warn!("An unknown task ID was detected in the block-producer."); - "unknown" - }, - }; - - tracing::error!( - task = task_name, - result = ?task_result, - "Block-producer task ended unexpectedly, aborting" - ); - - tasks.abort_all(); + // We could abort the other tasks here, but not much point as we're probably crashing the + // node. - Ok(()) + task_result + .map_err(|source| BlockProducerError::JoinError { task, source }) + .map(|(_, ())| Err(BlockProducerError::TaskFailedSuccesfully { task }))? } } diff --git a/crates/block-producer/src/store/mod.rs b/crates/block-producer/src/store/mod.rs index 476c05a07..2b34d75d4 100644 --- a/crates/block-producer/src/store/mod.rs +++ b/crates/block-producer/src/store/mod.rs @@ -168,9 +168,6 @@ impl DefaultStore { } } -// FIXME: remove the allow when the upstream clippy issue is fixed: -// https://github.com/rust-lang/rust-clippy/issues/12281 -#[allow(clippy::blocks_in_conditions)] #[async_trait] impl ApplyBlock for DefaultStore { #[instrument(target = "miden-block-producer", skip_all, err)] @@ -188,9 +185,6 @@ impl ApplyBlock for DefaultStore { } } -// FIXME: remove the allow when the upstream clippy issue is fixed: -// https://github.com/rust-lang/rust-clippy/issues/12281 -#[allow(clippy::blocks_in_conditions)] #[async_trait] impl Store for DefaultStore { #[instrument(target = "miden-block-producer", skip_all, err)] diff --git a/crates/rpc/src/server/api.rs b/crates/rpc/src/server/api.rs index 5c25bd480..56f97c788 100644 --- a/crates/rpc/src/server/api.rs +++ b/crates/rpc/src/server/api.rs @@ -56,9 +56,6 @@ impl RpcApi { } } -// FIXME: remove the allow when the upstream clippy issue is fixed: -// https://github.com/rust-lang/rust-clippy/issues/12281 -#[allow(clippy::blocks_in_conditions)] #[tonic::async_trait] impl api_server::Api for RpcApi { #[instrument( diff --git a/crates/store/src/db/mod.rs b/crates/store/src/db/mod.rs index 6e04e67b1..45c462cf4 100644 --- a/crates/store/src/db/mod.rs +++ b/crates/store/src/db/mod.rs @@ -443,9 +443,7 @@ impl Db { )?; let _ = allow_acquire.send(()); - acquire_done - .blocking_recv() - .map_err(DatabaseError::ApplyBlockFailedClosedChannel)?; + acquire_done.blocking_recv()?; transaction.commit()?; diff --git a/crates/store/src/db/sql.rs b/crates/store/src/db/sql.rs index 9980f40dd..8320f0db4 100644 --- a/crates/store/src/db/sql.rs +++ b/crates/store/src/db/sql.rs @@ -254,7 +254,7 @@ pub fn upsert_accounts( debug_assert_eq!(account_id, u64::from(account.id())); if account.hash() != update.new_state_hash() { - return Err(DatabaseError::ApplyBlockFailedAccountHashesMismatch { + return Err(DatabaseError::AccountHashesMismatch { calculated: account.hash(), expected: update.new_state_hash(), }); @@ -1151,7 +1151,7 @@ fn apply_delta( let actual_hash = account.hash(); if &actual_hash != final_state_hash { - return Err(DatabaseError::ApplyBlockFailedAccountHashesMismatch { + return Err(DatabaseError::AccountHashesMismatch { calculated: actual_hash, expected: *final_state_hash, }); diff --git a/crates/store/src/db/tests.rs b/crates/store/src/db/tests.rs index a79166003..a63f982f4 100644 --- a/crates/store/src/db/tests.rs +++ b/crates/store/src/db/tests.rs @@ -8,8 +8,8 @@ use miden_objects::{ ACCOUNT_ID_REGULAR_ACCOUNT_UPDATABLE_CODE_OFF_CHAIN, }, delta::AccountUpdateDetails, - Account, AccountCode, AccountDelta, AccountId, AccountStorage, AccountStorageDelta, - AccountVaultDelta, StorageSlot, + Account, AccountCode, AccountComponent, AccountDelta, AccountId, AccountStorage, + AccountStorageDelta, AccountType, AccountVaultDelta, StorageSlot, }, assets::{Asset, AssetVault, FungibleAsset, NonFungibleAsset, NonFungibleAssetDetails}, block::{BlockAccountUpdate, BlockNoteIndex, BlockNoteTree}, @@ -334,14 +334,6 @@ fn test_sql_public_account_details() { let non_fungible_faucet_id = AccountId::try_from(ACCOUNT_ID_NON_FUNGIBLE_FAUCET_ON_CHAIN).unwrap(); - let mut storage = AccountStorage::new( - std::iter::repeat(StorageSlot::Value(Word::default())).take(6).collect(), - ) - .unwrap(); - storage.set_item(1, num_to_word(1)).unwrap(); - storage.set_item(3, num_to_word(3)).unwrap(); - storage.set_item(5, num_to_word(5)).unwrap(); - let nft1 = Asset::NonFungible( NonFungibleAsset::new( &NonFungibleAssetDetails::new(non_fungible_faucet_id, vec![1, 2, 3]).unwrap(), @@ -349,6 +341,8 @@ fn test_sql_public_account_details() { .unwrap(), ); + let (code, storage) = mock_account_code_and_storage(account_id.account_type()); + let mut account = Account::from_parts( account_id, AssetVault::new(&[ @@ -357,7 +351,7 @@ fn test_sql_public_account_details() { ]) .unwrap(), storage, - mock_account_code(), + code, ZERO, ); @@ -983,12 +977,30 @@ fn insert_transactions(conn: &mut Connection) -> usize { count } -pub fn mock_account_code() -> AccountCode { - let account_code = "\ - export.account_procedure_1 - push.1.2 - add - end - "; - AccountCode::compile(account_code, TransactionKernel::assembler(), false).unwrap() +fn mock_account_code_and_storage(account_type: AccountType) -> (AccountCode, AccountStorage) { + let component_code = "\ + export.account_procedure_1 + push.1.2 + add + end + "; + + let component_storage = vec![ + StorageSlot::Value(Word::default()), + StorageSlot::Value(num_to_word(1)), + StorageSlot::Value(Word::default()), + StorageSlot::Value(num_to_word(3)), + StorageSlot::Value(Word::default()), + StorageSlot::Value(num_to_word(5)), + ]; + + let component = AccountComponent::compile( + component_code, + TransactionKernel::testing_assembler(), + component_storage, + ) + .unwrap() + .with_supported_type(account_type); + + Account::initialize_from_components(account_type, &[component]).unwrap() } diff --git a/crates/store/src/errors.rs b/crates/store/src/errors.rs index 700c9d3c5..bb563c36a 100644 --- a/crates/store/src/errors.rs +++ b/crates/store/src/errors.rs @@ -37,50 +37,56 @@ pub enum NullifierTreeError { #[derive(Debug, Error)] pub enum DatabaseError { - #[error("Missing database connection: {0}")] - MissingDbConnection(#[from] PoolError), - #[error("SQLite error: {0}")] - SqliteError(#[from] rusqlite::Error), - #[error("SQLite error: {0}")] - FromSqlError(#[from] FromSqlError), - #[error("Hex parsing error: {0}")] - FromHexError(#[from] hex::FromHexError), - #[error("I/O error: {0}")] - IoError(#[from] io::Error), + // ERRORS WITH AUTOMATIC CONVERSIONS FROM NESTED ERROR TYPES + // --------------------------------------------------------------------------------------------- #[error("Account error: {0}")] AccountError(#[from] AccountError), - #[error("Block error: {0}")] - BlockError(#[from] BlockError), - #[error("Note error: {0}")] - NoteError(#[from] NoteError), - #[error("Migration error: {0}")] - MigrationError(#[from] rusqlite_migration::Error), #[error("Account delta error: {0}")] AccountDeltaError(#[from] AccountDeltaError), - #[error("SQLite pool interaction task failed: {0}")] - InteractError(String), + #[error("Block error: {0}")] + BlockError(#[from] BlockError), + #[error("Closed channel: {0}")] + ClosedChannel(#[from] RecvError), #[error("Deserialization of BLOB data from database failed: {0}")] DeserializationError(DeserializationError), - #[error("Invalid Felt: {0}")] - InvalidFelt(String), - #[error("Block applying was broken because of closed channel on state side: {0}")] - ApplyBlockFailedClosedChannel(RecvError), + #[error("Hex parsing error: {0}")] + FromHexError(#[from] hex::FromHexError), + #[error("SQLite error: {0}")] + FromSqlError(#[from] FromSqlError), + #[error("I/O error: {0}")] + IoError(#[from] io::Error), + #[error("Migration error: {0}")] + MigrationError(#[from] rusqlite_migration::Error), + #[error("Missing database connection: {0}")] + MissingDbConnection(#[from] PoolError), + #[error("Note error: {0}")] + NoteError(#[from] NoteError), + #[error("SQLite error: {0}")] + SqliteError(#[from] rusqlite::Error), + + // OTHER ERRORS + // --------------------------------------------------------------------------------------------- + #[error("Account hashes mismatch (expected {expected}, but calculated is {calculated})")] + AccountHashesMismatch { + expected: RpoDigest, + calculated: RpoDigest, + }, #[error("Account {0} not found in the database")] AccountNotFoundInDb(AccountId), #[error("Accounts {0:?} not found in the database")] AccountsNotFoundInDb(Vec), #[error("Account {0} is not on the chain")] AccountNotOnChain(AccountId), - #[error("Failed to apply block because of public account final hashes mismatch (expected {expected}, \ - but calculated is {calculated}")] - ApplyBlockFailedAccountHashesMismatch { - expected: RpoDigest, - calculated: RpoDigest, - }, #[error("Block {0} not found in the database")] BlockNotFoundInDb(BlockNumber), - #[error("Unsupported database version. There is no migration chain from/to this version. Remove database files \ - and try again.")] + #[error("SQLite pool interaction task failed: {0}")] + InteractError(String), + #[error("Invalid Felt: {0}")] + InvalidFelt(String), + #[error( + "Unsupported database version. There is no migration chain from/to this version. \ + Remove all database files and try again." + )] UnsupportedDatabaseVersion, } @@ -132,12 +138,17 @@ pub enum DatabaseSetupError { #[derive(Debug, Error)] pub enum GenesisError { + // ERRORS WITH AUTOMATIC CONVERSIONS FROM NESTED ERROR TYPES + // --------------------------------------------------------------------------------------------- #[error("Database error: {0}")] DatabaseError(#[from] DatabaseError), #[error("Block error: {0}")] BlockError(#[from] BlockError), #[error("Merkle error: {0}")] MerkleError(#[from] MerkleError), + + // OTHER ERRORS + // --------------------------------------------------------------------------------------------- #[error("Apply block failed: {0}")] ApplyBlockFailed(String), #[error("Failed to read genesis file \"{genesis_filepath}\": {error}")] @@ -145,58 +156,74 @@ pub enum GenesisError { genesis_filepath: String, error: io::Error, }, - #[error("Failed to deserialize genesis file: {0}")] - GenesisFileDeserializationError(DeserializationError), #[error("Block header in store doesn't match block header in genesis file. Expected {expected_genesis_header:?}, but store contained {block_header_in_store:?}")] GenesisBlockHeaderMismatch { expected_genesis_header: Box, block_header_in_store: Box, }, + #[error("Failed to deserialize genesis file: {0}")] + GenesisFileDeserializationError(DeserializationError), #[error("Retrieving genesis block header failed: {0}")] SelectBlockHeaderByBlockNumError(Box), } // ENDPOINT ERRORS // ================================================================================================= +#[derive(Error, Debug)] +pub enum InvalidBlockError { + #[error("Duplicated nullifiers {0:?}")] + DuplicatedNullifiers(Vec), + #[error("Invalid output note type: {0:?}")] + InvalidOutputNoteType(Box), + #[error("Invalid tx hash: expected {expected}, but got {actual}")] + InvalidTxHash { expected: RpoDigest, actual: RpoDigest }, + #[error("Received invalid account tree root")] + NewBlockInvalidAccountRoot, + #[error("New block number must be 1 greater than the current block number")] + NewBlockInvalidBlockNum, + #[error("New block chain root is not consistent with chain MMR")] + NewBlockInvalidChainRoot, + #[error("Received invalid note root")] + NewBlockInvalidNoteRoot, + #[error("Received invalid nullifier root")] + NewBlockInvalidNullifierRoot, + #[error("New block `prev_hash` must match the chain's tip")] + NewBlockInvalidPrevHash, +} #[derive(Error, Debug)] pub enum ApplyBlockError { + // ERRORS WITH AUTOMATIC CONVERSIONS FROM NESTED ERROR TYPES + // --------------------------------------------------------------------------------------------- #[error("Database error: {0}")] DatabaseError(#[from] DatabaseError), #[error("I/O error: {0}")] IoError(#[from] io::Error), #[error("Task join error: {0}")] TokioJoinError(#[from] tokio::task::JoinError), + #[error("Invalid block error: {0}")] + InvalidBlockError(#[from] InvalidBlockError), + + // OTHER ERRORS + // --------------------------------------------------------------------------------------------- + #[error("Block applying was cancelled because of closed channel on database side: {0}")] + ClosedChannel(RecvError), #[error("Concurrent write detected")] ConcurrentWrite, - #[error("New block number must be 1 greater than the current block number")] - NewBlockInvalidBlockNum, - #[error("New block `prev_hash` must match the chain's tip")] - NewBlockInvalidPrevHash, - #[error("New block chain root is not consistent with chain MMR")] - NewBlockInvalidChainRoot, - #[error("Received invalid account tree root")] - NewBlockInvalidAccountRoot, - #[error("Received invalid note root")] - NewBlockInvalidNoteRoot, - #[error("Received invalid nullifier root")] - NewBlockInvalidNullifierRoot, - #[error("Duplicated nullifiers {0:?}")] - DuplicatedNullifiers(Vec), - #[error("Block applying was broken because of closed channel on database side: {0}")] - BlockApplyingBrokenBecauseOfClosedChannel(RecvError), - #[error("Failed to create notes tree: {0}")] - FailedToCreateNoteTree(MerkleError), #[error("Database doesn't have any block header data")] DbBlockHeaderEmpty, - #[error("Failed to get MMR peaks for forest ({forest}): {error}")] - FailedToGetMmrPeaksForForest { forest: usize, error: MmrError }, - #[error("Failed to update nullifier tree: {0}")] - FailedToUpdateNullifierTree(NullifierTreeError), - #[error("Invalid output note type: {0:?}")] - InvalidOutputNoteType(OutputNote), - #[error("Invalid tx hash: expected {expected}, but got {actual}")] - InvalidTxHash { expected: RpoDigest, actual: RpoDigest }, + #[error("Database update task failed: {0}")] + DbUpdateTaskFailed(String), +} + +impl From for Status { + fn from(err: ApplyBlockError) -> Self { + match err { + ApplyBlockError::InvalidBlockError(_) => Status::invalid_argument(err.to_string()), + + _ => Status::internal(err.to_string()), + } + } } #[derive(Error, Debug)] @@ -209,10 +236,10 @@ pub enum GetBlockHeaderError { #[derive(Error, Debug)] pub enum GetBlockInputsError { - #[error("Database error: {0}")] - DatabaseError(#[from] DatabaseError), #[error("Account error: {0}")] AccountError(#[from] AccountError), + #[error("Database error: {0}")] + DatabaseError(#[from] DatabaseError), #[error("Database doesn't have any block header data")] DbBlockHeaderEmpty, #[error("Failed to get MMR peaks for forest ({forest}): {error}")] diff --git a/crates/store/src/server/api.rs b/crates/store/src/server/api.rs index 51cb21ef4..3b6db4bd1 100644 --- a/crates/store/src/server/api.rs +++ b/crates/store/src/server/api.rs @@ -50,9 +50,6 @@ pub struct StoreApi { pub(super) state: Arc, } -// FIXME: remove the allow when the upstream clippy issue is fixed: -// https://github.com/rust-lang/rust-clippy/issues/12281 -#[allow(clippy::blocks_in_conditions)] #[tonic::async_trait] impl api_server::Api for StoreApi { // CLIENT ENDPOINTS @@ -372,8 +369,7 @@ impl api_server::Api for StoreApi { nullifier_count = block.nullifiers().len(), ); - // TODO: Why the error is swallowed here? Fix or add a comment with explanation. - let _ = self.state.apply_block(block).await; + self.state.apply_block(block).await?; Ok(Response::new(ApplyBlockResponse {})) } diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index 3ff4c4197..597219dac 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -41,13 +41,13 @@ use crate::{ db::{Db, NoteRecord, NoteSyncUpdate, NullifierInfo, StateSyncUpdate}, errors::{ ApplyBlockError, DatabaseError, GetBlockHeaderError, GetBlockInputsError, - GetNoteInclusionProofError, NoteSyncError, StateInitializationError, StateSyncError, + GetNoteInclusionProofError, InvalidBlockError, NoteSyncError, StateInitializationError, + StateSyncError, }, nullifier_tree::NullifierTree, types::{AccountId, BlockNumber}, COMPONENT, }; - // STRUCTURES // ================================================================================================ @@ -99,7 +99,9 @@ struct InnerState { impl InnerState { /// Returns the latest block number. fn latest_block_num(&self) -> BlockNumber { - (self.chain_mmr.forest() + 1).try_into().expect("block number overflow") + (self.chain_mmr.forest() - 1) + .try_into() + .expect("chain_mmr always has, at least, the genesis block") } } @@ -173,10 +175,11 @@ impl State { let tx_hash = block.compute_tx_hash(); if header.tx_hash() != tx_hash { - return Err(ApplyBlockError::InvalidTxHash { + return Err(InvalidBlockError::InvalidTxHash { expected: tx_hash, actual: header.tx_hash(), - }); + } + .into()); } let block_num = header.block_num(); @@ -190,10 +193,10 @@ impl State { .ok_or(ApplyBlockError::DbBlockHeaderEmpty)?; if block_num != prev_block.block_num() + 1 { - return Err(ApplyBlockError::NewBlockInvalidBlockNum); + return Err(InvalidBlockError::NewBlockInvalidBlockNum.into()); } if header.prev_hash() != prev_block.hash() { - return Err(ApplyBlockError::NewBlockInvalidPrevHash); + return Err(InvalidBlockError::NewBlockInvalidPrevHash.into()); } let block_data = block.to_bytes(); @@ -227,7 +230,7 @@ impl State { .cloned() .collect(); if !duplicate_nullifiers.is_empty() { - return Err(ApplyBlockError::DuplicatedNullifiers(duplicate_nullifiers)); + return Err(InvalidBlockError::DuplicatedNullifiers(duplicate_nullifiers).into()); } // compute updates for the in-memory data structures @@ -235,7 +238,7 @@ impl State { // new_block.chain_root must be equal to the chain MMR root prior to the update let peaks = inner.chain_mmr.peaks(); if peaks.hash_peaks() != header.chain_root() { - return Err(ApplyBlockError::NewBlockInvalidChainRoot); + return Err(InvalidBlockError::NewBlockInvalidChainRoot.into()); } // compute update for nullifier tree @@ -244,7 +247,7 @@ impl State { ); if nullifier_tree_update.root() != header.nullifier_root() { - return Err(ApplyBlockError::NewBlockInvalidNullifierRoot); + return Err(InvalidBlockError::NewBlockInvalidNullifierRoot.into()); } // compute update for account tree @@ -258,7 +261,7 @@ impl State { ); if account_tree_update.root() != header.account_root() { - return Err(ApplyBlockError::NewBlockInvalidAccountRoot); + return Err(InvalidBlockError::NewBlockInvalidAccountRoot.into()); } ( @@ -272,7 +275,7 @@ impl State { // build note tree let note_tree = block.build_note_tree(); if note_tree.root() != header.note_root() { - return Err(ApplyBlockError::NewBlockInvalidNoteRoot); + return Err(InvalidBlockError::NewBlockInvalidNoteRoot.into()); } let notes = block @@ -281,7 +284,11 @@ impl State { let details = match note { OutputNote::Full(note) => Some(note.to_bytes()), OutputNote::Header(_) => None, - note => return Err(ApplyBlockError::InvalidOutputNoteType(note.clone())), + note => { + return Err(InvalidBlockError::InvalidOutputNoteType(Box::new( + note.clone(), + ))) + }, }; let merkle_path = note_tree.get_note_path(note_index); @@ -295,7 +302,7 @@ impl State { merkle_path, }) }) - .collect::, ApplyBlockError>>()?; + .collect::, InvalidBlockError>>()?; // Signals the transaction is ready to be committed, and the write lock can be acquired let (allow_acquire, acquired_allowed) = oneshot::channel::<()>(); @@ -313,9 +320,7 @@ impl State { ); // Wait for the message from the DB update task, that we ready to commit the DB transaction - acquired_allowed - .await - .map_err(ApplyBlockError::BlockApplyingBrokenBecauseOfClosedChannel)?; + acquired_allowed.await.map_err(ApplyBlockError::ClosedChannel)?; // Awaiting the block saving task to complete without errors block_save_task.await??; @@ -339,13 +344,17 @@ impl State { // Notify the DB update task that the write lock has been acquired, so it can commit // the DB transaction - let _ = inform_acquire_done.send(()); + inform_acquire_done + .send(()) + .map_err(|_| ApplyBlockError::DbUpdateTaskFailed("Receiver was dropped".into()))?; // TODO: shutdown #91 // Await for successful commit of the DB transaction. If the commit fails, we mustn't // change in-memory state, so we return a block applying error and don't proceed with // in-memory updates. - db_update_task.await??; + db_update_task + .await? + .map_err(|err| ApplyBlockError::DbUpdateTaskFailed(err.to_string()))?; // Update the in-memory data structures after successful commit of the DB transaction inner From 54095db301390347efab9d4f9412a55656783d12 Mon Sep 17 00:00:00 2001 From: Varun Doshi <61531351+varun-doshi@users.noreply.github.com> Date: Mon, 2 Dec 2024 11:54:46 +0530 Subject: [PATCH 14/50] feat: https support in endpoint config (#556) Previously only http was supported. --- CHANGELOG.md | 1 + bin/node/src/config.rs | 9 ++++++--- crates/rpc/src/config.rs | 3 ++- crates/utils/src/config.rs | 27 +++++++++++++++++++++++++-- 4 files changed, 34 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ed4c12a1b..ccb6505c4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### Enhancements - Added `GetAccountProofs` endpoint (#506). +- Support Https in endpoint configuration (#556). ### Changes diff --git a/bin/node/src/config.rs b/bin/node/src/config.rs index 173496764..8e44ad064 100644 --- a/bin/node/src/config.rs +++ b/bin/node/src/config.rs @@ -74,7 +74,7 @@ impl NodeConfig { mod tests { use figment::Jail; use miden_node_store::config::StoreConfig; - use miden_node_utils::config::{load_config, Endpoint}; + use miden_node_utils::config::{load_config, Endpoint, Protocol}; use super::NodeConfig; use crate::{ @@ -93,10 +93,10 @@ mod tests { verify_tx_proofs = true [rpc] - endpoint = { host = "127.0.0.1", port = 8080 } + endpoint = { host = "127.0.0.1", port = 8080, protocol = "Http" } [store] - endpoint = { host = "127.0.0.1", port = 8080 } + endpoint = { host = "127.0.0.1", port = 8080, protocol = "Https" } database_filepath = "local.sqlite3" genesis_filepath = "genesis.dat" blockstore_dir = "blocks" @@ -112,6 +112,7 @@ mod tests { endpoint: Endpoint { host: "127.0.0.1".to_string(), port: 8080, + protocol: Protocol::default() }, verify_tx_proofs: true }, @@ -119,12 +120,14 @@ mod tests { endpoint: Endpoint { host: "127.0.0.1".to_string(), port: 8080, + protocol: Protocol::Http }, }, store: StoreConfig { endpoint: Endpoint { host: "127.0.0.1".to_string(), port: 8080, + protocol: Protocol::Https }, database_filepath: "local.sqlite3".into(), genesis_filepath: "genesis.dat".into(), diff --git a/crates/rpc/src/config.rs b/crates/rpc/src/config.rs index 7cd6e6448..07dac0778 100644 --- a/crates/rpc/src/config.rs +++ b/crates/rpc/src/config.rs @@ -1,7 +1,7 @@ use std::fmt::{Display, Formatter}; use miden_node_utils::config::{ - Endpoint, DEFAULT_BLOCK_PRODUCER_PORT, DEFAULT_NODE_RPC_PORT, DEFAULT_STORE_PORT, + Endpoint, Protocol, DEFAULT_BLOCK_PRODUCER_PORT, DEFAULT_NODE_RPC_PORT, DEFAULT_STORE_PORT, }; use serde::{Deserialize, Serialize}; @@ -39,6 +39,7 @@ impl Default for RpcConfig { endpoint: Endpoint { host: "0.0.0.0".to_string(), port: DEFAULT_NODE_RPC_PORT, + protocol: Protocol::default(), }, store_url: Endpoint::localhost(DEFAULT_STORE_PORT).to_string(), block_producer_url: Endpoint::localhost(DEFAULT_BLOCK_PRODUCER_PORT).to_string(), diff --git a/crates/utils/src/config.rs b/crates/utils/src/config.rs index c350fd36b..5c70315df 100644 --- a/crates/utils/src/config.rs +++ b/crates/utils/src/config.rs @@ -17,6 +17,12 @@ pub const DEFAULT_BLOCK_PRODUCER_PORT: u16 = 48046; pub const DEFAULT_STORE_PORT: u16 = 28943; pub const DEFAULT_FAUCET_SERVER_PORT: u16 = 8080; +#[derive(Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Debug, Serialize, Deserialize, Default)] +pub enum Protocol { + #[default] + Http, + Https, +} /// The `(host, port)` pair for the server's listening socket. #[derive(Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Debug, Serialize, Deserialize)] pub struct Endpoint { @@ -24,11 +30,18 @@ pub struct Endpoint { pub host: String, /// Port number used by the store. pub port: u16, + /// Protocol type: http or https. + #[serde(default)] + pub protocol: Protocol, } impl Endpoint { pub fn localhost(port: u16) -> Self { - Endpoint { host: "localhost".to_string(), port } + Endpoint { + host: "localhost".to_string(), + port, + protocol: Protocol::default(), + } } } @@ -41,7 +54,17 @@ impl ToSocketAddrs for Endpoint { impl Display for Endpoint { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - f.write_fmt(format_args!("http://{}:{}", self.host, self.port)) + let Endpoint { protocol, host, port } = self; + f.write_fmt(format_args!("{protocol}://{host}:{port}")) + } +} + +impl Display for Protocol { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + Protocol::Http => f.write_str("http"), + Protocol::Https => f.write_str("https"), + } } } From a686581c8ce36a9da3577471c3af0e4fd9fc8005 Mon Sep 17 00:00:00 2001 From: polydez <155382956+polydez@users.noreply.github.com> Date: Mon, 2 Dec 2024 22:02:19 +0500 Subject: [PATCH 15/50] chore: migrate code to the latest `miden-base` (#559) * refactor: remove unused dependencies, migrate code to the latest `next` of `miden-base` * chore: update `miden-base` * refactor: migrate to `assert_matches!` macro * chore: switch to the `next` branch of `miden-base` * refactor: add missed `assert_matches!` --- Cargo.lock | 123 ++++++++---------- Cargo.toml | 11 +- crates/block-producer/Cargo.toml | 5 +- .../src/batch_builder/tests/mod.rs | 9 +- .../src/block_builder/prover/tests.rs | 21 ++- .../block-producer/src/block_builder/tests.rs | 3 +- crates/block-producer/src/errors.rs | 26 ++-- .../src/state_view/tests/apply_block.rs | 15 ++- .../src/state_view/tests/verify_tx.rs | 51 +++++--- .../block-producer/src/txqueue/tests/mod.rs | 3 +- crates/proto/src/errors.rs | 4 +- crates/rpc/Cargo.toml | 9 -- crates/store/Cargo.toml | 9 +- crates/store/src/db/tests.rs | 61 ++++----- 14 files changed, 171 insertions(+), 179 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 184a24b62..0fd48b068 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -129,6 +129,12 @@ dependencies = [ "term", ] +[[package]] +name = "assert_matches" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b34d609dfbaf33d6889b2b7106d3ca345eacad44200913df5ba02bfd31d2ba9" + [[package]] name = "async-stream" version = "0.3.6" @@ -390,7 +396,7 @@ dependencies = [ "semver 1.0.23", "serde", "serde_json", - "thiserror", + "thiserror 1.0.68", ] [[package]] @@ -687,15 +693,6 @@ dependencies = [ "crypto-common", ] -[[package]] -name = "directories" -version = "5.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a49173b84e034382284f27f1af4dcbbd231ffa358c0fe316541a7337f376a35" -dependencies = [ - "dirs-sys", -] - [[package]] name = "dirs-next" version = "2.0.0" @@ -706,18 +703,6 @@ dependencies = [ "dirs-sys-next", ] -[[package]] -name = "dirs-sys" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c" -dependencies = [ - "libc", - "option-ext", - "redox_users", - "windows-sys 0.48.0", -] - [[package]] name = "dirs-sys-next" version = "0.1.2" @@ -1490,7 +1475,7 @@ dependencies = [ "rand_chacha", "serde", "static-files", - "thiserror", + "thiserror 2.0.3", "tokio", "toml", "tonic", @@ -1510,14 +1495,14 @@ dependencies = [ [[package]] name = "miden-lib" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "816404c10b0799f12d3b53b3a9baa9af99fa340fe1a579e0919bba57718fa97a" +version = "0.7.0" +source = "git+https://github.com/0xPolygonMiden/miden-base.git?branch=next#bc91c604862e2c6013b21eb989a3640d30cce25b" dependencies = [ "miden-assembly", "miden-objects", "miden-stdlib", "regex", + "walkdir", ] [[package]] @@ -1588,8 +1573,8 @@ dependencies = [ name = "miden-node-block-producer" version = "0.6.0" dependencies = [ + "assert_matches", "async-trait", - "figment", "itertools 0.13.0", "miden-air", "miden-lib", @@ -1602,13 +1587,11 @@ dependencies = [ "miden-tx", "rand_chacha", "serde", - "thiserror", + "thiserror 2.0.3", "tokio", "tokio-stream", - "toml", "tonic", "tracing", - "tracing-subscriber", "winterfell", ] @@ -1624,7 +1607,7 @@ dependencies = [ "prost", "prost-build", "protox", - "thiserror", + "thiserror 2.0.3", "tonic", "tonic-build", ] @@ -1633,49 +1616,37 @@ dependencies = [ name = "miden-node-rpc" version = "0.6.0" dependencies = [ - "directories", - "figment", - "hex", - "miden-node-block-producer", "miden-node-proto", - "miden-node-store", "miden-node-utils", "miden-objects", "miden-tx", - "prost", "serde", "tokio", "tokio-stream", - "toml", "tonic", "tonic-web", "tracing", - "tracing-subscriber", ] [[package]] name = "miden-node-store" version = "0.6.0" dependencies = [ + "assert_matches", "deadpool-sqlite", - "directories", - "figment", "hex", "miden-lib", "miden-node-proto", "miden-node-utils", "miden-objects", - "prost", "rusqlite", "rusqlite_migration", "serde", - "thiserror", + "thiserror 2.0.3", "tokio", "tokio-stream", - "toml", "tonic", "tracing", - "tracing-subscriber", ] [[package]] @@ -1696,7 +1667,7 @@ dependencies = [ "miden-objects", "rand", "serde", - "thiserror", + "thiserror 2.0.3", "tonic", "tracing", "tracing-forest", @@ -1707,9 +1678,8 @@ dependencies = [ [[package]] name = "miden-objects" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a6f54dde939928e438488b36651485a0e80057025b7e30343d4340a524a1651" +version = "0.7.0" +source = "git+https://github.com/0xPolygonMiden/miden-base.git?branch=next#bc91c604862e2c6013b21eb989a3640d30cce25b" dependencies = [ "getrandom", "miden-assembly", @@ -1718,6 +1688,7 @@ dependencies = [ "miden-processor", "miden-verifier", "rand", + "thiserror 2.0.3", "winter-rand-utils", ] @@ -1781,12 +1752,10 @@ dependencies = [ [[package]] name = "miden-tx" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d433744a3f02233ec53b151bfe88b7a008b4eae5dfd41bb38a9889eb67efaf7" +version = "0.7.0" +source = "git+https://github.com/0xPolygonMiden/miden-base.git?branch=next#bc91c604862e2c6013b21eb989a3640d30cce25b" dependencies = [ "async-trait", - "miden-assembly", "miden-lib", "miden-objects", "miden-processor", @@ -1794,8 +1763,6 @@ dependencies = [ "miden-verifier", "rand", "rand_chacha", - "regex", - "walkdir", "winter-maybe-async", ] @@ -1827,7 +1794,7 @@ dependencies = [ "supports-unicode", "terminal_size", "textwrap", - "thiserror", + "thiserror 1.0.68", "unicode-width 0.1.14", ] @@ -2042,12 +2009,6 @@ version = "1.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" -[[package]] -name = "option-ext" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" - [[package]] name = "overload" version = "0.1.1" @@ -2335,7 +2296,7 @@ dependencies = [ "prost-reflect", "prost-types", "protox-parse", - "thiserror", + "thiserror 1.0.68", ] [[package]] @@ -2347,7 +2308,7 @@ dependencies = [ "logos", "miette", "prost-types", - "thiserror", + "thiserror 1.0.68", ] [[package]] @@ -2441,7 +2402,7 @@ checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" dependencies = [ "getrandom", "libredox", - "thiserror", + "thiserror 1.0.68", ] [[package]] @@ -2904,7 +2865,16 @@ version = "1.0.68" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "02dd99dc800bbb97186339685293e1cc5d9df1f8fae2d0aecd9ff1c77efea892" dependencies = [ - "thiserror-impl", + "thiserror-impl 1.0.68", +] + +[[package]] +name = "thiserror" +version = "2.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c006c85c7651b3cf2ada4584faa36773bd07bac24acfb39f3c431b36d7e667aa" +dependencies = [ + "thiserror-impl 2.0.3", ] [[package]] @@ -2918,6 +2888,17 @@ dependencies = [ "syn", ] +[[package]] +name = "thiserror-impl" +version = "2.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f077553d607adc1caf65430528a576c757a71ed73944b66ebb58ef2bbd243568" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "thread_local" version = "1.1.8" @@ -3240,7 +3221,7 @@ checksum = "ee40835db14ddd1e3ba414292272eddde9dad04d3d4b65509656414d1c42592f" dependencies = [ "chrono", "smallvec", - "thiserror", + "thiserror 1.0.68", "tracing", "tracing-subscriber", ] @@ -3882,9 +3863,9 @@ dependencies = [ [[package]] name = "winter-utils" -version = "0.10.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76b116c8ade0172506f8bda32dc674cf6b230adc8516e5138a0173ae69158a4f" +checksum = "1f948e71ffd482aa13d0ec3349047f81ecdb89f3b3287577973dcbf092a25fb4" dependencies = [ "rayon", ] @@ -3904,9 +3885,9 @@ dependencies = [ [[package]] name = "winterfell" -version = "0.10.1" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39c8336dc6a035698780b8cc624f875e479bd6bf6e1846670f3ef4485c125882" +checksum = "bbf1ab01d2781f7d3f1bd5c12800905c5bbf62e06778672498be798006ac463a" dependencies = [ "winter-air", "winter-prover", diff --git a/Cargo.toml b/Cargo.toml index ce5eb685e..eb13182a8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,23 +25,22 @@ exclude = [".github/"] readme = "README.md" [workspace.dependencies] +assert_matches = { version = "1.5" } miden-air = { version = "0.11" } -miden-lib = { version = "0.6" } +miden-lib = { git = "https://github.com/0xPolygonMiden/miden-base.git", branch = "next" } miden-node-block-producer = { path = "crates/block-producer", version = "0.6" } -miden-node-faucet = { path = "bin/faucet", version = "0.6" } miden-node-proto = { path = "crates/proto", version = "0.6" } miden-node-rpc = { path = "crates/rpc", version = "0.6" } -miden-node-rpc-proto = { path = "crates/rpc-proto", version = "0.6" } miden-node-store = { path = "crates/store", version = "0.6" } miden-node-test-macro = { path = "crates/test-macro" } miden-node-utils = { path = "crates/utils", version = "0.6" } -miden-objects = { version = "0.6"} +miden-objects = { git = "https://github.com/0xPolygonMiden/miden-base.git", branch = "next" } miden-processor = { version = "0.11" } miden-stdlib = { version = "0.11", default-features = false } -miden-tx = { version = "0.6"} +miden-tx = { git = "https://github.com/0xPolygonMiden/miden-base.git", branch = "next" } prost = { version = "0.13" } rand = { version = "0.8" } -thiserror = { version = "1.0" } +thiserror = { version = "2.0", default-features = false } tokio = { version = "1.40", features = ["rt-multi-thread"] } tokio-stream = { version = "0.1" } tonic = { version = "0.12" } diff --git a/crates/block-producer/Cargo.toml b/crates/block-producer/Cargo.toml index ea6f32116..9c9aeefc7 100644 --- a/crates/block-producer/Cargo.toml +++ b/crates/block-producer/Cargo.toml @@ -16,7 +16,6 @@ tracing-forest = ["miden-node-utils/tracing-forest"] [dependencies] async-trait = { version = "0.1" } -figment = { version = "0.10", features = ["toml", "env"] } itertools = { version = "0.13" } miden-lib = { workspace = true } miden-node-proto = { workspace = true } @@ -29,13 +28,11 @@ serde = { version = "1.0", features = ["derive"] } thiserror = { workspace = true } tokio = { workspace = true, features = ["rt-multi-thread", "net", "macros", "sync", "time"] } tokio-stream = { workspace = true, features = ["net"] } -toml = { version = "0.8" } tonic = { workspace = true } tracing = { workspace = true } -tracing-subscriber = { workspace = true } [dev-dependencies] -figment = { version = "0.10", features = ["toml", "env", "test"] } +assert_matches = { workspace = true} miden-air = { workspace = true } miden-lib = { workspace = true, features = ["testing"] } miden-node-test-macro = { path = "../test-macro" } diff --git a/crates/block-producer/src/batch_builder/tests/mod.rs b/crates/block-producer/src/batch_builder/tests/mod.rs index 1e0012f1c..46b63f703 100644 --- a/crates/block-producer/src/batch_builder/tests/mod.rs +++ b/crates/block-producer/src/batch_builder/tests/mod.rs @@ -1,5 +1,6 @@ use std::iter; +use assert_matches::assert_matches; use miden_objects::{crypto::merkle::Mmr, Digest}; use tokio::sync::RwLock; @@ -254,7 +255,7 @@ async fn test_block_builder_no_missing_notes() { .block_builder .build_block(&batch_builder.ready_batches.read().await) .await; - assert_eq!(build_block_result, Ok(())); + assert_matches!(build_block_result, Ok(())); } #[tokio::test] @@ -307,9 +308,11 @@ async fn test_block_builder_fails_if_notes_are_missing() { let mut expected_missing_notes = vec![notes[4].id(), notes[5].id()]; expected_missing_notes.sort(); - assert_eq!( + assert_matches!( build_block_result, - Err(BuildBlockError::UnauthenticatedNotesNotFound(expected_missing_notes)) + Err(BuildBlockError::UnauthenticatedNotesNotFound(actual_missing_notes)) => { + assert_eq!(actual_missing_notes, expected_missing_notes); + } ); } diff --git a/crates/block-producer/src/block_builder/prover/tests.rs b/crates/block-producer/src/block_builder/prover/tests.rs index 3d776ffc6..56124065d 100644 --- a/crates/block-producer/src/block_builder/prover/tests.rs +++ b/crates/block-producer/src/block_builder/prover/tests.rs @@ -1,5 +1,6 @@ use std::{collections::BTreeMap, iter}; +use assert_matches::assert_matches; use miden_objects::{ accounts::{ account_id::testing::{ @@ -157,13 +158,17 @@ fn test_block_witness_validation_inconsistent_account_hashes() { let block_witness_result = BlockWitness::new(block_inputs_from_store, &batches); - assert_eq!( + assert_matches!( block_witness_result, Err(BuildBlockError::InconsistentAccountStateTransition( - account_id_1, - account_1_hash_store, - vec![account_1_hash_batches] - )) + account_id, + account_hash_store, + account_hash_batches + )) => { + assert_eq!(account_id, account_id_1); + assert_eq!(account_hash_store, account_1_hash_store); + assert_eq!(account_hash_batches, vec![account_1_hash_batches]); + } ); } @@ -668,9 +673,11 @@ fn test_block_witness_validation_inconsistent_nullifiers() { let block_witness_result = BlockWitness::new(block_inputs_from_store, &batches); - assert_eq!( + assert_matches!( block_witness_result, - Err(BuildBlockError::InconsistentNullifiers(vec![nullifier_1, nullifier_3])) + Err(BuildBlockError::InconsistentNullifiers(nullifiers)) => { + assert_eq!(nullifiers, vec![nullifier_1, nullifier_3]); + } ); } diff --git a/crates/block-producer/src/block_builder/tests.rs b/crates/block-producer/src/block_builder/tests.rs index c7c82629f..d8fbc3565 100644 --- a/crates/block-producer/src/block_builder/tests.rs +++ b/crates/block-producer/src/block_builder/tests.rs @@ -3,6 +3,7 @@ use std::sync::Arc; +use assert_matches::assert_matches; use miden_objects::{ accounts::{account_id::testing::ACCOUNT_ID_OFF_CHAIN_SENDER, AccountId}, Digest, Felt, @@ -78,5 +79,5 @@ async fn test_build_block_failure() { let result = block_builder.build_block(&Vec::new()).await; // Ensure that the store's `apply_block()` was called - assert!(matches!(result, Err(BuildBlockError::GetBlockInputsFailed(_)))); + assert_matches!(result, Err(BuildBlockError::GetBlockInputsFailed(_))); } diff --git a/crates/block-producer/src/errors.rs b/crates/block-producer/src/errors.rs index 717a29268..e31027a41 100644 --- a/crates/block-producer/src/errors.rs +++ b/crates/block-producer/src/errors.rs @@ -14,7 +14,7 @@ use thiserror::Error; // Transaction verification errors // ================================================================================================= -#[derive(Debug, PartialEq, Eq, Error)] +#[derive(Debug, Error)] pub enum VerifyTxError { /// Another transaction already consumed the notes with given nullifiers #[error("Input notes with given nullifiers were already consumed by another transaction")] @@ -52,7 +52,7 @@ pub enum VerifyTxError { // Transaction adding errors // ================================================================================================= -#[derive(Debug, PartialEq, Eq, Error)] +#[derive(Debug, Error)] pub enum AddTransactionError { #[error("Transaction verification failed: {0}")] VerificationFailed(#[from] VerifyTxError), @@ -66,18 +66,12 @@ pub enum AddTransactionError { /// These errors are returned from the batch builder to the transaction queue, instead of /// dropping the transactions, they are included into the error values, so that the transaction /// queue can re-queue them. -#[derive(Debug, PartialEq, Eq, Error)] +#[derive(Debug, Error)] pub enum BuildBatchError { - #[error( - "Too many input notes in the batch. Got: {0}, limit: {}", - MAX_INPUT_NOTES_PER_BATCH - )] + #[error("Too many input notes in the batch. Got: {0}, limit: {MAX_INPUT_NOTES_PER_BATCH}")] TooManyInputNotes(usize, Vec), - #[error( - "Too many notes created in the batch. Got: {0}, limit: {}", - MAX_OUTPUT_NOTES_PER_BATCH - )] + #[error("Too many notes created in the batch. Got: {0}, limit: {MAX_OUTPUT_NOTES_PER_BATCH}")] TooManyNotesCreated(usize, Vec), #[error( @@ -152,7 +146,7 @@ pub enum BlockProverError { // ================================================================================================= #[allow(clippy::enum_variant_names)] -#[derive(Debug, PartialEq, Eq, Error)] +#[derive(Debug, Error)] pub enum BlockInputsError { #[error("failed to parse protobuf message: {0}")] ConversionError(#[from] ConversionError), @@ -166,7 +160,7 @@ pub enum BlockInputsError { // ================================================================================================= #[allow(clippy::enum_variant_names)] -#[derive(Debug, PartialEq, Eq, Error)] +#[derive(Debug, Error)] pub enum NotePathsError { #[error("failed to parse protobuf message: {0}")] ConversionError(#[from] ConversionError), @@ -177,7 +171,7 @@ pub enum NotePathsError { // Block applying errors // ================================================================================================= -#[derive(Debug, PartialEq, Eq, Error)] +#[derive(Debug, Error)] pub enum ApplyBlockError { #[error("gRPC client failed with error: {0}")] GrpcClientError(String), @@ -186,7 +180,7 @@ pub enum ApplyBlockError { // Block building errors // ================================================================================================= -#[derive(Debug, PartialEq, Eq, Error)] +#[derive(Debug, Error)] pub enum BuildBlockError { #[error("failed to compute new block: {0}")] BlockProverFailed(#[from] BlockProverError), @@ -216,7 +210,7 @@ pub enum BuildBlockError { // Transaction inputs errors // ================================================================================================= -#[derive(Debug, PartialEq, Eq, Error)] +#[derive(Debug, Error)] pub enum TxInputsError { #[error("gRPC client failed with error: {0}")] GrpcClientError(String), diff --git a/crates/block-producer/src/state_view/tests/apply_block.rs b/crates/block-producer/src/state_view/tests/apply_block.rs index 9465ec951..b98371af5 100644 --- a/crates/block-producer/src/state_view/tests/apply_block.rs +++ b/crates/block-producer/src/state_view/tests/apply_block.rs @@ -7,6 +7,7 @@ use std::iter; +use assert_matches::assert_matches; use miden_objects::{accounts::delta::AccountUpdateDetails, block::BlockAccountUpdate}; use super::*; @@ -74,7 +75,9 @@ async fn test_apply_block_ab2() { // Verify transactions so it can be tracked in state view for tx in txs { let verify_tx_res = state_view.verify_tx(&tx).await; - assert_eq!(verify_tx_res, Ok(0)); + assert_matches!(verify_tx_res, Ok(block_height) => { + assert_eq!(block_height, 0); + }); } // All except the first account will go into the block. @@ -128,7 +131,9 @@ async fn test_apply_block_ab3() { // Verify transactions so it can be tracked in state view for tx in txs.clone() { let verify_tx_res = state_view.verify_tx(&tx).await; - assert_eq!(verify_tx_res, Ok(0)); + assert_matches!(verify_tx_res, Ok(block_height) => { + assert_eq!(block_height, 0); + }); } let block = MockBlockBuilder::new(&store) @@ -163,8 +168,10 @@ async fn test_apply_block_ab3() { .build(); let verify_tx_res = state_view.verify_tx(&tx_new).await; - assert_eq!( + assert_matches!( verify_tx_res, - Err(VerifyTxError::InputNotesAlreadyConsumed(txs[0].get_nullifiers().collect())) + Err(VerifyTxError::InputNotesAlreadyConsumed(nullifiers)) => { + assert_eq!(nullifiers, txs[0].get_nullifiers().collect::>()); + } ); } diff --git a/crates/block-producer/src/state_view/tests/verify_tx.rs b/crates/block-producer/src/state_view/tests/verify_tx.rs index c3f576193..d28a3cb9e 100644 --- a/crates/block-producer/src/state_view/tests/verify_tx.rs +++ b/crates/block-producer/src/state_view/tests/verify_tx.rs @@ -12,6 +12,7 @@ use std::iter; +use assert_matches::assert_matches; use miden_objects::notes::Note; use tokio::task::JoinSet; @@ -95,12 +96,17 @@ async fn test_verify_tx_vt1() { let verify_tx_result = state_view.verify_tx(&tx).await; - assert_eq!( + assert_matches!( verify_tx_result, Err(VerifyTxError::IncorrectAccountInitialHash { - tx_initial_account_hash: account.states[1], - current_account_hash: Some(account.states[0]), - }) + tx_initial_account_hash, + current_account_hash, + }) => { + assert_eq!(tx_initial_account_hash, account.states[1]); + assert_matches!(current_account_hash, Some(state) => { + assert_eq!(state, account.states[0]); + }); + } ); } @@ -152,9 +158,11 @@ async fn test_verify_tx_vt3() { let verify_tx_result = state_view.verify_tx(&tx).await; - assert_eq!( + assert_matches!( verify_tx_result, - Err(VerifyTxError::InputNotesAlreadyConsumed(vec![nullifier_in_store])) + Err(VerifyTxError::InputNotesAlreadyConsumed(nullifiers)) => { + assert_eq!(nullifiers, vec![nullifier_in_store]); + } ); } @@ -221,9 +229,11 @@ async fn test_verify_tx_vt5() { assert!(verify_tx1_result.is_ok()); let verify_tx2_result = state_view.verify_tx(&tx2).await; - assert_eq!( + assert_matches!( verify_tx2_result, - Err(VerifyTxError::InputNotesAlreadyConsumed(vec![nullifier_in_both_txs])) + Err(VerifyTxError::InputNotesAlreadyConsumed(nullifiers)) => { + assert_eq!(nullifiers, vec![nullifier_in_both_txs]); + } ); } @@ -250,16 +260,20 @@ async fn test_verify_tx_dangling_note_found_in_inflight_notes() { let tx1 = MockProvenTxBuilder::with_account_index(1).output_notes(output_notes).build(); let verify_tx1_result = state_view.verify_tx(&tx1).await; - assert_eq!(verify_tx1_result, Ok(0)); + assert_matches!(verify_tx1_result, Ok(block_height) => { + assert_eq!(block_height, 0); + }); let tx2 = MockProvenTxBuilder::with_account_index(2) .unauthenticated_notes(dangling_notes.clone()) .build(); let verify_tx2_result = state_view.verify_tx(&tx2).await; - assert_eq!( + assert_matches!( verify_tx2_result, - Ok(0), + Ok(block_height) => { + assert_eq!(block_height, 0); + }, "Dangling unauthenticated notes must be found in the in-flight notes after previous tx verification" ); } @@ -284,11 +298,11 @@ async fn test_verify_tx_stored_unauthenticated_notes() { let state_view = DefaultStateView::new(Arc::clone(&store), false); let verify_tx1_result = state_view.verify_tx(&tx1).await; - assert_eq!( + assert_matches!( verify_tx1_result, - Err(VerifyTxError::UnauthenticatedNotesNotFound( - dangling_notes.iter().map(Note::id).collect() - )), + Err(VerifyTxError::UnauthenticatedNotesNotFound(notes)) => { + assert_eq!(notes, dangling_notes.iter().map(Note::id).collect::>()); + }, "Dangling unauthenticated notes must not be found in the store by this moment" ); @@ -298,9 +312,10 @@ async fn test_verify_tx_stored_unauthenticated_notes() { store.apply_block(&block).await.unwrap(); let verify_tx1_result = state_view.verify_tx(&tx1).await; - assert_eq!( - verify_tx1_result, - Ok(0), + assert_matches!( + verify_tx1_result, Ok(block_height) => { + assert_eq!(block_height, 0); + }, "Dangling unauthenticated notes must be found in the store after block applying" ); } diff --git a/crates/block-producer/src/txqueue/tests/mod.rs b/crates/block-producer/src/txqueue/tests/mod.rs index 79ae32b21..f7b5ca6e3 100644 --- a/crates/block-producer/src/txqueue/tests/mod.rs +++ b/crates/block-producer/src/txqueue/tests/mod.rs @@ -1,3 +1,4 @@ +use assert_matches::assert_matches; use tokio::sync::mpsc::{self, error::TryRecvError}; use super::*; @@ -181,7 +182,7 @@ async fn test_tx_verify_failure() { .add_transaction(MockProvenTxBuilder::with_account_index(i).build()) .await; - assert!(matches!(r, Err(AddTransactionError::VerificationFailed(_)))); + assert_matches!(r, Err(AddTransactionError::VerificationFailed(_))); assert_eq!( Err(TryRecvError::Empty), receiver.try_recv(), diff --git a/crates/proto/src/errors.rs b/crates/proto/src/errors.rs index 7f06c7017..bf5967014 100644 --- a/crates/proto/src/errors.rs +++ b/crates/proto/src/errors.rs @@ -3,7 +3,7 @@ use std::{any::type_name, num::TryFromIntError}; use miden_objects::crypto::merkle::{SmtLeafError, SmtProofError}; use thiserror::Error; -#[derive(Debug, Clone, PartialEq, Error)] +#[derive(Debug, Error)] pub enum ConversionError { #[error("Hex error: {0}")] HexError(#[from] hex::FromHexError), @@ -28,8 +28,6 @@ pub enum ConversionError { }, } -impl Eq for ConversionError {} - pub trait MissingFieldHelper { fn missing_field(field_name: &'static str) -> ConversionError; } diff --git a/crates/rpc/Cargo.toml b/crates/rpc/Cargo.toml index 9ebf2cbcf..c8e8ab167 100644 --- a/crates/rpc/Cargo.toml +++ b/crates/rpc/Cargo.toml @@ -12,25 +12,16 @@ homepage.workspace = true repository.workspace = true [dependencies] -directories = { version = "5.0" } -figment = { version = "0.10", features = ["toml", "env"] } -hex = { version = "0.4" } -miden-node-block-producer = { workspace = true } miden-node-proto = { workspace = true } -miden-node-store = { workspace = true } miden-node-utils = { workspace = true } miden-objects = { workspace = true } miden-tx = { workspace = true } -prost = { workspace = true } serde = { version = "1.0", features = ["derive"] } tokio = { workspace = true, features = ["rt-multi-thread", "net", "macros"] } tokio-stream = { workspace = true, features = ["net"] } -toml = { version = "0.8" } tonic = { workspace = true } tonic-web = { version = "0.12" } tracing = { workspace = true } -tracing-subscriber = { workspace = true } [dev-dependencies] -figment = { version = "0.10", features = ["toml", "env", "test"] } miden-node-utils = { workspace = true, features = ["tracing-forest"] } diff --git a/crates/store/Cargo.toml b/crates/store/Cargo.toml index 1337fcd86..6ca51bb4b 100644 --- a/crates/store/Cargo.toml +++ b/crates/store/Cargo.toml @@ -13,26 +13,21 @@ repository.workspace = true [dependencies] deadpool-sqlite = { version = "0.9.0", features = ["rt_tokio_1"] } -directories = { version = "5.0" } -figment = { version = "0.10", features = ["toml", "env"] } hex = { version = "0.4" } miden-lib = { workspace = true } miden-node-proto = { workspace = true } miden-node-utils = { workspace = true } miden-objects = { workspace = true } -prost = { workspace = true } rusqlite = { version = "0.32.1", features = ["array", "buildtime_bindgen", "bundled"] } -rusqlite_migration = { version = "1.0" } +rusqlite_migration = { version = "1.3" } serde = { version = "1.0", features = ["derive"] } thiserror = { workspace = true } tokio = { workspace = true, features = ["fs", "net", "macros", "rt-multi-thread"] } tokio-stream = { workspace = true, features = ["net"] } -toml = { version = "0.8" } tonic = { workspace = true } tracing = { workspace = true } -tracing-subscriber = { workspace = true } [dev-dependencies] -figment = { version = "0.10", features = ["toml", "env", "test"] } +assert_matches = { workspace = true} miden-node-utils = { workspace = true, features = ["tracing-forest"] } miden-objects = { workspace = true, features = ["testing"] } diff --git a/crates/store/src/db/tests.rs b/crates/store/src/db/tests.rs index a63f982f4..9a4e3141b 100644 --- a/crates/store/src/db/tests.rs +++ b/crates/store/src/db/tests.rs @@ -1,21 +1,21 @@ +use assert_matches::assert_matches; use miden_lib::transaction::TransactionKernel; use miden_node_proto::domain::accounts::AccountSummary; use miden_objects::{ accounts::{ account_id::testing::{ ACCOUNT_ID_FUNGIBLE_FAUCET_ON_CHAIN, ACCOUNT_ID_NON_FUNGIBLE_FAUCET_ON_CHAIN, - ACCOUNT_ID_OFF_CHAIN_SENDER, ACCOUNT_ID_REGULAR_ACCOUNT_IMMUTABLE_CODE_ON_CHAIN, - ACCOUNT_ID_REGULAR_ACCOUNT_UPDATABLE_CODE_OFF_CHAIN, + ACCOUNT_ID_OFF_CHAIN_SENDER, ACCOUNT_ID_REGULAR_ACCOUNT_UPDATABLE_CODE_OFF_CHAIN, }, delta::AccountUpdateDetails, - Account, AccountCode, AccountComponent, AccountDelta, AccountId, AccountStorage, - AccountStorageDelta, AccountType, AccountVaultDelta, StorageSlot, + Account, AccountBuilder, AccountComponent, AccountDelta, AccountId, AccountStorageDelta, + AccountStorageMode, AccountType, AccountVaultDelta, StorageSlot, }, - assets::{Asset, AssetVault, FungibleAsset, NonFungibleAsset, NonFungibleAssetDetails}, + assets::{Asset, FungibleAsset, NonFungibleAsset, NonFungibleAssetDetails}, block::{BlockAccountUpdate, BlockNoteIndex, BlockNoteTree}, crypto::{hash::rpo::RpoDigest, merkle::MerklePath}, notes::{NoteExecutionHint, NoteId, NoteMetadata, NoteType, Nullifier}, - BlockHeader, Felt, FieldElement, Word, ONE, ZERO, + BlockHeader, Felt, FieldElement, Word, ZERO, }; use rusqlite::{vtab::array, Connection}; @@ -328,8 +328,6 @@ fn test_sql_public_account_details() { let block_num = 1; create_block(&mut conn, block_num); - let account_id = - AccountId::try_from(ACCOUNT_ID_REGULAR_ACCOUNT_IMMUTABLE_CODE_ON_CHAIN).unwrap(); let fungible_faucet_id = AccountId::try_from(ACCOUNT_ID_FUNGIBLE_FAUCET_ON_CHAIN).unwrap(); let non_fungible_faucet_id = AccountId::try_from(ACCOUNT_ID_NON_FUNGIBLE_FAUCET_ON_CHAIN).unwrap(); @@ -341,18 +339,10 @@ fn test_sql_public_account_details() { .unwrap(), ); - let (code, storage) = mock_account_code_and_storage(account_id.account_type()); - - let mut account = Account::from_parts( - account_id, - AssetVault::new(&[ - Asset::Fungible(FungibleAsset::new(fungible_faucet_id, 150).unwrap()), - nft1, - ]) - .unwrap(), - storage, - code, - ZERO, + let mut account = mock_account_code_and_storage( + AccountType::RegularAccountImmutableCode, + AccountStorageMode::Public, + [Asset::Fungible(FungibleAsset::new(fungible_faucet_id, 150).unwrap()), nft1], ); // test querying empty table @@ -363,7 +353,7 @@ fn test_sql_public_account_details() { let inserted = sql::upsert_accounts( &transaction, &[BlockAccountUpdate::new( - account_id, + account.id(), account.hash(), AccountUpdateDetails::New(account.clone()), vec![], @@ -395,7 +385,7 @@ fn test_sql_public_account_details() { let vault_delta = AccountVaultDelta::from_iters([nft2], [nft1]); - let delta = AccountDelta::new(storage_delta, vault_delta, Some(ONE)).unwrap(); + let delta = AccountDelta::new(storage_delta, vault_delta, Some(Felt::new(2))).unwrap(); account.apply_delta(&delta).unwrap(); @@ -403,7 +393,7 @@ fn test_sql_public_account_details() { let inserted = sql::upsert_accounts( &transaction, &[BlockAccountUpdate::new( - account_id, + account.id(), account.hash(), AccountUpdateDetails::Delta(delta.clone()), vec![], @@ -427,12 +417,14 @@ fn test_sql_public_account_details() { assert_eq!(account_read.nonce(), account.nonce()); // Cleared item was not serialized, check it and apply delta only with clear item second time: - assert_eq!(account_read.storage().get_item(3), Ok(RpoDigest::default())); + assert_matches!(account_read.storage().get_item(3), Ok(digest) => { + assert_eq!(digest, RpoDigest::default()); + }); let storage_delta = AccountStorageDelta::from_iters([3], [], []); account_read .apply_delta( - &AccountDelta::new(storage_delta, AccountVaultDelta::default(), Some(Felt::new(2))) + &AccountDelta::new(storage_delta, AccountVaultDelta::default(), Some(Felt::new(3))) .unwrap(), ) .unwrap(); @@ -456,7 +448,7 @@ fn test_sql_public_account_details() { let inserted = sql::upsert_accounts( &transaction, &[BlockAccountUpdate::new( - account_id, + account.id(), account.hash(), AccountUpdateDetails::Delta(delta2.clone()), vec![], @@ -480,7 +472,7 @@ fn test_sql_public_account_details() { assert_eq!(account_read.nonce(), account.nonce()); let read_deltas = - sql::select_account_deltas(&mut conn, account_id.into(), 0, block_num + 1).unwrap(); + sql::select_account_deltas(&mut conn, account.id().into(), 0, block_num + 1).unwrap(); assert_eq!(read_deltas, vec![delta, delta2]); } @@ -977,7 +969,11 @@ fn insert_transactions(conn: &mut Connection) -> usize { count } -fn mock_account_code_and_storage(account_type: AccountType) -> (AccountCode, AccountStorage) { +fn mock_account_code_and_storage( + account_type: AccountType, + storage_mode: AccountStorageMode, + assets: impl IntoIterator, +) -> Account { let component_code = "\ export.account_procedure_1 push.1.2 @@ -1002,5 +998,12 @@ fn mock_account_code_and_storage(account_type: AccountType) -> (AccountCode, Acc .unwrap() .with_supported_type(account_type); - Account::initialize_from_components(account_type, &[component]).unwrap() + AccountBuilder::new() + .init_seed([0; 32]) + .account_type(account_type) + .storage_mode(storage_mode) + .with_assets(assets) + .with_component(component) + .build_existing() + .unwrap() } From 3b1ac1f255dbb644a824fddd62b4b4af11f8d01f Mon Sep 17 00:00:00 2001 From: Mirko <48352201+Mirko-von-Leipzig@users.noreply.github.com> Date: Wed, 4 Dec 2024 12:37:52 +0200 Subject: [PATCH 16/50] fix(block-producer): handle reverted batches (#557) --- Cargo.lock | 17 +++ crates/block-producer/Cargo.toml | 1 + .../block-producer/src/mempool/batch_graph.rs | 7 +- .../src/mempool/dependency_graph.rs | 22 ++- crates/block-producer/src/mempool/mod.rs | 127 ++++++++++++++++-- crates/block-producer/src/server/mod.rs | 2 +- .../src/test_utils/proven_tx.rs | 22 +++ 7 files changed, 179 insertions(+), 19 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e5468123c..2dc59f74c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -677,6 +677,12 @@ dependencies = [ "syn", ] +[[package]] +name = "diff" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" + [[package]] name = "digest" version = "0.10.7" @@ -1600,6 +1606,7 @@ dependencies = [ "miden-processor", "miden-stdlib", "miden-tx", + "pretty_assertions", "rand", "rand_chacha", "serde", @@ -2206,6 +2213,16 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" +[[package]] +name = "pretty_assertions" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ae130e2f271fbc2ac3a40fb1d07180839cdbbe443c7a27e1e3c13c5cac0116d" +dependencies = [ + "diff", + "yansi", +] + [[package]] name = "prettyplease" version = "0.2.25" diff --git a/crates/block-producer/Cargo.toml b/crates/block-producer/Cargo.toml index f6d987b2e..ad3173583 100644 --- a/crates/block-producer/Cargo.toml +++ b/crates/block-producer/Cargo.toml @@ -42,6 +42,7 @@ miden-lib = { workspace = true, features = ["testing"] } miden-node-test-macro = { path = "../test-macro" } miden-objects = { workspace = true, features = ["testing"] } miden-tx = { workspace = true, features = ["testing"] } +pretty_assertions = "1.4.1" rand_chacha = { version = "0.3", default-features = false } tokio = { workspace = true, features = ["test-util"] } winterfell = { version = "0.10" } diff --git a/crates/block-producer/src/mempool/batch_graph.rs b/crates/block-producer/src/mempool/batch_graph.rs index dea63d992..c489cddde 100644 --- a/crates/block-producer/src/mempool/batch_graph.rs +++ b/crates/block-producer/src/mempool/batch_graph.rs @@ -50,7 +50,7 @@ use crate::batch_builder::batch::TransactionBatch; /// │ ◄────┘ /// └───────────┘ /// ``` -#[derive(Default, Debug, Clone)] +#[derive(Default, Debug, Clone, PartialEq)] pub struct BatchGraph { /// Tracks the interdependencies between batches. inner: DependencyGraph, @@ -239,6 +239,11 @@ impl BatchGraph { batches } + + /// Returns `true` if the graph contains the given batch. + pub fn contains(&self, id: &BatchJobId) -> bool { + self.batches.contains_key(id) + } } #[cfg(any(test, doctest))] diff --git a/crates/block-producer/src/mempool/dependency_graph.rs b/crates/block-producer/src/mempool/dependency_graph.rs index 9814d0059..5fb2766a8 100644 --- a/crates/block-producer/src/mempool/dependency_graph.rs +++ b/crates/block-producer/src/mempool/dependency_graph.rs @@ -1,6 +1,6 @@ use std::{ collections::{BTreeMap, BTreeSet}, - fmt::Display, + fmt::{Debug, Display}, }; // DEPENDENCY GRAPH @@ -35,7 +35,7 @@ use std::{ /// │ ◄────┘ /// └───────────┘ /// ``` -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Clone, PartialEq, Eq)] pub struct DependencyGraph { /// Node's who's data is still pending. pending: BTreeSet, @@ -61,6 +61,22 @@ pub struct DependencyGraph { processed: BTreeSet, } +impl Debug for DependencyGraph +where + K: Debug, +{ + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("DependencyGraph") + .field("pending", &self.pending) + .field("vertices", &self.vertices.keys()) + .field("processed", &self.processed) + .field("roots", &self.roots) + .field("parents", &self.parents) + .field("children", &self.children) + .finish() + } +} + #[derive(thiserror::Error, Debug, Clone, PartialEq, Eq)] pub enum GraphError { #[error("Node {0} already exists")] @@ -99,7 +115,7 @@ impl Default for DependencyGraph { } } -impl DependencyGraph { +impl DependencyGraph { /// Inserts a new pending node into the graph. /// /// # Errors diff --git a/crates/block-producer/src/mempool/mod.rs b/crates/block-producer/src/mempool/mod.rs index c56b72208..3ebc98622 100644 --- a/crates/block-producer/src/mempool/mod.rs +++ b/crates/block-producer/src/mempool/mod.rs @@ -177,6 +177,7 @@ impl BlockBudget { pub type SharedMempool = Arc>; +#[derive(Clone, Debug, PartialEq)] pub struct Mempool { /// The latest inflight state of each account. /// @@ -198,19 +199,28 @@ pub struct Mempool { /// The current inflight block, if any. block_in_progress: Option>, - batch_budget: BatchBudget, block_budget: BlockBudget, + batch_budget: BatchBudget, } impl Mempool { - /// Creates a new [Mempool] with the provided configuration. - pub fn new( + /// Creates a new [SharedMempool] with the provided configuration. + pub fn shared( chain_tip: BlockNumber, batch_budget: BatchBudget, block_budget: BlockBudget, state_retention: usize, ) -> SharedMempool { - Arc::new(Mutex::new(Self { + Arc::new(Mutex::new(Self::new(chain_tip, batch_budget, block_budget, state_retention))) + } + + fn new( + chain_tip: BlockNumber, + batch_budget: BatchBudget, + block_budget: BlockBudget, + state_retention: usize, + ) -> Mempool { + Self { chain_tip, batch_budget, block_budget, @@ -219,7 +229,7 @@ impl Mempool { transactions: Default::default(), batches: Default::default(), next_batch_id: Default::default(), - })) + } } /// Adds a transaction to the mempool. @@ -271,27 +281,34 @@ impl Mempool { /// /// Transactions are placed back in the queue. pub fn batch_failed(&mut self, batch: BatchJobId) { - let removed_batches = - self.batches.remove_batches([batch].into()).expect("Batch was not present"); - - // Its possible to receive failures for batches which were already removed - // as part of a prior failure. Early exit to prevent logging these no-ops. - if removed_batches.is_empty() { + // Batch may already have been removed as part of a parent batches failure. + if !self.batches.contains(&batch) { return; } - let batches = removed_batches.keys().copied().collect::>(); - let transactions = removed_batches.into_values().flatten().collect(); + let removed_batches = + self.batches.remove_batches([batch].into()).expect("Batch was not present"); + + let transactions = removed_batches.values().flatten().copied().collect(); self.transactions .requeue_transactions(transactions) .expect("Transaction should requeue"); - tracing::warn!(%batch, descendents=?batches, "Batch failed, dropping all inflight descendent batches, impacted transactions are back in queue."); + tracing::warn!( + %batch, + descendents=?removed_batches.keys(), + "Batch failed, dropping all inflight descendent batches, impacted transactions are back in queue." + ); } /// Marks a batch as proven if it exists. pub fn batch_proved(&mut self, batch_id: BatchJobId, batch: TransactionBatch) { + // Batch may have been removed as part of a parent batches failure. + if !self.batches.contains(&batch_id) { + return; + } + self.batches.submit_proof(batch_id, batch).expect("Batch proof should submit"); } @@ -356,3 +373,85 @@ impl Mempool { self.state.revert_transactions(transactions); } } + +#[cfg(test)] +mod tests { + use pretty_assertions::assert_eq; + + use super::*; + use crate::test_utils::MockProvenTxBuilder; + + impl Mempool { + fn for_tests() -> Self { + Self::new(BlockNumber::new(0), Default::default(), Default::default(), 5) + } + } + + // BATCH REVERSION TESTS + // ================================================================================================ + + #[test] + fn children_of_reverted_batches_are_ignored() { + // Batches are proved concurrently. This makes it possible for a child job to complete after + // the parent has been reverted (and therefore reverting the child job). Such a child job + // should be ignored. + let txs = MockProvenTxBuilder::sequential(); + + let mut uut = Mempool::for_tests(); + uut.add_transaction(txs[0].clone()).unwrap(); + let (parent_batch, batch_txs) = uut.select_batch().unwrap(); + assert_eq!(batch_txs, vec![txs[0].clone()]); + + uut.add_transaction(txs[1].clone()).unwrap(); + let (child_batch_a, batch_txs) = uut.select_batch().unwrap(); + assert_eq!(batch_txs, vec![txs[1].clone()]); + + uut.add_transaction(txs[2].clone()).unwrap(); + let (child_batch_b, batch_txs) = uut.select_batch().unwrap(); + assert_eq!(batch_txs, vec![txs[2].clone()]); + + // Child batch jobs are now dangling. + uut.batch_failed(parent_batch); + let reference = uut.clone(); + + // Success or failure of the child job should effectively do nothing. + uut.batch_failed(child_batch_a); + assert_eq!(uut, reference); + + let proof = TransactionBatch::new( + vec![txs[2].raw_proven_transaction().clone()], + Default::default(), + ) + .unwrap(); + uut.batch_proved(child_batch_b, proof); + assert_eq!(uut, reference); + } + + #[test] + fn reverted_batch_transactions_are_requeued() { + let txs = MockProvenTxBuilder::sequential(); + + let mut uut = Mempool::for_tests(); + uut.add_transaction(txs[0].clone()).unwrap(); + uut.select_batch().unwrap(); + + uut.add_transaction(txs[1].clone()).unwrap(); + let (failed_batch, _) = uut.select_batch().unwrap(); + + uut.add_transaction(txs[2].clone()).unwrap(); + uut.select_batch().unwrap(); + + // Middle batch failed, so it and its child transaction should be re-entered into the queue. + uut.batch_failed(failed_batch); + + let mut reference = Mempool::for_tests(); + reference.add_transaction(txs[0].clone()).unwrap(); + reference.select_batch().unwrap(); + reference.add_transaction(txs[1].clone()).unwrap(); + reference.add_transaction(txs[2].clone()).unwrap(); + reference.next_batch_id.increment(); + reference.next_batch_id.increment(); + + assert_eq!(uut, reference); + } +} diff --git a/crates/block-producer/src/server/mod.rs b/crates/block-producer/src/server/mod.rs index dbd32ed30..7f79d7fd4 100644 --- a/crates/block-producer/src/server/mod.rs +++ b/crates/block-producer/src/server/mod.rs @@ -96,7 +96,7 @@ impl BlockProducer { chain_tip, } = self; - let mempool = Mempool::new(chain_tip, batch_budget, block_budget, state_retention); + let mempool = Mempool::shared(chain_tip, batch_budget, block_budget, state_retention); // Spawn rpc server and batch and block provers. // diff --git a/crates/block-producer/src/test_utils/proven_tx.rs b/crates/block-producer/src/test_utils/proven_tx.rs index cf804d8d7..271466341 100644 --- a/crates/block-producer/src/test_utils/proven_tx.rs +++ b/crates/block-producer/src/test_utils/proven_tx.rs @@ -1,5 +1,6 @@ use std::ops::Range; +use itertools::Itertools; use miden_air::HashFunction; use miden_objects::{ accounts::AccountId, @@ -8,9 +9,11 @@ use miden_objects::{ vm::ExecutionProof, Digest, Felt, Hasher, ONE, }; +use rand::Rng; use winterfell::Proof; use super::MockPrivateAccount; +use crate::domain::transaction::AuthenticatedTransaction; pub struct MockProvenTxBuilder { account_id: AccountId, @@ -29,6 +32,25 @@ impl MockProvenTxBuilder { Self::with_account(mock_account.id, mock_account.states[0], mock_account.states[1]) } + /// Generates 3 random, sequential transactions acting on the same account. + pub fn sequential() -> [AuthenticatedTransaction; 3] { + let mut rng = rand::thread_rng(); + let mock_account: MockPrivateAccount<4> = rng.gen::().into(); + + (0..3) + .map(|i| { + Self::with_account( + mock_account.id, + mock_account.states[i], + mock_account.states[i + 1], + ) + }) + .map(|tx| AuthenticatedTransaction::from_inner(tx.build())) + .collect_vec() + .try_into() + .expect("Sizes should match") + } + pub fn with_account( account_id: AccountId, initial_account_hash: Digest, From 93999178625f395ef557f1cdd075f0e90d6d0656 Mon Sep 17 00:00:00 2001 From: Mirko <48352201+Mirko-von-Leipzig@users.noreply.github.com> Date: Wed, 4 Dec 2024 13:17:50 +0200 Subject: [PATCH 17/50] feat(block-producer): merge in next (#561) --- CHANGELOG.md | 1 + Cargo.lock | 139 ++++++++---------- Cargo.toml | 11 +- bin/node/src/config.rs | 9 +- crates/block-producer/Cargo.toml | 5 +- .../src/batch_builder/tests/mod.rs | 9 +- .../src/block_builder/prover/tests.rs | 21 ++- .../block-producer/src/block_builder/tests.rs | 3 +- crates/block-producer/src/errors.rs | 16 +- .../src/mempool/inflight_state/mod.rs | 25 ++-- crates/proto/src/errors.rs | 4 +- crates/rpc/Cargo.toml | 9 -- crates/rpc/src/config.rs | 3 +- crates/store/Cargo.toml | 13 +- crates/store/src/db/tests.rs | 61 ++++---- crates/utils/src/config.rs | 27 +++- 16 files changed, 183 insertions(+), 173 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ed4c12a1b..ccb6505c4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### Enhancements - Added `GetAccountProofs` endpoint (#506). +- Support Https in endpoint configuration (#556). ### Changes diff --git a/Cargo.lock b/Cargo.lock index 2dc59f74c..d18f90e80 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -129,6 +129,12 @@ dependencies = [ "term", ] +[[package]] +name = "assert_matches" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b34d609dfbaf33d6889b2b7106d3ca345eacad44200913df5ba02bfd31d2ba9" + [[package]] name = "async-stream" version = "0.3.6" @@ -390,7 +396,7 @@ dependencies = [ "semver 1.0.23", "serde", "serde_json", - "thiserror", + "thiserror 1.0.68", ] [[package]] @@ -619,9 +625,9 @@ dependencies = [ [[package]] name = "deadpool-sqlite" -version = "0.8.1" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f9cc6210316f8b7ced394e2a5d2833ce7097fb28afb5881299c61bc18e8e0e9" +checksum = "656f14fc1ab819c65f332045ea7cb38841bbe551f3b2bc7e3abefb559af4155c" dependencies = [ "deadpool", "deadpool-sync", @@ -693,15 +699,6 @@ dependencies = [ "crypto-common", ] -[[package]] -name = "directories" -version = "5.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a49173b84e034382284f27f1af4dcbbd231ffa358c0fe316541a7337f376a35" -dependencies = [ - "dirs-sys", -] - [[package]] name = "dirs-next" version = "2.0.0" @@ -712,18 +709,6 @@ dependencies = [ "dirs-sys-next", ] -[[package]] -name = "dirs-sys" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c" -dependencies = [ - "libc", - "option-ext", - "redox_users", - "windows-sys 0.48.0", -] - [[package]] name = "dirs-sys-next" version = "0.1.2" @@ -1308,9 +1293,9 @@ dependencies = [ [[package]] name = "libsqlite3-sys" -version = "0.28.0" +version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c10584274047cb335c23d3e61bcef8e323adae7c5c8c760540f73610177fc3f" +checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149" dependencies = [ "bindgen", "cc", @@ -1496,7 +1481,7 @@ dependencies = [ "rand_chacha", "serde", "static-files", - "thiserror", + "thiserror 2.0.3", "tokio", "toml", "tonic", @@ -1516,14 +1501,14 @@ dependencies = [ [[package]] name = "miden-lib" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "816404c10b0799f12d3b53b3a9baa9af99fa340fe1a579e0919bba57718fa97a" +version = "0.7.0" +source = "git+https://github.com/0xPolygonMiden/miden-base.git?branch=next#bc91c604862e2c6013b21eb989a3640d30cce25b" dependencies = [ "miden-assembly", "miden-objects", "miden-stdlib", "regex", + "walkdir", ] [[package]] @@ -1594,8 +1579,8 @@ dependencies = [ name = "miden-node-block-producer" version = "0.6.0" dependencies = [ + "assert_matches", "async-trait", - "figment", "itertools 0.13.0", "miden-air", "miden-lib", @@ -1610,13 +1595,11 @@ dependencies = [ "rand", "rand_chacha", "serde", - "thiserror", + "thiserror 2.0.3", "tokio", "tokio-stream", - "toml", "tonic", "tracing", - "tracing-subscriber", "winterfell", ] @@ -1632,7 +1615,7 @@ dependencies = [ "prost", "prost-build", "protox", - "thiserror", + "thiserror 2.0.3", "tonic", "tonic-build", ] @@ -1641,49 +1624,37 @@ dependencies = [ name = "miden-node-rpc" version = "0.6.0" dependencies = [ - "directories", - "figment", - "hex", - "miden-node-block-producer", "miden-node-proto", - "miden-node-store", "miden-node-utils", "miden-objects", "miden-tx", - "prost", "serde", "tokio", "tokio-stream", - "toml", "tonic", "tonic-web", "tracing", - "tracing-subscriber", ] [[package]] name = "miden-node-store" version = "0.6.0" dependencies = [ + "assert_matches", "deadpool-sqlite", - "directories", - "figment", "hex", "miden-lib", "miden-node-proto", "miden-node-utils", "miden-objects", - "prost", "rusqlite", "rusqlite_migration", "serde", - "thiserror", + "thiserror 2.0.3", "tokio", "tokio-stream", - "toml", "tonic", "tracing", - "tracing-subscriber", ] [[package]] @@ -1704,7 +1675,7 @@ dependencies = [ "miden-objects", "rand", "serde", - "thiserror", + "thiserror 2.0.3", "tonic", "tracing", "tracing-forest", @@ -1715,9 +1686,8 @@ dependencies = [ [[package]] name = "miden-objects" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a6f54dde939928e438488b36651485a0e80057025b7e30343d4340a524a1651" +version = "0.7.0" +source = "git+https://github.com/0xPolygonMiden/miden-base.git?branch=next#bc91c604862e2c6013b21eb989a3640d30cce25b" dependencies = [ "getrandom", "miden-assembly", @@ -1726,6 +1696,7 @@ dependencies = [ "miden-processor", "miden-verifier", "rand", + "thiserror 2.0.3", "winter-rand-utils", ] @@ -1789,12 +1760,10 @@ dependencies = [ [[package]] name = "miden-tx" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d433744a3f02233ec53b151bfe88b7a008b4eae5dfd41bb38a9889eb67efaf7" +version = "0.7.0" +source = "git+https://github.com/0xPolygonMiden/miden-base.git?branch=next#bc91c604862e2c6013b21eb989a3640d30cce25b" dependencies = [ "async-trait", - "miden-assembly", "miden-lib", "miden-objects", "miden-processor", @@ -1802,8 +1771,6 @@ dependencies = [ "miden-verifier", "rand", "rand_chacha", - "regex", - "walkdir", "winter-maybe-async", ] @@ -1835,7 +1802,7 @@ dependencies = [ "supports-unicode", "terminal_size", "textwrap", - "thiserror", + "thiserror 1.0.68", "unicode-width 0.1.14", ] @@ -2050,12 +2017,6 @@ version = "1.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" -[[package]] -name = "option-ext" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" - [[package]] name = "overload" version = "0.1.1" @@ -2353,7 +2314,7 @@ dependencies = [ "prost-reflect", "prost-types", "protox-parse", - "thiserror", + "thiserror 1.0.68", ] [[package]] @@ -2365,7 +2326,7 @@ dependencies = [ "logos", "miette", "prost-types", - "thiserror", + "thiserror 1.0.68", ] [[package]] @@ -2459,7 +2420,7 @@ checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" dependencies = [ "getrandom", "libredox", - "thiserror", + "thiserror 1.0.68", ] [[package]] @@ -2508,9 +2469,9 @@ checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "rusqlite" -version = "0.31.0" +version = "0.32.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b838eba278d213a8beaf485bd313fd580ca4505a00d5871caeb1457c55322cae" +checksum = "7753b721174eb8ff87a9a0e799e2d7bc3749323e773db92e0984debb00019d6e" dependencies = [ "bitflags", "fallible-iterator", @@ -2522,9 +2483,9 @@ dependencies = [ [[package]] name = "rusqlite_migration" -version = "1.2.0" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55709bc01054c69e2f1cefdc886642b5e6376a8db3c86f761be0c423eebf178b" +checksum = "923b42e802f7dc20a0a6b5e097ba7c83fe4289da07e49156fecf6af08aa9cd1c" dependencies = [ "log", "rusqlite", @@ -2922,7 +2883,16 @@ version = "1.0.68" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "02dd99dc800bbb97186339685293e1cc5d9df1f8fae2d0aecd9ff1c77efea892" dependencies = [ - "thiserror-impl", + "thiserror-impl 1.0.68", +] + +[[package]] +name = "thiserror" +version = "2.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c006c85c7651b3cf2ada4584faa36773bd07bac24acfb39f3c431b36d7e667aa" +dependencies = [ + "thiserror-impl 2.0.3", ] [[package]] @@ -2936,6 +2906,17 @@ dependencies = [ "syn", ] +[[package]] +name = "thiserror-impl" +version = "2.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f077553d607adc1caf65430528a576c757a71ed73944b66ebb58ef2bbd243568" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "thread_local" version = "1.1.8" @@ -3258,7 +3239,7 @@ checksum = "ee40835db14ddd1e3ba414292272eddde9dad04d3d4b65509656414d1c42592f" dependencies = [ "chrono", "smallvec", - "thiserror", + "thiserror 1.0.68", "tracing", "tracing-subscriber", ] @@ -3900,9 +3881,9 @@ dependencies = [ [[package]] name = "winter-utils" -version = "0.10.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76b116c8ade0172506f8bda32dc674cf6b230adc8516e5138a0173ae69158a4f" +checksum = "1f948e71ffd482aa13d0ec3349047f81ecdb89f3b3287577973dcbf092a25fb4" dependencies = [ "rayon", ] @@ -3922,9 +3903,9 @@ dependencies = [ [[package]] name = "winterfell" -version = "0.10.1" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39c8336dc6a035698780b8cc624f875e479bd6bf6e1846670f3ef4485c125882" +checksum = "bbf1ab01d2781f7d3f1bd5c12800905c5bbf62e06778672498be798006ac463a" dependencies = [ "winter-air", "winter-prover", diff --git a/Cargo.toml b/Cargo.toml index ce5eb685e..eb13182a8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,23 +25,22 @@ exclude = [".github/"] readme = "README.md" [workspace.dependencies] +assert_matches = { version = "1.5" } miden-air = { version = "0.11" } -miden-lib = { version = "0.6" } +miden-lib = { git = "https://github.com/0xPolygonMiden/miden-base.git", branch = "next" } miden-node-block-producer = { path = "crates/block-producer", version = "0.6" } -miden-node-faucet = { path = "bin/faucet", version = "0.6" } miden-node-proto = { path = "crates/proto", version = "0.6" } miden-node-rpc = { path = "crates/rpc", version = "0.6" } -miden-node-rpc-proto = { path = "crates/rpc-proto", version = "0.6" } miden-node-store = { path = "crates/store", version = "0.6" } miden-node-test-macro = { path = "crates/test-macro" } miden-node-utils = { path = "crates/utils", version = "0.6" } -miden-objects = { version = "0.6"} +miden-objects = { git = "https://github.com/0xPolygonMiden/miden-base.git", branch = "next" } miden-processor = { version = "0.11" } miden-stdlib = { version = "0.11", default-features = false } -miden-tx = { version = "0.6"} +miden-tx = { git = "https://github.com/0xPolygonMiden/miden-base.git", branch = "next" } prost = { version = "0.13" } rand = { version = "0.8" } -thiserror = { version = "1.0" } +thiserror = { version = "2.0", default-features = false } tokio = { version = "1.40", features = ["rt-multi-thread"] } tokio-stream = { version = "0.1" } tonic = { version = "0.12" } diff --git a/bin/node/src/config.rs b/bin/node/src/config.rs index 173496764..8e44ad064 100644 --- a/bin/node/src/config.rs +++ b/bin/node/src/config.rs @@ -74,7 +74,7 @@ impl NodeConfig { mod tests { use figment::Jail; use miden_node_store::config::StoreConfig; - use miden_node_utils::config::{load_config, Endpoint}; + use miden_node_utils::config::{load_config, Endpoint, Protocol}; use super::NodeConfig; use crate::{ @@ -93,10 +93,10 @@ mod tests { verify_tx_proofs = true [rpc] - endpoint = { host = "127.0.0.1", port = 8080 } + endpoint = { host = "127.0.0.1", port = 8080, protocol = "Http" } [store] - endpoint = { host = "127.0.0.1", port = 8080 } + endpoint = { host = "127.0.0.1", port = 8080, protocol = "Https" } database_filepath = "local.sqlite3" genesis_filepath = "genesis.dat" blockstore_dir = "blocks" @@ -112,6 +112,7 @@ mod tests { endpoint: Endpoint { host: "127.0.0.1".to_string(), port: 8080, + protocol: Protocol::default() }, verify_tx_proofs: true }, @@ -119,12 +120,14 @@ mod tests { endpoint: Endpoint { host: "127.0.0.1".to_string(), port: 8080, + protocol: Protocol::Http }, }, store: StoreConfig { endpoint: Endpoint { host: "127.0.0.1".to_string(), port: 8080, + protocol: Protocol::Https }, database_filepath: "local.sqlite3".into(), genesis_filepath: "genesis.dat".into(), diff --git a/crates/block-producer/Cargo.toml b/crates/block-producer/Cargo.toml index ad3173583..4a93628bd 100644 --- a/crates/block-producer/Cargo.toml +++ b/crates/block-producer/Cargo.toml @@ -16,7 +16,6 @@ tracing-forest = ["miden-node-utils/tracing-forest"] [dependencies] async-trait = { version = "0.1" } -figment = { version = "0.10", features = ["toml", "env"] } itertools = { version = "0.13" } miden-lib = { workspace = true } miden-node-proto = { workspace = true } @@ -30,13 +29,11 @@ serde = { version = "1.0", features = ["derive"] } thiserror = { workspace = true } tokio = { workspace = true, features = ["rt-multi-thread", "net", "macros", "sync", "time"] } tokio-stream = { workspace = true, features = ["net"] } -toml = { version = "0.8" } tonic = { workspace = true } tracing = { workspace = true } -tracing-subscriber = { workspace = true } [dev-dependencies] -figment = { version = "0.10", features = ["toml", "env", "test"] } +assert_matches = { workspace = true} miden-air = { workspace = true } miden-lib = { workspace = true, features = ["testing"] } miden-node-test-macro = { path = "../test-macro" } diff --git a/crates/block-producer/src/batch_builder/tests/mod.rs b/crates/block-producer/src/batch_builder/tests/mod.rs index 1e0012f1c..46b63f703 100644 --- a/crates/block-producer/src/batch_builder/tests/mod.rs +++ b/crates/block-producer/src/batch_builder/tests/mod.rs @@ -1,5 +1,6 @@ use std::iter; +use assert_matches::assert_matches; use miden_objects::{crypto::merkle::Mmr, Digest}; use tokio::sync::RwLock; @@ -254,7 +255,7 @@ async fn test_block_builder_no_missing_notes() { .block_builder .build_block(&batch_builder.ready_batches.read().await) .await; - assert_eq!(build_block_result, Ok(())); + assert_matches!(build_block_result, Ok(())); } #[tokio::test] @@ -307,9 +308,11 @@ async fn test_block_builder_fails_if_notes_are_missing() { let mut expected_missing_notes = vec![notes[4].id(), notes[5].id()]; expected_missing_notes.sort(); - assert_eq!( + assert_matches!( build_block_result, - Err(BuildBlockError::UnauthenticatedNotesNotFound(expected_missing_notes)) + Err(BuildBlockError::UnauthenticatedNotesNotFound(actual_missing_notes)) => { + assert_eq!(actual_missing_notes, expected_missing_notes); + } ); } diff --git a/crates/block-producer/src/block_builder/prover/tests.rs b/crates/block-producer/src/block_builder/prover/tests.rs index a28f86f9f..28d011cff 100644 --- a/crates/block-producer/src/block_builder/prover/tests.rs +++ b/crates/block-producer/src/block_builder/prover/tests.rs @@ -1,5 +1,6 @@ use std::{collections::BTreeMap, iter}; +use assert_matches::assert_matches; use miden_objects::{ accounts::{ account_id::testing::{ @@ -157,13 +158,17 @@ fn test_block_witness_validation_inconsistent_account_hashes() { let block_witness_result = BlockWitness::new(block_inputs_from_store, &batches); - assert_eq!( + assert_matches!( block_witness_result, Err(BuildBlockError::InconsistentAccountStateTransition( - account_id_1, - account_1_hash_store, - vec![account_1_hash_batches] - )) + account_id, + account_hash_store, + account_hash_batches + )) => { + assert_eq!(account_id, account_id_1); + assert_eq!(account_hash_store, account_1_hash_store); + assert_eq!(account_hash_batches, vec![account_1_hash_batches]); + } ); } @@ -668,9 +673,11 @@ fn test_block_witness_validation_inconsistent_nullifiers() { let block_witness_result = BlockWitness::new(block_inputs_from_store, &batches); - assert_eq!( + assert_matches!( block_witness_result, - Err(BuildBlockError::InconsistentNullifiers(vec![nullifier_1, nullifier_3])) + Err(BuildBlockError::InconsistentNullifiers(nullifiers)) => { + assert_eq!(nullifiers, vec![nullifier_1, nullifier_3]); + } ); } diff --git a/crates/block-producer/src/block_builder/tests.rs b/crates/block-producer/src/block_builder/tests.rs index c7c82629f..d8fbc3565 100644 --- a/crates/block-producer/src/block_builder/tests.rs +++ b/crates/block-producer/src/block_builder/tests.rs @@ -3,6 +3,7 @@ use std::sync::Arc; +use assert_matches::assert_matches; use miden_objects::{ accounts::{account_id::testing::ACCOUNT_ID_OFF_CHAIN_SENDER, AccountId}, Digest, Felt, @@ -78,5 +79,5 @@ async fn test_build_block_failure() { let result = block_builder.build_block(&Vec::new()).await; // Ensure that the store's `apply_block()` was called - assert!(matches!(result, Err(BuildBlockError::GetBlockInputsFailed(_)))); + assert_matches!(result, Err(BuildBlockError::GetBlockInputsFailed(_))); } diff --git a/crates/block-producer/src/errors.rs b/crates/block-producer/src/errors.rs index 9ff3a76bb..937bbefe1 100644 --- a/crates/block-producer/src/errors.rs +++ b/crates/block-producer/src/errors.rs @@ -30,7 +30,7 @@ pub enum BlockProducerError { // Transaction verification errors // ================================================================================================= -#[derive(Debug, PartialEq, Eq, Error)] +#[derive(Debug, Error)] pub enum VerifyTxError { /// Another transaction already consumed the notes with given nullifiers #[error("Input notes with given nullifiers were already consumed by another transaction")] @@ -71,7 +71,7 @@ pub enum VerifyTxError { // Transaction adding errors // ================================================================================================= -#[derive(Debug, PartialEq, Eq, Error)] +#[derive(Debug, Error)] pub enum AddTransactionError { #[error("Transaction verification failed: {0}")] VerificationFailed(#[from] VerifyTxError), @@ -109,7 +109,7 @@ impl From for tonic::Status { // ================================================================================================= /// Error encountered while building a batch. -#[derive(Debug, PartialEq, Eq, Error)] +#[derive(Debug, Error)] pub enum BuildBatchError { #[error("Duplicated unauthenticated transaction input note ID in the batch: {0}")] DuplicateUnauthenticatedNote(NoteId), @@ -137,7 +137,7 @@ pub enum BuildBatchError { // Block prover errors // ================================================================================================= -#[derive(Debug, PartialEq, Eq, Error)] +#[derive(Debug, Error)] pub enum BlockProverError { #[error("Received invalid merkle path")] InvalidMerklePaths(MerkleError), @@ -151,7 +151,7 @@ pub enum BlockProverError { // ================================================================================================= #[allow(clippy::enum_variant_names)] -#[derive(Debug, PartialEq, Eq, Error)] +#[derive(Debug, Error)] pub enum BlockInputsError { #[error("failed to parse protobuf message: {0}")] ConversionError(#[from] ConversionError), @@ -164,7 +164,7 @@ pub enum BlockInputsError { // Block applying errors // ================================================================================================= -#[derive(Debug, PartialEq, Eq, Error)] +#[derive(Debug, Error)] pub enum ApplyBlockError { #[error("gRPC client failed with error: {0}")] GrpcClientError(String), @@ -173,7 +173,7 @@ pub enum ApplyBlockError { // Block building errors // ================================================================================================= -#[derive(Debug, PartialEq, Eq, Error)] +#[derive(Debug, Error)] pub enum BuildBlockError { #[error("failed to compute new block: {0}")] BlockProverFailed(#[from] BlockProverError), @@ -203,7 +203,7 @@ pub enum BuildBlockError { // Transaction inputs errors // ================================================================================================= -#[derive(Debug, PartialEq, Eq, Error)] +#[derive(Debug, Error)] pub enum TxInputsError { #[error("gRPC client failed with error: {0}")] GrpcClientError(String), diff --git a/crates/block-producer/src/mempool/inflight_state/mod.rs b/crates/block-producer/src/mempool/inflight_state/mod.rs index e26c1b73e..2f4836621 100644 --- a/crates/block-producer/src/mempool/inflight_state/mod.rs +++ b/crates/block-producer/src/mempool/inflight_state/mod.rs @@ -352,6 +352,7 @@ impl OutputNoteState { #[cfg(test)] mod tests { + use assert_matches::assert_matches; use miden_objects::Digest; use super::*; @@ -384,9 +385,11 @@ mod tests { let err = uut.add_transaction(&AuthenticatedTransaction::from_inner(tx2)).unwrap_err(); - assert_eq!( + assert_matches!( err, - VerifyTxError::InputNotesAlreadyConsumed(vec![mock_note(note_seed).nullifier()]).into() + AddTransactionError::VerificationFailed(VerifyTxError::InputNotesAlreadyConsumed( + notes + )) if notes == vec![mock_note(note_seed).nullifier()] ); } @@ -408,7 +411,12 @@ mod tests { let err = uut.add_transaction(&AuthenticatedTransaction::from_inner(tx1)).unwrap_err(); - assert_eq!(err, VerifyTxError::OutputNotesAlreadyExist(vec![note.id()]).into()); + assert_matches!( + err, + AddTransactionError::VerificationFailed(VerifyTxError::OutputNotesAlreadyExist( + notes + )) if notes == vec![note.id()] + ); } #[test] @@ -423,13 +431,12 @@ mod tests { .add_transaction(&AuthenticatedTransaction::from_inner(tx).with_store_state(states[2])) .unwrap_err(); - assert_eq!( + assert_matches!( err, - VerifyTxError::IncorrectAccountInitialHash { - tx_initial_account_hash: states[0], - current_account_hash: states[2].into() - } - .into() + AddTransactionError::VerificationFailed(VerifyTxError::IncorrectAccountInitialHash { + tx_initial_account_hash: init_state, + current_account_hash: current_state, + }) if init_state == states[0] && current_state == states[2].into() ); } diff --git a/crates/proto/src/errors.rs b/crates/proto/src/errors.rs index 7f06c7017..bf5967014 100644 --- a/crates/proto/src/errors.rs +++ b/crates/proto/src/errors.rs @@ -3,7 +3,7 @@ use std::{any::type_name, num::TryFromIntError}; use miden_objects::crypto::merkle::{SmtLeafError, SmtProofError}; use thiserror::Error; -#[derive(Debug, Clone, PartialEq, Error)] +#[derive(Debug, Error)] pub enum ConversionError { #[error("Hex error: {0}")] HexError(#[from] hex::FromHexError), @@ -28,8 +28,6 @@ pub enum ConversionError { }, } -impl Eq for ConversionError {} - pub trait MissingFieldHelper { fn missing_field(field_name: &'static str) -> ConversionError; } diff --git a/crates/rpc/Cargo.toml b/crates/rpc/Cargo.toml index 9ebf2cbcf..c8e8ab167 100644 --- a/crates/rpc/Cargo.toml +++ b/crates/rpc/Cargo.toml @@ -12,25 +12,16 @@ homepage.workspace = true repository.workspace = true [dependencies] -directories = { version = "5.0" } -figment = { version = "0.10", features = ["toml", "env"] } -hex = { version = "0.4" } -miden-node-block-producer = { workspace = true } miden-node-proto = { workspace = true } -miden-node-store = { workspace = true } miden-node-utils = { workspace = true } miden-objects = { workspace = true } miden-tx = { workspace = true } -prost = { workspace = true } serde = { version = "1.0", features = ["derive"] } tokio = { workspace = true, features = ["rt-multi-thread", "net", "macros"] } tokio-stream = { workspace = true, features = ["net"] } -toml = { version = "0.8" } tonic = { workspace = true } tonic-web = { version = "0.12" } tracing = { workspace = true } -tracing-subscriber = { workspace = true } [dev-dependencies] -figment = { version = "0.10", features = ["toml", "env", "test"] } miden-node-utils = { workspace = true, features = ["tracing-forest"] } diff --git a/crates/rpc/src/config.rs b/crates/rpc/src/config.rs index 7cd6e6448..07dac0778 100644 --- a/crates/rpc/src/config.rs +++ b/crates/rpc/src/config.rs @@ -1,7 +1,7 @@ use std::fmt::{Display, Formatter}; use miden_node_utils::config::{ - Endpoint, DEFAULT_BLOCK_PRODUCER_PORT, DEFAULT_NODE_RPC_PORT, DEFAULT_STORE_PORT, + Endpoint, Protocol, DEFAULT_BLOCK_PRODUCER_PORT, DEFAULT_NODE_RPC_PORT, DEFAULT_STORE_PORT, }; use serde::{Deserialize, Serialize}; @@ -39,6 +39,7 @@ impl Default for RpcConfig { endpoint: Endpoint { host: "0.0.0.0".to_string(), port: DEFAULT_NODE_RPC_PORT, + protocol: Protocol::default(), }, store_url: Endpoint::localhost(DEFAULT_STORE_PORT).to_string(), block_producer_url: Endpoint::localhost(DEFAULT_BLOCK_PRODUCER_PORT).to_string(), diff --git a/crates/store/Cargo.toml b/crates/store/Cargo.toml index bb9a63103..6ca51bb4b 100644 --- a/crates/store/Cargo.toml +++ b/crates/store/Cargo.toml @@ -12,27 +12,22 @@ homepage.workspace = true repository.workspace = true [dependencies] -deadpool-sqlite = { version = "0.8", features = ["rt_tokio_1"] } -directories = { version = "5.0" } -figment = { version = "0.10", features = ["toml", "env"] } +deadpool-sqlite = { version = "0.9.0", features = ["rt_tokio_1"] } hex = { version = "0.4" } miden-lib = { workspace = true } miden-node-proto = { workspace = true } miden-node-utils = { workspace = true } miden-objects = { workspace = true } -prost = { workspace = true } -rusqlite = { version = "0.31", features = ["array", "buildtime_bindgen", "bundled"] } -rusqlite_migration = { version = "1.0" } +rusqlite = { version = "0.32.1", features = ["array", "buildtime_bindgen", "bundled"] } +rusqlite_migration = { version = "1.3" } serde = { version = "1.0", features = ["derive"] } thiserror = { workspace = true } tokio = { workspace = true, features = ["fs", "net", "macros", "rt-multi-thread"] } tokio-stream = { workspace = true, features = ["net"] } -toml = { version = "0.8" } tonic = { workspace = true } tracing = { workspace = true } -tracing-subscriber = { workspace = true } [dev-dependencies] -figment = { version = "0.10", features = ["toml", "env", "test"] } +assert_matches = { workspace = true} miden-node-utils = { workspace = true, features = ["tracing-forest"] } miden-objects = { workspace = true, features = ["testing"] } diff --git a/crates/store/src/db/tests.rs b/crates/store/src/db/tests.rs index a63f982f4..9a4e3141b 100644 --- a/crates/store/src/db/tests.rs +++ b/crates/store/src/db/tests.rs @@ -1,21 +1,21 @@ +use assert_matches::assert_matches; use miden_lib::transaction::TransactionKernel; use miden_node_proto::domain::accounts::AccountSummary; use miden_objects::{ accounts::{ account_id::testing::{ ACCOUNT_ID_FUNGIBLE_FAUCET_ON_CHAIN, ACCOUNT_ID_NON_FUNGIBLE_FAUCET_ON_CHAIN, - ACCOUNT_ID_OFF_CHAIN_SENDER, ACCOUNT_ID_REGULAR_ACCOUNT_IMMUTABLE_CODE_ON_CHAIN, - ACCOUNT_ID_REGULAR_ACCOUNT_UPDATABLE_CODE_OFF_CHAIN, + ACCOUNT_ID_OFF_CHAIN_SENDER, ACCOUNT_ID_REGULAR_ACCOUNT_UPDATABLE_CODE_OFF_CHAIN, }, delta::AccountUpdateDetails, - Account, AccountCode, AccountComponent, AccountDelta, AccountId, AccountStorage, - AccountStorageDelta, AccountType, AccountVaultDelta, StorageSlot, + Account, AccountBuilder, AccountComponent, AccountDelta, AccountId, AccountStorageDelta, + AccountStorageMode, AccountType, AccountVaultDelta, StorageSlot, }, - assets::{Asset, AssetVault, FungibleAsset, NonFungibleAsset, NonFungibleAssetDetails}, + assets::{Asset, FungibleAsset, NonFungibleAsset, NonFungibleAssetDetails}, block::{BlockAccountUpdate, BlockNoteIndex, BlockNoteTree}, crypto::{hash::rpo::RpoDigest, merkle::MerklePath}, notes::{NoteExecutionHint, NoteId, NoteMetadata, NoteType, Nullifier}, - BlockHeader, Felt, FieldElement, Word, ONE, ZERO, + BlockHeader, Felt, FieldElement, Word, ZERO, }; use rusqlite::{vtab::array, Connection}; @@ -328,8 +328,6 @@ fn test_sql_public_account_details() { let block_num = 1; create_block(&mut conn, block_num); - let account_id = - AccountId::try_from(ACCOUNT_ID_REGULAR_ACCOUNT_IMMUTABLE_CODE_ON_CHAIN).unwrap(); let fungible_faucet_id = AccountId::try_from(ACCOUNT_ID_FUNGIBLE_FAUCET_ON_CHAIN).unwrap(); let non_fungible_faucet_id = AccountId::try_from(ACCOUNT_ID_NON_FUNGIBLE_FAUCET_ON_CHAIN).unwrap(); @@ -341,18 +339,10 @@ fn test_sql_public_account_details() { .unwrap(), ); - let (code, storage) = mock_account_code_and_storage(account_id.account_type()); - - let mut account = Account::from_parts( - account_id, - AssetVault::new(&[ - Asset::Fungible(FungibleAsset::new(fungible_faucet_id, 150).unwrap()), - nft1, - ]) - .unwrap(), - storage, - code, - ZERO, + let mut account = mock_account_code_and_storage( + AccountType::RegularAccountImmutableCode, + AccountStorageMode::Public, + [Asset::Fungible(FungibleAsset::new(fungible_faucet_id, 150).unwrap()), nft1], ); // test querying empty table @@ -363,7 +353,7 @@ fn test_sql_public_account_details() { let inserted = sql::upsert_accounts( &transaction, &[BlockAccountUpdate::new( - account_id, + account.id(), account.hash(), AccountUpdateDetails::New(account.clone()), vec![], @@ -395,7 +385,7 @@ fn test_sql_public_account_details() { let vault_delta = AccountVaultDelta::from_iters([nft2], [nft1]); - let delta = AccountDelta::new(storage_delta, vault_delta, Some(ONE)).unwrap(); + let delta = AccountDelta::new(storage_delta, vault_delta, Some(Felt::new(2))).unwrap(); account.apply_delta(&delta).unwrap(); @@ -403,7 +393,7 @@ fn test_sql_public_account_details() { let inserted = sql::upsert_accounts( &transaction, &[BlockAccountUpdate::new( - account_id, + account.id(), account.hash(), AccountUpdateDetails::Delta(delta.clone()), vec![], @@ -427,12 +417,14 @@ fn test_sql_public_account_details() { assert_eq!(account_read.nonce(), account.nonce()); // Cleared item was not serialized, check it and apply delta only with clear item second time: - assert_eq!(account_read.storage().get_item(3), Ok(RpoDigest::default())); + assert_matches!(account_read.storage().get_item(3), Ok(digest) => { + assert_eq!(digest, RpoDigest::default()); + }); let storage_delta = AccountStorageDelta::from_iters([3], [], []); account_read .apply_delta( - &AccountDelta::new(storage_delta, AccountVaultDelta::default(), Some(Felt::new(2))) + &AccountDelta::new(storage_delta, AccountVaultDelta::default(), Some(Felt::new(3))) .unwrap(), ) .unwrap(); @@ -456,7 +448,7 @@ fn test_sql_public_account_details() { let inserted = sql::upsert_accounts( &transaction, &[BlockAccountUpdate::new( - account_id, + account.id(), account.hash(), AccountUpdateDetails::Delta(delta2.clone()), vec![], @@ -480,7 +472,7 @@ fn test_sql_public_account_details() { assert_eq!(account_read.nonce(), account.nonce()); let read_deltas = - sql::select_account_deltas(&mut conn, account_id.into(), 0, block_num + 1).unwrap(); + sql::select_account_deltas(&mut conn, account.id().into(), 0, block_num + 1).unwrap(); assert_eq!(read_deltas, vec![delta, delta2]); } @@ -977,7 +969,11 @@ fn insert_transactions(conn: &mut Connection) -> usize { count } -fn mock_account_code_and_storage(account_type: AccountType) -> (AccountCode, AccountStorage) { +fn mock_account_code_and_storage( + account_type: AccountType, + storage_mode: AccountStorageMode, + assets: impl IntoIterator, +) -> Account { let component_code = "\ export.account_procedure_1 push.1.2 @@ -1002,5 +998,12 @@ fn mock_account_code_and_storage(account_type: AccountType) -> (AccountCode, Acc .unwrap() .with_supported_type(account_type); - Account::initialize_from_components(account_type, &[component]).unwrap() + AccountBuilder::new() + .init_seed([0; 32]) + .account_type(account_type) + .storage_mode(storage_mode) + .with_assets(assets) + .with_component(component) + .build_existing() + .unwrap() } diff --git a/crates/utils/src/config.rs b/crates/utils/src/config.rs index c350fd36b..5c70315df 100644 --- a/crates/utils/src/config.rs +++ b/crates/utils/src/config.rs @@ -17,6 +17,12 @@ pub const DEFAULT_BLOCK_PRODUCER_PORT: u16 = 48046; pub const DEFAULT_STORE_PORT: u16 = 28943; pub const DEFAULT_FAUCET_SERVER_PORT: u16 = 8080; +#[derive(Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Debug, Serialize, Deserialize, Default)] +pub enum Protocol { + #[default] + Http, + Https, +} /// The `(host, port)` pair for the server's listening socket. #[derive(Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Debug, Serialize, Deserialize)] pub struct Endpoint { @@ -24,11 +30,18 @@ pub struct Endpoint { pub host: String, /// Port number used by the store. pub port: u16, + /// Protocol type: http or https. + #[serde(default)] + pub protocol: Protocol, } impl Endpoint { pub fn localhost(port: u16) -> Self { - Endpoint { host: "localhost".to_string(), port } + Endpoint { + host: "localhost".to_string(), + port, + protocol: Protocol::default(), + } } } @@ -41,7 +54,17 @@ impl ToSocketAddrs for Endpoint { impl Display for Endpoint { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - f.write_fmt(format_args!("http://{}:{}", self.host, self.port)) + let Endpoint { protocol, host, port } = self; + f.write_fmt(format_args!("{protocol}://{host}:{port}")) + } +} + +impl Display for Protocol { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + Protocol::Http => f.write_str("http"), + Protocol::Https => f.write_str("https"), + } } } From 536bcd00a718ff094c0694d1a0b3a0935aafd743 Mon Sep 17 00:00:00 2001 From: Mirko <48352201+Mirko-von-Leipzig@users.noreply.github.com> Date: Wed, 4 Dec 2024 13:31:36 +0200 Subject: [PATCH 18/50] chore: update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ccb6505c4..2a2352883 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ - Added `GetAccountProofs` endpoint (#506). - Support Https in endpoint configuration (#556). +- Upgrade `block-producer` from FIFO queue to mempool dependency graph (#562). ### Changes From 0f061bbe6cf0af3c6bcd1ffbb80e28ecc65ef086 Mon Sep 17 00:00:00 2001 From: polydez <155382956+polydez@users.noreply.github.com> Date: Thu, 5 Dec 2024 18:08:42 +0500 Subject: [PATCH 19/50] Refactor and store account deltas by separate fields in the database (#554) * feat: prepare DB structure for new delta format * chore: remove unused dependencies * fix: delta merging bugs * fix: remove obsolete check in test * fix: update directory name to correspond common format * refactor: rename DB tables * refactor: add `WITHOUT ROWID` to tables where ROWID is unnecessary * refactor: don't require allocations and clones for `bulk_insert` * fix: get rid of `unsafe` in `NonFungibleAsset` construction from `Word` * refactor: extract utils to separate module inside `sql` * refactor: rename `account_nonce_updates` table to `account_deltas` * docs: improve function description for `select_account_state_delta` * fix: compilation errors * refactor: rename `account_hash_update_from_row` to `account_summary_from_row` * refactor: rename `account_hash_update_from_row` to `account_summary_from_row` * refactor: get rid of `u32_to_value` * refactor: remove unnecessary indexes * refactor: multiple inserts instead of `bulk_insert` * refactor: multiple queries instead of single query with unions * refactor: implement `insert_sql` macro * docs: improve doc comment for `insert_sql` macro --- crates/store/src/db/migrations.rs | 2 +- crates/store/src/db/migrations/001-init.sql | 56 +- crates/store/src/db/mod.rs | 13 +- crates/store/src/db/settings.rs | 2 +- crates/store/src/db/{sql.rs => sql/mod.rs} | 500 +++++++++++------- ...ct_notes_since_block_by_tag_and_sender.sql | 29 + crates/store/src/db/sql/utils.rs | 142 +++++ crates/store/src/db/tests.rs | 94 ++-- crates/store/src/errors.rs | 2 + crates/store/src/server/api.rs | 5 +- crates/store/src/state.rs | 12 +- 11 files changed, 589 insertions(+), 268 deletions(-) rename crates/store/src/db/{sql.rs => sql/mod.rs} (75%) create mode 100644 crates/store/src/db/sql/queries/select_notes_since_block_by_tag_and_sender.sql create mode 100644 crates/store/src/db/sql/utils.rs diff --git a/crates/store/src/db/migrations.rs b/crates/store/src/db/migrations.rs index 8ccbcdaf7..0336685de 100644 --- a/crates/store/src/db/migrations.rs +++ b/crates/store/src/db/migrations.rs @@ -6,7 +6,7 @@ use rusqlite_migration::{Migrations, SchemaVersion, M}; use tracing::{debug, error, info, instrument}; use crate::{ - db::{settings::Settings, sql::schema_version}, + db::{settings::Settings, sql::utils::schema_version}, errors::DatabaseError, COMPONENT, }; diff --git a/crates/store/src/db/migrations/001-init.sql b/crates/store/src/db/migrations/001-init.sql index d5f9c8df7..293b9ca4d 100644 --- a/crates/store/src/db/migrations/001-init.sql +++ b/crates/store/src/db/migrations/001-init.sql @@ -41,7 +41,7 @@ CREATE TABLE CONSTRAINT notes_block_num_is_u32 CHECK (block_num BETWEEN 0 AND 0xFFFFFFFF), CONSTRAINT notes_batch_index_is_u32 CHECK (batch_index BETWEEN 0 AND 0xFFFFFFFF), CONSTRAINT notes_note_index_is_u32 CHECK (note_index BETWEEN 0 AND 0xFFFFFFFF) -) STRICT; +) STRICT, WITHOUT ROWID; CREATE TABLE accounts @@ -61,11 +61,61 @@ CREATE TABLE ( account_id INTEGER NOT NULL, block_num INTEGER NOT NULL, - delta BLOB NOT NULL, + nonce INTEGER NOT NULL, PRIMARY KEY (account_id, block_num), + FOREIGN KEY (account_id) REFERENCES accounts(account_id), FOREIGN KEY (block_num) REFERENCES block_headers(block_num) -) STRICT; +) STRICT, WITHOUT ROWID; + +CREATE TABLE + account_storage_slot_updates +( + account_id INTEGER NOT NULL, + block_num INTEGER NOT NULL, + slot INTEGER NOT NULL, + value BLOB NOT NULL, + + PRIMARY KEY (account_id, block_num, slot), + FOREIGN KEY (account_id, block_num) REFERENCES account_deltas (account_id, block_num) +) STRICT, WITHOUT ROWID; + +CREATE TABLE + account_storage_map_updates +( + account_id INTEGER NOT NULL, + block_num INTEGER NOT NULL, + slot INTEGER NOT NULL, + key BLOB NOT NULL, + value BLOB NOT NULL, + + PRIMARY KEY (account_id, block_num, slot, key), + FOREIGN KEY (account_id, block_num) REFERENCES account_deltas (account_id, block_num) +) STRICT, WITHOUT ROWID; + +CREATE TABLE + account_fungible_asset_deltas +( + account_id INTEGER NOT NULL, + block_num INTEGER NOT NULL, + faucet_id INTEGER NOT NULL, + delta INTEGER NOT NULL, + + PRIMARY KEY (account_id, block_num, faucet_id), + FOREIGN KEY (account_id, block_num) REFERENCES account_deltas (account_id, block_num) +) STRICT, WITHOUT ROWID; + +CREATE TABLE + account_non_fungible_asset_updates +( + account_id INTEGER NOT NULL, + block_num INTEGER NOT NULL, + vault_key BLOB NOT NULL, + is_remove INTEGER NOT NULL, -- 0 - add, 1 - remove + + PRIMARY KEY (account_id, block_num, vault_key), + FOREIGN KEY (account_id, block_num) REFERENCES account_deltas (account_id, block_num) +) STRICT, WITHOUT ROWID; CREATE TABLE nullifiers diff --git a/crates/store/src/db/mod.rs b/crates/store/src/db/mod.rs index 45c462cf4..554ecf794 100644 --- a/crates/store/src/db/mod.rs +++ b/crates/store/src/db/mod.rs @@ -457,20 +457,23 @@ impl Db { Ok(()) } - /// Loads account deltas from the DB for given account ID and block range. + /// Merges all account deltas from the DB for given account ID and block range. /// Note, that `from_block` is exclusive and `to_block` is inclusive. - pub(crate) async fn select_account_state_deltas( + /// + /// Returns `Ok(None)` if no deltas were found in the DB for the specified account within + /// the given block range. + pub(crate) async fn select_account_state_delta( &self, account_id: AccountId, from_block: BlockNumber, to_block: BlockNumber, - ) -> Result> { + ) -> Result> { self.pool .get() .await .map_err(DatabaseError::MissingDbConnection)? - .interact(move |conn| -> Result> { - sql::select_account_deltas(conn, account_id, from_block, to_block) + .interact(move |conn| -> Result> { + sql::select_account_delta(conn, account_id, from_block, to_block) }) .await .map_err(|err| DatabaseError::InteractError(err.to_string()))? diff --git a/crates/store/src/db/settings.rs b/crates/store/src/db/settings.rs index 14b2a4ada..6fdb41272 100644 --- a/crates/store/src/db/settings.rs +++ b/crates/store/src/db/settings.rs @@ -1,6 +1,6 @@ use rusqlite::{params, types::FromSql, Connection, OptionalExtension, Result, ToSql}; -use crate::db::sql::table_exists; +use crate::db::sql::utils::table_exists; pub struct Settings; diff --git a/crates/store/src/db/sql.rs b/crates/store/src/db/sql/mod.rs similarity index 75% rename from crates/store/src/db/sql.rs rename to crates/store/src/db/sql/mod.rs index 8320f0db4..5161ac386 100644 --- a/crates/store/src/db/sql.rs +++ b/crates/store/src/db/sql/mod.rs @@ -1,36 +1,42 @@ //! Wrapper functions for SQL statements. +#[macro_use] +pub(crate) mod utils; + use std::{ borrow::Cow, - collections::{BTreeMap, BTreeSet}, + collections::{btree_map::Entry, BTreeMap, BTreeSet}, rc::Rc, }; use miden_node_proto::domain::accounts::{AccountInfo, AccountSummary}; use miden_objects::{ - accounts::{delta::AccountUpdateDetails, Account, AccountDelta}, + accounts::{ + delta::AccountUpdateDetails, AccountDelta, AccountStorageDelta, AccountVaultDelta, + FungibleAssetDelta, NonFungibleAssetDelta, NonFungibleDeltaAction, StorageMapDelta, + }, + assets::NonFungibleAsset, block::{BlockAccountUpdate, BlockNoteIndex}, crypto::{hash::rpo::RpoDigest, merkle::MerklePath}, notes::{NoteId, NoteInclusionProof, NoteMetadata, NoteType, Nullifier}, transaction::TransactionId, utils::serde::{Deserializable, Serializable}, - BlockHeader, -}; -use rusqlite::{ - params, - types::{Value, ValueRef}, - Connection, OptionalExtension, Transaction, + BlockHeader, Digest, Word, }; +use rusqlite::{params, types::Value, Connection, Transaction}; use super::{ NoteRecord, NoteSyncRecord, NoteSyncUpdate, NullifierInfo, Result, StateSyncUpdate, TransactionSummary, }; use crate::{ + db::sql::utils::{ + account_info_from_row, account_summary_from_row, apply_delta, column_value_as_u64, + get_nullifier_prefix, u64_to_value, + }, errors::{DatabaseError, NoteSyncError, StateSyncError}, types::{AccountId, BlockNumber}, }; - // ACCOUNT QUERIES // ================================================================================================ @@ -118,7 +124,7 @@ pub fn select_accounts_by_block_range( let mut result = Vec::new(); while let Some(row) = rows.next()? { - result.push(account_hash_update_from_row(row)?) + result.push(account_summary_from_row(row)?) } Ok(result) @@ -184,7 +190,8 @@ pub fn select_accounts_by_ids( Ok(result) } -/// Select account deltas by account id and block range from the DB using the given [Connection]. +/// Selects and merges account deltas by account id and block range from the DB using the given +/// [Connection]. /// /// # Note: /// @@ -192,33 +199,176 @@ pub fn select_accounts_by_ids( /// /// # Returns /// -/// The account deltas, or an error. -pub fn select_account_deltas( +/// The resulting account delta, or an error. +pub fn select_account_delta( conn: &mut Connection, account_id: AccountId, block_start: BlockNumber, block_end: BlockNumber, -) -> Result> { - let mut stmt = conn.prepare_cached( +) -> Result> { + let mut select_nonce_stmt = conn.prepare_cached( " SELECT - delta + nonce FROM account_deltas WHERE account_id = ?1 AND block_num > ?2 AND block_num <= ?3 ORDER BY - block_num ASC + block_num DESC + LIMIT 1 ", )?; - let mut rows = stmt.query(params![u64_to_value(account_id), block_start, block_end])?; - let mut result = Vec::new(); + let mut select_slot_updates_stmt = conn.prepare_cached( + " + SELECT + slot, value + FROM + account_storage_slot_updates AS a + WHERE + account_id = ?1 AND + block_num > ?2 AND + block_num <= ?3 AND + NOT EXISTS( + SELECT 1 + FROM account_storage_slot_updates AS b + WHERE + b.account_id = ?1 AND + a.slot = b.slot AND + a.block_num < b.block_num AND + b.block_num <= ?3 + ) + ", + )?; + + let mut select_storage_map_updates_stmt = conn.prepare_cached( + " + SELECT + slot, key, value + FROM + account_storage_map_updates AS a + WHERE + account_id = ?1 AND + block_num > ?2 AND + block_num <= ?3 AND + NOT EXISTS( + SELECT 1 + FROM account_storage_map_updates AS b + WHERE + b.account_id = ?1 AND + a.slot = b.slot AND + a.key = b.key AND + a.block_num < b.block_num AND + b.block_num <= ?3 + ) + ", + )?; + + let mut select_fungible_asset_deltas_stmt = conn.prepare_cached( + " + SELECT + faucet_id, SUM(delta) + FROM + account_fungible_asset_deltas + WHERE + account_id = ?1 AND + block_num > ?2 AND + block_num <= ?3 + GROUP BY + faucet_id + ", + )?; + + let mut select_non_fungible_asset_updates_stmt = conn.prepare_cached( + " + SELECT + block_num, vault_key, is_remove + FROM + account_non_fungible_asset_updates + WHERE + account_id = ?1 AND + block_num > ?2 AND + block_num <= ?3 + ORDER BY + block_num + ", + )?; + + let account_id = u64_to_value(account_id); + let nonce = match select_nonce_stmt + .query_row(params![account_id, block_start, block_end], |row| row.get::<_, u64>(0)) + { + Ok(nonce) => nonce.try_into().map_err(DatabaseError::InvalidFelt)?, + Err(rusqlite::Error::QueryReturnedNoRows) => return Ok(None), + Err(e) => return Err(e.into()), + }; + + let mut storage_scalars = BTreeMap::new(); + let mut rows = select_slot_updates_stmt.query(params![account_id, block_start, block_end])?; while let Some(row) = rows.next()? { - let delta = AccountDelta::read_from_bytes(row.get_ref(0)?.as_blob()?)?; - result.push(delta); + let slot = row.get(0)?; + let value_data = row.get_ref(1)?.as_blob()?; + let value = Word::read_from_bytes(value_data)?; + storage_scalars.insert(slot, value); } - Ok(result) + + let mut storage_maps = BTreeMap::new(); + let mut rows = + select_storage_map_updates_stmt.query(params![account_id, block_start, block_end])?; + while let Some(row) = rows.next()? { + let slot = row.get(0)?; + let key_data = row.get_ref(1)?.as_blob()?; + let key = Digest::read_from_bytes(key_data)?; + let value_data = row.get_ref(2)?.as_blob()?; + let value = Word::read_from_bytes(value_data)?; + + match storage_maps.entry(slot) { + Entry::Vacant(entry) => { + entry.insert(StorageMapDelta::new(BTreeMap::from([(key, value)]))); + }, + Entry::Occupied(mut entry) => { + entry.get_mut().insert(key, value); + }, + } + } + + let mut fungible = BTreeMap::new(); + let mut rows = + select_fungible_asset_deltas_stmt.query(params![account_id, block_start, block_end])?; + while let Some(row) = rows.next()? { + let faucet_id: u64 = row.get(0)?; + let value = row.get(1)?; + fungible.insert(faucet_id.try_into()?, value); + } + + let mut non_fungible_delta = NonFungibleAssetDelta::default(); + let mut rows = select_non_fungible_asset_updates_stmt.query(params![ + account_id, + block_start, + block_end + ])?; + while let Some(row) = rows.next()? { + let vault_key_data = row.get_ref(1)?.as_blob()?; + let vault_key = Word::read_from_bytes(vault_key_data)?; + let asset = NonFungibleAsset::try_from(vault_key) + .map_err(|err| DatabaseError::DataCorrupted(err.to_string()))?; + let action: usize = row.get(2)?; + match action { + 0 => non_fungible_delta.add(asset)?, + 1 => non_fungible_delta.remove(asset)?, + _ => { + return Err(DatabaseError::DataCorrupted(format!( + "Invalid non-fungible asset delta action: {action}" + ))) + }, + } + } + + let storage = AccountStorageDelta::new(storage_scalars, storage_maps)?; + let vault = AccountVaultDelta::new(FungibleAssetDelta::new(fungible)?, non_fungible_delta); + + Ok(Some(AccountDelta::new(storage, vault, Some(nonce))?)) } /// Inserts or updates accounts to the DB using the given [Transaction]. @@ -239,17 +389,14 @@ pub fn upsert_accounts( let mut upsert_stmt = transaction.prepare_cached( "INSERT OR REPLACE INTO accounts (account_id, account_hash, block_num, details) VALUES (?1, ?2, ?3, ?4);", )?; - let mut insert_delta_stmt = transaction.prepare_cached( - "INSERT INTO account_deltas (account_id, block_num, delta) VALUES (?1, ?2, ?3);", - )?; let mut select_details_stmt = transaction.prepare_cached("SELECT details FROM accounts WHERE account_id = ?1;")?; let mut count = 0; for update in accounts.iter() { let account_id = update.account_id().into(); - let full_account = match update.details() { - AccountUpdateDetails::Private => None, + let (full_account, insert_delta) = match update.details() { + AccountUpdateDetails::Private => (None, None), AccountUpdateDetails::New(account) => { debug_assert_eq!(account_id, u64::from(account.id())); @@ -260,7 +407,9 @@ pub fn upsert_accounts( }); } - Some(Cow::Borrowed(account)) + let insert_delta = AccountDelta::from(account.clone()); + + (Some(Cow::Borrowed(account)), Some(Cow::Owned(insert_delta))) }, AccountUpdateDetails::Delta(delta) => { let mut rows = select_details_stmt.query(params![u64_to_value(account_id)])?; @@ -268,16 +417,10 @@ pub fn upsert_accounts( return Err(DatabaseError::AccountNotFoundInDb(account_id)); }; - insert_delta_stmt.execute(params![ - u64_to_value(account_id), - block_num, - delta.to_bytes() - ])?; - let account = apply_delta(account_id, &row.get_ref(0)?, delta, &update.new_state_hash())?; - Some(Cow::Owned(account)) + (Some(Cow::Owned(account)), Some(Cow::Borrowed(delta))) }, }; @@ -290,12 +433,111 @@ pub fn upsert_accounts( debug_assert_eq!(inserted, 1); + if let Some(delta) = insert_delta { + insert_account_delta(transaction, account_id, block_num, &delta)?; + } + count += inserted; } Ok(count) } +/// Inserts account delta to the DB using the given [Transaction]. +fn insert_account_delta( + transaction: &Transaction, + account_id: AccountId, + block_number: BlockNumber, + delta: &AccountDelta, +) -> Result<()> { + let mut insert_acc_delta_stmt = + transaction.prepare_cached(insert_sql!(account_deltas { account_id, block_num, nonce }))?; + + let mut insert_slot_update_stmt = + transaction.prepare_cached(insert_sql!(account_storage_slot_updates { + account_id, + block_num, + slot, + value, + }))?; + + let mut insert_storage_map_update_stmt = + transaction.prepare_cached(insert_sql!(account_storage_map_updates { + account_id, + block_num, + slot, + key, + value, + }))?; + + let mut insert_fungible_asset_delta_stmt = + transaction.prepare_cached(insert_sql!(account_fungible_asset_deltas { + account_id, + block_num, + faucet_id, + delta, + }))?; + + let mut insert_non_fungible_asset_update_stmt = + transaction.prepare_cached(insert_sql!(account_non_fungible_asset_updates { + account_id, + block_num, + vault_key, + is_remove, + }))?; + + insert_acc_delta_stmt.execute(params![ + u64_to_value(account_id), + block_number, + delta.nonce().map(Into::::into).unwrap_or_default() + ])?; + + for (&slot, value) in delta.storage().values() { + insert_slot_update_stmt.execute(params![ + u64_to_value(account_id), + block_number, + slot, + value.to_bytes() + ])?; + } + + for (&slot, map_delta) in delta.storage().maps() { + for (key, value) in map_delta.leaves() { + insert_storage_map_update_stmt.execute(params![ + u64_to_value(account_id), + block_number, + slot, + key.to_bytes(), + value.to_bytes(), + ])?; + } + } + + for (&faucet_id, &delta) in delta.vault().fungible().iter() { + insert_fungible_asset_delta_stmt.execute(params![ + u64_to_value(account_id), + block_number, + u64_to_value(faucet_id.into()), + delta, + ])?; + } + + for (&asset, action) in delta.vault().non_fungible().iter() { + let is_remove = match action { + NonFungibleDeltaAction::Add => 0, + NonFungibleDeltaAction::Remove => 1, + }; + insert_non_fungible_asset_update_stmt.execute(params![ + u64_to_value(account_id), + block_number, + asset.vault_key().to_bytes(), + is_remove, + ])?; + } + + Ok(()) +} + // NULLIFIER QUERIES // ================================================================================================ @@ -363,7 +605,7 @@ pub fn select_nullifiers_by_block_range( nullifier_prefixes: &[u32], ) -> Result> { let nullifier_prefixes: Vec = - nullifier_prefixes.iter().copied().map(u32_to_value).collect(); + nullifier_prefixes.iter().copied().map(Into::into).collect(); let mut stmt = conn.prepare_cached( " @@ -412,7 +654,7 @@ pub fn select_nullifiers_by_prefix( assert_eq!(prefix_len, 16, "Only 16-bit prefixes are supported"); let nullifier_prefixes: Vec = - nullifier_prefixes.iter().copied().map(u32_to_value).collect(); + nullifier_prefixes.iter().copied().map(Into::into).collect(); let mut stmt = conn.prepare_cached( " @@ -521,28 +763,19 @@ pub fn select_all_notes(conn: &mut Connection) -> Result> { /// The [Transaction] object is not consumed. It's up to the caller to commit or rollback the /// transaction. pub fn insert_notes(transaction: &Transaction, notes: &[NoteRecord]) -> Result { - let mut stmt = transaction.prepare_cached( - " - INSERT INTO - notes - ( - block_num, - batch_index, - note_index, - note_id, - note_type, - sender, - tag, - aux, - execution_hint, - merkle_path, - details - ) - VALUES - ( - ?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11 - );", - )?; + let mut stmt = transaction.prepare_cached(insert_sql!(notes { + block_num, + batch_index, + note_index, + note_id, + note_type, + sender, + tag, + aux, + execution_hint, + merkle_path, + details, + }))?; let mut count = 0; for note in notes.iter() { @@ -583,43 +816,10 @@ pub fn select_notes_since_block_by_tag_and_sender( account_ids: &[AccountId], block_num: BlockNumber, ) -> Result> { - let mut stmt = conn.prepare_cached( - " - SELECT - block_num, - batch_index, - note_index, - note_id, - note_type, - sender, - tag, - aux, - execution_hint, - merkle_path - FROM - notes - WHERE - -- find the next block which contains at least one note with a matching tag or sender - block_num = ( - SELECT - block_num - FROM - notes - WHERE - (tag IN rarray(?1) OR sender IN rarray(?2)) AND - block_num > ?3 - ORDER BY - block_num ASC - LIMIT - 1 - ) AND - -- filter the block's notes and return only the ones matching the requested tags - -- or senders - (tag IN rarray(?1) OR sender IN rarray(?2)); - ", - )?; + let mut stmt = conn + .prepare_cached(include_str!("queries/select_notes_since_block_by_tag_and_sender.sql"))?; - let tags: Vec = tags.iter().copied().map(u32_to_value).collect(); + let tags: Vec = tags.iter().copied().map(Into::into).collect(); let account_ids: Vec = account_ids.iter().copied().map(u64_to_value).collect(); let mut rows = stmt.query(params![Rc::new(tags), Rc::new(account_ids), block_num])?; @@ -848,7 +1048,7 @@ pub fn select_block_headers( ) -> Result> { let mut headers = Vec::with_capacity(blocks.len()); - let blocks: Vec = blocks.iter().copied().map(u32_to_value).collect(); + let blocks: Vec = blocks.iter().copied().map(Into::into).collect(); let mut stmt = conn .prepare_cached("SELECT block_header FROM block_headers WHERE block_num IN rarray(?1);")?; let mut rows = stmt.query(params![Rc::new(blocks)])?; @@ -1051,111 +1251,3 @@ pub fn apply_block( count += insert_nullifiers_for_block(transaction, nullifiers, block_header.block_num())?; Ok(count) } - -// UTILITIES -// ================================================================================================ - -/// Returns the high 16 bits of the provided nullifier. -pub(crate) fn get_nullifier_prefix(nullifier: &Nullifier) -> u32 { - (nullifier.most_significant_felt().as_int() >> 48) as u32 -} - -/// Checks if a table exists in the database. -pub(crate) fn table_exists(conn: &Connection, table_name: &str) -> rusqlite::Result { - Ok(conn - .query_row( - "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = $1", - params![table_name], - |_| Ok(()), - ) - .optional()? - .is_some()) -} - -/// Returns the schema version of the database. -pub(crate) fn schema_version(conn: &Connection) -> rusqlite::Result { - conn.query_row("SELECT * FROM pragma_schema_version", [], |row| row.get(0)) -} - -/// Converts a `u64` into a [Value]. -/// -/// Sqlite uses `i64` as its internal representation format. Note that the `as` operator performs a -/// lossless conversion from `u64` to `i64`. -fn u64_to_value(v: u64) -> Value { - Value::Integer(v as i64) -} - -/// Converts a `u32` into a [Value]. -/// -/// Sqlite uses `i64` as its internal representation format. -fn u32_to_value(v: u32) -> Value { - let v: i64 = v.into(); - Value::Integer(v) -} - -/// Gets a `u64` value from the database. -/// -/// Sqlite uses `i64` as its internal representation format, and so when retrieving -/// we need to make sure we cast as `u64` to get the original value -fn column_value_as_u64( - row: &rusqlite::Row<'_>, - index: I, -) -> rusqlite::Result { - let value: i64 = row.get(index)?; - Ok(value as u64) -} - -/// Constructs `AccountSummary` from the row of `accounts` table. -/// -/// Note: field ordering must be the same, as in `accounts` table! -fn account_hash_update_from_row(row: &rusqlite::Row<'_>) -> Result { - let account_id = column_value_as_u64(row, 0)?; - let account_hash_data = row.get_ref(1)?.as_blob()?; - let account_hash = RpoDigest::read_from_bytes(account_hash_data)?; - let block_num = row.get(2)?; - - Ok(AccountSummary { - account_id: account_id.try_into()?, - account_hash, - block_num, - }) -} - -/// Constructs `AccountInfo` from the row of `accounts` table. -/// -/// Note: field ordering must be the same, as in `accounts` table! -fn account_info_from_row(row: &rusqlite::Row<'_>) -> Result { - let update = account_hash_update_from_row(row)?; - - let details = row.get_ref(3)?.as_blob_or_null()?; - let details = details.map(Account::read_from_bytes).transpose()?; - - Ok(AccountInfo { summary: update, details }) -} - -/// Deserializes account and applies account delta. -fn apply_delta( - account_id: u64, - value: &ValueRef<'_>, - delta: &AccountDelta, - final_state_hash: &RpoDigest, -) -> Result { - let account = value.as_blob_or_null()?; - let account = account.map(Account::read_from_bytes).transpose()?; - - let Some(mut account) = account else { - return Err(DatabaseError::AccountNotOnChain(account_id)); - }; - - account.apply_delta(delta)?; - - let actual_hash = account.hash(); - if &actual_hash != final_state_hash { - return Err(DatabaseError::AccountHashesMismatch { - calculated: actual_hash, - expected: *final_state_hash, - }); - } - - Ok(account) -} diff --git a/crates/store/src/db/sql/queries/select_notes_since_block_by_tag_and_sender.sql b/crates/store/src/db/sql/queries/select_notes_since_block_by_tag_and_sender.sql new file mode 100644 index 000000000..4307e52ad --- /dev/null +++ b/crates/store/src/db/sql/queries/select_notes_since_block_by_tag_and_sender.sql @@ -0,0 +1,29 @@ +-- Selects new notes matching the tags and account IDs search criteria. +SELECT + block_num, + batch_index, + note_index, + note_id, + note_type, + sender, + tag, + aux, + execution_hint, + merkle_path +FROM + notes +WHERE + -- find the next block which contains at least one note with a matching tag or sender + block_num = ( + SELECT + block_num + FROM + notes + WHERE + (tag IN rarray(?1) OR sender IN rarray(?2)) AND + block_num > ?3 + ORDER BY + block_num ASC + LIMIT 1) AND + -- filter the block's notes and return only the ones matching the requested tags or senders + (tag IN rarray(?1) OR sender IN rarray(?2)) diff --git a/crates/store/src/db/sql/utils.rs b/crates/store/src/db/sql/utils.rs new file mode 100644 index 000000000..79bd1b198 --- /dev/null +++ b/crates/store/src/db/sql/utils.rs @@ -0,0 +1,142 @@ +use miden_node_proto::domain::accounts::{AccountInfo, AccountSummary}; +use miden_objects::{ + accounts::{Account, AccountDelta}, + crypto::hash::rpo::RpoDigest, + notes::Nullifier, + utils::Deserializable, +}; +use rusqlite::{ + params, + types::{Value, ValueRef}, + Connection, OptionalExtension, +}; + +use crate::errors::DatabaseError; + +/// Returns the high 16 bits of the provided nullifier. +pub fn get_nullifier_prefix(nullifier: &Nullifier) -> u32 { + (nullifier.most_significant_felt().as_int() >> 48) as u32 +} + +/// Checks if a table exists in the database. +pub fn table_exists(conn: &Connection, table_name: &str) -> rusqlite::Result { + Ok(conn + .query_row( + "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = $1", + params![table_name], + |_| Ok(()), + ) + .optional()? + .is_some()) +} + +/// Returns the schema version of the database. +pub fn schema_version(conn: &Connection) -> rusqlite::Result { + conn.query_row("SELECT * FROM pragma_schema_version", [], |row| row.get(0)) +} + +/// Auxiliary macro which substitutes `$src` token by `$dst` expression. +macro_rules! subst { + ($src:tt, $dst:expr) => { + $dst + }; +} + +/// Generates a simple insert SQL statement with parameters for the provided table name and fields. +/// +/// # Usage: +/// +/// ``` +/// insert_sql!(users { id, first_name, last_name, age }); +/// ``` +/// which generates: +/// "INSERT INTO users (id, first_name, last_name, age) VALUES (?, ?, ?, ?)" +macro_rules! insert_sql { + ($table:ident { $first_field:ident $(, $($field:ident),+)? $(,)? }) => { + concat!( + stringify!(INSERT INTO $table), + " (", + stringify!($first_field), + $($(concat!(", ", stringify!($field))),+ ,)? + ") VALUES (", + subst!($first_field, "?"), + $($(subst!($field, ", ?")),+ ,)? + ")" + ) + }; +} + +/// Converts a `u64` into a [Value]. +/// +/// Sqlite uses `i64` as its internal representation format. Note that the `as` operator performs a +/// lossless conversion from `u64` to `i64`. +pub fn u64_to_value(v: u64) -> Value { + Value::Integer(v as i64) +} + +/// Gets a `u64` value from the database. +/// +/// Sqlite uses `i64` as its internal representation format, and so when retrieving +/// we need to make sure we cast as `u64` to get the original value +pub fn column_value_as_u64( + row: &rusqlite::Row<'_>, + index: I, +) -> rusqlite::Result { + let value: i64 = row.get(index)?; + Ok(value as u64) +} + +/// Constructs `AccountSummary` from the row of `accounts` table. +/// +/// Note: field ordering must be the same, as in `accounts` table! +pub fn account_summary_from_row(row: &rusqlite::Row<'_>) -> crate::db::Result { + let account_id = column_value_as_u64(row, 0)?; + let account_hash_data = row.get_ref(1)?.as_blob()?; + let account_hash = RpoDigest::read_from_bytes(account_hash_data)?; + let block_num = row.get(2)?; + + Ok(AccountSummary { + account_id: account_id.try_into()?, + account_hash, + block_num, + }) +} + +/// Constructs `AccountInfo` from the row of `accounts` table. +/// +/// Note: field ordering must be the same, as in `accounts` table! +pub fn account_info_from_row(row: &rusqlite::Row<'_>) -> crate::db::Result { + let update = account_summary_from_row(row)?; + + let details = row.get_ref(3)?.as_blob_or_null()?; + let details = details.map(Account::read_from_bytes).transpose()?; + + Ok(AccountInfo { summary: update, details }) +} + +/// Deserializes account and applies account delta. +pub fn apply_delta( + account_id: u64, + value: &ValueRef<'_>, + delta: &AccountDelta, + final_state_hash: &RpoDigest, +) -> crate::db::Result { + let account = value.as_blob_or_null()?; + let account = account.map(Account::read_from_bytes).transpose()?; + + let Some(mut account) = account else { + return Err(DatabaseError::AccountNotOnChain(account_id)); + }; + + account.apply_delta(delta)?; + + let actual_hash = account.hash(); + if &actual_hash != final_state_hash { + return Err(DatabaseError::AccountHashesMismatch { + calculated: actual_hash, + expected: *final_state_hash, + }); + } + + Ok(account) +} diff --git a/crates/store/src/db/tests.rs b/crates/store/src/db/tests.rs index 9a4e3141b..d4d3199f7 100644 --- a/crates/store/src/db/tests.rs +++ b/crates/store/src/db/tests.rs @@ -1,4 +1,3 @@ -use assert_matches::assert_matches; use miden_lib::transaction::TransactionKernel; use miden_node_proto::domain::accounts::AccountSummary; use miden_objects::{ @@ -325,8 +324,7 @@ fn test_sql_select_accounts() { fn test_sql_public_account_details() { let mut conn = create_db(); - let block_num = 1; - create_block(&mut conn, block_num); + create_block(&mut conn, 1); let fungible_faucet_id = AccountId::try_from(ACCOUNT_ID_FUNGIBLE_FAUCET_ON_CHAIN).unwrap(); let non_fungible_faucet_id = @@ -358,7 +356,7 @@ fn test_sql_public_account_details() { AccountUpdateDetails::New(account.clone()), vec![], )], - block_num, + 1, ) .unwrap(); @@ -373,6 +371,12 @@ fn test_sql_public_account_details() { let account_read = accounts_in_db.pop().unwrap().details.unwrap(); assert_eq!(account_read, account); + create_block(&mut conn, 2); + + let read_delta = sql::select_account_delta(&mut conn, account.id().into(), 1, 2).unwrap(); + + assert_eq!(read_delta, None); + let storage_delta = AccountStorageDelta::from_iters([3], [(4, num_to_word(5)), (5, num_to_word(6))], []); @@ -385,9 +389,9 @@ fn test_sql_public_account_details() { let vault_delta = AccountVaultDelta::from_iters([nft2], [nft1]); - let delta = AccountDelta::new(storage_delta, vault_delta, Some(Felt::new(2))).unwrap(); + let mut delta2 = AccountDelta::new(storage_delta, vault_delta, Some(Felt::new(2))).unwrap(); - account.apply_delta(&delta).unwrap(); + account.apply_delta(&delta2).unwrap(); let transaction = conn.transaction().unwrap(); let inserted = sql::upsert_accounts( @@ -395,10 +399,10 @@ fn test_sql_public_account_details() { &[BlockAccountUpdate::new( account.id(), account.hash(), - AccountUpdateDetails::Delta(delta.clone()), + AccountUpdateDetails::Delta(delta2.clone()), vec![], )], - block_num, + 2, ) .unwrap(); @@ -410,39 +414,28 @@ fn test_sql_public_account_details() { assert_eq!(accounts_in_db.len(), 1, "One element must have been inserted"); - let mut account_read = accounts_in_db.pop().unwrap().details.unwrap(); + let account_read = accounts_in_db.pop().unwrap().details.unwrap(); assert_eq!(account_read.id(), account.id()); assert_eq!(account_read.vault(), account.vault()); assert_eq!(account_read.nonce(), account.nonce()); + assert_eq!(account_read.storage(), account.storage()); - // Cleared item was not serialized, check it and apply delta only with clear item second time: - assert_matches!(account_read.storage().get_item(3), Ok(digest) => { - assert_eq!(digest, RpoDigest::default()); - }); - - let storage_delta = AccountStorageDelta::from_iters([3], [], []); - account_read - .apply_delta( - &AccountDelta::new(storage_delta, AccountVaultDelta::default(), Some(Felt::new(3))) - .unwrap(), - ) - .unwrap(); + let read_delta = sql::select_account_delta(&mut conn, account.id().into(), 1, 2).unwrap(); + assert_eq!(read_delta.as_ref(), Some(&delta2)); - assert_eq!(account_read.storage(), account.storage()); + create_block(&mut conn, 3); - let storage_delta2 = AccountStorageDelta::from_iters([5], [], []); + let storage_delta3 = AccountStorageDelta::from_iters([5], [], []); - let delta2 = AccountDelta::new( - storage_delta2, + let delta3 = AccountDelta::new( + storage_delta3, AccountVaultDelta::from_iters([nft1], []), Some(Felt::new(3)), ) .unwrap(); - account.apply_delta(&delta2).unwrap(); - - create_block(&mut conn, block_num + 1); + account.apply_delta(&delta3).unwrap(); let transaction = conn.transaction().unwrap(); let inserted = sql::upsert_accounts( @@ -450,10 +443,10 @@ fn test_sql_public_account_details() { &[BlockAccountUpdate::new( account.id(), account.hash(), - AccountUpdateDetails::Delta(delta2.clone()), + AccountUpdateDetails::Delta(delta3.clone()), vec![], )], - block_num + 1, + 3, ) .unwrap(); @@ -471,10 +464,11 @@ fn test_sql_public_account_details() { assert_eq!(account_read.vault(), account.vault()); assert_eq!(account_read.nonce(), account.nonce()); - let read_deltas = - sql::select_account_deltas(&mut conn, account.id().into(), 0, block_num + 1).unwrap(); + let read_delta = sql::select_account_delta(&mut conn, account.id().into(), 1, 3).unwrap(); + + delta2.merge(delta3).unwrap(); - assert_eq!(read_deltas, vec![delta, delta2]); + assert_eq!(read_delta, Some(delta2)); } #[test] @@ -498,7 +492,7 @@ fn test_sql_select_nullifiers_by_block_range() { &mut conn, 0, u32::MAX, - &[sql::get_nullifier_prefix(&nullifier1)], + &[sql::utils::get_nullifier_prefix(&nullifier1)], ) .unwrap(); assert_eq!( @@ -526,7 +520,7 @@ fn test_sql_select_nullifiers_by_block_range() { &mut conn, 0, u32::MAX, - &[sql::get_nullifier_prefix(&nullifier1)], + &[sql::utils::get_nullifier_prefix(&nullifier1)], ) .unwrap(); assert_eq!( @@ -540,7 +534,7 @@ fn test_sql_select_nullifiers_by_block_range() { &mut conn, 0, u32::MAX, - &[sql::get_nullifier_prefix(&nullifier2)], + &[sql::utils::get_nullifier_prefix(&nullifier2)], ) .unwrap(); assert_eq!( @@ -556,7 +550,10 @@ fn test_sql_select_nullifiers_by_block_range() { &mut conn, 0, 1, - &[sql::get_nullifier_prefix(&nullifier1), sql::get_nullifier_prefix(&nullifier2)], + &[ + sql::utils::get_nullifier_prefix(&nullifier1), + sql::utils::get_nullifier_prefix(&nullifier2), + ], ) .unwrap(); assert_eq!( @@ -572,7 +569,10 @@ fn test_sql_select_nullifiers_by_block_range() { &mut conn, 1, u32::MAX, - &[sql::get_nullifier_prefix(&nullifier1), sql::get_nullifier_prefix(&nullifier2)], + &[ + sql::utils::get_nullifier_prefix(&nullifier1), + sql::utils::get_nullifier_prefix(&nullifier2), + ], ) .unwrap(); assert_eq!( @@ -589,7 +589,10 @@ fn test_sql_select_nullifiers_by_block_range() { &mut conn, 2, 2, - &[sql::get_nullifier_prefix(&nullifier1), sql::get_nullifier_prefix(&nullifier2)], + &[ + sql::utils::get_nullifier_prefix(&nullifier1), + sql::utils::get_nullifier_prefix(&nullifier2), + ], ) .unwrap(); assert!(nullifiers.is_empty()); @@ -615,7 +618,7 @@ fn test_select_nullifiers_by_prefix() { let nullifiers = sql::select_nullifiers_by_prefix( &mut conn, PREFIX_LEN, - &[sql::get_nullifier_prefix(&nullifier1)], + &[sql::utils::get_nullifier_prefix(&nullifier1)], ) .unwrap(); assert_eq!( @@ -642,7 +645,7 @@ fn test_select_nullifiers_by_prefix() { let nullifiers = sql::select_nullifiers_by_prefix( &mut conn, PREFIX_LEN, - &[sql::get_nullifier_prefix(&nullifier1)], + &[sql::utils::get_nullifier_prefix(&nullifier1)], ) .unwrap(); assert_eq!( @@ -655,7 +658,7 @@ fn test_select_nullifiers_by_prefix() { let nullifiers = sql::select_nullifiers_by_prefix( &mut conn, PREFIX_LEN, - &[sql::get_nullifier_prefix(&nullifier2)], + &[sql::utils::get_nullifier_prefix(&nullifier2)], ) .unwrap(); assert_eq!( @@ -670,7 +673,10 @@ fn test_select_nullifiers_by_prefix() { let nullifiers = sql::select_nullifiers_by_prefix( &mut conn, PREFIX_LEN, - &[sql::get_nullifier_prefix(&nullifier1), sql::get_nullifier_prefix(&nullifier2)], + &[ + sql::utils::get_nullifier_prefix(&nullifier1), + sql::utils::get_nullifier_prefix(&nullifier2), + ], ) .unwrap(); assert_eq!( @@ -691,7 +697,7 @@ fn test_select_nullifiers_by_prefix() { let nullifiers = sql::select_nullifiers_by_prefix( &mut conn, PREFIX_LEN, - &[sql::get_nullifier_prefix(&num_to_nullifier(3 << 48))], + &[sql::utils::get_nullifier_prefix(&num_to_nullifier(3 << 48))], ) .unwrap(); assert!(nullifiers.is_empty()); diff --git a/crates/store/src/errors.rs b/crates/store/src/errors.rs index bb563c36a..2465da52f 100644 --- a/crates/store/src/errors.rs +++ b/crates/store/src/errors.rs @@ -79,6 +79,8 @@ pub enum DatabaseError { AccountNotOnChain(AccountId), #[error("Block {0} not found in the database")] BlockNotFoundInDb(BlockNumber), + #[error("Data corrupted: {0}")] + DataCorrupted(String), #[error("SQLite pool interaction task failed: {0}")] InteractError(String), #[error("Invalid Felt: {0}")] diff --git a/crates/store/src/server/api.rs b/crates/store/src/server/api.rs index 3b6db4bd1..4b9903046 100644 --- a/crates/store/src/server/api.rs +++ b/crates/store/src/server/api.rs @@ -529,9 +529,10 @@ impl api_server::Api for StoreApi { request.from_block_num, request.to_block_num, ) - .await?; + .await? + .map(|delta| delta.to_bytes()); - Ok(Response::new(GetAccountStateDeltaResponse { delta: Some(delta.to_bytes()) })) + Ok(Response::new(GetAccountStateDeltaResponse { delta })) } // TESTING ENDPOINTS diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index 597219dac..caaf0b5c0 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -765,14 +765,10 @@ impl State { account_id: AccountId, from_block: BlockNumber, to_block: BlockNumber, - ) -> Result { - let deltas = self.db.select_account_state_deltas(account_id, from_block, to_block).await?; - - deltas - .into_iter() - .try_fold(AccountDelta::default(), |mut accumulator, delta| { - accumulator.merge(delta).map(|_| accumulator) - }) + ) -> Result, DatabaseError> { + self.db + .select_account_state_delta(account_id, from_block, to_block) + .await .map_err(Into::into) } From 25160eaf48c4a78272ec8c5b0815de337602388b Mon Sep 17 00:00:00 2001 From: Mirko von Leipzig Date: Thu, 5 Dec 2024 11:22:19 +0200 Subject: [PATCH 20/50] feat(block-producer): address mempool review comments --- crates/block-producer/Cargo.toml | 4 +- .../block-producer/src/batch_builder/mod.rs | 156 ++- .../src/batch_builder/tests/mod.rs | 329 ------ .../block-producer/src/block_builder/mod.rs | 4 - .../src/block_builder/prover/block_witness.rs | 5 +- .../block-producer/src/block_builder/tests.rs | 83 -- crates/block-producer/src/errors.rs | 10 + .../block-producer/src/mempool/batch_graph.rs | 14 +- .../src/mempool/dependency_graph.rs | 1000 ----------------- .../block-producer/src/mempool/graph/mod.rs | 419 +++++++ .../block-producer/src/mempool/graph/tests.rs | 577 ++++++++++ crates/block-producer/src/mempool/mod.rs | 2 +- .../src/mempool/transaction_graph.rs | 2 +- crates/block-producer/src/server/mod.rs | 33 +- crates/block-producer/src/store/mod.rs | 2 +- 15 files changed, 1156 insertions(+), 1484 deletions(-) delete mode 100644 crates/block-producer/src/batch_builder/tests/mod.rs delete mode 100644 crates/block-producer/src/block_builder/tests.rs delete mode 100644 crates/block-producer/src/mempool/dependency_graph.rs create mode 100644 crates/block-producer/src/mempool/graph/mod.rs create mode 100644 crates/block-producer/src/mempool/graph/tests.rs diff --git a/crates/block-producer/Cargo.toml b/crates/block-producer/Cargo.toml index 4a93628bd..52c898f8c 100644 --- a/crates/block-producer/Cargo.toml +++ b/crates/block-producer/Cargo.toml @@ -24,7 +24,7 @@ miden-objects = { workspace = true } miden-processor = { workspace = true } miden-stdlib = { workspace = true } miden-tx = { workspace = true } -rand = { version = "0.8.5" } +rand = { version = "0.8" } serde = { version = "1.0", features = ["derive"] } thiserror = { workspace = true } tokio = { workspace = true, features = ["rt-multi-thread", "net", "macros", "sync", "time"] } @@ -39,7 +39,7 @@ miden-lib = { workspace = true, features = ["testing"] } miden-node-test-macro = { path = "../test-macro" } miden-objects = { workspace = true, features = ["testing"] } miden-tx = { workspace = true, features = ["testing"] } -pretty_assertions = "1.4.1" +pretty_assertions = "1.4" rand_chacha = { version = "0.3", default-features = false } tokio = { workspace = true, features = ["test-util"] } winterfell = { version = "0.10" } diff --git a/crates/block-producer/src/batch_builder/mod.rs b/crates/block-producer/src/batch_builder/mod.rs index 30070b58e..e6399ae9c 100644 --- a/crates/block-producer/src/batch_builder/mod.rs +++ b/crates/block-producer/src/batch_builder/mod.rs @@ -10,10 +10,6 @@ use crate::{ COMPONENT, SERVER_BUILD_BATCH_FREQUENCY, }; -// FIXME: fix the batch builder tests. -// #[cfg(test)] -// mod tests; - pub mod batch; pub use batch::TransactionBatch; use miden_node_utils::formatting::{format_array, format_blake3_digest}; @@ -23,6 +19,11 @@ use crate::errors::BuildBatchError; // BATCH BUILDER // ================================================================================================ +/// Builds [TransactionBatch] from sets of transactions. +/// +/// Transaction sets are pulled from the [Mempool] at a configurable interval, and passed to a pool +/// of provers for proof generation. Proving is currently unimplemented and is instead simulated via +/// the given proof time and failure rate. pub struct BatchBuilder { pub batch_interval: Duration, pub workers: NonZeroUsize, @@ -62,12 +63,13 @@ impl BatchBuilder { let mut interval = tokio::time::interval(self.batch_interval); interval.set_missed_tick_behavior(time::MissedTickBehavior::Delay); - let mut inflight = WorkerPool::new(self.simulated_proof_time, self.failure_rate); + let mut worker_pool = + WorkerPool::new(self.workers, self.simulated_proof_time, self.failure_rate); loop { tokio::select! { _ = interval.tick() => { - if inflight.len() >= self.workers.get() { + if !worker_pool.has_capacity() { tracing::info!("All batch workers occupied."); continue; } @@ -80,21 +82,16 @@ impl BatchBuilder { continue; }; - inflight.spawn(batch_id, transactions); + worker_pool.spawn(batch_id, transactions).expect("Worker capacity was checked"); }, - result = inflight.join_next() => { + result = worker_pool.join_next() => { let mut mempool = mempool.lock().await; match result { - Err(err) => { - tracing::warn!(%err, "Batch job panic'd.") - // TODO: somehow embed the batch ID into the join error, though this doesn't seem possible? - // mempool.batch_failed(batch_id); - }, - Ok(Err((batch_id, err))) => { + Err((batch_id, err)) => { tracing::warn!(%batch_id, %err, "Batch job failed."); mempool.batch_failed(batch_id); }, - Ok(Ok((batch_id, batch))) => { + Ok((batch_id, batch)) => { mempool.batch_proved(batch_id, batch); } } @@ -109,63 +106,132 @@ impl BatchBuilder { type BatchResult = Result<(BatchJobId, TransactionBatch), (BatchJobId, BuildBatchError)>; -/// Wrapper around tokio's JoinSet that remains pending if the set is empty, +/// Represents a pool of batch provers. +/// +/// Effectively a wrapper around tokio's JoinSet that remains pending if the set is empty, /// instead of returning None. struct WorkerPool { in_progress: JoinSet, simulated_proof_time: Range, failure_rate: f32, + /// Maximum number of workers allowed. + capacity: NonZeroUsize, + /// Maps spawned tasks to their job ID. + /// + /// This allows us to map panic'd tasks to the job ID. Uses [Vec] because the task ID does not + /// implement [Ord]. Given that the expected capacity is relatively low, this has no real + /// impact beyond ergonomics. + task_map: Vec<(tokio::task::Id, BatchJobId)>, } impl WorkerPool { - fn new(simulated_proof_time: Range, failure_rate: f32) -> Self { + fn new( + capacity: NonZeroUsize, + simulated_proof_time: Range, + failure_rate: f32, + ) -> Self { Self { simulated_proof_time, failure_rate, - in_progress: JoinSet::new(), + capacity, + in_progress: JoinSet::default(), + task_map: Default::default(), } } - async fn join_next(&mut self) -> Result { + /// Returns the next batch proof result. + /// + /// Will return pending if there are no jobs in progress (unlike tokio's [JoinSet::join_next] + /// which returns an option). + async fn join_next(&mut self) -> BatchResult { if self.in_progress.is_empty() { - std::future::pending().await - } else { - // Cannot be None as its not empty. - self.in_progress.join_next().await.unwrap() + return std::future::pending().await; } + + let result = self + .in_progress + .join_next() + .await + .expect("JoinSet::join_next must be Some as the set is not empty") + .map_err(|join_err| { + // Map task ID to job ID as otherwise the caller can't tell which batch failed. + // + // Note that the mapping cleanup happens lower down. + let batch_id = self + .task_map + .iter() + .find(|(task_id, _)| &join_err.id() == task_id) + .expect("Task ID should be in the task map") + .1; + + (batch_id, join_err.into()) + }) + .and_then(|x| x); + + // Cleanup task mapping by removing the result's task. This is inefficient but does not + // matter as the capacity is expected to be low. + let job_id = match &result { + Ok((id, _)) => id, + Err((id, _)) => id, + }; + self.task_map.retain(|(_, elem_job_id)| elem_job_id != job_id); + + result } - fn len(&self) -> usize { - self.in_progress.len() + /// Returns `true` if there is a worker available. + fn has_capacity(&self) -> bool { + self.in_progress.len() < self.capacity.get() } - fn spawn(&mut self, id: BatchJobId, transactions: Vec) { - self.in_progress.spawn({ - // Select a random work duration from the given proof range. - let simulated_proof_time = - rand::thread_rng().gen_range(self.simulated_proof_time.clone()); + /// Spawns a new batch proving task on the worker pool. + /// + /// # Errors + /// + /// Returns an error if no workers are available which can be checked using + /// [has_capacity](Self::has_capacity). + fn spawn( + &mut self, + id: BatchJobId, + transactions: Vec, + ) -> Result<(), ()> { + if !self.has_capacity() { + return Err(()); + } + + let task_id = self + .in_progress + .spawn({ + // Select a random work duration from the given proof range. + let simulated_proof_time = + rand::thread_rng().gen_range(self.simulated_proof_time.clone()); + + // Randomly fail batches at the configured rate. + // + // Note: Rng::gen rolls between [0, 1.0) for f32, so this works as expected. + let failed = rand::thread_rng().gen::() < self.failure_rate; - // Randomly fail batches at the configured rate. - // - // Note: Rng::gen rolls between [0, 1.0) for f32, so this works as expected. - let failed = rand::thread_rng().gen::() < self.failure_rate; + async move { + tracing::debug!("Begin proving batch."); - async move { - tracing::debug!("Begin proving batch."); + let batch = Self::build_batch(transactions).map_err(|err| (id, err))?; - let batch = Self::build_batch(transactions).map_err(|err| (id, err))?; + tokio::time::sleep(simulated_proof_time).await; + if failed { + tracing::debug!("Batch proof failure injected."); + return Err((id, BuildBatchError::InjectedFailure)); + } + + tracing::debug!("Batch proof completed."); - tokio::time::sleep(simulated_proof_time).await; - if failed { - tracing::debug!("Batch proof failure injected."); - return Err((id, BuildBatchError::InjectedFailure)); + Ok((id, batch)) } + }) + .id(); - tracing::debug!("Batch proof completed."); + self.task_map.push((task_id, id)); - Ok((id, batch)) - } - }); + Ok(()) } #[instrument(target = "miden-block-producer", skip_all, err, fields(batch_id))] diff --git a/crates/block-producer/src/batch_builder/tests/mod.rs b/crates/block-producer/src/batch_builder/tests/mod.rs deleted file mode 100644 index 46b63f703..000000000 --- a/crates/block-producer/src/batch_builder/tests/mod.rs +++ /dev/null @@ -1,329 +0,0 @@ -use std::iter; - -use assert_matches::assert_matches; -use miden_objects::{crypto::merkle::Mmr, Digest}; -use tokio::sync::RwLock; - -use super::*; -use crate::{ - block_builder::DefaultBlockBuilder, - errors::BuildBlockError, - test_utils::{ - note::mock_note, MockPrivateAccount, MockProvenTxBuilder, MockStoreSuccessBuilder, - }, -}; -// STRUCTS -// ================================================================================================ - -#[derive(Default)] -struct BlockBuilderSuccess { - batch_groups: SharedRwVec>, - num_empty_batches_received: Arc>, -} - -#[async_trait] -impl BlockBuilder for BlockBuilderSuccess { - async fn build_block(&self, batches: &[TransactionBatch]) -> Result<(), BuildBlockError> { - if batches.is_empty() { - *self.num_empty_batches_received.write().await += 1; - } else { - self.batch_groups.write().await.push(batches.to_vec()); - } - - Ok(()) - } -} - -#[derive(Default)] -struct BlockBuilderFailure; - -#[async_trait] -impl BlockBuilder for BlockBuilderFailure { - async fn build_block(&self, _batches: &[TransactionBatch]) -> Result<(), BuildBlockError> { - Err(BuildBlockError::TooManyBatchesInBlock(0)) - } -} - -// TESTS -// ================================================================================================ - -/// Tests that the number of batches in a block doesn't exceed `max_batches_per_block` -#[tokio::test] -#[miden_node_test_macro::enable_logging] -async fn test_block_size_doesnt_exceed_limit() { - let block_frequency = Duration::from_millis(20); - let max_batches_per_block = 2; - - let store = Arc::new(MockStoreSuccessBuilder::from_accounts(iter::empty()).build()); - let block_builder = Arc::new(BlockBuilderSuccess::default()); - - let batch_builder = Arc::new(DefaultBatchBuilder::new( - store, - block_builder.clone(), - DefaultBatchBuilderOptions { block_frequency, max_batches_per_block }, - )); - - // Add 3 batches in internal queue (remember: 2 batches/block) - { - let mut batch_group = - vec![dummy_tx_batch(0, 2), dummy_tx_batch(10, 2), dummy_tx_batch(20, 2)]; - - batch_builder.ready_batches.write().await.append(&mut batch_group); - } - - // start batch builder - tokio::spawn(batch_builder.run()); - - // Wait for 2 blocks to be produced - time::sleep(block_frequency * 3).await; - - // Ensure the block builder received 2 batches of the expected size - { - let batch_groups = block_builder.batch_groups.read().await; - - assert_eq!(batch_groups.len(), 2); - assert_eq!(batch_groups[0].len(), max_batches_per_block); - assert_eq!(batch_groups[1].len(), 1); - } -} - -/// Tests that `BlockBuilder::build_block()` is still called when there are no transactions -#[tokio::test] -#[miden_node_test_macro::enable_logging] -async fn test_build_block_called_when_no_batches() { - let block_frequency = Duration::from_millis(20); - let max_batches_per_block = 2; - - let store = Arc::new(MockStoreSuccessBuilder::from_accounts(iter::empty()).build()); - let block_builder = Arc::new(BlockBuilderSuccess::default()); - - let batch_builder = Arc::new(DefaultBatchBuilder::new( - store, - block_builder.clone(), - DefaultBatchBuilderOptions { block_frequency, max_batches_per_block }, - )); - - // start batch builder - tokio::spawn(batch_builder.run()); - - // Wait for at least 1 block to be produced - time::sleep(block_frequency * 2).await; - - // Ensure the block builder received at least 1 empty batch Note: we check `> 0` instead of an - // exact number to make the test flaky in case timings change in the implementation - assert!(*block_builder.num_empty_batches_received.read().await > 0); -} - -/// Tests that if `BlockBuilder::build_block()` fails, then batches are added back on the queue -#[tokio::test] -#[miden_node_test_macro::enable_logging] -async fn test_batches_added_back_to_queue_on_block_build_failure() { - let block_frequency = Duration::from_millis(20); - let max_batches_per_block = 2; - - let store = Arc::new(MockStoreSuccessBuilder::from_accounts(iter::empty()).build()); - let block_builder = Arc::new(BlockBuilderFailure); - - let batch_builder = Arc::new(DefaultBatchBuilder::new( - store, - block_builder.clone(), - DefaultBatchBuilderOptions { block_frequency, max_batches_per_block }, - )); - - let internal_ready_batches = batch_builder.ready_batches.clone(); - - // Add 3 batches in internal queue - { - let mut batch_group = - vec![dummy_tx_batch(0, 2), dummy_tx_batch(10, 2), dummy_tx_batch(20, 2)]; - - batch_builder.ready_batches.write().await.append(&mut batch_group); - } - - // start batch builder - tokio::spawn(batch_builder.run()); - - // Wait for 2 blocks to failed to be produced - time::sleep(block_frequency * 2 + (block_frequency / 2)).await; - - // Ensure the transaction batches are all still on the queue - assert_eq!(internal_ready_batches.read().await.len(), 3); -} - -#[tokio::test] -async fn test_batch_builder_find_dangling_notes() { - let store = Arc::new(MockStoreSuccessBuilder::from_accounts(iter::empty()).build()); - let block_builder = Arc::new(BlockBuilderSuccess::default()); - - let batch_builder = Arc::new(DefaultBatchBuilder::new( - store, - block_builder, - DefaultBatchBuilderOptions { - block_frequency: Duration::from_millis(20), - max_batches_per_block: 2, - }, - )); - - // An account with 5 states so that we can simulate running 2 transactions against it. - let account = MockPrivateAccount::<3>::from(1); - - let note_1 = mock_note(1); - let note_2 = mock_note(2); - let tx1 = MockProvenTxBuilder::with_account(account.id, account.states[0], account.states[1]) - .output_notes(vec![OutputNote::Full(note_1.clone())]) - .build(); - let tx2 = MockProvenTxBuilder::with_account(account.id, account.states[1], account.states[2]) - .unauthenticated_notes(vec![note_1.clone()]) - .output_notes(vec![OutputNote::Full(note_2.clone())]) - .build(); - - let txs = vec![tx1, tx2]; - - let dangling_notes = batch_builder.find_dangling_notes(&txs).await; - assert_eq!(dangling_notes, vec![], "Note must be presented in the same batch"); - - batch_builder.build_batch(txs.clone()).await.unwrap(); - - let dangling_notes = batch_builder.find_dangling_notes(&txs).await; - assert_eq!(dangling_notes, vec![], "Note must be presented in the same batch"); - - let note_3 = mock_note(3); - - let tx1 = MockProvenTxBuilder::with_account(account.id, account.states[0], account.states[1]) - .unauthenticated_notes(vec![note_2.clone()]) - .build(); - let tx2 = MockProvenTxBuilder::with_account(account.id, account.states[1], account.states[2]) - .unauthenticated_notes(vec![note_3.clone()]) - .build(); - - let txs = vec![tx1, tx2]; - - let dangling_notes = batch_builder.find_dangling_notes(&txs).await; - assert_eq!( - dangling_notes, - vec![note_3.id()], - "Only one dangling node must be found before block is built" - ); - - batch_builder.try_build_block().await; - - let dangling_notes = batch_builder.find_dangling_notes(&txs).await; - assert_eq!( - dangling_notes, - vec![note_2.id(), note_3.id()], - "Two dangling notes must be found after block is built" - ); -} - -#[tokio::test] -async fn test_block_builder_no_missing_notes() { - let account_1: MockPrivateAccount<3> = MockPrivateAccount::from(1); - let account_2: MockPrivateAccount<3> = MockPrivateAccount::from(2); - let store = Arc::new( - MockStoreSuccessBuilder::from_accounts( - [account_1, account_2].iter().map(|account| (account.id, account.states[0])), - ) - .build(), - ); - let block_builder = Arc::new(DefaultBlockBuilder::new(Arc::clone(&store), Arc::clone(&store))); - let batch_builder = Arc::new(DefaultBatchBuilder::new( - store, - Arc::clone(&block_builder), - DefaultBatchBuilderOptions { - block_frequency: Duration::from_millis(20), - max_batches_per_block: 2, - }, - )); - - let note_1 = mock_note(1); - let note_2 = mock_note(2); - - let tx1 = MockProvenTxBuilder::with_account_index(1) - .output_notes(vec![OutputNote::Full(note_1.clone())]) - .build(); - - let tx2 = MockProvenTxBuilder::with_account_index(2) - .unauthenticated_notes(vec![note_1.clone()]) - .output_notes(vec![OutputNote::Full(note_2.clone())]) - .build(); - - let txs = vec![tx1, tx2]; - - batch_builder.build_batch(txs.clone()).await.unwrap(); - - let build_block_result = batch_builder - .block_builder - .build_block(&batch_builder.ready_batches.read().await) - .await; - assert_matches!(build_block_result, Ok(())); -} - -#[tokio::test] -async fn test_block_builder_fails_if_notes_are_missing() { - let accounts: Vec<_> = (1..=4).map(MockPrivateAccount::<3>::from).collect(); - let notes: Vec<_> = (1..=6).map(mock_note).collect(); - // We require mmr for the note authentication to succeed. - // - // We also need two blocks worth of mmr because the mock store skips genesis. - let mut mmr = Mmr::new(); - mmr.add(Digest::new([1u32.into(), 2u32.into(), 3u32.into(), 4u32.into()])); - mmr.add(Digest::new([1u32.into(), 2u32.into(), 3u32.into(), 4u32.into()])); - - let store = Arc::new( - MockStoreSuccessBuilder::from_accounts( - accounts.iter().map(|account| (account.id, account.states[0])), - ) - .initial_notes([vec![OutputNote::Full(notes[0].clone())]].iter()) - .initial_chain_mmr(mmr) - .build(), - ); - let block_builder = Arc::new(DefaultBlockBuilder::new(Arc::clone(&store), Arc::clone(&store))); - let batch_builder = Arc::new(DefaultBatchBuilder::new( - store, - Arc::clone(&block_builder), - DefaultBatchBuilderOptions { - block_frequency: Duration::from_millis(20), - max_batches_per_block: 2, - }, - )); - - let tx1 = MockProvenTxBuilder::with_account_index(1) - .output_notes(vec![OutputNote::Full(notes[1].clone())]) - .build(); - - let tx2 = MockProvenTxBuilder::with_account_index(2) - .unauthenticated_notes(vec![notes[0].clone()]) - .output_notes(vec![OutputNote::Full(notes[2].clone()), OutputNote::Full(notes[3].clone())]) - .build(); - - let tx3 = MockProvenTxBuilder::with_account_index(3) - .unauthenticated_notes(notes.iter().skip(1).cloned().collect()) - .build(); - - let txs = vec![tx1, tx2, tx3]; - - let batch = TransactionBatch::new(txs.clone(), Default::default()).unwrap(); - let build_block_result = batch_builder.block_builder.build_block(&[batch]).await; - - let mut expected_missing_notes = vec![notes[4].id(), notes[5].id()]; - expected_missing_notes.sort(); - - assert_matches!( - build_block_result, - Err(BuildBlockError::UnauthenticatedNotesNotFound(actual_missing_notes)) => { - assert_eq!(actual_missing_notes, expected_missing_notes); - } - ); -} - -// HELPERS -// ================================================================================================ - -fn dummy_tx_batch(starting_account_index: u32, num_txs_in_batch: usize) -> TransactionBatch { - let txs = (0..num_txs_in_batch) - .map(|index| { - MockProvenTxBuilder::with_account_index(starting_account_index + index as u32).build() - }) - .collect(); - TransactionBatch::new(txs, Default::default()).unwrap() -} diff --git a/crates/block-producer/src/block_builder/mod.rs b/crates/block-producer/src/block_builder/mod.rs index a8c017c7f..bb739131d 100644 --- a/crates/block-producer/src/block_builder/mod.rs +++ b/crates/block-producer/src/block_builder/mod.rs @@ -23,10 +23,6 @@ pub(crate) mod prover; use self::prover::{block_witness::BlockWitness, BlockProver}; -// FIXME: reimplement the tests. -// #[cfg(test)] -// mod tests; - // BLOCK BUILDER // ================================================================================================= diff --git a/crates/block-producer/src/block_builder/prover/block_witness.rs b/crates/block-producer/src/block_builder/prover/block_witness.rs index af837339c..10e37c00a 100644 --- a/crates/block-producer/src/block_builder/prover/block_witness.rs +++ b/crates/block-producer/src/block_builder/prover/block_witness.rs @@ -7,7 +7,7 @@ use miden_objects::{ notes::Nullifier, transaction::TransactionId, vm::{AdviceInputs, StackInputs}, - BlockHeader, Digest, Felt, BLOCK_NOTE_TREE_DEPTH, ZERO, + BlockHeader, Digest, Felt, BLOCK_NOTE_TREE_DEPTH, MAX_BATCHES_PER_BLOCK, ZERO, }; use crate::{ @@ -35,6 +35,9 @@ impl BlockWitness { mut block_inputs: BlockInputs, batches: &[TransactionBatch], ) -> Result<(Self, Vec), BuildBlockError> { + // This limit should be enforced by the mempool. + assert!(batches.len() <= MAX_BATCHES_PER_BLOCK); + Self::validate_nullifiers(&block_inputs, batches)?; let batch_created_notes_roots = batches diff --git a/crates/block-producer/src/block_builder/tests.rs b/crates/block-producer/src/block_builder/tests.rs deleted file mode 100644 index d8fbc3565..000000000 --- a/crates/block-producer/src/block_builder/tests.rs +++ /dev/null @@ -1,83 +0,0 @@ -// block builder tests (higher level) -// `apply_block()` is called - -use std::sync::Arc; - -use assert_matches::assert_matches; -use miden_objects::{ - accounts::{account_id::testing::ACCOUNT_ID_OFF_CHAIN_SENDER, AccountId}, - Digest, Felt, -}; - -use crate::{ - batch_builder::TransactionBatch, - block_builder::{BlockBuilder, BuildBlockError, DefaultBlockBuilder}, - test_utils::{MockProvenTxBuilder, MockStoreFailure, MockStoreSuccessBuilder}, -}; - -/// Tests that `build_block()` succeeds when the transaction batches are not empty -#[tokio::test] -#[miden_node_test_macro::enable_logging] -async fn test_apply_block_called_nonempty_batches() { - let account_id = AccountId::new_unchecked(Felt::new(ACCOUNT_ID_OFF_CHAIN_SENDER)); - let account_initial_hash: Digest = - [Felt::new(1u64), Felt::new(1u64), Felt::new(1u64), Felt::new(1u64)].into(); - let store = Arc::new( - MockStoreSuccessBuilder::from_accounts(std::iter::once((account_id, account_initial_hash))) - .build(), - ); - - let block_builder = DefaultBlockBuilder::new(store.clone(), store.clone()); - - let batches: Vec = { - let batch_1 = { - let tx = MockProvenTxBuilder::with_account( - account_id, - account_initial_hash, - [Felt::new(2u64), Felt::new(2u64), Felt::new(2u64), Felt::new(2u64)].into(), - ) - .build(); - - TransactionBatch::new(vec![tx], Default::default()).unwrap() - }; - - vec![batch_1] - }; - block_builder.build_block(&batches).await.unwrap(); - - // Ensure that the store's `apply_block()` was called - assert_eq!(*store.num_apply_block_called.read().await, 1); -} - -/// Tests that `build_block()` succeeds when the transaction batches are empty -#[tokio::test] -#[miden_node_test_macro::enable_logging] -async fn test_apply_block_called_empty_batches() { - let account_id = AccountId::new_unchecked(Felt::new(ACCOUNT_ID_OFF_CHAIN_SENDER)); - let account_hash: Digest = - [Felt::new(1u64), Felt::new(1u64), Felt::new(1u64), Felt::new(1u64)].into(); - let store = Arc::new( - MockStoreSuccessBuilder::from_accounts(std::iter::once((account_id, account_hash))).build(), - ); - - let block_builder = DefaultBlockBuilder::new(store.clone(), store.clone()); - - block_builder.build_block(&Vec::new()).await.unwrap(); - - // Ensure that the store's `apply_block()` was called - assert_eq!(*store.num_apply_block_called.read().await, 1); -} - -/// Tests that `build_block()` fails when `get_block_inputs()` fails -#[tokio::test] -#[miden_node_test_macro::enable_logging] -async fn test_build_block_failure() { - let store = Arc::new(MockStoreFailure); - - let block_builder = DefaultBlockBuilder::new(store.clone(), store.clone()); - - let result = block_builder.build_block(&Vec::new()).await; - - // Ensure that the store's `apply_block()` was called - assert_matches!(result, Err(BuildBlockError::GetBlockInputsFailed(_))); -} diff --git a/crates/block-producer/src/errors.rs b/crates/block-producer/src/errors.rs index 937bbefe1..b2e9b13c7 100644 --- a/crates/block-producer/src/errors.rs +++ b/crates/block-producer/src/errors.rs @@ -25,6 +25,13 @@ pub enum BlockProducerError { /// A block-producer task panic'd. #[error("error joining {task} task")] JoinError { task: &'static str, source: JoinError }, + + /// A block-producer task reported a transport error. + #[error("task {task} had a transport error")] + TonicTransportError { + task: &'static str, + source: tonic::transport::Error, + }, } // Transaction verification errors @@ -132,6 +139,9 @@ pub enum BuildBatchError { #[error("Nothing actually went wrong, failure was injected on purpose")] InjectedFailure, + + #[error("Batch proving task panic'd")] + JoinError(#[from] tokio::task::JoinError), } // Block prover errors diff --git a/crates/block-producer/src/mempool/batch_graph.rs b/crates/block-producer/src/mempool/batch_graph.rs index c489cddde..62b4c568c 100644 --- a/crates/block-producer/src/mempool/batch_graph.rs +++ b/crates/block-producer/src/mempool/batch_graph.rs @@ -3,7 +3,7 @@ use std::collections::{BTreeMap, BTreeSet}; use miden_objects::transaction::TransactionId; use super::{ - dependency_graph::{DependencyGraph, GraphError}, + graph::{DependencyGraph, GraphError}, BatchJobId, BlockBudget, BudgetStatus, }; use crate::batch_builder::batch::TransactionBatch; @@ -135,7 +135,7 @@ impl BatchGraph { /// /// # Returns /// - /// Returns all removes batches and their transactions. + /// Returns all removed batches and their transactions. /// /// # Errors /// @@ -185,6 +185,8 @@ impl BatchGraph { &mut self, batch_ids: BTreeSet, ) -> Result, GraphError> { + // This clone could be elided by moving this call to the end. This would lose the atomic + // property of this method though its unclear what value (if any) that has. self.inner.prune_processed(batch_ids.clone())?; let mut transactions = Vec::new(); @@ -224,13 +226,17 @@ impl BatchGraph { while let Some(batch_id) = self.inner.roots().first().copied() { // SAFETY: Since it was a root batch, it must definitely have a processed batch // associated with it. - let batch = self.inner.get(&batch_id).expect("root should be in graph").clone(); + let batch = self.inner.get(&batch_id).expect("root should be in graph"); // Adhere to block's budget. - if budget.check_then_subtract(&batch) == BudgetStatus::Exceeded { + if budget.check_then_subtract(batch) == BudgetStatus::Exceeded { break; } + // Clone is required to avoid multiple borrows of self. We delay this clone until after + // the budget check, which is why this looks so out of place. + let batch = batch.clone(); + // SAFETY: This is definitely a root since we just selected it from the set of roots. self.inner.process_root(batch_id).expect("root should be processed"); diff --git a/crates/block-producer/src/mempool/dependency_graph.rs b/crates/block-producer/src/mempool/dependency_graph.rs deleted file mode 100644 index 5fb2766a8..000000000 --- a/crates/block-producer/src/mempool/dependency_graph.rs +++ /dev/null @@ -1,1000 +0,0 @@ -use std::{ - collections::{BTreeMap, BTreeSet}, - fmt::{Debug, Display}, -}; - -// DEPENDENCY GRAPH -// ================================================================================================ - -/// A dependency graph structure where nodes are inserted, and then made available for processing -/// once all parent nodes have been processed. -/// -/// Forms the basis of our transaction and batch dependency graphs. -/// -/// # Node lifecycle -/// ```text -/// │ -/// │ -/// insert_pending│ -/// ┌─────▼─────┐ -/// │ pending │────┐ -/// └─────┬─────┘ │ -/// │ │ -/// promote_pending│ │ -/// ┌─────▼─────┐ │ -/// ┌──────────► in queue │────│ -/// │ └─────┬─────┘ │ -/// revert_processed│ │ │ -/// │ process_root│ │ -/// │ ┌─────▼─────┐ │ -/// └──────────┼ processed │────│ -/// └─────┬─────┘ │ -/// │ │ -/// prune_processed│ │purge_subgraphs -/// ┌─────▼─────┐ │ -/// │ ◄────┘ -/// └───────────┘ -/// ``` -#[derive(Clone, PartialEq, Eq)] -pub struct DependencyGraph { - /// Node's who's data is still pending. - pending: BTreeSet, - - /// Each node's data. - vertices: BTreeMap, - - /// Each node's parents. This is redundant with `children`, - /// but we require both for efficient lookups. - parents: BTreeMap>, - - /// Each node's children. This is redundant with `parents`, - /// but we require both for efficient lookups. - children: BTreeMap>, - - /// Nodes that are available to process next. - /// - /// Effectively this is the set of nodes which are - /// unprocessed and whose parent's _are_ all processed. - roots: BTreeSet, - - /// Set of nodes that are already processed. - processed: BTreeSet, -} - -impl Debug for DependencyGraph -where - K: Debug, -{ - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("DependencyGraph") - .field("pending", &self.pending) - .field("vertices", &self.vertices.keys()) - .field("processed", &self.processed) - .field("roots", &self.roots) - .field("parents", &self.parents) - .field("children", &self.children) - .finish() - } -} - -#[derive(thiserror::Error, Debug, Clone, PartialEq, Eq)] -pub enum GraphError { - #[error("Node {0} already exists")] - DuplicateKey(K), - - #[error("Parents not found: {0:?}")] - MissingParents(BTreeSet), - - #[error("Nodes not found: {0:?}")] - UnknownNodes(BTreeSet), - - #[error("Nodes were not yet processed: {0:?}")] - UnprocessedNodes(BTreeSet), - - #[error("Nodes would be left dangling: {0:?}")] - DanglingNodes(BTreeSet), - - #[error("Node {0} is not a root node")] - InvalidRootNode(K), - - #[error("Node {0} is not a pending node")] - InvalidPendingNode(K), -} - -/// This cannot be derived without enforcing `Default` bounds on K and V. -impl Default for DependencyGraph { - fn default() -> Self { - Self { - vertices: Default::default(), - pending: Default::default(), - parents: Default::default(), - children: Default::default(), - roots: Default::default(), - processed: Default::default(), - } - } -} - -impl DependencyGraph { - /// Inserts a new pending node into the graph. - /// - /// # Errors - /// - /// Errors if the node already exists, or if any of the parents are not part of the graph. - /// - /// This method is atomic. - pub fn insert_pending(&mut self, key: K, parents: BTreeSet) -> Result<(), GraphError> { - if self.contains(&key) { - return Err(GraphError::DuplicateKey(key)); - } - - let missing_parents = parents - .iter() - .filter(|parent| !self.contains(parent)) - .copied() - .collect::>(); - if !missing_parents.is_empty() { - return Err(GraphError::MissingParents(missing_parents)); - } - - // Inform parents of their new child. - for parent in &parents { - self.children.entry(*parent).or_default().insert(key); - } - self.pending.insert(key); - self.parents.insert(key, parents); - self.children.insert(key, Default::default()); - - Ok(()) - } - - /// Promotes a pending node, associating it with the provided value and allowing it to be - /// considered for processing. - /// - /// # Errors - /// - /// Errors if the given node is not pending. - /// - /// This method is atomic. - pub fn promote_pending(&mut self, key: K, value: V) -> Result<(), GraphError> { - if !self.pending.remove(&key) { - return Err(GraphError::InvalidPendingNode(key)); - } - - self.vertices.insert(key, value); - self.try_make_root(key); - - Ok(()) - } - - /// Reverts the nodes __and their descendents__, requeueing them for processing. - /// - /// Descendents which are pending remain unchanged. - /// - /// # Errors - /// - /// Returns an error if any of the given nodes: - /// - /// - are not part of the graph, or - /// - were not previously processed - /// - /// This method is atomic. - pub fn revert_subgraphs(&mut self, keys: BTreeSet) -> Result<(), GraphError> { - let missing_nodes = keys - .iter() - .filter(|key| !self.vertices.contains_key(key)) - .copied() - .collect::>(); - if !missing_nodes.is_empty() { - return Err(GraphError::UnknownNodes(missing_nodes)); - } - let unprocessed = keys.difference(&self.processed).copied().collect::>(); - if !unprocessed.is_empty() { - return Err(GraphError::UnprocessedNodes(unprocessed)); - } - - let mut reverted = BTreeSet::new(); - let mut to_revert = keys.clone(); - - while let Some(key) = to_revert.pop_first() { - self.processed.remove(&key); - - let unprocessed_children = self - .children - .get(&key) - .map(|children| children.difference(&reverted)) - .into_iter() - .flatten() - // We should not revert children which are pending. - .filter(|child| self.vertices.contains_key(child)) - .copied(); - - to_revert.extend(unprocessed_children); - - reverted.insert(key); - } - - // Only the original keys and the current roots need to be considered as roots. - // - // The children of the input keys are disqualified by definition (they're descendents), - // and current roots must be re-evaluated since their parents may have been requeued. - std::mem::take(&mut self.roots) - .into_iter() - .chain(keys) - .for_each(|key| self.try_make_root(key)); - - Ok(()) - } - - /// Removes a set of previously processed nodes from the graph. - /// - /// This is used to bound the size of the graph by removing nodes once they are no longer - /// required. - /// - /// # Errors - /// - /// Errors if - /// - any node is unknown - /// - any node is __not__ processed - /// - any parent node would be left unpruned - /// - /// The last point implies that all parents of the given nodes must either be part of the set, - /// or already been pruned. - /// - /// This method is atomic. - pub fn prune_processed(&mut self, keys: BTreeSet) -> Result, GraphError> { - let missing_nodes = - keys.iter().filter(|key| !self.contains(key)).copied().collect::>(); - if !missing_nodes.is_empty() { - return Err(GraphError::UnknownNodes(missing_nodes)); - } - - let unprocessed = keys.difference(&self.processed).copied().collect::>(); - if !unprocessed.is_empty() { - return Err(GraphError::UnprocessedNodes(unprocessed)); - } - - // No parent may be left dangling i.e. all parents must be part of this prune set. - let dangling = keys - .iter() - .flat_map(|key| self.parents.get(key)) - .flatten() - .filter(|parent| !keys.contains(parent)) - .copied() - .collect::>(); - if !dangling.is_empty() { - return Err(GraphError::DanglingNodes(dangling)); - } - - let mut pruned = Vec::with_capacity(keys.len()); - - for key in keys { - let value = self.vertices.remove(&key).expect("Checked in precondition"); - pruned.push(value); - self.processed.remove(&key); - self.parents.remove(&key); - - let children = self.children.remove(&key).unwrap_or_default(); - - // Remove edges from children to this node. - for child in children { - if let Some(child) = self.parents.get_mut(&child) { - child.remove(&key); - } - } - } - - Ok(pruned) - } - - /// Removes the set of nodes __and all descendents__ from the graph, returning all removed - /// nodes. This __includes__ pending nodes. - /// - /// # Returns - /// - /// All nodes removed. - /// - /// # Errors - /// - /// Returns an error if any of the given nodes does not exist. - /// - /// This method is atomic. - pub fn purge_subgraphs(&mut self, keys: BTreeSet) -> Result, GraphError> { - let missing_nodes = - keys.iter().filter(|key| !self.contains(key)).copied().collect::>(); - if !missing_nodes.is_empty() { - return Err(GraphError::UnknownNodes(missing_nodes)); - } - - let visited = keys.clone(); - let mut to_remove = keys; - let mut removed = BTreeSet::new(); - - while let Some(key) = to_remove.pop_first() { - self.vertices.remove(&key); - self.pending.remove(&key); - removed.insert(key); - - self.processed.remove(&key); - self.roots.remove(&key); - - // Children must also be purged. Take care not to visit them twice which is - // possible since children can have multiple purged parents. - let unvisited_children = self.children.remove(&key).unwrap_or_default(); - let unvisited_children = unvisited_children.difference(&visited); - to_remove.extend(unvisited_children); - - // Inform parents that this child no longer exists. - let parents = self.parents.remove(&key).unwrap_or_default(); - for parent in parents { - if let Some(parent) = self.children.get_mut(&parent) { - parent.remove(&key); - } - } - } - - Ok(removed) - } - - /// Adds the node to the `roots` list _IFF_ all of its parents are processed. - /// - /// # SAFETY - /// - /// This method assumes the node exists. Caller is responsible for ensuring this is true. - fn try_make_root(&mut self, key: K) { - if self.pending.contains(&key) { - return; - } - debug_assert!( - self.vertices.contains_key(&key), - "Potential root {key} must exist in the graph" - ); - debug_assert!( - !self.processed.contains(&key), - "Potential root {key} cannot already be processed" - ); - - let all_parents_processed = self - .parents - .get(&key) - .into_iter() - .flatten() - .all(|parent| self.processed.contains(parent)); - - if all_parents_processed { - self.roots.insert(key); - } - } - - /// Returns the set of nodes that are ready for processing. - /// - /// Nodes can be selected from here and marked as processed using [`Self::process_root`]. - pub fn roots(&self) -> &BTreeSet { - &self.roots - } - - /// Marks a root node as processed, removing it from the roots list. - /// - /// The node's children are [evaluated](Self::try_make_root) as possible roots. - /// - /// # Error - /// - /// Errors if the node is not in the roots list. - /// - /// This method is atomic. - pub fn process_root(&mut self, key: K) -> Result<(), GraphError> { - if !self.roots.remove(&key) { - return Err(GraphError::InvalidRootNode(key)); - } - - self.processed.insert(key); - - self.children - .get(&key) - .cloned() - .unwrap_or_default() - .into_iter() - .for_each(|child| self.try_make_root(child)); - - Ok(()) - } - - /// Returns the value of a node. - pub fn get(&self, key: &K) -> Option<&V> { - self.vertices.get(key) - } - - /// Returns the parents of the node, or [None] if the node does not exist. - pub fn parents(&self, key: &K) -> Option<&BTreeSet> { - self.parents.get(key) - } - - /// Returns true if the node exists, in either the pending or non-pending sets. - fn contains(&self, key: &K) -> bool { - self.pending.contains(key) || self.vertices.contains_key(key) - } -} - -// TESTS -// ================================================================================================ - -#[cfg(test)] -mod tests { - use super::*; - - // TEST UTILITIES - // ================================================================================================ - - /// Simplified graph variant where a node's key always equals its value. This is done to make - /// generating test values simpler. - type TestGraph = DependencyGraph; - - impl TestGraph { - /// Alias for inserting a node with no parents. - fn insert_with_no_parents(&mut self, node: u32) -> Result<(), GraphError> { - self.insert_with_parents(node, Default::default()) - } - - /// Alias for inserting a node with a single parent. - fn insert_with_parent(&mut self, node: u32, parent: u32) -> Result<(), GraphError> { - self.insert_with_parents(node, [parent].into()) - } - - /// Alias for inserting a node with multiple parents. - fn insert_with_parents( - &mut self, - node: u32, - parents: BTreeSet, - ) -> Result<(), GraphError> { - self.insert_pending(node, parents) - } - - /// Alias for promoting nodes with the same value as the key. - fn promote(&mut self, nodes: impl IntoIterator) -> Result<(), GraphError> { - for node in nodes { - self.promote_pending(node, node)?; - } - Ok(()) - } - - /// Promotes all nodes in the pending list with value=key. - fn promote_all(&mut self) { - // SAFETY: these are definitely pending nodes. - self.promote(self.pending.clone()).unwrap(); - } - - /// Calls process_root until all nodes have been processed. - fn process_all(&mut self) { - while let Some(root) = self.roots().first().copied() { - // SAFETY: this is definitely a root since we just took it from there :) - self.process_root(root).unwrap(); - } - } - } - - // PROMOTE TESTS - // ================================================================================================ - - #[test] - fn promoted_nodes_are_considered_for_root() { - //! Ensure that a promoted node is added to the root list if all parents are already - //! processed. - let parent_a = 1; - let parent_b = 2; - let child_a = 3; - let child_b = 4; - let child_c = 5; - - let mut uut = TestGraph::default(); - uut.insert_with_no_parents(parent_a).unwrap(); - uut.insert_with_no_parents(parent_b).unwrap(); - uut.promote_all(); - - // Only process one parent so that some children remain unrootable. - uut.process_root(parent_a).unwrap(); - - uut.insert_with_parent(child_a, parent_a).unwrap(); - uut.insert_with_parent(child_b, parent_b).unwrap(); - uut.insert_with_parents(child_c, [parent_a, parent_b].into()).unwrap(); - - uut.promote_all(); - - // Only child_a should be added (in addition to the parents), since the other children - // are dependent on parent_b which is incomplete. - let expected_roots = [parent_b, child_a].into(); - - assert_eq!(uut.roots, expected_roots); - } - - #[test] - fn pending_nodes_are_not_considered_for_root() { - //! Ensure that an unpromoted node is _not_ added to the root list even if all parents are - //! already processed. - let parent_a = 1; - let parent_b = 2; - let child_a = 3; - let child_b = 4; - let child_c = 5; - - let mut uut = TestGraph::default(); - uut.insert_with_no_parents(parent_a).unwrap(); - uut.insert_with_no_parents(parent_b).unwrap(); - uut.promote_all(); - uut.process_all(); - - uut.insert_with_parent(child_a, parent_a).unwrap(); - uut.insert_with_parent(child_b, parent_b).unwrap(); - uut.insert_with_parents(child_c, [parent_a, parent_b].into()).unwrap(); - - uut.promote([child_b]).unwrap(); - - // Only child b is valid as it was promoted. - let expected = [child_b].into(); - - assert_eq!(uut.roots, expected); - } - - #[test] - fn promoted_nodes_are_moved() { - let mut uut = TestGraph::default(); - uut.insert_with_no_parents(123).unwrap(); - - assert!(uut.pending.contains(&123)); - assert!(!uut.vertices.contains_key(&123)); - - uut.promote_pending(123, 123).unwrap(); - - assert!(!uut.pending.contains(&123)); - assert!(uut.vertices.contains_key(&123)); - } - - #[test] - fn promote_rejects_already_promoted_nodes() { - let mut uut = TestGraph::default(); - uut.insert_with_no_parents(123).unwrap(); - uut.promote_all(); - - let err = uut.promote_pending(123, 123).unwrap_err(); - let expected = GraphError::InvalidPendingNode(123); - assert_eq!(err, expected); - } - - #[test] - fn promote_rejects_unknown_nodes() { - let err = TestGraph::default().promote_pending(123, 123).unwrap_err(); - let expected = GraphError::InvalidPendingNode(123); - assert_eq!(err, expected); - } - - // INSERT TESTS - // ================================================================================================ - - #[test] - fn insert_with_known_parents_succeeds() { - let parent_a = 10; - let parent_b = 20; - let grandfather = 123; - let uncle = 222; - - let mut uut = TestGraph::default(); - uut.insert_with_no_parents(grandfather).unwrap(); - uut.insert_with_no_parents(parent_a).unwrap(); - uut.insert_with_parent(parent_b, grandfather).unwrap(); - uut.insert_with_parent(uncle, grandfather).unwrap(); - uut.insert_with_parents(1, [parent_a, parent_b].into()).unwrap(); - } - - #[test] - fn insert_duplicate_is_rejected() { - //! Ensure that inserting a duplicate node - //! - results in an error, and - //! - does not mutate the state (atomicity) - const KEY: u32 = 123; - let mut uut = TestGraph::default(); - uut.insert_with_no_parents(KEY).unwrap(); - - let err = uut.insert_with_no_parents(KEY).unwrap_err(); - let expected = GraphError::DuplicateKey(KEY); - assert_eq!(err, expected); - - let mut atomic_reference = TestGraph::default(); - atomic_reference.insert_with_no_parents(KEY).unwrap(); - assert_eq!(uut, atomic_reference); - } - - #[test] - fn insert_with_all_parents_missing_is_rejected() { - //! Ensure that inserting a node with unknown parents - //! - results in an error, and - //! - does not mutate the state (atomicity) - const MISSING: [u32; 4] = [1, 2, 3, 4]; - let mut uut = TestGraph::default(); - - let err = uut.insert_with_parents(0xABC, MISSING.into()).unwrap_err(); - let expected = GraphError::MissingParents(MISSING.into()); - assert_eq!(err, expected); - - let atomic_reference = TestGraph::default(); - assert_eq!(uut, atomic_reference); - } - - #[test] - fn insert_with_some_parents_missing_is_rejected() { - //! Ensure that inserting a node with unknown parents - //! - results in an error, and - //! - does not mutate the state (atomicity) - const MISSING: u32 = 123; - let mut uut = TestGraph::default(); - - uut.insert_with_no_parents(1).unwrap(); - uut.insert_with_no_parents(2).unwrap(); - uut.insert_with_no_parents(3).unwrap(); - - let atomic_reference = uut.clone(); - - let err = uut.insert_with_parents(0xABC, [1, 2, 3, MISSING].into()).unwrap_err(); - let expected = GraphError::MissingParents([MISSING].into()); - assert_eq!(err, expected); - assert_eq!(uut, atomic_reference); - } - - // REVERT TESTS - // ================================================================================================ - - #[test] - fn reverting_unprocessed_nodes_is_rejected() { - let mut uut = TestGraph::default(); - uut.insert_with_no_parents(1).unwrap(); - uut.insert_with_no_parents(2).unwrap(); - uut.insert_with_no_parents(3).unwrap(); - uut.promote_all(); - uut.process_root(1).unwrap(); - - let err = uut.revert_subgraphs([1, 2, 3].into()).unwrap_err(); - let expected = GraphError::UnprocessedNodes([2, 3].into()); - - assert_eq!(err, expected); - } - - #[test] - fn reverting_unknown_nodes_is_rejected() { - let err = TestGraph::default().revert_subgraphs([1].into()).unwrap_err(); - let expected = GraphError::UnknownNodes([1].into()); - assert_eq!(err, expected); - } - - #[test] - fn reverting_resets_the_entire_subgraph() { - //! Reverting should reset the state to before any of the nodes where processed. - let grandparent = 1; - let parent_a = 2; - let parent_b = 3; - let child_a = 4; - let child_b = 5; - let child_c = 6; - - let disjoint = 7; - - let mut uut = TestGraph::default(); - uut.insert_with_no_parents(grandparent).unwrap(); - uut.insert_with_no_parents(disjoint).unwrap(); - uut.insert_with_parent(parent_a, grandparent).unwrap(); - uut.insert_with_parent(parent_b, grandparent).unwrap(); - uut.insert_with_parent(child_a, parent_a).unwrap(); - uut.insert_with_parent(child_b, parent_b).unwrap(); - uut.insert_with_parents(child_c, [parent_a, parent_b].into()).unwrap(); - - uut.promote([disjoint, grandparent, parent_a, parent_b, child_a, child_c]) - .unwrap(); - uut.process_root(disjoint).unwrap(); - - let reference = uut.clone(); - - uut.process_all(); - uut.revert_subgraphs([grandparent].into()).unwrap(); - - assert_eq!(uut, reference); - } - - #[test] - fn reverting_reevaluates_roots() { - //! Node reverting from processed to unprocessed should cause the root nodes to be - //! re-evaluated. Only nodes with all parents processed should remain in the set. - let disjoint_parent = 1; - let disjoint_child = 2; - - let parent_a = 3; - let parent_b = 4; - let child_a = 5; - let child_b = 6; - - let partially_disjoin_child = 7; - - let mut uut = TestGraph::default(); - // This pair of nodes should not be impacted by the reverted subgraph. - uut.insert_with_no_parents(disjoint_parent).unwrap(); - uut.insert_with_parent(disjoint_child, disjoint_parent).unwrap(); - - uut.insert_with_no_parents(parent_a).unwrap(); - uut.insert_with_no_parents(parent_b).unwrap(); - uut.insert_with_parent(child_a, parent_a).unwrap(); - uut.insert_with_parent(child_b, parent_b).unwrap(); - uut.insert_with_parents(partially_disjoin_child, [disjoint_parent, parent_a].into()) - .unwrap(); - - // Since we are reverting the other parents, we expect the roots to match the current state. - uut.promote_all(); - uut.process_root(disjoint_parent).unwrap(); - let reference = uut.roots().clone(); - - uut.process_root(parent_a).unwrap(); - uut.process_root(parent_b).unwrap(); - uut.revert_subgraphs([parent_a, parent_b].into()).unwrap(); - - assert_eq!(uut.roots(), &reference); - } - - // PRUNING TESTS - // ================================================================================================ - - #[test] - fn pruned_nodes_are_nonextant() { - //! Checks that processed and then pruned nodes behave as if they never existed in the - //! graph. We test this by comparing it to a reference graph created without these ancestor - //! nodes. - let ancestor_a = 1; - let ancestor_b = 2; - - let child_a = 3; - let child_b = 4; - let child_both = 5; - - let mut uut = TestGraph::default(); - uut.insert_with_no_parents(ancestor_a).unwrap(); - uut.insert_with_no_parents(ancestor_b).unwrap(); - uut.insert_with_parent(child_a, ancestor_a).unwrap(); - uut.insert_with_parent(child_b, ancestor_b).unwrap(); - uut.insert_with_parents(child_both, [ancestor_a, ancestor_b].into()).unwrap(); - uut.promote_all(); - - uut.process_root(ancestor_a).unwrap(); - uut.process_root(ancestor_b).unwrap(); - uut.prune_processed([ancestor_a, ancestor_b].into()).unwrap(); - - let mut reference = TestGraph::default(); - reference.insert_with_no_parents(child_a).unwrap(); - reference.insert_with_no_parents(child_b).unwrap(); - reference.insert_with_no_parents(child_both).unwrap(); - reference.promote_all(); - - assert_eq!(uut, reference); - } - - #[test] - fn pruning_unknown_nodes_is_rejected() { - let err = TestGraph::default().prune_processed([1].into()).unwrap_err(); - let expected = GraphError::UnknownNodes([1].into()); - assert_eq!(err, expected); - } - - #[test] - fn pruning_unprocessed_nodes_is_rejected() { - let mut uut = TestGraph::default(); - uut.insert_with_no_parents(1).unwrap(); - uut.promote_all(); - - let err = uut.prune_processed([1].into()).unwrap_err(); - let expected = GraphError::UnprocessedNodes([1].into()); - assert_eq!(err, expected); - } - - #[test] - fn pruning_cannot_leave_parents_dangling() { - //! Pruning processed nodes must always prune all parent nodes as well. No parent node may - //! be left behind. - let dangling = 1; - let pruned = 2; - let mut uut = TestGraph::default(); - uut.insert_with_no_parents(dangling).unwrap(); - uut.insert_with_parent(pruned, dangling).unwrap(); - uut.promote_all(); - uut.process_all(); - - let err = uut.prune_processed([pruned].into()).unwrap_err(); - let expected = GraphError::DanglingNodes([dangling].into()); - assert_eq!(err, expected); - } - - // PURGING TESTS - // ================================================================================================ - - #[test] - fn purging_subgraph_handles_internal_nodes() { - //! Purging a subgraph should correctly handle nodes already deleted within that subgraph. - //! - //! This is a concern for errors as we are deleting parts of the subgraph while we are - //! iterating through the nodes to purge. This means its likely a node will already have - //! been deleted before processing it as an input. - //! - //! We can force this to occur by re-ordering the inputs relative to the actual dependency - //! order. This means this test is a bit weaker because it relies on implementation details. - - let ancestor_a = 1; - let ancestor_b = 2; - let parent_a = 3; - let parent_b = 4; - let child_a = 5; - // This should be purged prior to parent_a. Relies on the fact that we are iterating over a - // btree which is ordered by value. - let child_b = 0; - let child_c = 6; - - let mut uut = TestGraph::default(); - uut.insert_with_no_parents(ancestor_a).unwrap(); - uut.insert_with_no_parents(ancestor_b).unwrap(); - uut.insert_with_parent(parent_a, ancestor_a).unwrap(); - uut.insert_with_parent(parent_b, ancestor_b).unwrap(); - uut.insert_with_parents(child_a, [ancestor_a, parent_a].into()).unwrap(); - uut.insert_with_parents(child_b, [parent_a, parent_b].into()).unwrap(); - uut.insert_with_parent(child_c, parent_b).unwrap(); - - uut.purge_subgraphs([child_b, parent_a].into()).unwrap(); - - let mut reference = TestGraph::default(); - reference.insert_with_no_parents(ancestor_a).unwrap(); - reference.insert_with_no_parents(ancestor_b).unwrap(); - reference.insert_with_parent(parent_b, ancestor_b).unwrap(); - reference.insert_with_parent(child_c, parent_b).unwrap(); - - assert_eq!(uut, reference); - } - - #[test] - fn purging_removes_all_descendents() { - let ancestor_a = 1; - let ancestor_b = 2; - let parent_a = 3; - let parent_b = 4; - let child_a = 5; - let child_b = 6; - let child_c = 7; - - let mut uut = TestGraph::default(); - uut.insert_with_no_parents(ancestor_a).unwrap(); - uut.insert_with_no_parents(ancestor_b).unwrap(); - uut.insert_with_parent(parent_a, ancestor_a).unwrap(); - uut.insert_with_parent(parent_b, ancestor_b).unwrap(); - uut.insert_with_parents(child_a, [ancestor_a, parent_a].into()).unwrap(); - uut.insert_with_parents(child_b, [parent_a, parent_b].into()).unwrap(); - uut.insert_with_parent(child_c, parent_b).unwrap(); - - uut.purge_subgraphs([parent_a].into()).unwrap(); - - let mut reference = TestGraph::default(); - reference.insert_with_no_parents(ancestor_a).unwrap(); - reference.insert_with_no_parents(ancestor_b).unwrap(); - reference.insert_with_parent(parent_b, ancestor_b).unwrap(); - reference.insert_with_parent(child_c, parent_b).unwrap(); - - assert_eq!(uut, reference); - } - - // PROCESSING TESTS - // ================================================================================================ - - #[test] - fn process_root_evaluates_children_as_roots() { - let parent_a = 1; - let parent_b = 2; - let child_a = 3; - let child_b = 4; - let child_c = 5; - - let mut uut = TestGraph::default(); - uut.insert_with_no_parents(parent_a).unwrap(); - uut.insert_with_no_parents(parent_b).unwrap(); - uut.insert_with_parent(child_a, parent_a).unwrap(); - uut.insert_with_parent(child_b, parent_b).unwrap(); - uut.insert_with_parents(child_c, [parent_a, parent_b].into()).unwrap(); - uut.promote_all(); - - uut.process_root(parent_a).unwrap(); - assert_eq!(uut.roots(), &[parent_b, child_a].into()); - } - - #[test] - fn process_root_rejects_non_root_node() { - let mut uut = TestGraph::default(); - uut.insert_with_no_parents(1).unwrap(); - uut.insert_with_parent(2, 1).unwrap(); - uut.promote_all(); - - let err = uut.process_root(2).unwrap_err(); - let expected = GraphError::InvalidRootNode(2); - assert_eq!(err, expected); - } - - #[test] - fn process_root_cannot_reprocess_same_node() { - let mut uut = TestGraph::default(); - uut.insert_with_no_parents(1).unwrap(); - uut.promote_all(); - uut.process_root(1).unwrap(); - - let err = uut.process_root(1).unwrap_err(); - let expected = GraphError::InvalidRootNode(1); - assert_eq!(err, expected); - } - - #[test] - fn processing_a_queue_graph() { - //! Creates a queue graph and ensures that nodes processed in FIFO order. - let nodes = (0..10).collect::>(); - - let mut uut = TestGraph::default(); - uut.insert_with_no_parents(nodes[0]).unwrap(); - for pairs in nodes.windows(2) { - let (parent, id) = (pairs[0], pairs[1]); - uut.insert_with_parent(id, parent).unwrap(); - } - uut.promote_all(); - - let mut ordered_roots = Vec::::new(); - for _ in &nodes { - let current_roots = uut.roots().clone(); - ordered_roots.extend(¤t_roots); - - for root in current_roots { - uut.process_root(root).unwrap(); - } - } - - assert_eq!(ordered_roots, nodes); - } - - #[test] - fn processing_and_root_tracking() { - //! Creates a somewhat arbitrarily connected graph and ensures that roots are tracked as - //! expected as the they are processed. - let ancestor_a = 1; - let ancestor_b = 2; - let parent_a = 3; - let parent_b = 4; - let child_a = 5; - let child_b = 6; - let child_c = 7; - - let mut uut = TestGraph::default(); - uut.insert_with_no_parents(ancestor_a).unwrap(); - uut.insert_with_no_parents(ancestor_b).unwrap(); - uut.insert_with_parent(parent_a, ancestor_a).unwrap(); - uut.insert_with_parent(parent_b, ancestor_b).unwrap(); - uut.insert_with_parents(child_a, [ancestor_a, parent_a].into()).unwrap(); - uut.insert_with_parents(child_b, [parent_a, parent_b].into()).unwrap(); - uut.insert_with_parent(child_c, parent_b).unwrap(); - uut.promote_all(); - - assert_eq!(uut.roots(), &[ancestor_a, ancestor_b].into()); - - uut.process_root(ancestor_a).unwrap(); - assert_eq!(uut.roots(), &[ancestor_b, parent_a].into()); - - uut.process_root(ancestor_b).unwrap(); - assert_eq!(uut.roots(), &[parent_a, parent_b].into()); - - uut.process_root(parent_a).unwrap(); - assert_eq!(uut.roots(), &[parent_b, child_a].into()); - - uut.process_root(parent_b).unwrap(); - assert_eq!(uut.roots(), &[child_a, child_b, child_c].into()); - - uut.process_root(child_a).unwrap(); - assert_eq!(uut.roots(), &[child_b, child_c].into()); - - uut.process_root(child_b).unwrap(); - assert_eq!(uut.roots(), &[child_c].into()); - - uut.process_root(child_c).unwrap(); - assert!(uut.roots().is_empty()); - } -} diff --git a/crates/block-producer/src/mempool/graph/mod.rs b/crates/block-producer/src/mempool/graph/mod.rs new file mode 100644 index 000000000..d9199909d --- /dev/null +++ b/crates/block-producer/src/mempool/graph/mod.rs @@ -0,0 +1,419 @@ +use std::{ + collections::{BTreeMap, BTreeSet}, + fmt::{Debug, Display}, +}; + +#[cfg(test)] +mod tests; + +// DEPENDENCY GRAPH +// ================================================================================================ + +/// A dependency graph structure where nodes are inserted, and then made available for processing +/// once all parent nodes have been processed. +/// +/// Forms the basis of our transaction and batch dependency graphs. +/// +/// # Node lifecycle +/// ```text +/// │ +/// │ +/// insert_pending│ +/// ┌─────▼─────┐ +/// │ pending │────┐ +/// └─────┬─────┘ │ +/// │ │ +/// promote_pending│ │ +/// ┌─────▼─────┐ │ +/// ┌──────────► in queue │────│ +/// │ └─────┬─────┘ │ +/// revert_processed│ │ │ +/// │ process_root│ │ +/// │ ┌─────▼─────┐ │ +/// └──────────┼ processed │────│ +/// └─────┬─────┘ │ +/// │ │ +/// prune_processed│ │purge_subgraphs +/// ┌─────▼─────┐ │ +/// │ ◄────┘ +/// └───────────┘ +/// ``` +#[derive(Clone, PartialEq, Eq)] +pub struct DependencyGraph { + /// Node's who's data is still pending. + pending: BTreeSet, + + /// Each node's data. + vertices: BTreeMap, + + /// Each node's parents. This is redundant with `children`, + /// but we require both for efficient lookups. + parents: BTreeMap>, + + /// Each node's children. This is redundant with `parents`, + /// but we require both for efficient lookups. + children: BTreeMap>, + + /// Nodes that are available to process next. + /// + /// Effectively this is the set of nodes which are + /// unprocessed and whose parent's _are_ all processed. + roots: BTreeSet, + + /// Set of nodes that are already processed. + processed: BTreeSet, +} + +impl Debug for DependencyGraph +where + K: Debug, +{ + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("DependencyGraph") + .field("pending", &self.pending) + .field("vertices", &self.vertices.keys()) + .field("processed", &self.processed) + .field("roots", &self.roots) + .field("parents", &self.parents) + .field("children", &self.children) + .finish() + } +} + +#[derive(thiserror::Error, Debug, Clone, PartialEq, Eq)] +pub enum GraphError { + #[error("Node {0} already exists")] + DuplicateKey(K), + + #[error("Parents not found: {0:?}")] + MissingParents(BTreeSet), + + #[error("Nodes not found: {0:?}")] + UnknownNodes(BTreeSet), + + #[error("Nodes were not yet processed: {0:?}")] + UnprocessedNodes(BTreeSet), + + #[error("Nodes would be left dangling: {0:?}")] + DanglingNodes(BTreeSet), + + #[error("Node {0} is not a root node")] + InvalidRootNode(K), + + #[error("Node {0} is not a pending node")] + InvalidPendingNode(K), +} + +/// This cannot be derived without enforcing `Default` bounds on K and V. +impl Default for DependencyGraph { + fn default() -> Self { + Self { + vertices: Default::default(), + pending: Default::default(), + parents: Default::default(), + children: Default::default(), + roots: Default::default(), + processed: Default::default(), + } + } +} + +impl DependencyGraph { + /// Inserts a new pending node into the graph. + /// + /// # Errors + /// + /// Errors if the node already exists, or if any of the parents are not part of the graph. + /// + /// This method is atomic. + pub fn insert_pending(&mut self, key: K, parents: BTreeSet) -> Result<(), GraphError> { + if self.contains(&key) { + return Err(GraphError::DuplicateKey(key)); + } + + let missing_parents = parents + .iter() + .filter(|parent| !self.contains(parent)) + .copied() + .collect::>(); + if !missing_parents.is_empty() { + return Err(GraphError::MissingParents(missing_parents)); + } + + // Inform parents of their new child. + for parent in &parents { + self.children.entry(*parent).or_default().insert(key); + } + self.pending.insert(key); + self.parents.insert(key, parents); + self.children.insert(key, Default::default()); + + Ok(()) + } + + /// Promotes a pending node, associating it with the provided value and allowing it to be + /// considered for processing. + /// + /// # Errors + /// + /// Errors if the given node is not pending. + /// + /// This method is atomic. + pub fn promote_pending(&mut self, key: K, value: V) -> Result<(), GraphError> { + if !self.pending.remove(&key) { + return Err(GraphError::InvalidPendingNode(key)); + } + + self.vertices.insert(key, value); + self.try_make_root(key); + + Ok(()) + } + + /// Reverts the nodes __and their descendents__, requeueing them for processing. + /// + /// Descendents which are pending remain unchanged. + /// + /// # Errors + /// + /// Returns an error if any of the given nodes: + /// + /// - are not part of the graph, or + /// - were not previously processed + /// + /// This method is atomic. + pub fn revert_subgraphs(&mut self, keys: BTreeSet) -> Result<(), GraphError> { + let missing_nodes = keys + .iter() + .filter(|key| !self.vertices.contains_key(key)) + .copied() + .collect::>(); + if !missing_nodes.is_empty() { + return Err(GraphError::UnknownNodes(missing_nodes)); + } + let unprocessed = keys.difference(&self.processed).copied().collect::>(); + if !unprocessed.is_empty() { + return Err(GraphError::UnprocessedNodes(unprocessed)); + } + + let mut reverted = BTreeSet::new(); + let mut to_revert = keys.clone(); + + while let Some(key) = to_revert.pop_first() { + self.processed.remove(&key); + + let unprocessed_children = self + .children + .get(&key) + .map(|children| children.difference(&reverted)) + .into_iter() + .flatten() + // We should not revert children which are pending. + .filter(|child| self.vertices.contains_key(child)) + .copied(); + + to_revert.extend(unprocessed_children); + + reverted.insert(key); + } + + // Only the original keys and the current roots need to be considered as roots. + // + // The children of the input keys are disqualified by definition (they're descendents), + // and current roots must be re-evaluated since their parents may have been requeued. + std::mem::take(&mut self.roots) + .into_iter() + .chain(keys) + .for_each(|key| self.try_make_root(key)); + + Ok(()) + } + + /// Removes a set of previously processed nodes from the graph. + /// + /// This is used to bound the size of the graph by removing nodes once they are no longer + /// required. + /// + /// # Errors + /// + /// Errors if + /// - any node is unknown + /// - any node is __not__ processed + /// - any parent node would be left unpruned + /// + /// The last point implies that all parents of the given nodes must either be part of the set, + /// or already been pruned. + /// + /// This method is atomic. + pub fn prune_processed(&mut self, keys: BTreeSet) -> Result, GraphError> { + let missing_nodes = + keys.iter().filter(|key| !self.contains(key)).copied().collect::>(); + if !missing_nodes.is_empty() { + return Err(GraphError::UnknownNodes(missing_nodes)); + } + + let unprocessed = keys.difference(&self.processed).copied().collect::>(); + if !unprocessed.is_empty() { + return Err(GraphError::UnprocessedNodes(unprocessed)); + } + + // No parent may be left dangling i.e. all parents must be part of this prune set. + let dangling = keys + .iter() + .flat_map(|key| self.parents.get(key)) + .flatten() + .filter(|parent| !keys.contains(parent)) + .copied() + .collect::>(); + if !dangling.is_empty() { + return Err(GraphError::DanglingNodes(dangling)); + } + + let mut pruned = Vec::with_capacity(keys.len()); + + for key in keys { + let value = self.vertices.remove(&key).expect("Checked in precondition"); + pruned.push(value); + self.processed.remove(&key); + self.parents.remove(&key); + + let children = self.children.remove(&key).unwrap_or_default(); + + // Remove edges from children to this node. + for child in children { + if let Some(child) = self.parents.get_mut(&child) { + child.remove(&key); + } + } + } + + Ok(pruned) + } + + /// Removes the set of nodes __and all descendents__ from the graph, returning all removed + /// nodes. This __includes__ pending nodes. + /// + /// # Returns + /// + /// All nodes removed. + /// + /// # Errors + /// + /// Returns an error if any of the given nodes does not exist. + /// + /// This method is atomic. + pub fn purge_subgraphs(&mut self, keys: BTreeSet) -> Result, GraphError> { + let missing_nodes = + keys.iter().filter(|key| !self.contains(key)).copied().collect::>(); + if !missing_nodes.is_empty() { + return Err(GraphError::UnknownNodes(missing_nodes)); + } + + let visited = keys.clone(); + let mut to_remove = keys; + let mut removed = BTreeSet::new(); + + while let Some(key) = to_remove.pop_first() { + self.vertices.remove(&key); + self.pending.remove(&key); + removed.insert(key); + + self.processed.remove(&key); + self.roots.remove(&key); + + // Children must also be purged. Take care not to visit them twice which is + // possible since children can have multiple purged parents. + let unvisited_children = self.children.remove(&key).unwrap_or_default(); + let unvisited_children = unvisited_children.difference(&visited); + to_remove.extend(unvisited_children); + + // Inform parents that this child no longer exists. + let parents = self.parents.remove(&key).unwrap_or_default(); + for parent in parents { + if let Some(parent) = self.children.get_mut(&parent) { + parent.remove(&key); + } + } + } + + Ok(removed) + } + + /// Adds the node to the `roots` list _IFF_ all of its parents are processed. + /// + /// # SAFETY + /// + /// This method assumes the node exists. Caller is responsible for ensuring this is true. + fn try_make_root(&mut self, key: K) { + if self.pending.contains(&key) { + return; + } + debug_assert!( + self.vertices.contains_key(&key), + "Potential root {key} must exist in the graph" + ); + debug_assert!( + !self.processed.contains(&key), + "Potential root {key} cannot already be processed" + ); + + let all_parents_processed = self + .parents + .get(&key) + .into_iter() + .flatten() + .all(|parent| self.processed.contains(parent)); + + if all_parents_processed { + self.roots.insert(key); + } + } + + /// Returns the set of nodes that are ready for processing. + /// + /// Nodes can be selected from here and marked as processed using [`Self::process_root`]. + pub fn roots(&self) -> &BTreeSet { + &self.roots + } + + /// Marks a root node as processed, removing it from the roots list. + /// + /// The node's children are [evaluated](Self::try_make_root) as possible roots. + /// + /// # Error + /// + /// Errors if the node is not in the roots list. + /// + /// This method is atomic. + pub fn process_root(&mut self, key: K) -> Result<(), GraphError> { + if !self.roots.remove(&key) { + return Err(GraphError::InvalidRootNode(key)); + } + + self.processed.insert(key); + + self.children + .get(&key) + .cloned() + .unwrap_or_default() + .into_iter() + .for_each(|child| self.try_make_root(child)); + + Ok(()) + } + + /// Returns the value of a node. + pub fn get(&self, key: &K) -> Option<&V> { + self.vertices.get(key) + } + + /// Returns the parents of the node, or [None] if the node does not exist. + pub fn parents(&self, key: &K) -> Option<&BTreeSet> { + self.parents.get(key) + } + + /// Returns true if the node exists, in either the pending or non-pending sets. + fn contains(&self, key: &K) -> bool { + self.pending.contains(key) || self.vertices.contains_key(key) + } +} diff --git a/crates/block-producer/src/mempool/graph/tests.rs b/crates/block-producer/src/mempool/graph/tests.rs new file mode 100644 index 000000000..acee736f2 --- /dev/null +++ b/crates/block-producer/src/mempool/graph/tests.rs @@ -0,0 +1,577 @@ +use super::*; + +// TEST UTILITIES +// ================================================================================================ + +/// Simplified graph variant where a node's key always equals its value. This is done to make +/// generating test values simpler. +type TestGraph = DependencyGraph; + +impl TestGraph { + /// Alias for inserting a node with no parents. + fn insert_with_no_parents(&mut self, node: u32) -> Result<(), GraphError> { + self.insert_with_parents(node, Default::default()) + } + + /// Alias for inserting a node with a single parent. + fn insert_with_parent(&mut self, node: u32, parent: u32) -> Result<(), GraphError> { + self.insert_with_parents(node, [parent].into()) + } + + /// Alias for inserting a node with multiple parents. + fn insert_with_parents( + &mut self, + node: u32, + parents: BTreeSet, + ) -> Result<(), GraphError> { + self.insert_pending(node, parents) + } + + /// Alias for promoting nodes with the same value as the key. + fn promote(&mut self, nodes: impl IntoIterator) -> Result<(), GraphError> { + for node in nodes { + self.promote_pending(node, node)?; + } + Ok(()) + } + + /// Promotes all nodes in the pending list with value=key. + fn promote_all(&mut self) { + // SAFETY: these are definitely pending nodes. + self.promote(self.pending.clone()).unwrap(); + } + + /// Calls process_root until all nodes have been processed. + fn process_all(&mut self) { + while let Some(root) = self.roots().first().copied() { + // SAFETY: this is definitely a root since we just took it from there :) + self.process_root(root).unwrap(); + } + } +} + +// PROMOTE TESTS +// ================================================================================================ + +#[test] +fn promoted_nodes_are_considered_for_root() { + //! Ensure that a promoted node is added to the root list if all parents are already + //! processed. + let parent_a = 1; + let parent_b = 2; + let child_a = 3; + let child_b = 4; + let child_c = 5; + + let mut uut = TestGraph::default(); + uut.insert_with_no_parents(parent_a).unwrap(); + uut.insert_with_no_parents(parent_b).unwrap(); + uut.promote_all(); + + // Only process one parent so that some children remain unrootable. + uut.process_root(parent_a).unwrap(); + + uut.insert_with_parent(child_a, parent_a).unwrap(); + uut.insert_with_parent(child_b, parent_b).unwrap(); + uut.insert_with_parents(child_c, [parent_a, parent_b].into()).unwrap(); + + uut.promote_all(); + + // Only child_a should be added (in addition to the parents), since the other children + // are dependent on parent_b which is incomplete. + let expected_roots = [parent_b, child_a].into(); + + assert_eq!(uut.roots, expected_roots); +} + +#[test] +fn pending_nodes_are_not_considered_for_root() { + //! Ensure that an unpromoted node is _not_ added to the root list even if all parents are + //! already processed. + let parent_a = 1; + let parent_b = 2; + let child_a = 3; + let child_b = 4; + let child_c = 5; + + let mut uut = TestGraph::default(); + uut.insert_with_no_parents(parent_a).unwrap(); + uut.insert_with_no_parents(parent_b).unwrap(); + uut.promote_all(); + uut.process_all(); + + uut.insert_with_parent(child_a, parent_a).unwrap(); + uut.insert_with_parent(child_b, parent_b).unwrap(); + uut.insert_with_parents(child_c, [parent_a, parent_b].into()).unwrap(); + + uut.promote([child_b]).unwrap(); + + // Only child b is valid as it was promoted. + let expected = [child_b].into(); + + assert_eq!(uut.roots, expected); +} + +#[test] +fn promoted_nodes_are_moved() { + let mut uut = TestGraph::default(); + uut.insert_with_no_parents(123).unwrap(); + + assert!(uut.pending.contains(&123)); + assert!(!uut.vertices.contains_key(&123)); + + uut.promote_pending(123, 123).unwrap(); + + assert!(!uut.pending.contains(&123)); + assert!(uut.vertices.contains_key(&123)); +} + +#[test] +fn promote_rejects_already_promoted_nodes() { + let mut uut = TestGraph::default(); + uut.insert_with_no_parents(123).unwrap(); + uut.promote_all(); + + let err = uut.promote_pending(123, 123).unwrap_err(); + let expected = GraphError::InvalidPendingNode(123); + assert_eq!(err, expected); +} + +#[test] +fn promote_rejects_unknown_nodes() { + let err = TestGraph::default().promote_pending(123, 123).unwrap_err(); + let expected = GraphError::InvalidPendingNode(123); + assert_eq!(err, expected); +} + +// INSERT TESTS +// ================================================================================================ + +#[test] +fn insert_with_known_parents_succeeds() { + let parent_a = 10; + let parent_b = 20; + let grandfather = 123; + let uncle = 222; + + let mut uut = TestGraph::default(); + uut.insert_with_no_parents(grandfather).unwrap(); + uut.insert_with_no_parents(parent_a).unwrap(); + uut.insert_with_parent(parent_b, grandfather).unwrap(); + uut.insert_with_parent(uncle, grandfather).unwrap(); + uut.insert_with_parents(1, [parent_a, parent_b].into()).unwrap(); +} + +#[test] +fn insert_duplicate_is_rejected() { + //! Ensure that inserting a duplicate node + //! - results in an error, and + //! - does not mutate the state (atomicity) + const KEY: u32 = 123; + let mut uut = TestGraph::default(); + uut.insert_with_no_parents(KEY).unwrap(); + + let err = uut.insert_with_no_parents(KEY).unwrap_err(); + let expected = GraphError::DuplicateKey(KEY); + assert_eq!(err, expected); + + let mut atomic_reference = TestGraph::default(); + atomic_reference.insert_with_no_parents(KEY).unwrap(); + assert_eq!(uut, atomic_reference); +} + +#[test] +fn insert_with_all_parents_missing_is_rejected() { + //! Ensure that inserting a node with unknown parents + //! - results in an error, and + //! - does not mutate the state (atomicity) + const MISSING: [u32; 4] = [1, 2, 3, 4]; + let mut uut = TestGraph::default(); + + let err = uut.insert_with_parents(0xABC, MISSING.into()).unwrap_err(); + let expected = GraphError::MissingParents(MISSING.into()); + assert_eq!(err, expected); + + let atomic_reference = TestGraph::default(); + assert_eq!(uut, atomic_reference); +} + +#[test] +fn insert_with_some_parents_missing_is_rejected() { + //! Ensure that inserting a node with unknown parents + //! - results in an error, and + //! - does not mutate the state (atomicity) + const MISSING: u32 = 123; + let mut uut = TestGraph::default(); + + uut.insert_with_no_parents(1).unwrap(); + uut.insert_with_no_parents(2).unwrap(); + uut.insert_with_no_parents(3).unwrap(); + + let atomic_reference = uut.clone(); + + let err = uut.insert_with_parents(0xABC, [1, 2, 3, MISSING].into()).unwrap_err(); + let expected = GraphError::MissingParents([MISSING].into()); + assert_eq!(err, expected); + assert_eq!(uut, atomic_reference); +} + +// REVERT TESTS +// ================================================================================================ + +#[test] +fn reverting_unprocessed_nodes_is_rejected() { + let mut uut = TestGraph::default(); + uut.insert_with_no_parents(1).unwrap(); + uut.insert_with_no_parents(2).unwrap(); + uut.insert_with_no_parents(3).unwrap(); + uut.promote_all(); + uut.process_root(1).unwrap(); + + let err = uut.revert_subgraphs([1, 2, 3].into()).unwrap_err(); + let expected = GraphError::UnprocessedNodes([2, 3].into()); + + assert_eq!(err, expected); +} + +#[test] +fn reverting_unknown_nodes_is_rejected() { + let err = TestGraph::default().revert_subgraphs([1].into()).unwrap_err(); + let expected = GraphError::UnknownNodes([1].into()); + assert_eq!(err, expected); +} + +#[test] +fn reverting_resets_the_entire_subgraph() { + //! Reverting should reset the state to before any of the nodes where processed. + let grandparent = 1; + let parent_a = 2; + let parent_b = 3; + let child_a = 4; + let child_b = 5; + let child_c = 6; + + let disjoint = 7; + + let mut uut = TestGraph::default(); + uut.insert_with_no_parents(grandparent).unwrap(); + uut.insert_with_no_parents(disjoint).unwrap(); + uut.insert_with_parent(parent_a, grandparent).unwrap(); + uut.insert_with_parent(parent_b, grandparent).unwrap(); + uut.insert_with_parent(child_a, parent_a).unwrap(); + uut.insert_with_parent(child_b, parent_b).unwrap(); + uut.insert_with_parents(child_c, [parent_a, parent_b].into()).unwrap(); + + uut.promote([disjoint, grandparent, parent_a, parent_b, child_a, child_c]) + .unwrap(); + uut.process_root(disjoint).unwrap(); + + let reference = uut.clone(); + + uut.process_all(); + uut.revert_subgraphs([grandparent].into()).unwrap(); + + assert_eq!(uut, reference); +} + +#[test] +fn reverting_reevaluates_roots() { + //! Node reverting from processed to unprocessed should cause the root nodes to be + //! re-evaluated. Only nodes with all parents processed should remain in the set. + let disjoint_parent = 1; + let disjoint_child = 2; + + let parent_a = 3; + let parent_b = 4; + let child_a = 5; + let child_b = 6; + + let partially_disjoin_child = 7; + + let mut uut = TestGraph::default(); + // This pair of nodes should not be impacted by the reverted subgraph. + uut.insert_with_no_parents(disjoint_parent).unwrap(); + uut.insert_with_parent(disjoint_child, disjoint_parent).unwrap(); + + uut.insert_with_no_parents(parent_a).unwrap(); + uut.insert_with_no_parents(parent_b).unwrap(); + uut.insert_with_parent(child_a, parent_a).unwrap(); + uut.insert_with_parent(child_b, parent_b).unwrap(); + uut.insert_with_parents(partially_disjoin_child, [disjoint_parent, parent_a].into()) + .unwrap(); + + // Since we are reverting the other parents, we expect the roots to match the current state. + uut.promote_all(); + uut.process_root(disjoint_parent).unwrap(); + let reference = uut.roots().clone(); + + uut.process_root(parent_a).unwrap(); + uut.process_root(parent_b).unwrap(); + uut.revert_subgraphs([parent_a, parent_b].into()).unwrap(); + + assert_eq!(uut.roots(), &reference); +} + +// PRUNING TESTS +// ================================================================================================ + +#[test] +fn pruned_nodes_are_nonextant() { + //! Checks that processed and then pruned nodes behave as if they never existed in the + //! graph. We test this by comparing it to a reference graph created without these ancestor + //! nodes. + let ancestor_a = 1; + let ancestor_b = 2; + + let child_a = 3; + let child_b = 4; + let child_both = 5; + + let mut uut = TestGraph::default(); + uut.insert_with_no_parents(ancestor_a).unwrap(); + uut.insert_with_no_parents(ancestor_b).unwrap(); + uut.insert_with_parent(child_a, ancestor_a).unwrap(); + uut.insert_with_parent(child_b, ancestor_b).unwrap(); + uut.insert_with_parents(child_both, [ancestor_a, ancestor_b].into()).unwrap(); + uut.promote_all(); + + uut.process_root(ancestor_a).unwrap(); + uut.process_root(ancestor_b).unwrap(); + uut.prune_processed([ancestor_a, ancestor_b].into()).unwrap(); + + let mut reference = TestGraph::default(); + reference.insert_with_no_parents(child_a).unwrap(); + reference.insert_with_no_parents(child_b).unwrap(); + reference.insert_with_no_parents(child_both).unwrap(); + reference.promote_all(); + + assert_eq!(uut, reference); +} + +#[test] +fn pruning_unknown_nodes_is_rejected() { + let err = TestGraph::default().prune_processed([1].into()).unwrap_err(); + let expected = GraphError::UnknownNodes([1].into()); + assert_eq!(err, expected); +} + +#[test] +fn pruning_unprocessed_nodes_is_rejected() { + let mut uut = TestGraph::default(); + uut.insert_with_no_parents(1).unwrap(); + uut.promote_all(); + + let err = uut.prune_processed([1].into()).unwrap_err(); + let expected = GraphError::UnprocessedNodes([1].into()); + assert_eq!(err, expected); +} + +#[test] +fn pruning_cannot_leave_parents_dangling() { + //! Pruning processed nodes must always prune all parent nodes as well. No parent node may + //! be left behind. + let dangling = 1; + let pruned = 2; + let mut uut = TestGraph::default(); + uut.insert_with_no_parents(dangling).unwrap(); + uut.insert_with_parent(pruned, dangling).unwrap(); + uut.promote_all(); + uut.process_all(); + + let err = uut.prune_processed([pruned].into()).unwrap_err(); + let expected = GraphError::DanglingNodes([dangling].into()); + assert_eq!(err, expected); +} + +// PURGING TESTS +// ================================================================================================ + +#[test] +fn purging_subgraph_handles_internal_nodes() { + //! Purging a subgraph should correctly handle nodes already deleted within that subgraph. + //! + //! This is a concern for errors as we are deleting parts of the subgraph while we are + //! iterating through the nodes to purge. This means its likely a node will already have + //! been deleted before processing it as an input. + //! + //! We can force this to occur by re-ordering the inputs relative to the actual dependency + //! order. This means this test is a bit weaker because it relies on implementation details. + + let ancestor_a = 1; + let ancestor_b = 2; + let parent_a = 3; + let parent_b = 4; + let child_a = 5; + // This should be purged prior to parent_a. Relies on the fact that we are iterating over a + // btree which is ordered by value. + let child_b = 0; + let child_c = 6; + + let mut uut = TestGraph::default(); + uut.insert_with_no_parents(ancestor_a).unwrap(); + uut.insert_with_no_parents(ancestor_b).unwrap(); + uut.insert_with_parent(parent_a, ancestor_a).unwrap(); + uut.insert_with_parent(parent_b, ancestor_b).unwrap(); + uut.insert_with_parents(child_a, [ancestor_a, parent_a].into()).unwrap(); + uut.insert_with_parents(child_b, [parent_a, parent_b].into()).unwrap(); + uut.insert_with_parent(child_c, parent_b).unwrap(); + + uut.purge_subgraphs([child_b, parent_a].into()).unwrap(); + + let mut reference = TestGraph::default(); + reference.insert_with_no_parents(ancestor_a).unwrap(); + reference.insert_with_no_parents(ancestor_b).unwrap(); + reference.insert_with_parent(parent_b, ancestor_b).unwrap(); + reference.insert_with_parent(child_c, parent_b).unwrap(); + + assert_eq!(uut, reference); +} + +#[test] +fn purging_removes_all_descendents() { + let ancestor_a = 1; + let ancestor_b = 2; + let parent_a = 3; + let parent_b = 4; + let child_a = 5; + let child_b = 6; + let child_c = 7; + + let mut uut = TestGraph::default(); + uut.insert_with_no_parents(ancestor_a).unwrap(); + uut.insert_with_no_parents(ancestor_b).unwrap(); + uut.insert_with_parent(parent_a, ancestor_a).unwrap(); + uut.insert_with_parent(parent_b, ancestor_b).unwrap(); + uut.insert_with_parents(child_a, [ancestor_a, parent_a].into()).unwrap(); + uut.insert_with_parents(child_b, [parent_a, parent_b].into()).unwrap(); + uut.insert_with_parent(child_c, parent_b).unwrap(); + + uut.purge_subgraphs([parent_a].into()).unwrap(); + + let mut reference = TestGraph::default(); + reference.insert_with_no_parents(ancestor_a).unwrap(); + reference.insert_with_no_parents(ancestor_b).unwrap(); + reference.insert_with_parent(parent_b, ancestor_b).unwrap(); + reference.insert_with_parent(child_c, parent_b).unwrap(); + + assert_eq!(uut, reference); +} + +// PROCESSING TESTS +// ================================================================================================ + +#[test] +fn process_root_evaluates_children_as_roots() { + let parent_a = 1; + let parent_b = 2; + let child_a = 3; + let child_b = 4; + let child_c = 5; + + let mut uut = TestGraph::default(); + uut.insert_with_no_parents(parent_a).unwrap(); + uut.insert_with_no_parents(parent_b).unwrap(); + uut.insert_with_parent(child_a, parent_a).unwrap(); + uut.insert_with_parent(child_b, parent_b).unwrap(); + uut.insert_with_parents(child_c, [parent_a, parent_b].into()).unwrap(); + uut.promote_all(); + + uut.process_root(parent_a).unwrap(); + assert_eq!(uut.roots(), &[parent_b, child_a].into()); +} + +#[test] +fn process_root_rejects_non_root_node() { + let mut uut = TestGraph::default(); + uut.insert_with_no_parents(1).unwrap(); + uut.insert_with_parent(2, 1).unwrap(); + uut.promote_all(); + + let err = uut.process_root(2).unwrap_err(); + let expected = GraphError::InvalidRootNode(2); + assert_eq!(err, expected); +} + +#[test] +fn process_root_cannot_reprocess_same_node() { + let mut uut = TestGraph::default(); + uut.insert_with_no_parents(1).unwrap(); + uut.promote_all(); + uut.process_root(1).unwrap(); + + let err = uut.process_root(1).unwrap_err(); + let expected = GraphError::InvalidRootNode(1); + assert_eq!(err, expected); +} + +#[test] +fn processing_a_queue_graph() { + //! Creates a queue graph and ensures that nodes processed in FIFO order. + let nodes = (0..10).collect::>(); + + let mut uut = TestGraph::default(); + uut.insert_with_no_parents(nodes[0]).unwrap(); + for pairs in nodes.windows(2) { + let (parent, id) = (pairs[0], pairs[1]); + uut.insert_with_parent(id, parent).unwrap(); + } + uut.promote_all(); + + let mut ordered_roots = Vec::::new(); + for _ in &nodes { + let current_roots = uut.roots().clone(); + ordered_roots.extend(¤t_roots); + + for root in current_roots { + uut.process_root(root).unwrap(); + } + } + + assert_eq!(ordered_roots, nodes); +} + +#[test] +fn processing_and_root_tracking() { + //! Creates a somewhat arbitrarily connected graph and ensures that roots are tracked as + //! expected as the they are processed. + let ancestor_a = 1; + let ancestor_b = 2; + let parent_a = 3; + let parent_b = 4; + let child_a = 5; + let child_b = 6; + let child_c = 7; + + let mut uut = TestGraph::default(); + uut.insert_with_no_parents(ancestor_a).unwrap(); + uut.insert_with_no_parents(ancestor_b).unwrap(); + uut.insert_with_parent(parent_a, ancestor_a).unwrap(); + uut.insert_with_parent(parent_b, ancestor_b).unwrap(); + uut.insert_with_parents(child_a, [ancestor_a, parent_a].into()).unwrap(); + uut.insert_with_parents(child_b, [parent_a, parent_b].into()).unwrap(); + uut.insert_with_parent(child_c, parent_b).unwrap(); + uut.promote_all(); + + assert_eq!(uut.roots(), &[ancestor_a, ancestor_b].into()); + + uut.process_root(ancestor_a).unwrap(); + assert_eq!(uut.roots(), &[ancestor_b, parent_a].into()); + + uut.process_root(ancestor_b).unwrap(); + assert_eq!(uut.roots(), &[parent_a, parent_b].into()); + + uut.process_root(parent_a).unwrap(); + assert_eq!(uut.roots(), &[parent_b, child_a].into()); + + uut.process_root(parent_b).unwrap(); + assert_eq!(uut.roots(), &[child_a, child_b, child_c].into()); + + uut.process_root(child_a).unwrap(); + assert_eq!(uut.roots(), &[child_b, child_c].into()); + + uut.process_root(child_b).unwrap(); + assert_eq!(uut.roots(), &[child_c].into()); + + uut.process_root(child_c).unwrap(); + assert!(uut.roots().is_empty()); +} diff --git a/crates/block-producer/src/mempool/mod.rs b/crates/block-producer/src/mempool/mod.rs index 3ebc98622..00e96e2ad 100644 --- a/crates/block-producer/src/mempool/mod.rs +++ b/crates/block-producer/src/mempool/mod.rs @@ -18,7 +18,7 @@ use crate::{ }; mod batch_graph; -mod dependency_graph; +mod graph; mod inflight_state; mod transaction_graph; diff --git a/crates/block-producer/src/mempool/transaction_graph.rs b/crates/block-producer/src/mempool/transaction_graph.rs index ced50cdd1..5709bfdeb 100644 --- a/crates/block-producer/src/mempool/transaction_graph.rs +++ b/crates/block-producer/src/mempool/transaction_graph.rs @@ -3,7 +3,7 @@ use std::collections::BTreeSet; use miden_objects::transaction::TransactionId; use super::{ - dependency_graph::{DependencyGraph, GraphError}, + graph::{DependencyGraph, GraphError}, BatchBudget, BudgetStatus, }; use crate::domain::transaction::AuthenticatedTransaction; diff --git a/crates/block-producer/src/server/mod.rs b/crates/block-producer/src/server/mod.rs index 7f79d7fd4..08e72137e 100644 --- a/crates/block-producer/src/server/mod.rs +++ b/crates/block-producer/src/server/mod.rs @@ -106,27 +106,30 @@ impl BlockProducer { // any complete or fail, we can shutdown the rest (somewhat) gracefully. let mut tasks = tokio::task::JoinSet::new(); - // TODO: improve the error situationship. let batch_builder_id = tasks .spawn({ let mempool = mempool.clone(); - async { batch_builder.run(mempool).await } + async { + batch_builder.run(mempool).await; + Ok(()) + } }) .id(); let block_builder_id = tasks .spawn({ let mempool = mempool.clone(); - async { block_builder.run(mempool).await } - }) - .id(); - let rpc_id = tasks - .spawn(async move { - BlockProducerRpcServer::new(mempool, store) - .serve(rpc_listener) - .await - .expect("block-producer failed") + async { + block_builder.run(mempool).await; + Ok(()) + } }) .id(); + let rpc_id = + tasks + .spawn(async move { + BlockProducerRpcServer::new(mempool, store).serve(rpc_listener).await + }) + .id(); let task_ids = HashMap::from([ (batch_builder_id, "batch-builder"), @@ -140,7 +143,7 @@ impl BlockProducer { let task_result = tasks.join_next_with_id().await.unwrap(); let task_id = match &task_result { - Ok((id, ())) => *id, + Ok((id, _)) => *id, Err(err) => err.id(), }; let task = task_ids.get(&task_id).unwrap_or(&"unknown"); @@ -150,7 +153,11 @@ impl BlockProducer { task_result .map_err(|source| BlockProducerError::JoinError { task, source }) - .map(|(_, ())| Err(BlockProducerError::TaskFailedSuccesfully { task }))? + .map(|(_, result)| match result { + Ok(_) => Err(BlockProducerError::TaskFailedSuccesfully { task }), + Err(source) => Err(BlockProducerError::TonicTransportError { task, source }), + }) + .and_then(|x| x) } } diff --git a/crates/block-producer/src/store/mod.rs b/crates/block-producer/src/store/mod.rs index 2b34d75d4..a6af90e0f 100644 --- a/crates/block-producer/src/store/mod.rs +++ b/crates/block-producer/src/store/mod.rs @@ -155,7 +155,7 @@ impl DefaultStore { /// Returns the latest block's header from the store. pub async fn latest_header(&self) -> Result { - // TODO: fixup the errors types. + // TODO: Consolidate the error types returned by the store (and its trait). let response = self .store .clone() From 848b3b835b0356c46abf1a014096aed339ff21a4 Mon Sep 17 00:00:00 2001 From: Mirko <48352201+Mirko-von-Leipzig@users.noreply.github.com> Date: Mon, 9 Dec 2024 11:17:41 +0200 Subject: [PATCH 21/50] chore(block-producer): lower block interval from 10s to 5s (#569) --- crates/block-producer/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/block-producer/src/lib.rs b/crates/block-producer/src/lib.rs index cf82a3570..16efd97b2 100644 --- a/crates/block-producer/src/lib.rs +++ b/crates/block-producer/src/lib.rs @@ -24,7 +24,7 @@ pub const COMPONENT: &str = "miden-block-producer"; const SERVER_MAX_TXS_PER_BATCH: usize = 2; /// The frequency at which blocks are produced -const SERVER_BLOCK_FREQUENCY: Duration = Duration::from_secs(10); +const SERVER_BLOCK_FREQUENCY: Duration = Duration::from_secs(5); /// The frequency at which batches are built const SERVER_BUILD_BATCH_FREQUENCY: Duration = Duration::from_secs(2); From 0f6b69af58189755d0ec9b13f994d3ef75317485 Mon Sep 17 00:00:00 2001 From: Mirko <48352201+Mirko-von-Leipzig@users.noreply.github.com> Date: Mon, 9 Dec 2024 11:18:23 +0200 Subject: [PATCH 22/50] chore: remove test_ prefix stutter (#568) --- bin/node/src/commands/genesis/mod.rs | 5 ++-- .../block-producer/src/batch_builder/batch.rs | 10 +++---- .../src/block_builder/prover/tests.rs | 28 +++++++++---------- .../src/state_view/tests/apply_block.rs | 6 ++-- .../src/state_view/tests/verify_tx.rs | 18 ++++++------ .../block-producer/src/txqueue/tests/mod.rs | 6 ++-- crates/proto/src/domain/digest.rs | 4 +-- crates/store/src/db/tests.rs | 26 ++++++++--------- crates/store/src/nullifier_tree.rs | 4 +-- 9 files changed, 53 insertions(+), 54 deletions(-) diff --git a/bin/node/src/commands/genesis/mod.rs b/bin/node/src/commands/genesis/mod.rs index 8e9d7f7f3..5c285b837 100644 --- a/bin/node/src/commands/genesis/mod.rs +++ b/bin/node/src/commands/genesis/mod.rs @@ -189,11 +189,10 @@ mod tests { use miden_node_store::genesis::GenesisState; use miden_objects::{accounts::AccountData, utils::serde::Deserializable}; - use super::make_genesis; use crate::DEFAULT_GENESIS_FILE_PATH; #[test] - fn test_make_genesis() { + fn make_genesis() { let genesis_inputs_file_path = PathBuf::from("genesis.toml"); // node genesis configuration @@ -217,7 +216,7 @@ mod tests { let genesis_dat_file_path = PathBuf::from(DEFAULT_GENESIS_FILE_PATH); // run make_genesis to generate genesis.dat and accounts folder and files - make_genesis(&genesis_inputs_file_path, &genesis_dat_file_path, &true).unwrap(); + super::make_genesis(&genesis_inputs_file_path, &genesis_dat_file_path, &true).unwrap(); let a0_file_path = PathBuf::from("accounts/faucet.mac"); diff --git a/crates/block-producer/src/batch_builder/batch.rs b/crates/block-producer/src/batch_builder/batch.rs index 225112d00..bdede064d 100644 --- a/crates/block-producer/src/batch_builder/batch.rs +++ b/crates/block-producer/src/batch_builder/batch.rs @@ -280,7 +280,7 @@ mod tests { }; #[test] - fn test_output_note_tracker_duplicate_output_notes() { + fn output_note_tracker_duplicate_output_notes() { let mut txs = mock_proven_txs(); let result = OutputNoteTracker::new(&txs); @@ -306,7 +306,7 @@ mod tests { } #[test] - fn test_output_note_tracker_remove_in_place_consumed_note() { + fn output_note_tracker_remove_in_place_consumed_note() { let txs = mock_proven_txs(); let mut tracker = OutputNoteTracker::new(&txs).unwrap(); @@ -329,7 +329,7 @@ mod tests { } #[test] - fn test_duplicate_unauthenticated_notes() { + fn duplicate_unauthenticated_notes() { let mut txs = mock_proven_txs(); let duplicate_note = mock_note(5); txs.push(mock_proven_tx(4, vec![duplicate_note.clone()], vec![mock_output_note(9)])); @@ -342,7 +342,7 @@ mod tests { } #[test] - fn test_consume_notes_in_place() { + fn consume_notes_in_place() { let mut txs = mock_proven_txs(); let note_to_consume = mock_note(3); txs.push(mock_proven_tx( @@ -385,7 +385,7 @@ mod tests { } #[test] - fn test_convert_unauthenticated_note_to_authenticated() { + fn convert_unauthenticated_note_to_authenticated() { let txs = mock_proven_txs(); let found_unauthenticated_notes = BTreeMap::from_iter([( mock_note(5).id(), diff --git a/crates/block-producer/src/block_builder/prover/tests.rs b/crates/block-producer/src/block_builder/prover/tests.rs index 28d011cff..1f3953c7f 100644 --- a/crates/block-producer/src/block_builder/prover/tests.rs +++ b/crates/block-producer/src/block_builder/prover/tests.rs @@ -38,7 +38,7 @@ use crate::{ /// /// The store will contain accounts 1 & 2, while the transaction batches will contain 2 & 3. #[test] -fn test_block_witness_validation_inconsistent_account_ids() { +fn block_witness_validation_inconsistent_account_ids() { let account_id_1 = AccountId::new_unchecked(Felt::new(ACCOUNT_ID_OFF_CHAIN_SENDER)); let account_id_2 = AccountId::new_unchecked(Felt::new(ACCOUNT_ID_OFF_CHAIN_SENDER + 1)); let account_id_3 = AccountId::new_unchecked(Felt::new(ACCOUNT_ID_OFF_CHAIN_SENDER + 2)); @@ -97,7 +97,7 @@ fn test_block_witness_validation_inconsistent_account_ids() { /// /// Only account 1 will have a different state hash #[test] -fn test_block_witness_validation_inconsistent_account_hashes() { +fn block_witness_validation_inconsistent_account_hashes() { let account_id_1 = AccountId::new_unchecked(Felt::new(ACCOUNT_ID_REGULAR_ACCOUNT_UPDATABLE_CODE_OFF_CHAIN)); let account_id_2 = AccountId::new_unchecked(Felt::new(ACCOUNT_ID_OFF_CHAIN_SENDER)); @@ -178,7 +178,7 @@ fn test_block_witness_validation_inconsistent_account_hashes() { /// themselves: `[tx_x0, tx_y1], [tx_y0, tx_x1]`. This test ensures that the witness is /// produced correctly as if for a single batch: `[tx_x0, tx_x1, tx_y0, tx_y1]`. #[test] -fn test_block_witness_multiple_batches_per_account() { +fn block_witness_multiple_batches_per_account() { let x_account_id = AccountId::new_unchecked(Felt::new(ACCOUNT_ID_REGULAR_ACCOUNT_UPDATABLE_CODE_OFF_CHAIN)); let y_account_id = AccountId::new_unchecked(Felt::new(ACCOUNT_ID_OFF_CHAIN_SENDER)); @@ -274,7 +274,7 @@ fn test_block_witness_multiple_batches_per_account() { /// We assume an initial store with 5 accounts, and all will be updated. #[tokio::test] #[miden_node_test_macro::enable_logging] -async fn test_compute_account_root_success() { +async fn compute_account_root_success() { // Set up account states // --------------------------------------------------------------------------------------------- let account_ids = [ @@ -374,7 +374,7 @@ async fn test_compute_account_root_success() { /// Test that the current account root is returned if the batches are empty #[tokio::test] #[miden_node_test_macro::enable_logging] -async fn test_compute_account_root_empty_batches() { +async fn compute_account_root_empty_batches() { // Set up account states // --------------------------------------------------------------------------------------------- let account_ids = [ @@ -431,7 +431,7 @@ async fn test_compute_account_root_empty_batches() { /// contains no batches #[tokio::test] #[miden_node_test_macro::enable_logging] -async fn test_compute_note_root_empty_batches_success() { +async fn compute_note_root_empty_batches_success() { // Set up store // --------------------------------------------------------------------------------------------- @@ -463,7 +463,7 @@ async fn test_compute_note_root_empty_batches_success() { /// which contains at least 1 batch. #[tokio::test] #[miden_node_test_macro::enable_logging] -async fn test_compute_note_root_empty_notes_success() { +async fn compute_note_root_empty_notes_success() { // Set up store // --------------------------------------------------------------------------------------------- @@ -498,7 +498,7 @@ async fn test_compute_note_root_empty_notes_success() { /// many batches. #[tokio::test] #[miden_node_test_macro::enable_logging] -async fn test_compute_note_root_success() { +async fn compute_note_root_success() { let account_ids = [ AccountId::new_unchecked(Felt::new(ACCOUNT_ID_OFF_CHAIN_SENDER)), AccountId::new_unchecked(Felt::new(ACCOUNT_ID_OFF_CHAIN_SENDER + 1)), @@ -605,7 +605,7 @@ async fn test_compute_note_root_success() { /// /// The transaction batches will contain nullifiers 1 & 2, while the store will contain 2 & 3. #[test] -fn test_block_witness_validation_inconsistent_nullifiers() { +fn block_witness_validation_inconsistent_nullifiers() { let batches: Vec = { let batch_1 = { let tx = MockProvenTxBuilder::with_account_index(0).nullifiers_range(0..1).build(); @@ -684,7 +684,7 @@ fn test_block_witness_validation_inconsistent_nullifiers() { /// Tests that the block kernel returns the expected nullifier tree when no nullifiers are present /// in the transaction #[tokio::test] -async fn test_compute_nullifier_root_empty_success() { +async fn compute_nullifier_root_empty_success() { let batches: Vec = { let batch_1 = { let tx = MockProvenTxBuilder::with_account_index(0).build(); @@ -738,7 +738,7 @@ async fn test_compute_nullifier_root_empty_success() { /// Tests that the block kernel returns the expected nullifier tree when multiple nullifiers are /// present in the transaction #[tokio::test] -async fn test_compute_nullifier_root_success() { +async fn compute_nullifier_root_success() { let batches: Vec = { let batch_1 = { let tx = MockProvenTxBuilder::with_account_index(0).nullifiers_range(0..1).build(); @@ -810,7 +810,7 @@ async fn test_compute_nullifier_root_success() { /// Test that the chain mmr root is as expected if the batches are empty #[tokio::test] #[miden_node_test_macro::enable_logging] -async fn test_compute_chain_mmr_root_empty_mmr() { +async fn compute_chain_mmr_root_empty_mmr() { let store = MockStoreSuccessBuilder::from_batches(iter::empty()).build(); let expected_block_header = build_expected_block_header(&store, &[]).await; @@ -822,7 +822,7 @@ async fn test_compute_chain_mmr_root_empty_mmr() { /// add header to non-empty MMR (1 peak), and check that we get the expected commitment #[tokio::test] #[miden_node_test_macro::enable_logging] -async fn test_compute_chain_mmr_root_mmr_1_peak() { +async fn compute_chain_mmr_root_mmr_1_peak() { let initial_chain_mmr = { let mut mmr = Mmr::new(); mmr.add(Digest::default()); @@ -843,7 +843,7 @@ async fn test_compute_chain_mmr_root_mmr_1_peak() { /// add header to an MMR with 17 peaks, and check that we get the expected commitment #[tokio::test] #[miden_node_test_macro::enable_logging] -async fn test_compute_chain_mmr_root_mmr_17_peaks() { +async fn compute_chain_mmr_root_mmr_17_peaks() { let initial_chain_mmr = { let mut mmr = Mmr::new(); for _ in 0..(2_u32.pow(17) - 1) { diff --git a/crates/block-producer/src/state_view/tests/apply_block.rs b/crates/block-producer/src/state_view/tests/apply_block.rs index b98371af5..5bec67698 100644 --- a/crates/block-producer/src/state_view/tests/apply_block.rs +++ b/crates/block-producer/src/state_view/tests/apply_block.rs @@ -16,7 +16,7 @@ use crate::test_utils::{block::MockBlockBuilder, MockStoreSuccessBuilder}; /// Tests requirement AB1 #[tokio::test] #[miden_node_test_macro::enable_logging] -async fn test_apply_block_ab1() { +async fn apply_block_ab1() { let account: MockPrivateAccount<3> = MockPrivateAccount::from(0); let store = Arc::new( @@ -57,7 +57,7 @@ async fn test_apply_block_ab1() { /// Tests requirement AB2 #[tokio::test] #[miden_node_test_macro::enable_logging] -async fn test_apply_block_ab2() { +async fn apply_block_ab2() { let (txs, accounts): (Vec<_>, Vec<_>) = get_txs_and_accounts(0, 3).unzip(); let store = Arc::new( @@ -113,7 +113,7 @@ async fn test_apply_block_ab2() { /// Tests requirement AB3 #[tokio::test] #[miden_node_test_macro::enable_logging] -async fn test_apply_block_ab3() { +async fn apply_block_ab3() { let (txs, accounts): (Vec<_>, Vec<_>) = get_txs_and_accounts(0, 3).unzip(); let store = Arc::new( diff --git a/crates/block-producer/src/state_view/tests/verify_tx.rs b/crates/block-producer/src/state_view/tests/verify_tx.rs index d28a3cb9e..6df558dbb 100644 --- a/crates/block-producer/src/state_view/tests/verify_tx.rs +++ b/crates/block-producer/src/state_view/tests/verify_tx.rs @@ -23,7 +23,7 @@ use crate::test_utils::{block::MockBlockBuilder, note::mock_note, MockStoreSucce /// notes all verify successfully #[tokio::test] #[miden_node_test_macro::enable_logging] -async fn test_verify_tx_happy_path() { +async fn verify_tx_happy_path() { let (txs, accounts): (Vec, Vec) = get_txs_and_accounts(0, 3).unzip(); @@ -49,7 +49,7 @@ async fn test_verify_tx_happy_path() { /// In this test, all calls to `verify_tx()` are concurrent #[tokio::test] #[miden_node_test_macro::enable_logging] -async fn test_verify_tx_happy_path_concurrent() { +async fn verify_tx_happy_path_concurrent() { let (txs, accounts): (Vec, Vec) = get_txs_and_accounts(0, 3).unzip(); @@ -79,7 +79,7 @@ async fn test_verify_tx_happy_path_concurrent() { /// Verifies requirement VT1 #[tokio::test] #[miden_node_test_macro::enable_logging] -async fn test_verify_tx_vt1() { +async fn verify_tx_vt1() { let account = MockPrivateAccount::<3>::from(1); let store = Arc::new( @@ -113,7 +113,7 @@ async fn test_verify_tx_vt1() { /// Verifies requirement VT2 #[tokio::test] #[miden_node_test_macro::enable_logging] -async fn test_verify_tx_vt2() { +async fn verify_tx_vt2() { let account_not_in_store: MockPrivateAccount<3> = MockPrivateAccount::from(0); // Notice: account is not added to the store @@ -137,7 +137,7 @@ async fn test_verify_tx_vt2() { /// Verifies requirement VT3 #[tokio::test] #[miden_node_test_macro::enable_logging] -async fn test_verify_tx_vt3() { +async fn verify_tx_vt3() { let account: MockPrivateAccount<3> = MockPrivateAccount::from(1); let nullifier_in_store = nullifier_by_index(0); @@ -169,7 +169,7 @@ async fn test_verify_tx_vt3() { /// Verifies requirement VT4 #[tokio::test] #[miden_node_test_macro::enable_logging] -async fn test_verify_tx_vt4() { +async fn verify_tx_vt4() { let account: MockPrivateAccount<3> = MockPrivateAccount::from(1); let store = Arc::new( @@ -196,7 +196,7 @@ async fn test_verify_tx_vt4() { /// Verifies requirement VT5 #[tokio::test] #[miden_node_test_macro::enable_logging] -async fn test_verify_tx_vt5() { +async fn verify_tx_vt5() { let account_1: MockPrivateAccount<3> = MockPrivateAccount::from(1); let account_2: MockPrivateAccount<3> = MockPrivateAccount::from(2); let nullifier_in_both_txs = nullifier_by_index(0); @@ -241,7 +241,7 @@ async fn test_verify_tx_vt5() { /// notes #[tokio::test] #[miden_node_test_macro::enable_logging] -async fn test_verify_tx_dangling_note_found_in_inflight_notes() { +async fn verify_tx_dangling_note_found_in_inflight_notes() { let account_1: MockPrivateAccount<3> = MockPrivateAccount::from(1); let account_2: MockPrivateAccount<3> = MockPrivateAccount::from(2); let store = Arc::new( @@ -282,7 +282,7 @@ async fn test_verify_tx_dangling_note_found_in_inflight_notes() { /// in-flight notes nor in the store #[tokio::test] #[miden_node_test_macro::enable_logging] -async fn test_verify_tx_stored_unauthenticated_notes() { +async fn verify_tx_stored_unauthenticated_notes() { let account_1: MockPrivateAccount<3> = MockPrivateAccount::from(1); let store = Arc::new( MockStoreSuccessBuilder::from_accounts( diff --git a/crates/block-producer/src/txqueue/tests/mod.rs b/crates/block-producer/src/txqueue/tests/mod.rs index f7b5ca6e3..bedd46ced 100644 --- a/crates/block-producer/src/txqueue/tests/mod.rs +++ b/crates/block-producer/src/txqueue/tests/mod.rs @@ -69,7 +69,7 @@ impl BatchBuilder for BatchBuilderFailure { /// be built in some batch #[tokio::test(start_paused = true)] #[miden_node_test_macro::enable_logging] -async fn test_build_batch_success() { +async fn build_batch_success() { let build_batch_frequency = Duration::from_millis(5); let batch_size = 3; let (sender, mut receiver) = mpsc::unbounded_channel::(); @@ -160,7 +160,7 @@ async fn test_build_batch_success() { /// Tests that when transactions fail to verify, they are not added to the queue #[tokio::test(start_paused = true)] #[miden_node_test_macro::enable_logging] -async fn test_tx_verify_failure() { +async fn tx_verify_failure() { let build_batch_frequency = Duration::from_millis(5); let batch_size = 3; @@ -194,7 +194,7 @@ async fn test_tx_verify_failure() { /// Tests that when batch building fails, transactions are added back to the ready queue #[tokio::test] #[miden_node_test_macro::enable_logging] -async fn test_build_batch_failure() { +async fn build_batch_failure() { let build_batch_frequency = Duration::from_millis(30); let batch_size = 3; diff --git a/crates/proto/src/domain/digest.rs b/crates/proto/src/domain/digest.rs index 6be724a00..fb396cb99 100644 --- a/crates/proto/src/domain/digest.rs +++ b/crates/proto/src/domain/digest.rs @@ -210,7 +210,7 @@ mod test { use crate::generated::digest::Digest; #[test] - fn test_hex_digest() { + fn hex_digest() { let digest = Digest { d0: 3488802789098113751, d1: 5271242459988994564, @@ -229,7 +229,7 @@ mod test { proptest! { #[test] - fn test_encode_decode( + fn encode_decode( d0: u64, d1: u64, d2: u64, diff --git a/crates/store/src/db/tests.rs b/crates/store/src/db/tests.rs index d4d3199f7..f7260f761 100644 --- a/crates/store/src/db/tests.rs +++ b/crates/store/src/db/tests.rs @@ -49,7 +49,7 @@ fn create_block(conn: &mut Connection, block_num: u32) { } #[test] -fn test_sql_insert_nullifiers_for_block() { +fn sql_insert_nullifiers_for_block() { let mut conn = create_db(); let nullifiers = [num_to_nullifier(1 << 48)]; @@ -95,7 +95,7 @@ fn test_sql_insert_nullifiers_for_block() { } #[test] -fn test_sql_insert_transactions() { +fn sql_insert_transactions() { let mut conn = create_db(); let count = insert_transactions(&mut conn); @@ -104,7 +104,7 @@ fn test_sql_insert_transactions() { } #[test] -fn test_sql_select_transactions() { +fn sql_select_transactions() { fn query_transactions(conn: &mut Connection) -> Vec { sql::select_transactions_by_accounts_and_block_range(conn, 0, 2, &[1]).unwrap() } @@ -125,7 +125,7 @@ fn test_sql_select_transactions() { } #[test] -fn test_sql_select_nullifiers() { +fn sql_select_nullifiers() { let mut conn = create_db(); let block_num = 1; @@ -151,7 +151,7 @@ fn test_sql_select_nullifiers() { } #[test] -fn test_sql_select_notes() { +fn sql_select_notes() { let mut conn = create_db(); let block_num = 1; @@ -191,7 +191,7 @@ fn test_sql_select_notes() { } #[test] -fn test_sql_select_notes_different_execution_hints() { +fn sql_select_notes_different_execution_hints() { let mut conn = create_db(); let block_num = 1; @@ -278,7 +278,7 @@ fn test_sql_select_notes_different_execution_hints() { } #[test] -fn test_sql_select_accounts() { +fn sql_select_accounts() { let mut conn = create_db(); let block_num = 1; @@ -321,7 +321,7 @@ fn test_sql_select_accounts() { } #[test] -fn test_sql_public_account_details() { +fn sql_public_account_details() { let mut conn = create_db(); create_block(&mut conn, 1); @@ -472,7 +472,7 @@ fn test_sql_public_account_details() { } #[test] -fn test_sql_select_nullifiers_by_block_range() { +fn sql_select_nullifiers_by_block_range() { let mut conn = create_db(); // test empty table @@ -599,7 +599,7 @@ fn test_sql_select_nullifiers_by_block_range() { } #[test] -fn test_select_nullifiers_by_prefix() { +fn select_nullifiers_by_prefix() { let mut conn = create_db(); const PREFIX_LEN: u32 = 16; // test empty table @@ -704,7 +704,7 @@ fn test_select_nullifiers_by_prefix() { } #[test] -fn test_db_block_header() { +fn db_block_header() { let mut conn = create_db(); // test querying empty table @@ -776,7 +776,7 @@ fn test_db_block_header() { } #[test] -fn test_db_account() { +fn db_account() { let mut conn = create_db(); let block_num = 1; @@ -830,7 +830,7 @@ fn test_db_account() { } #[test] -fn test_notes() { +fn notes() { let mut conn = create_db(); let block_num_1 = 1; diff --git a/crates/store/src/nullifier_tree.rs b/crates/store/src/nullifier_tree.rs index 04f868015..0d6ed342c 100644 --- a/crates/store/src/nullifier_tree.rs +++ b/crates/store/src/nullifier_tree.rs @@ -90,7 +90,7 @@ mod tests { use super::NullifierTree; #[test] - fn test_leaf_value_encoding() { + fn leaf_value_encoding() { let block_num = 123; let nullifier_value = NullifierTree::block_num_to_leaf_value(block_num); @@ -98,7 +98,7 @@ mod tests { } #[test] - fn test_leaf_value_decoding() { + fn leaf_value_decoding() { let block_num = 123; let nullifier_value = [Felt::from(block_num), ZERO, ZERO, ZERO]; let decoded_block_num = NullifierTree::leaf_value_to_block_num(nullifier_value); From 2e3fa34f6ea2fe0742287ff6278c2103317769fc Mon Sep 17 00:00:00 2001 From: Mirko <48352201+Mirko-von-Leipzig@users.noreply.github.com> Date: Tue, 10 Dec 2024 09:36:15 +0200 Subject: [PATCH 23/50] refactor(block-producer): remove Store traits (#571) --- .../block-producer/src/block_builder/mod.rs | 11 +-- .../src/block_builder/prover/tests.rs | 1 - crates/block-producer/src/server/mod.rs | 10 +-- crates/block-producer/src/store/mod.rs | 75 ++++++------------- crates/block-producer/src/test_utils/block.rs | 1 - crates/block-producer/src/test_utils/mod.rs | 2 +- crates/block-producer/src/test_utils/store.rs | 46 +----------- 7 files changed, 37 insertions(+), 109 deletions(-) diff --git a/crates/block-producer/src/block_builder/mod.rs b/crates/block-producer/src/block_builder/mod.rs index bb739131d..dea87dc03 100644 --- a/crates/block-producer/src/block_builder/mod.rs +++ b/crates/block-producer/src/block_builder/mod.rs @@ -12,11 +12,8 @@ use tokio::time::Duration; use tracing::{debug, info, instrument}; use crate::{ - batch_builder::batch::TransactionBatch, - errors::BuildBlockError, - mempool::SharedMempool, - store::{ApplyBlock, DefaultStore, Store}, - COMPONENT, SERVER_BLOCK_FREQUENCY, + batch_builder::batch::TransactionBatch, errors::BuildBlockError, mempool::SharedMempool, + store::StoreClient, COMPONENT, SERVER_BLOCK_FREQUENCY, }; pub(crate) mod prover; @@ -36,12 +33,12 @@ pub struct BlockBuilder { /// Note: this _must_ be sign positive and less than 1.0. pub failure_rate: f32, - pub store: DefaultStore, + pub store: StoreClient, pub block_kernel: BlockProver, } impl BlockBuilder { - pub fn new(store: DefaultStore) -> Self { + pub fn new(store: StoreClient) -> Self { Self { block_interval: SERVER_BLOCK_FREQUENCY, // Note: The range cannot be empty. diff --git a/crates/block-producer/src/block_builder/prover/tests.rs b/crates/block-producer/src/block_builder/prover/tests.rs index 1f3953c7f..d8bd8851c 100644 --- a/crates/block-producer/src/block_builder/prover/tests.rs +++ b/crates/block-producer/src/block_builder/prover/tests.rs @@ -23,7 +23,6 @@ use super::*; use crate::{ batch_builder::TransactionBatch, block::{AccountWitness, BlockInputs}, - store::Store, test_utils::{ block::{build_actual_block_header, build_expected_block_header, MockBlockBuilder}, MockProvenTxBuilder, MockStoreSuccessBuilder, diff --git a/crates/block-producer/src/server/mod.rs b/crates/block-producer/src/server/mod.rs index 08e72137e..c2cf192e8 100644 --- a/crates/block-producer/src/server/mod.rs +++ b/crates/block-producer/src/server/mod.rs @@ -21,7 +21,7 @@ use crate::{ domain::transaction::AuthenticatedTransaction, errors::{AddTransactionError, BlockProducerError, VerifyTxError}, mempool::{BatchBudget, BlockBudget, BlockNumber, Mempool, SharedMempool}, - store::{DefaultStore, Store}, + store::StoreClient, COMPONENT, SERVER_MEMPOOL_STATE_RETENTION, }; @@ -38,7 +38,7 @@ pub struct BlockProducer { block_budget: BlockBudget, state_retention: usize, rpc_listener: TcpListener, - store: DefaultStore, + store: StoreClient, chain_tip: BlockNumber, } @@ -49,7 +49,7 @@ impl BlockProducer { pub async fn init(config: BlockProducerConfig) -> Result { info!(target: COMPONENT, %config, "Initializing server"); - let store = DefaultStore::new( + let store = StoreClient::new( store_client::ApiClient::connect(config.store_url.to_string()) .await .map_err(|err| ApiError::DatabaseConnectionFailed(err.to_string()))?, @@ -171,7 +171,7 @@ struct BlockProducerRpcServer { /// the block-producers usage of the mempool. mempool: Mutex, - store: DefaultStore, + store: StoreClient, } #[tonic::async_trait] @@ -189,7 +189,7 @@ impl api_server::Api for BlockProducerRpcServer { } impl BlockProducerRpcServer { - pub fn new(mempool: SharedMempool, store: DefaultStore) -> Self { + pub fn new(mempool: SharedMempool, store: StoreClient) -> Self { Self { mempool: Mutex::new(mempool), store } } diff --git a/crates/block-producer/src/store/mod.rs b/crates/block-producer/src/store/mod.rs index a6af90e0f..38691d350 100644 --- a/crates/block-producer/src/store/mod.rs +++ b/crates/block-producer/src/store/mod.rs @@ -4,7 +4,6 @@ use std::{ num::NonZeroU32, }; -use async_trait::async_trait; use itertools::Itertools; use miden_node_proto::{ errors::{ConversionError, MissingFieldHelper}, @@ -32,31 +31,6 @@ use tracing::{debug, info, instrument}; pub use crate::errors::{ApplyBlockError, BlockInputsError, TxInputsError}; use crate::{block::BlockInputs, COMPONENT}; -// STORE TRAIT -// ================================================================================================ - -#[async_trait] -pub trait Store: ApplyBlock { - /// Returns information needed from the store to verify a given proven transaction. - async fn get_tx_inputs( - &self, - proven_tx: &ProvenTransaction, - ) -> Result; - - /// Returns information needed from the store to build a block. - async fn get_block_inputs( - &self, - updated_accounts: impl Iterator + Send, - produced_nullifiers: impl Iterator + Send, - notes: impl Iterator + Send, - ) -> Result; -} - -#[async_trait] -pub trait ApplyBlock: Send + Sync + 'static { - async fn apply_block(&self, block: &Block) -> Result<(), ApplyBlockError>; -} - // TRANSACTION INPUTS // ================================================================================================ @@ -139,15 +113,18 @@ impl TryFrom for TransactionInputs { } } -// DEFAULT STORE IMPLEMENTATION +// STORE CLIENT // ================================================================================================ +/// Interface to the store's gRPC API. +/// +/// Essentially just a thin wrapper around the generated gRPC client which improves type safety. #[derive(Clone)] -pub struct DefaultStore { +pub struct StoreClient { store: store_client::ApiClient, } -impl DefaultStore { +impl StoreClient { /// TODO: this should probably take store connection string and create a connection internally pub fn new(store: store_client::ApiClient) -> Self { Self { store } @@ -166,29 +143,9 @@ impl DefaultStore { BlockHeader::try_from(response.block_header.unwrap()).map_err(|err| err.to_string()) } -} -#[async_trait] -impl ApplyBlock for DefaultStore { #[instrument(target = "miden-block-producer", skip_all, err)] - async fn apply_block(&self, block: &Block) -> Result<(), ApplyBlockError> { - let request = tonic::Request::new(ApplyBlockRequest { block: block.to_bytes() }); - - let _ = self - .store - .clone() - .apply_block(request) - .await - .map_err(|status| ApplyBlockError::GrpcClientError(status.message().to_string()))?; - - Ok(()) - } -} - -#[async_trait] -impl Store for DefaultStore { - #[instrument(target = "miden-block-producer", skip_all, err)] - async fn get_tx_inputs( + pub async fn get_tx_inputs( &self, proven_tx: &ProvenTransaction, ) -> Result { @@ -230,7 +187,7 @@ impl Store for DefaultStore { Ok(tx_inputs) } - async fn get_block_inputs( + pub async fn get_block_inputs( &self, updated_accounts: impl Iterator + Send, produced_nullifiers: impl Iterator + Send, @@ -250,6 +207,20 @@ impl Store for DefaultStore { .map_err(|err| BlockInputsError::GrpcClientError(err.message().to_string()))? .into_inner(); - Ok(store_response.try_into()?) + store_response.try_into() + } + + #[instrument(target = "miden-block-producer", skip_all, err)] + pub async fn apply_block(&self, block: &Block) -> Result<(), ApplyBlockError> { + let request = tonic::Request::new(ApplyBlockRequest { block: block.to_bytes() }); + + let _ = self + .store + .clone() + .apply_block(request) + .await + .map_err(|status| ApplyBlockError::GrpcClientError(status.message().to_string()))?; + + Ok(()) } } diff --git a/crates/block-producer/src/test_utils/block.rs b/crates/block-producer/src/test_utils/block.rs index 35abbe6a0..d81376caf 100644 --- a/crates/block-producer/src/test_utils/block.rs +++ b/crates/block-producer/src/test_utils/block.rs @@ -13,7 +13,6 @@ use crate::{ batch_builder::TransactionBatch, block::BlockInputs, block_builder::prover::{block_witness::BlockWitness, BlockProver}, - store::Store, }; /// Constructs the block we expect to be built given the store state, and a set of transaction diff --git a/crates/block-producer/src/test_utils/mod.rs b/crates/block-producer/src/test_utils/mod.rs index e2385e747..b246839d2 100644 --- a/crates/block-producer/src/test_utils/mod.rs +++ b/crates/block-producer/src/test_utils/mod.rs @@ -13,7 +13,7 @@ pub use proven_tx::{mock_proven_tx, MockProvenTxBuilder}; mod store; -pub use store::{MockStoreFailure, MockStoreSuccess, MockStoreSuccessBuilder}; +pub use store::{MockStoreSuccess, MockStoreSuccessBuilder}; mod account; diff --git a/crates/block-producer/src/test_utils/store.rs b/crates/block-producer/src/test_utils/store.rs index c04ff3923..b6ebe629a 100644 --- a/crates/block-producer/src/test_utils/store.rs +++ b/crates/block-producer/src/test_utils/store.rs @@ -4,7 +4,6 @@ use std::{ ops::Not, }; -use async_trait::async_trait; use miden_node_proto::domain::{blocks::BlockInclusionProof, notes::NoteAuthenticationInfo}; use miden_objects::{ block::{Block, NoteBatch}, @@ -19,9 +18,7 @@ use super::*; use crate::{ batch_builder::TransactionBatch, block::{AccountWitness, BlockInputs}, - store::{ - ApplyBlock, ApplyBlockError, BlockInputsError, Store, TransactionInputs, TxInputsError, - }, + store::{ApplyBlockError, BlockInputsError, TransactionInputs, TxInputsError}, test_utils::block::{ block_output_notes, flatten_output_notes, note_created_smt_from_note_batches, }, @@ -185,11 +182,8 @@ impl MockStoreSuccess { locked_accounts.root() } -} -#[async_trait] -impl ApplyBlock for MockStoreSuccess { - async fn apply_block(&self, block: &Block) -> Result<(), ApplyBlockError> { + pub async fn apply_block(&self, block: &Block) -> Result<(), ApplyBlockError> { // Intentionally, we take and hold both locks, to prevent calls to `get_tx_inputs()` from // going through while we're updating the store's data structure let mut locked_accounts = self.accounts.write().await; @@ -240,11 +234,8 @@ impl ApplyBlock for MockStoreSuccess { Ok(()) } -} -#[async_trait] -impl Store for MockStoreSuccess { - async fn get_tx_inputs( + pub async fn get_tx_inputs( &self, proven_tx: &ProvenTransaction, ) -> Result { @@ -290,7 +281,7 @@ impl Store for MockStoreSuccess { }) } - async fn get_block_inputs( + pub async fn get_block_inputs( &self, updated_accounts: impl Iterator + Send, produced_nullifiers: impl Iterator + Send, @@ -352,32 +343,3 @@ impl Store for MockStoreSuccess { }) } } - -#[derive(Default)] -pub struct MockStoreFailure; - -#[async_trait] -impl ApplyBlock for MockStoreFailure { - async fn apply_block(&self, _block: &Block) -> Result<(), ApplyBlockError> { - Err(ApplyBlockError::GrpcClientError(String::new())) - } -} - -#[async_trait] -impl Store for MockStoreFailure { - async fn get_tx_inputs( - &self, - _proven_tx: &ProvenTransaction, - ) -> Result { - Err(TxInputsError::Dummy) - } - - async fn get_block_inputs( - &self, - _updated_accounts: impl Iterator + Send, - _produced_nullifiers: impl Iterator + Send, - _notes: impl Iterator + Send, - ) -> Result { - Err(BlockInputsError::GrpcClientError(String::new())) - } -} From aaaef512528839453191818aecb286dd04dda168 Mon Sep 17 00:00:00 2001 From: Mirko <48352201+Mirko-von-Leipzig@users.noreply.github.com> Date: Tue, 10 Dec 2024 09:37:18 +0200 Subject: [PATCH 24/50] refactor(block-producer): order batches within a block (#572) --- crates/block-producer/src/block_builder/mod.rs | 2 +- crates/block-producer/src/mempool/batch_graph.rs | 12 ++++++------ crates/block-producer/src/mempool/mod.rs | 14 ++++++-------- 3 files changed, 13 insertions(+), 15 deletions(-) diff --git a/crates/block-producer/src/block_builder/mod.rs b/crates/block-producer/src/block_builder/mod.rs index dea87dc03..69b1c54e7 100644 --- a/crates/block-producer/src/block_builder/mod.rs +++ b/crates/block-producer/src/block_builder/mod.rs @@ -69,7 +69,7 @@ impl BlockBuilder { interval.tick().await; let (block_number, batches) = mempool.lock().await.select_block(); - let batches = batches.into_values().collect::>(); + let batches = batches.into_iter().map(|(_, batch)| batch).collect::>(); let mut result = self.build_block(&batches).await; let proving_duration = rand::thread_rng().gen_range(self.simulated_proof_time.clone()); diff --git a/crates/block-producer/src/mempool/batch_graph.rs b/crates/block-producer/src/mempool/batch_graph.rs index 62b4c568c..9d6a614c6 100644 --- a/crates/block-producer/src/mempool/batch_graph.rs +++ b/crates/block-producer/src/mempool/batch_graph.rs @@ -217,11 +217,11 @@ impl BatchGraph { /// Selects the next set of batches ready for inclusion in a block while adhering to the given /// budget. - pub fn select_block( - &mut self, - mut budget: BlockBudget, - ) -> BTreeMap { - let mut batches = BTreeMap::new(); + /// + /// Note that batch order should be maintained to allow for inter-batch dependencies to be + /// correctly resolved. + pub fn select_block(&mut self, mut budget: BlockBudget) -> Vec<(BatchJobId, TransactionBatch)> { + let mut batches = Vec::with_capacity(budget.batches); while let Some(batch_id) = self.inner.roots().first().copied() { // SAFETY: Since it was a root batch, it must definitely have a processed batch @@ -240,7 +240,7 @@ impl BatchGraph { // SAFETY: This is definitely a root since we just selected it from the set of roots. self.inner.process_root(batch_id).expect("root should be processed"); - batches.insert(batch_id, batch); + batches.push((batch_id, batch)); } batches diff --git a/crates/block-producer/src/mempool/mod.rs b/crates/block-producer/src/mempool/mod.rs index 00e96e2ad..5ba1deafb 100644 --- a/crates/block-producer/src/mempool/mod.rs +++ b/crates/block-producer/src/mempool/mod.rs @@ -1,8 +1,4 @@ -use std::{ - collections::{BTreeMap, BTreeSet}, - fmt::Display, - sync::Arc, -}; +use std::{collections::BTreeSet, fmt::Display, sync::Arc}; use batch_graph::BatchGraph; use inflight_state::InflightState; @@ -314,16 +310,18 @@ impl Mempool { /// Select batches for the next block. /// - /// May return an empty set if no batches are ready. + /// Note that the set of batches + /// - may be empty if none are available, and + /// - may contain dependencies and therefore the order must be maintained /// /// # Panics /// /// Panics if there is already a block in flight. - pub fn select_block(&mut self) -> (BlockNumber, BTreeMap) { + pub fn select_block(&mut self) -> (BlockNumber, Vec<(BatchJobId, TransactionBatch)>) { assert!(self.block_in_progress.is_none(), "Cannot have two blocks inflight."); let batches = self.batches.select_block(self.block_budget); - self.block_in_progress = Some(batches.keys().cloned().collect()); + self.block_in_progress = Some(batches.iter().map(|(id, _)| *id).collect()); (self.chain_tip.next(), batches) } From 28c20d0fc9c7e003ff348a8d165af2769f578570 Mon Sep 17 00:00:00 2001 From: Mirko <48352201+Mirko-von-Leipzig@users.noreply.github.com> Date: Tue, 10 Dec 2024 09:37:37 +0200 Subject: [PATCH 25/50] feat(block-producer): remove deep clone of tx data (#570) This was previously added in #544 but was mistakenly dropped by a rebase. --- .../block-producer/src/batch_builder/batch.rs | 86 +++++++++++-------- .../block-producer/src/batch_builder/mod.rs | 8 +- .../src/block_builder/prover/tests.rs | 36 ++++---- crates/block-producer/src/mempool/mod.rs | 7 +- crates/block-producer/src/test_utils/batch.rs | 4 +- 5 files changed, 69 insertions(+), 72 deletions(-) diff --git a/crates/block-producer/src/batch_builder/batch.rs b/crates/block-producer/src/batch_builder/batch.rs index bdede064d..00e535c89 100644 --- a/crates/block-producer/src/batch_builder/batch.rs +++ b/crates/block-producer/src/batch_builder/batch.rs @@ -70,9 +70,13 @@ impl TransactionBatch { // CONSTRUCTORS // -------------------------------------------------------------------------------------------- - /// Returns a new [TransactionBatch] instantiated from the provided vector of proven - /// transactions. If a map of unauthenticated notes found in the store is provided, it is used - /// for transforming unauthenticated notes into authenticated notes. + /// Returns a new [TransactionBatch] built from the provided transactions. If a map of + /// unauthenticated notes found in the store is provided, it is used for transforming + /// unauthenticated notes into authenticated notes. + /// + /// The tx input takes an `IntoIterator` of a reference, which effectively allows for cheap + /// cloning of the iterator. Or put differently, we want something similar to `impl + /// Iterator + Clone` which this provides. /// /// # Errors /// @@ -81,17 +85,21 @@ impl TransactionBatch { /// in the batch. /// - Hashes for corresponding input notes and output notes don't match. #[instrument(target = "miden-block-producer", name = "new_batch", skip_all, err)] - pub fn new( - txs: Vec, + pub fn new<'a, I>( + txs: impl IntoIterator, found_unauthenticated_notes: NoteAuthenticationInfo, - ) -> Result { - let id = Self::compute_id(&txs); + ) -> Result + where + I: Iterator + Clone, + { + let tx_iter = txs.into_iter(); + let id = Self::compute_id(tx_iter.clone()); // Populate batch output notes and updated accounts. - let mut output_notes = OutputNoteTracker::new(&txs)?; + let mut output_notes = OutputNoteTracker::new(tx_iter.clone())?; let mut updated_accounts = BTreeMap::::new(); let mut unauthenticated_input_notes = BTreeSet::new(); - for tx in &txs { + for tx in tx_iter.clone() { // Merge account updates so that state transitions A->B->C become A->C. match updated_accounts.entry(tx.account_id()) { Entry::Vacant(vacant) => { @@ -121,26 +129,28 @@ impl TransactionBatch { // note `x` (i.e., have a circular dependency between transactions), but this is not // a problem. let mut input_notes = vec![]; - for input_note in txs.iter().flat_map(|tx| tx.input_notes().iter()) { - // Header is presented only for unauthenticated input notes. - let input_note = match input_note.header() { - Some(input_note_header) => { - if output_notes.remove_note(input_note_header)? { - continue; - } - - // If an unauthenticated note was found in the store, transform it to an - // authenticated one (i.e. erase additional note details - // except the nullifier) - if found_unauthenticated_notes.contains_note(&input_note_header.id()) { - InputNoteCommitment::from(input_note.nullifier()) - } else { - input_note.clone() - } - }, - None => input_note.clone(), - }; - input_notes.push(input_note) + for tx in tx_iter { + for input_note in tx.input_notes().iter() { + // Header is presented only for unauthenticated input notes. + let input_note = match input_note.header() { + Some(input_note_header) => { + if output_notes.remove_note(input_note_header)? { + continue; + } + + // If an unauthenticated note was found in the store, transform it to an + // authenticated one (i.e. erase additional note details + // except the nullifier) + if found_unauthenticated_notes.contains_note(&input_note_header.id()) { + InputNoteCommitment::from(input_note.nullifier()) + } else { + input_note.clone() + } + }, + None => input_note.clone(), + }; + input_notes.push(input_note) + } } let output_notes = output_notes.into_notes(); @@ -208,8 +218,8 @@ impl TransactionBatch { // HELPER FUNCTIONS // -------------------------------------------------------------------------------------------- - fn compute_id(txs: &[ProvenTransaction]) -> BatchId { - let mut buf = Vec::with_capacity(32 * txs.len()); + fn compute_id<'a>(txs: impl Iterator) -> BatchId { + let mut buf = Vec::with_capacity(32 * txs.size_hint().0); for tx in txs { buf.extend_from_slice(&tx.id().as_bytes()); } @@ -224,7 +234,7 @@ struct OutputNoteTracker { } impl OutputNoteTracker { - fn new(txs: &[ProvenTransaction]) -> Result { + fn new<'a>(txs: impl Iterator) -> Result { let mut output_notes = vec![]; let mut output_note_index = BTreeMap::new(); for tx in txs { @@ -283,7 +293,7 @@ mod tests { fn output_note_tracker_duplicate_output_notes() { let mut txs = mock_proven_txs(); - let result = OutputNoteTracker::new(&txs); + let result = OutputNoteTracker::new(txs.iter()); assert!( result.is_ok(), "Creation of output note tracker was not expected to fail: {result:?}" @@ -297,7 +307,7 @@ mod tests { vec![duplicate_output_note.clone(), mock_output_note(8), mock_output_note(4)], )); - match OutputNoteTracker::new(&txs) { + match OutputNoteTracker::new(txs.iter()) { Err(BuildBatchError::DuplicateOutputNote(note_id)) => { assert_eq!(note_id, duplicate_output_note.id()) }, @@ -308,7 +318,7 @@ mod tests { #[test] fn output_note_tracker_remove_in_place_consumed_note() { let txs = mock_proven_txs(); - let mut tracker = OutputNoteTracker::new(&txs).unwrap(); + let mut tracker = OutputNoteTracker::new(txs.iter()).unwrap(); let note_to_remove = mock_note(4); @@ -333,7 +343,7 @@ mod tests { let mut txs = mock_proven_txs(); let duplicate_note = mock_note(5); txs.push(mock_proven_tx(4, vec![duplicate_note.clone()], vec![mock_output_note(9)])); - match TransactionBatch::new(txs, Default::default()) { + match TransactionBatch::new(&txs, Default::default()) { Err(BuildBatchError::DuplicateUnauthenticatedNote(note_id)) => { assert_eq!(note_id, duplicate_note.id()) }, @@ -351,7 +361,7 @@ mod tests { vec![mock_output_note(9), mock_output_note(10)], )); - let batch = TransactionBatch::new(txs, Default::default()).unwrap(); + let batch = TransactionBatch::new(&txs, Default::default()).unwrap(); // One of the unauthenticated notes must be removed from the batch due to the consumption // of the corresponding output note @@ -395,7 +405,7 @@ mod tests { note_proofs: found_unauthenticated_notes, block_proofs: Default::default(), }; - let batch = TransactionBatch::new(txs, found_unauthenticated_notes).unwrap(); + let batch = TransactionBatch::new(&txs, found_unauthenticated_notes).unwrap(); let expected_input_notes = vec![mock_unauthenticated_note_commitment(1), mock_note(5).nullifier().into()]; diff --git a/crates/block-producer/src/batch_builder/mod.rs b/crates/block-producer/src/batch_builder/mod.rs index e6399ae9c..9cb6ee0f3 100644 --- a/crates/block-producer/src/batch_builder/mod.rs +++ b/crates/block-producer/src/batch_builder/mod.rs @@ -243,13 +243,7 @@ impl WorkerPool { info!(target: COMPONENT, num_txs, "Building a transaction batch"); debug!(target: COMPONENT, txs = %format_array(txs.iter().map(|tx| tx.id().to_hex()))); - // TODO: This is a deep clone which can be avoided by change batch building to using - // refs or arcs. - let txs = txs - .iter() - .map(AuthenticatedTransaction::raw_proven_transaction) - .cloned() - .collect(); + let txs = txs.iter().map(AuthenticatedTransaction::raw_proven_transaction); // TODO: Found unauthenticated notes are no longer required.. potentially? let batch = TransactionBatch::new(txs, Default::default())?; diff --git a/crates/block-producer/src/block_builder/prover/tests.rs b/crates/block-producer/src/block_builder/prover/tests.rs index d8bd8851c..e8463f3c3 100644 --- a/crates/block-producer/src/block_builder/prover/tests.rs +++ b/crates/block-producer/src/block_builder/prover/tests.rs @@ -69,7 +69,7 @@ fn block_witness_validation_inconsistent_account_ids() { ) .build(); - TransactionBatch::new(vec![tx], Default::default()).unwrap() + TransactionBatch::new([&tx], Default::default()).unwrap() }; let batch_2 = { @@ -80,7 +80,7 @@ fn block_witness_validation_inconsistent_account_ids() { ) .build(); - TransactionBatch::new(vec![tx], Default::default()).unwrap() + TransactionBatch::new([&tx], Default::default()).unwrap() }; vec![batch_1, batch_2] @@ -132,7 +132,7 @@ fn block_witness_validation_inconsistent_account_hashes() { let batches = { let batch_1 = TransactionBatch::new( - vec![MockProvenTxBuilder::with_account( + [&MockProvenTxBuilder::with_account( account_id_1, account_1_hash_batches, Digest::default(), @@ -142,7 +142,7 @@ fn block_witness_validation_inconsistent_account_hashes() { ) .unwrap(); let batch_2 = TransactionBatch::new( - vec![MockProvenTxBuilder::with_account( + [&MockProvenTxBuilder::with_account( account_id_2, Digest::default(), Digest::default(), @@ -233,12 +233,8 @@ fn block_witness_multiple_batches_per_account() { }; let batches = { - let batch_1 = - TransactionBatch::new(vec![x_txs[0].clone(), y_txs[1].clone()], Default::default()) - .unwrap(); - let batch_2 = - TransactionBatch::new(vec![y_txs[0].clone(), x_txs[1].clone()], Default::default()) - .unwrap(); + let batch_1 = TransactionBatch::new([&x_txs[0], &y_txs[1]], Default::default()).unwrap(); + let batch_2 = TransactionBatch::new([&y_txs[0], &x_txs[1]], Default::default()).unwrap(); vec![batch_1, batch_2] }; @@ -334,8 +330,8 @@ async fn compute_account_root_success() { }) .collect(); - let batch_1 = TransactionBatch::new(txs[..2].to_vec(), Default::default()).unwrap(); - let batch_2 = TransactionBatch::new(txs[2..].to_vec(), Default::default()).unwrap(); + let batch_1 = TransactionBatch::new(&txs[..2], Default::default()).unwrap(); + let batch_2 = TransactionBatch::new(&txs[2..], Default::default()).unwrap(); vec![batch_1, batch_2] }; @@ -552,8 +548,8 @@ async fn compute_note_root_success() { }) .collect(); - let batch_1 = TransactionBatch::new(txs[..2].to_vec(), Default::default()).unwrap(); - let batch_2 = TransactionBatch::new(txs[2..].to_vec(), Default::default()).unwrap(); + let batch_1 = TransactionBatch::new(&txs[..2], Default::default()).unwrap(); + let batch_2 = TransactionBatch::new(&txs[2..], Default::default()).unwrap(); vec![batch_1, batch_2] }; @@ -609,13 +605,13 @@ fn block_witness_validation_inconsistent_nullifiers() { let batch_1 = { let tx = MockProvenTxBuilder::with_account_index(0).nullifiers_range(0..1).build(); - TransactionBatch::new(vec![tx], Default::default()).unwrap() + TransactionBatch::new([&tx], Default::default()).unwrap() }; let batch_2 = { let tx = MockProvenTxBuilder::with_account_index(1).nullifiers_range(1..2).build(); - TransactionBatch::new(vec![tx], Default::default()).unwrap() + TransactionBatch::new([&tx], Default::default()).unwrap() }; vec![batch_1, batch_2] @@ -688,13 +684,13 @@ async fn compute_nullifier_root_empty_success() { let batch_1 = { let tx = MockProvenTxBuilder::with_account_index(0).build(); - TransactionBatch::new(vec![tx], Default::default()).unwrap() + TransactionBatch::new([&tx], Default::default()).unwrap() }; let batch_2 = { let tx = MockProvenTxBuilder::with_account_index(1).build(); - TransactionBatch::new(vec![tx], Default::default()).unwrap() + TransactionBatch::new([&tx], Default::default()).unwrap() }; vec![batch_1, batch_2] @@ -742,13 +738,13 @@ async fn compute_nullifier_root_success() { let batch_1 = { let tx = MockProvenTxBuilder::with_account_index(0).nullifiers_range(0..1).build(); - TransactionBatch::new(vec![tx], Default::default()).unwrap() + TransactionBatch::new([&tx], Default::default()).unwrap() }; let batch_2 = { let tx = MockProvenTxBuilder::with_account_index(1).nullifiers_range(1..2).build(); - TransactionBatch::new(vec![tx], Default::default()).unwrap() + TransactionBatch::new([&tx], Default::default()).unwrap() }; vec![batch_1, batch_2] diff --git a/crates/block-producer/src/mempool/mod.rs b/crates/block-producer/src/mempool/mod.rs index 5ba1deafb..670033d34 100644 --- a/crates/block-producer/src/mempool/mod.rs +++ b/crates/block-producer/src/mempool/mod.rs @@ -416,11 +416,8 @@ mod tests { uut.batch_failed(child_batch_a); assert_eq!(uut, reference); - let proof = TransactionBatch::new( - vec![txs[2].raw_proven_transaction().clone()], - Default::default(), - ) - .unwrap(); + let proof = + TransactionBatch::new([txs[2].raw_proven_transaction()], Default::default()).unwrap(); uut.batch_proved(child_batch_b, proof); assert_eq!(uut, reference); } diff --git a/crates/block-producer/src/test_utils/batch.rs b/crates/block-producer/src/test_utils/batch.rs index 2717a3a79..53a572b9d 100644 --- a/crates/block-producer/src/test_utils/batch.rs +++ b/crates/block-producer/src/test_utils/batch.rs @@ -24,7 +24,7 @@ impl TransactionBatchConstructor for TransactionBatch { }) .collect(); - Self::new(txs, Default::default()).unwrap() + Self::new(&txs, Default::default()).unwrap() } fn from_txs(starting_account_index: u32, num_txs_in_batch: u64) -> Self { @@ -36,6 +36,6 @@ impl TransactionBatchConstructor for TransactionBatch { }) .collect(); - Self::new(txs, Default::default()).unwrap() + Self::new(&txs, Default::default()).unwrap() } } From 30f18d1446168bc2d1eedba6b7f57e7b878614cd Mon Sep 17 00:00:00 2001 From: Mirko <48352201+Mirko-von-Leipzig@users.noreply.github.com> Date: Wed, 11 Dec 2024 08:52:36 +0200 Subject: [PATCH 26/50] chore: move new changelog entries to unreleased (#579) --- CHANGELOG.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a2352883..85647670d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,13 +1,18 @@ # Changelog -## v0.6.0 (2024-11-05) +## Unreleased ### Enhancements -- Added `GetAccountProofs` endpoint (#506). - Support Https in endpoint configuration (#556). - Upgrade `block-producer` from FIFO queue to mempool dependency graph (#562). +## v0.6.0 (2024-11-05) + +### Enhancements + +- Added `GetAccountProofs` endpoint (#506). + ### Changes - [BREAKING] Added `kernel_root` to block header's protobuf message definitions (#496). From 5b8531b08bba54d18bc8af204137007fe98d3645 Mon Sep 17 00:00:00 2001 From: Mirko <48352201+Mirko-von-Leipzig@users.noreply.github.com> Date: Wed, 11 Dec 2024 08:53:10 +0200 Subject: [PATCH 27/50] chore(proto): replace miette with anyhow (#576) --- Cargo.lock | 10 +--------- crates/proto/Cargo.toml | 2 +- crates/proto/build.rs | 21 +++++++++++---------- 3 files changed, 13 insertions(+), 20 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d18f90e80..9f69727cd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1607,10 +1607,10 @@ dependencies = [ name = "miden-node-proto" version = "0.6.0" dependencies = [ + "anyhow", "hex", "miden-node-utils", "miden-objects", - "miette", "proptest", "prost", "prost-build", @@ -1792,16 +1792,8 @@ version = "7.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4edc8853320c2a0dab800fbda86253c8938f6ea88510dc92c5f1ed20e794afc1" dependencies = [ - "backtrace", - "backtrace-ext", "cfg-if", "miette-derive", - "owo-colors", - "supports-color", - "supports-hyperlinks", - "supports-unicode", - "terminal_size", - "textwrap", "thiserror 1.0.68", "unicode-width 0.1.14", ] diff --git a/crates/proto/Cargo.toml b/crates/proto/Cargo.toml index 4caf9fba0..de58f3512 100644 --- a/crates/proto/Cargo.toml +++ b/crates/proto/Cargo.toml @@ -23,7 +23,7 @@ tonic = { workspace = true } proptest = { version = "1.5" } [build-dependencies] -miette = { version = "7.2", features = ["fancy"] } +anyhow = { version = "1.0" } prost = { workspace = true } prost-build = { version = "0.13" } protox = { version = "0.7" } diff --git a/crates/proto/build.rs b/crates/proto/build.rs index 5f9c13384..b1382f2ca 100644 --- a/crates/proto/build.rs +++ b/crates/proto/build.rs @@ -3,14 +3,14 @@ use std::{ path::{Path, PathBuf}, }; -use miette::IntoDiagnostic; +use anyhow::Context; use protox::prost::Message; /// Generates Rust protobuf bindings from .proto files in the root directory. /// /// This is done only if BUILD_PROTO environment variable is set to `1` to avoid running the script /// on crates.io where repo-level .proto files are not available. -fn main() -> miette::Result<()> { +fn main() -> anyhow::Result<()> { println!("cargo::rerun-if-changed=../../proto"); println!("cargo::rerun-if-env-changed=BUILD_PROTO"); @@ -24,21 +24,21 @@ fn main() -> miette::Result<()> { let dst_dir = crate_root.join("src").join("generated"); // Remove all existing files. - fs::remove_dir_all(&dst_dir).into_diagnostic()?; - fs::create_dir(&dst_dir).into_diagnostic()?; + fs::remove_dir_all(&dst_dir).context("removing existing files")?; + fs::create_dir(&dst_dir).context("creating destination folder")?; // Compute the directory of the `proto` definitions - let cwd: PathBuf = env::current_dir().into_diagnostic()?; + let cwd: PathBuf = env::current_dir().context("current directory")?; let cwd = cwd .parent() .and_then(|p| p.parent()) - .ok_or_else(|| miette::miette!("Failed to navigate two directories up"))?; + .context("navigating to grandparent directory")?; let proto_dir: PathBuf = cwd.join("proto"); // Compute the compiler's target file path. - let out = env::var("OUT_DIR").into_diagnostic()?; + let out = env::var("OUT_DIR").context("env::OUT_DIR not set")?; let file_descriptor_path = PathBuf::from(out).join("file_descriptor_set.bin"); // Compile the proto file for all servers APIs @@ -49,7 +49,8 @@ fn main() -> miette::Result<()> { ]; let includes = &[proto_dir]; let file_descriptors = protox::compile(protos, includes)?; - fs::write(&file_descriptor_path, file_descriptors.encode_to_vec()).into_diagnostic()?; + fs::write(&file_descriptor_path, file_descriptors.encode_to_vec()) + .context("writing file descriptors")?; let mut prost_config = prost_build::Config::new(); prost_config.skip_debug(["AccountId", "Digest"]); @@ -60,9 +61,9 @@ fn main() -> miette::Result<()> { .skip_protoc_run() .out_dir(&dst_dir) .compile_protos_with_config(prost_config, protos, includes) - .into_diagnostic()?; + .context("compiling protobufs")?; - generate_mod_rs(&dst_dir).into_diagnostic()?; + generate_mod_rs(&dst_dir).context("generating mod.rs")?; Ok(()) } From c9cc21144b905b660b3b5eb021421aadd39f9420 Mon Sep 17 00:00:00 2001 From: Mirko <48352201+Mirko-von-Leipzig@users.noreply.github.com> Date: Wed, 11 Dec 2024 11:36:24 +0200 Subject: [PATCH 28/50] refactor(block-producer): cleanup StoreClient (#574) Instruments all functions and consolidates errors into a single enum. --- crates/block-producer/src/block.rs | 6 +- .../block-producer/src/block_builder/mod.rs | 8 ++- crates/block-producer/src/errors.rs | 46 ++++---------- crates/block-producer/src/store/mod.rs | 62 +++++++------------ crates/block-producer/src/test_utils/store.rs | 9 +-- crates/proto/src/errors.rs | 2 + 6 files changed, 49 insertions(+), 84 deletions(-) diff --git a/crates/block-producer/src/block.rs b/crates/block-producer/src/block.rs index 27715fed7..5d47d3efd 100644 --- a/crates/block-producer/src/block.rs +++ b/crates/block-producer/src/block.rs @@ -13,8 +13,6 @@ use miden_objects::{ BlockHeader, Digest, }; -use crate::store::BlockInputsError; - // BLOCK INPUTS // ================================================================================================ @@ -44,12 +42,12 @@ pub struct AccountWitness { } impl TryFrom for BlockInputs { - type Error = BlockInputsError; + type Error = ConversionError; fn try_from(response: GetBlockInputsResponse) -> Result { let block_header: BlockHeader = response .block_header - .ok_or(GetBlockInputsResponse::missing_field(stringify!(block_header)))? + .ok_or(miden_node_proto::generated::block::BlockHeader::missing_field("block_header"))? .try_into()?; let chain_peaks = { diff --git a/crates/block-producer/src/block_builder/mod.rs b/crates/block-producer/src/block_builder/mod.rs index 69b1c54e7..545b95705 100644 --- a/crates/block-producer/src/block_builder/mod.rs +++ b/crates/block-producer/src/block_builder/mod.rs @@ -135,7 +135,8 @@ impl BlockBuilder { produced_nullifiers.iter(), dangling_notes.iter(), ) - .await?; + .await + .map_err(BuildBlockError::GetBlockInputsFailed)?; let missing_notes: Vec<_> = dangling_notes .difference(&block_inputs.found_unauthenticated_notes.note_ids()) @@ -160,7 +161,10 @@ impl BlockBuilder { info!(target: COMPONENT, block_num, %block_hash, "block built"); debug!(target: COMPONENT, ?block); - self.store.apply_block(&block).await?; + self.store + .apply_block(&block) + .await + .map_err(BuildBlockError::ApplyBlockFailed)?; info!(target: COMPONENT, block_num, %block_hash, "block committed"); diff --git a/crates/block-producer/src/errors.rs b/crates/block-producer/src/errors.rs index b2e9b13c7..27f7b0564 100644 --- a/crates/block-producer/src/errors.rs +++ b/crates/block-producer/src/errors.rs @@ -2,7 +2,7 @@ use miden_node_proto::errors::ConversionError; use miden_node_utils::formatting::format_opt; use miden_objects::{ accounts::AccountId, - crypto::merkle::{MerkleError, MmrError}, + crypto::merkle::MerkleError, notes::{NoteId, Nullifier}, transaction::TransactionId, AccountDeltaError, Digest, TransactionInputError, @@ -65,7 +65,7 @@ pub enum VerifyTxError { /// TODO: Make this an "internal error". Q: Should we have a single `InternalError` enum for /// all internal errors that can occur across the system? #[error("Failed to retrieve transaction inputs from the store: {0}")] - StoreConnectionFailed(#[from] TxInputsError), + StoreConnectionFailed(#[from] StoreError), #[error("Transaction input error: {0}")] TransactionInputError(#[from] TransactionInputError), @@ -157,29 +157,6 @@ pub enum BlockProverError { InvalidRootOutput(&'static str), } -// Block inputs errors -// ================================================================================================= - -#[allow(clippy::enum_variant_names)] -#[derive(Debug, Error)] -pub enum BlockInputsError { - #[error("failed to parse protobuf message: {0}")] - ConversionError(#[from] ConversionError), - #[error("MmrPeaks error: {0}")] - MmrPeaksError(#[from] MmrError), - #[error("gRPC client failed with error: {0}")] - GrpcClientError(String), -} - -// Block applying errors -// ================================================================================================= - -#[derive(Debug, Error)] -pub enum ApplyBlockError { - #[error("gRPC client failed with error: {0}")] - GrpcClientError(String), -} - // Block building errors // ================================================================================================= @@ -188,9 +165,9 @@ pub enum BuildBlockError { #[error("failed to compute new block: {0}")] BlockProverFailed(#[from] BlockProverError), #[error("failed to apply block: {0}")] - ApplyBlockFailed(#[from] ApplyBlockError), + ApplyBlockFailed(#[source] StoreError), #[error("failed to get block inputs from store: {0}")] - GetBlockInputsFailed(#[from] BlockInputsError), + GetBlockInputsFailed(#[source] StoreError), #[error("store did not produce data for account: {0}")] MissingAccountInput(AccountId), #[error("store produced extra account data. Offending accounts: {0:?}")] @@ -210,17 +187,16 @@ pub enum BuildBlockError { InjectedFailure, } -// Transaction inputs errors +// Store errors // ================================================================================================= +/// Errors returned by the [StoreClient](crate::store::StoreClient). #[derive(Debug, Error)] -pub enum TxInputsError { - #[error("gRPC client failed with error: {0}")] - GrpcClientError(String), +pub enum StoreError { + #[error("gRPC client error")] + GrpcClientError(#[from] tonic::Status), #[error("malformed response from store: {0}")] MalformedResponse(String), - #[error("failed to parse protobuf message: {0}")] - ConversionError(#[from] ConversionError), - #[error("dummy")] - Dummy, + #[error("failed to parse response")] + DeserializationError(#[from] ConversionError), } diff --git a/crates/block-producer/src/store/mod.rs b/crates/block-producer/src/store/mod.rs index 38691d350..3b9ec7aa7 100644 --- a/crates/block-producer/src/store/mod.rs +++ b/crates/block-producer/src/store/mod.rs @@ -28,8 +28,7 @@ use miden_processor::crypto::RpoDigest; use tonic::transport::Channel; use tracing::{debug, info, instrument}; -pub use crate::errors::{ApplyBlockError, BlockInputsError, TxInputsError}; -use crate::{block::BlockInputs, COMPONENT}; +use crate::{block::BlockInputs, errors::StoreError, COMPONENT}; // TRANSACTION INPUTS // ================================================================================================ @@ -121,34 +120,37 @@ impl TryFrom for TransactionInputs { /// Essentially just a thin wrapper around the generated gRPC client which improves type safety. #[derive(Clone)] pub struct StoreClient { - store: store_client::ApiClient, + inner: store_client::ApiClient, } impl StoreClient { /// TODO: this should probably take store connection string and create a connection internally pub fn new(store: store_client::ApiClient) -> Self { - Self { store } + Self { inner: store } } /// Returns the latest block's header from the store. - pub async fn latest_header(&self) -> Result { - // TODO: Consolidate the error types returned by the store (and its trait). + #[instrument(target = "miden-block-producer", skip_all, err)] + pub async fn latest_header(&self) -> Result { let response = self - .store + .inner .clone() .get_block_header_by_number(tonic::Request::new(Default::default())) - .await - .map_err(|err| err.to_string())? - .into_inner(); - - BlockHeader::try_from(response.block_header.unwrap()).map_err(|err| err.to_string()) + .await? + .into_inner() + .block_header + .ok_or(miden_node_proto::generated::block::BlockHeader::missing_field( + "block_header", + ))?; + + BlockHeader::try_from(response).map_err(Into::into) } #[instrument(target = "miden-block-producer", skip_all, err)] pub async fn get_tx_inputs( &self, proven_tx: &ProvenTransaction, - ) -> Result { + ) -> Result { let message = GetTransactionInputsRequest { account_id: Some(proven_tx.account_id().into()), nullifiers: proven_tx.get_nullifiers().map(Into::into).collect(), @@ -162,20 +164,14 @@ impl StoreClient { debug!(target: COMPONENT, ?message); let request = tonic::Request::new(message); - let response = self - .store - .clone() - .get_transaction_inputs(request) - .await - .map_err(|status| TxInputsError::GrpcClientError(status.message().to_string()))? - .into_inner(); + let response = self.inner.clone().get_transaction_inputs(request).await?.into_inner(); debug!(target: COMPONENT, ?response); let tx_inputs: TransactionInputs = response.try_into()?; if tx_inputs.account_id != proven_tx.account_id() { - return Err(TxInputsError::MalformedResponse(format!( + return Err(StoreError::MalformedResponse(format!( "incorrect account id returned from store. Got: {}, expected: {}", tx_inputs.account_id, proven_tx.account_id() @@ -187,40 +183,28 @@ impl StoreClient { Ok(tx_inputs) } + #[instrument(target = "miden-block-producer", skip_all, err)] pub async fn get_block_inputs( &self, updated_accounts: impl Iterator + Send, produced_nullifiers: impl Iterator + Send, notes: impl Iterator + Send, - ) -> Result { + ) -> Result { let request = tonic::Request::new(GetBlockInputsRequest { account_ids: updated_accounts.map(Into::into).collect(), nullifiers: produced_nullifiers.map(digest::Digest::from).collect(), unauthenticated_notes: notes.map(digest::Digest::from).collect(), }); - let store_response = self - .store - .clone() - .get_block_inputs(request) - .await - .map_err(|err| BlockInputsError::GrpcClientError(err.message().to_string()))? - .into_inner(); + let store_response = self.inner.clone().get_block_inputs(request).await?.into_inner(); - store_response.try_into() + store_response.try_into().map_err(Into::into) } #[instrument(target = "miden-block-producer", skip_all, err)] - pub async fn apply_block(&self, block: &Block) -> Result<(), ApplyBlockError> { + pub async fn apply_block(&self, block: &Block) -> Result<(), StoreError> { let request = tonic::Request::new(ApplyBlockRequest { block: block.to_bytes() }); - let _ = self - .store - .clone() - .apply_block(request) - .await - .map_err(|status| ApplyBlockError::GrpcClientError(status.message().to_string()))?; - - Ok(()) + self.inner.clone().apply_block(request).await.map(|_| ()).map_err(Into::into) } } diff --git a/crates/block-producer/src/test_utils/store.rs b/crates/block-producer/src/test_utils/store.rs index b6ebe629a..4b3b40a95 100644 --- a/crates/block-producer/src/test_utils/store.rs +++ b/crates/block-producer/src/test_utils/store.rs @@ -18,7 +18,8 @@ use super::*; use crate::{ batch_builder::TransactionBatch, block::{AccountWitness, BlockInputs}, - store::{ApplyBlockError, BlockInputsError, TransactionInputs, TxInputsError}, + errors::StoreError, + store::TransactionInputs, test_utils::block::{ block_output_notes, flatten_output_notes, note_created_smt_from_note_batches, }, @@ -183,7 +184,7 @@ impl MockStoreSuccess { locked_accounts.root() } - pub async fn apply_block(&self, block: &Block) -> Result<(), ApplyBlockError> { + pub async fn apply_block(&self, block: &Block) -> Result<(), StoreError> { // Intentionally, we take and hold both locks, to prevent calls to `get_tx_inputs()` from // going through while we're updating the store's data structure let mut locked_accounts = self.accounts.write().await; @@ -238,7 +239,7 @@ impl MockStoreSuccess { pub async fn get_tx_inputs( &self, proven_tx: &ProvenTransaction, - ) -> Result { + ) -> Result { let locked_accounts = self.accounts.read().await; let locked_produced_nullifiers = self.produced_nullifiers.read().await; @@ -286,7 +287,7 @@ impl MockStoreSuccess { updated_accounts: impl Iterator + Send, produced_nullifiers: impl Iterator + Send, notes: impl Iterator + Send, - ) -> Result { + ) -> Result { let locked_accounts = self.accounts.read().await; let locked_produced_nullifiers = self.produced_nullifiers.read().await; diff --git a/crates/proto/src/errors.rs b/crates/proto/src/errors.rs index bf5967014..ae3acab58 100644 --- a/crates/proto/src/errors.rs +++ b/crates/proto/src/errors.rs @@ -26,6 +26,8 @@ pub enum ConversionError { entity: &'static str, field_name: &'static str, }, + #[error("MMR error: {0}")] + MmrError(#[from] miden_objects::crypto::merkle::MmrError), } pub trait MissingFieldHelper { From 2775f5ab6a0a6f4d9d7670268d01209f46b0d73c Mon Sep 17 00:00:00 2001 From: Mirko <48352201+Mirko-von-Leipzig@users.noreply.github.com> Date: Wed, 11 Dec 2024 11:51:34 +0200 Subject: [PATCH 29/50] refactor(block-producer): replace BatchJobId with BatchId (#577) --- .../block-producer/src/batch_builder/batch.rs | 69 ++++++++++------ .../block-producer/src/batch_builder/mod.rs | 28 +++---- .../block-producer/src/block_builder/mod.rs | 5 +- .../block-producer/src/mempool/batch_graph.rs | 79 +++++++------------ crates/block-producer/src/mempool/mod.rs | 61 ++++---------- 5 files changed, 106 insertions(+), 136 deletions(-) diff --git a/crates/block-producer/src/batch_builder/batch.rs b/crates/block-producer/src/batch_builder/batch.rs index 00e535c89..b06dcc5a3 100644 --- a/crates/block-producer/src/batch_builder/batch.rs +++ b/crates/block-producer/src/batch_builder/batch.rs @@ -1,9 +1,11 @@ use std::{ + borrow::Borrow, collections::{btree_map::Entry, BTreeMap, BTreeSet}, mem, }; use miden_node_proto::domain::notes::NoteAuthenticationInfo; +use miden_node_utils::formatting::format_blake3_digest; use miden_objects::{ accounts::{delta::AccountUpdateDetails, AccountId}, batches::BatchNoteTree, @@ -16,23 +18,36 @@ use tracing::instrument; use crate::errors::BuildBatchError; -pub type BatchId = Blake3Digest<32>; - -// TRANSACTION BATCH +// BATCH ID // ================================================================================================ -/// A batch of transactions that share a common proof. -/// -/// Note: Until recursive proofs are available in the Miden VM, we don't include the common proof. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct TransactionBatch { - id: BatchId, - updated_accounts: BTreeMap, - input_notes: Vec, - output_notes_smt: BatchNoteTree, - output_notes: Vec, +/// Uniquely identifies a [TransactionBatch]. +#[derive(Debug, Copy, Clone, Eq, Ord, PartialEq, PartialOrd)] +pub struct BatchId(Blake3Digest<32>); + +impl BatchId { + /// Calculates a batch ID from the given set of transactions. + pub fn compute(txs: impl Iterator) -> Self + where + T: Borrow, + { + let mut buf = Vec::with_capacity(32 * txs.size_hint().0); + for tx in txs { + buf.extend_from_slice(&tx.borrow().as_bytes()); + } + Self(Blake3_256::hash(&buf)) + } } +impl std::fmt::Display for BatchId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&format_blake3_digest(self.0)) + } +} + +// ACCOUNT UPDATE +// ================================================================================================ + #[derive(Debug, Clone, PartialEq, Eq)] pub struct AccountUpdate { pub init_state: Digest, @@ -66,6 +81,21 @@ impl AccountUpdate { } } +// TRANSACTION BATCH +// ================================================================================================ + +/// A batch of transactions that share a common proof. +/// +/// Note: Until recursive proofs are available in the Miden VM, we don't include the common proof. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TransactionBatch { + id: BatchId, + updated_accounts: BTreeMap, + input_notes: Vec, + output_notes_smt: BatchNoteTree, + output_notes: Vec, +} + impl TransactionBatch { // CONSTRUCTORS // -------------------------------------------------------------------------------------------- @@ -93,7 +123,7 @@ impl TransactionBatch { I: Iterator + Clone, { let tx_iter = txs.into_iter(); - let id = Self::compute_id(tx_iter.clone()); + let id = BatchId::compute(tx_iter.clone().map(ProvenTransaction::id)); // Populate batch output notes and updated accounts. let mut output_notes = OutputNoteTracker::new(tx_iter.clone())?; @@ -214,17 +244,6 @@ impl TransactionBatch { pub fn output_notes(&self) -> &Vec { &self.output_notes } - - // HELPER FUNCTIONS - // -------------------------------------------------------------------------------------------- - - fn compute_id<'a>(txs: impl Iterator) -> BatchId { - let mut buf = Vec::with_capacity(32 * txs.size_hint().0); - for tx in txs { - buf.extend_from_slice(&tx.id().as_bytes()); - } - Blake3_256::hash(&buf) - } } #[derive(Debug)] diff --git a/crates/block-producer/src/batch_builder/mod.rs b/crates/block-producer/src/batch_builder/mod.rs index 9cb6ee0f3..86d43c1f8 100644 --- a/crates/block-producer/src/batch_builder/mod.rs +++ b/crates/block-producer/src/batch_builder/mod.rs @@ -1,18 +1,18 @@ use std::{num::NonZeroUsize, ops::Range, time::Duration}; +use batch::BatchId; use rand::Rng; use tokio::{task::JoinSet, time}; use tracing::{debug, info, instrument, Span}; use crate::{ - domain::transaction::AuthenticatedTransaction, - mempool::{BatchJobId, SharedMempool}, - COMPONENT, SERVER_BUILD_BATCH_FREQUENCY, + domain::transaction::AuthenticatedTransaction, mempool::SharedMempool, COMPONENT, + SERVER_BUILD_BATCH_FREQUENCY, }; pub mod batch; pub use batch::TransactionBatch; -use miden_node_utils::formatting::{format_array, format_blake3_digest}; +use miden_node_utils::formatting::format_array; use crate::errors::BuildBatchError; @@ -91,8 +91,8 @@ impl BatchBuilder { tracing::warn!(%batch_id, %err, "Batch job failed."); mempool.batch_failed(batch_id); }, - Ok((batch_id, batch)) => { - mempool.batch_proved(batch_id, batch); + Ok(batch) => { + mempool.batch_proved(batch); } } } @@ -104,7 +104,7 @@ impl BatchBuilder { // BATCH WORKER // ================================================================================================ -type BatchResult = Result<(BatchJobId, TransactionBatch), (BatchJobId, BuildBatchError)>; +type BatchResult = Result; /// Represents a pool of batch provers. /// @@ -121,7 +121,7 @@ struct WorkerPool { /// This allows us to map panic'd tasks to the job ID. Uses [Vec] because the task ID does not /// implement [Ord]. Given that the expected capacity is relatively low, this has no real /// impact beyond ergonomics. - task_map: Vec<(tokio::task::Id, BatchJobId)>, + task_map: Vec<(tokio::task::Id, BatchId)>, } impl WorkerPool { @@ -171,10 +171,10 @@ impl WorkerPool { // Cleanup task mapping by removing the result's task. This is inefficient but does not // matter as the capacity is expected to be low. let job_id = match &result { - Ok((id, _)) => id, - Err((id, _)) => id, + Ok(batch) => batch.id(), + Err((id, _)) => *id, }; - self.task_map.retain(|(_, elem_job_id)| elem_job_id != job_id); + self.task_map.retain(|(_, elem_job_id)| *elem_job_id != job_id); result } @@ -192,7 +192,7 @@ impl WorkerPool { /// [has_capacity](Self::has_capacity). fn spawn( &mut self, - id: BatchJobId, + id: BatchId, transactions: Vec, ) -> Result<(), ()> { if !self.has_capacity() { @@ -224,7 +224,7 @@ impl WorkerPool { tracing::debug!("Batch proof completed."); - Ok((id, batch)) + Ok(batch) } }) .id(); @@ -247,7 +247,7 @@ impl WorkerPool { // TODO: Found unauthenticated notes are no longer required.. potentially? let batch = TransactionBatch::new(txs, Default::default())?; - Span::current().record("batch_id", format_blake3_digest(batch.id())); + Span::current().record("batch_id", batch.id().to_string()); info!(target: COMPONENT, "Transaction batch built"); Ok(batch) diff --git a/crates/block-producer/src/block_builder/mod.rs b/crates/block-producer/src/block_builder/mod.rs index 545b95705..404a132d9 100644 --- a/crates/block-producer/src/block_builder/mod.rs +++ b/crates/block-producer/src/block_builder/mod.rs @@ -1,6 +1,6 @@ use std::{collections::BTreeSet, ops::Range}; -use miden_node_utils::formatting::{format_array, format_blake3_digest}; +use miden_node_utils::formatting::format_array; use miden_objects::{ accounts::AccountId, block::Block, @@ -69,7 +69,6 @@ impl BlockBuilder { interval.tick().await; let (block_number, batches) = mempool.lock().await.select_block(); - let batches = batches.into_iter().map(|(_, batch)| batch).collect::>(); let mut result = self.build_block(&batches).await; let proving_duration = rand::thread_rng().gen_range(self.simulated_proof_time.clone()); @@ -96,7 +95,7 @@ impl BlockBuilder { info!( target: COMPONENT, num_batches = batches.len(), - batches = %format_array(batches.iter().map(|batch| format_blake3_digest(batch.id()))), + batches = %format_array(batches.iter().map(|batch| batch.id())), ); let updated_account_set: BTreeSet = batches diff --git a/crates/block-producer/src/mempool/batch_graph.rs b/crates/block-producer/src/mempool/batch_graph.rs index 9d6a614c6..14dae83da 100644 --- a/crates/block-producer/src/mempool/batch_graph.rs +++ b/crates/block-producer/src/mempool/batch_graph.rs @@ -4,9 +4,9 @@ use miden_objects::transaction::TransactionId; use super::{ graph::{DependencyGraph, GraphError}, - BatchJobId, BlockBudget, BudgetStatus, + BlockBudget, BudgetStatus, }; -use crate::batch_builder::batch::TransactionBatch; +use crate::batch_builder::batch::{BatchId, TransactionBatch}; // BATCH GRAPH // ================================================================================================ @@ -53,19 +53,19 @@ use crate::batch_builder::batch::TransactionBatch; #[derive(Default, Debug, Clone, PartialEq)] pub struct BatchGraph { /// Tracks the interdependencies between batches. - inner: DependencyGraph, + inner: DependencyGraph, /// Maps each transaction to its batch, allowing for reverse lookups. /// /// Incoming batches are defined entirely in terms of transactions, including parent edges. /// This let's us transform these parent transactions into the relevant parent batches. - transactions: BTreeMap, + transactions: BTreeMap, /// Maps each batch to its transaction set. /// /// Required because the dependency graph is defined in terms of batches. This let's us /// translate between batches and their transactions when required. - batches: BTreeMap>, + batches: BTreeMap>, } #[derive(thiserror::Error, Debug, Clone, PartialEq, Eq)] @@ -75,7 +75,7 @@ pub enum BatchInsertError { #[error("Unknown parent transaction {0}")] UnknownParentTransaction(TransactionId), #[error(transparent)] - GraphError(#[from] GraphError), + GraphError(#[from] GraphError), } impl BatchGraph { @@ -85,6 +85,10 @@ impl BatchGraph { /// includes transactions within the same batch i.e. a transaction and parent transaction may /// both be in this batch. /// + /// # Returns + /// + /// The new batch's ID. + /// /// # Errors /// /// Returns an error if: @@ -93,10 +97,9 @@ impl BatchGraph { /// - any parent transactions are _not_ in the graph pub fn insert( &mut self, - id: BatchJobId, transactions: Vec, mut parents: BTreeSet, - ) -> Result<(), BatchInsertError> { + ) -> Result { let duplicates = transactions .iter() .filter(|tx| self.transactions.contains_key(tx)) @@ -121,6 +124,7 @@ impl BatchGraph { }) .collect::>()?; + let id = BatchId::compute(transactions.iter()); self.inner.insert_pending(id, parent_batches)?; for tx in transactions.iter().copied() { @@ -128,7 +132,7 @@ impl BatchGraph { } self.batches.insert(id, transactions); - Ok(()) + Ok(id) } /// Removes the batches and their descendants from the graph. @@ -142,8 +146,8 @@ impl BatchGraph { /// Returns an error if any of the batches are not currently in the graph. pub fn remove_batches( &mut self, - batch_ids: BTreeSet, - ) -> Result>, GraphError> { + batch_ids: BTreeSet, + ) -> Result>, GraphError> { // This returns all descendent batches as well. let batch_ids = self.inner.purge_subgraphs(batch_ids)?; @@ -183,8 +187,8 @@ impl BatchGraph { /// The last point implies that batches should be removed in block order. pub fn prune_committed( &mut self, - batch_ids: BTreeSet, - ) -> Result, GraphError> { + batch_ids: BTreeSet, + ) -> Result, GraphError> { // This clone could be elided by moving this call to the end. This would lose the atomic // property of this method though its unclear what value (if any) that has. self.inner.prune_processed(batch_ids.clone())?; @@ -207,12 +211,8 @@ impl BatchGraph { /// # Errors /// /// Returns an error if the batch is not in the graph or if it was already previously proven. - pub fn submit_proof( - &mut self, - id: BatchJobId, - batch: TransactionBatch, - ) -> Result<(), GraphError> { - self.inner.promote_pending(id, batch) + pub fn submit_proof(&mut self, batch: TransactionBatch) -> Result<(), GraphError> { + self.inner.promote_pending(batch.id(), batch) } /// Selects the next set of batches ready for inclusion in a block while adhering to the given @@ -220,7 +220,7 @@ impl BatchGraph { /// /// Note that batch order should be maintained to allow for inter-batch dependencies to be /// correctly resolved. - pub fn select_block(&mut self, mut budget: BlockBudget) -> Vec<(BatchJobId, TransactionBatch)> { + pub fn select_block(&mut self, mut budget: BlockBudget) -> Vec { let mut batches = Vec::with_capacity(budget.batches); while let Some(batch_id) = self.inner.roots().first().copied() { @@ -240,14 +240,14 @@ impl BatchGraph { // SAFETY: This is definitely a root since we just selected it from the set of roots. self.inner.process_root(batch_id).expect("root should be processed"); - batches.push((batch_id, batch)); + batches.push(batch); } batches } /// Returns `true` if the graph contains the given batch. - pub fn contains(&self, id: &BatchJobId) -> bool { + pub fn contains(&self, id: &BatchId) -> bool { self.batches.contains_key(id) } } @@ -260,18 +260,6 @@ mod tests { // INSERT TESTS // ================================================================================================ - #[test] - fn insert_rejects_duplicate_batch_ids() { - let id = BatchJobId::new(1); - let mut uut = BatchGraph::default(); - - uut.insert(id, Default::default(), Default::default()).unwrap(); - let err = uut.insert(id, Default::default(), Default::default()).unwrap_err(); - let expected = BatchInsertError::GraphError(GraphError::DuplicateKey(id)); - - assert_eq!(err, expected); - } - #[test] fn insert_rejects_duplicate_transactions() { let mut rng = Random::with_random_seed(); @@ -280,10 +268,8 @@ mod tests { let mut uut = BatchGraph::default(); - uut.insert(BatchJobId::new(1), vec![tx_dup], Default::default()).unwrap(); - let err = uut - .insert(BatchJobId::new(2), vec![tx_dup, tx_non_dup], Default::default()) - .unwrap_err(); + uut.insert(vec![tx_dup], Default::default()).unwrap(); + let err = uut.insert(vec![tx_dup, tx_non_dup], Default::default()).unwrap_err(); let expected = BatchInsertError::DuplicateTransactions([tx_dup].into()); assert_eq!(err, expected); @@ -297,7 +283,7 @@ mod tests { let mut uut = BatchGraph::default(); - let err = uut.insert(BatchJobId::new(2), vec![tx], [missing].into()).unwrap_err(); + let err = uut.insert(vec![tx], [missing].into()).unwrap_err(); let expected = BatchInsertError::UnknownParentTransaction(missing); assert_eq!(err, expected); @@ -311,7 +297,7 @@ mod tests { let child = rng.draw_tx_id(); let mut uut = BatchGraph::default(); - uut.insert(BatchJobId::new(2), vec![parent, child], [parent].into()).unwrap(); + uut.insert(vec![parent, child], [parent].into()).unwrap(); } // PURGE_SUBGRAPHS TESTS @@ -326,16 +312,11 @@ mod tests { let child_batch_txs = (0..5).map(|_| rng.draw_tx_id()).collect::>(); let disjoint_batch_txs = (0..5).map(|_| rng.draw_tx_id()).collect(); - let parent_batch_id = BatchJobId::new(0); - let child_batch_id = BatchJobId::new(1); - let disjoint_batch_id = BatchJobId::new(2); - let mut uut = BatchGraph::default(); - uut.insert(parent_batch_id, parent_batch_txs.clone(), Default::default()) - .unwrap(); - uut.insert(child_batch_id, child_batch_txs.clone(), [parent_batch_txs[0]].into()) - .unwrap(); - uut.insert(disjoint_batch_id, disjoint_batch_txs, Default::default()).unwrap(); + let parent_batch_id = uut.insert(parent_batch_txs.clone(), Default::default()).unwrap(); + let child_batch_id = + uut.insert(child_batch_txs.clone(), [parent_batch_txs[0]].into()).unwrap(); + uut.insert(disjoint_batch_txs, Default::default()).unwrap(); let result = uut.remove_batches([parent_batch_id].into()).unwrap(); let expected = diff --git a/crates/block-producer/src/mempool/mod.rs b/crates/block-producer/src/mempool/mod.rs index 670033d34..3e329b7ff 100644 --- a/crates/block-producer/src/mempool/mod.rs +++ b/crates/block-producer/src/mempool/mod.rs @@ -9,8 +9,10 @@ use tokio::sync::Mutex; use transaction_graph::TransactionGraph; use crate::{ - batch_builder::batch::TransactionBatch, domain::transaction::AuthenticatedTransaction, - errors::AddTransactionError, SERVER_MAX_BATCHES_PER_BLOCK, SERVER_MAX_TXS_PER_BATCH, + batch_builder::batch::{BatchId, TransactionBatch}, + domain::transaction::AuthenticatedTransaction, + errors::AddTransactionError, + SERVER_MAX_BATCHES_PER_BLOCK, SERVER_MAX_TXS_PER_BATCH, }; mod batch_graph; @@ -18,26 +20,6 @@ mod graph; mod inflight_state; mod transaction_graph; -#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] -pub struct BatchJobId(u64); - -impl Display for BatchJobId { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - self.0.fmt(f) - } -} - -impl BatchJobId { - pub fn increment(&mut self) { - self.0 += 1; - } - - #[cfg(test)] - pub fn new(value: u64) -> Self { - Self(value) - } -} - #[derive(Debug, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] pub struct BlockNumber(u32); @@ -186,14 +168,11 @@ pub struct Mempool { /// Inflight batches. batches: BatchGraph, - /// The next batches ID. - next_batch_id: BatchJobId, - /// The current block height of the chain. chain_tip: BlockNumber, /// The current inflight block, if any. - block_in_progress: Option>, + block_in_progress: Option>, block_budget: BlockBudget, batch_budget: BatchBudget, @@ -224,7 +203,6 @@ impl Mempool { block_in_progress: Default::default(), transactions: Default::default(), batches: Default::default(), - next_batch_id: Default::default(), } } @@ -256,19 +234,14 @@ impl Mempool { /// Transactions are returned in a valid execution ordering. /// /// Returns `None` if no transactions are available. - pub fn select_batch(&mut self) -> Option<(BatchJobId, Vec)> { + pub fn select_batch(&mut self) -> Option<(BatchId, Vec)> { let (batch, parents) = self.transactions.select_batch(self.batch_budget); if batch.is_empty() { return None; } - let tx_ids = batch.iter().map(AuthenticatedTransaction::id).collect(); - - let batch_id = self.next_batch_id; - self.next_batch_id.increment(); + let tx_ids = batch.iter().map(AuthenticatedTransaction::id).collect::>(); - self.batches - .insert(batch_id, tx_ids, parents) - .expect("Selected batch should insert"); + let batch_id = self.batches.insert(tx_ids, parents).expect("Selected batch should insert"); Some((batch_id, batch)) } @@ -276,7 +249,7 @@ impl Mempool { /// Drops the failed batch and all of its descendants. /// /// Transactions are placed back in the queue. - pub fn batch_failed(&mut self, batch: BatchJobId) { + pub fn batch_failed(&mut self, batch: BatchId) { // Batch may already have been removed as part of a parent batches failure. if !self.batches.contains(&batch) { return; @@ -299,13 +272,13 @@ impl Mempool { } /// Marks a batch as proven if it exists. - pub fn batch_proved(&mut self, batch_id: BatchJobId, batch: TransactionBatch) { + pub fn batch_proved(&mut self, batch: TransactionBatch) { // Batch may have been removed as part of a parent batches failure. - if !self.batches.contains(&batch_id) { + if !self.batches.contains(&batch.id()) { return; } - self.batches.submit_proof(batch_id, batch).expect("Batch proof should submit"); + self.batches.submit_proof(batch).expect("Batch proof should submit"); } /// Select batches for the next block. @@ -317,11 +290,11 @@ impl Mempool { /// # Panics /// /// Panics if there is already a block in flight. - pub fn select_block(&mut self) -> (BlockNumber, Vec<(BatchJobId, TransactionBatch)>) { + pub fn select_block(&mut self) -> (BlockNumber, Vec) { assert!(self.block_in_progress.is_none(), "Cannot have two blocks inflight."); let batches = self.batches.select_block(self.block_budget); - self.block_in_progress = Some(batches.iter().map(|(id, _)| *id).collect()); + self.block_in_progress = Some(batches.iter().map(TransactionBatch::id).collect()); (self.chain_tip.next(), batches) } @@ -405,7 +378,7 @@ mod tests { assert_eq!(batch_txs, vec![txs[1].clone()]); uut.add_transaction(txs[2].clone()).unwrap(); - let (child_batch_b, batch_txs) = uut.select_batch().unwrap(); + let (_, batch_txs) = uut.select_batch().unwrap(); assert_eq!(batch_txs, vec![txs[2].clone()]); // Child batch jobs are now dangling. @@ -418,7 +391,7 @@ mod tests { let proof = TransactionBatch::new([txs[2].raw_proven_transaction()], Default::default()).unwrap(); - uut.batch_proved(child_batch_b, proof); + uut.batch_proved(proof); assert_eq!(uut, reference); } @@ -444,8 +417,6 @@ mod tests { reference.select_batch().unwrap(); reference.add_transaction(txs[1].clone()).unwrap(); reference.add_transaction(txs[2].clone()).unwrap(); - reference.next_batch_id.increment(); - reference.next_batch_id.increment(); assert_eq!(uut, reference); } From 9986d74a3d8c13b82d71f23489af0450dcee48b5 Mon Sep 17 00:00:00 2001 From: Mirko <48352201+Mirko-von-Leipzig@users.noreply.github.com> Date: Wed, 11 Dec 2024 12:20:19 +0200 Subject: [PATCH 30/50] feat(block-producer): instrument mempool (#578) Also updates `tracing` so we can use `target = const` in `#[instrument]` --- Cargo.lock | 12 +++--- .../block-producer/src/batch_builder/batch.rs | 4 +- .../block-producer/src/batch_builder/mod.rs | 2 +- .../block-producer/src/block_builder/mod.rs | 2 +- crates/block-producer/src/mempool/mod.rs | 10 ++++- crates/block-producer/src/server/api.rs | 2 +- crates/block-producer/src/server/mod.rs | 2 +- crates/block-producer/src/state_view/mod.rs | 6 +-- crates/block-producer/src/store/mod.rs | 8 ++-- crates/rpc/src/server/api.rs | 22 +++++------ crates/store/src/db/migrations.rs | 2 +- crates/store/src/db/mod.rs | 36 +++++++++--------- crates/store/src/server/api.rs | 38 +++++++++---------- crates/store/src/state.rs | 20 +++++----- 14 files changed, 87 insertions(+), 79 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9f69727cd..2d91e4f56 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3192,9 +3192,9 @@ checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "tracing" -version = "0.1.40" +version = "0.1.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" +checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" dependencies = [ "log", "pin-project-lite", @@ -3204,9 +3204,9 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.27" +version = "0.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" +checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" dependencies = [ "proc-macro2", "quote", @@ -3215,9 +3215,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.32" +version = "0.1.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" +checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c" dependencies = [ "once_cell", "valuable", diff --git a/crates/block-producer/src/batch_builder/batch.rs b/crates/block-producer/src/batch_builder/batch.rs index b06dcc5a3..b6230b5ac 100644 --- a/crates/block-producer/src/batch_builder/batch.rs +++ b/crates/block-producer/src/batch_builder/batch.rs @@ -16,7 +16,7 @@ use miden_objects::{ }; use tracing::instrument; -use crate::errors::BuildBatchError; +use crate::{errors::BuildBatchError, COMPONENT}; // BATCH ID // ================================================================================================ @@ -114,7 +114,7 @@ impl TransactionBatch { /// - There are duplicated output notes or unauthenticated notes found across all transactions /// in the batch. /// - Hashes for corresponding input notes and output notes don't match. - #[instrument(target = "miden-block-producer", name = "new_batch", skip_all, err)] + #[instrument(target = COMPONENT, name = "new_batch", skip_all, err)] pub fn new<'a, I>( txs: impl IntoIterator, found_unauthenticated_notes: NoteAuthenticationInfo, diff --git a/crates/block-producer/src/batch_builder/mod.rs b/crates/block-producer/src/batch_builder/mod.rs index 86d43c1f8..2bfc7b151 100644 --- a/crates/block-producer/src/batch_builder/mod.rs +++ b/crates/block-producer/src/batch_builder/mod.rs @@ -234,7 +234,7 @@ impl WorkerPool { Ok(()) } - #[instrument(target = "miden-block-producer", skip_all, err, fields(batch_id))] + #[instrument(target = COMPONENT, skip_all, err, fields(batch_id))] fn build_batch( txs: Vec, ) -> Result { diff --git a/crates/block-producer/src/block_builder/mod.rs b/crates/block-producer/src/block_builder/mod.rs index 404a132d9..d007c34ac 100644 --- a/crates/block-producer/src/block_builder/mod.rs +++ b/crates/block-producer/src/block_builder/mod.rs @@ -90,7 +90,7 @@ impl BlockBuilder { } } - #[instrument(target = "miden-block-producer", skip_all, err)] + #[instrument(target = COMPONENT, skip_all, err)] async fn build_block(&self, batches: &[TransactionBatch]) -> Result<(), BuildBlockError> { info!( target: COMPONENT, diff --git a/crates/block-producer/src/mempool/mod.rs b/crates/block-producer/src/mempool/mod.rs index 3e329b7ff..8b1d756b1 100644 --- a/crates/block-producer/src/mempool/mod.rs +++ b/crates/block-producer/src/mempool/mod.rs @@ -6,13 +6,14 @@ use miden_objects::{ MAX_ACCOUNTS_PER_BATCH, MAX_INPUT_NOTES_PER_BATCH, MAX_OUTPUT_NOTES_PER_BATCH, }; use tokio::sync::Mutex; +use tracing::instrument; use transaction_graph::TransactionGraph; use crate::{ batch_builder::batch::{BatchId, TransactionBatch}, domain::transaction::AuthenticatedTransaction, errors::AddTransactionError, - SERVER_MAX_BATCHES_PER_BLOCK, SERVER_MAX_TXS_PER_BATCH, + COMPONENT, SERVER_MAX_BATCHES_PER_BLOCK, SERVER_MAX_TXS_PER_BATCH, }; mod batch_graph; @@ -215,6 +216,7 @@ impl Mempool { /// # Errors /// /// Returns an error if the transaction's initial conditions don't match the current state. + #[instrument(target = COMPONENT, skip_all, fields(tx=%transaction.id()))] pub fn add_transaction( &mut self, transaction: AuthenticatedTransaction, @@ -234,6 +236,7 @@ impl Mempool { /// Transactions are returned in a valid execution ordering. /// /// Returns `None` if no transactions are available. + #[instrument(target = COMPONENT, skip_all)] pub fn select_batch(&mut self) -> Option<(BatchId, Vec)> { let (batch, parents) = self.transactions.select_batch(self.batch_budget); if batch.is_empty() { @@ -249,6 +252,7 @@ impl Mempool { /// Drops the failed batch and all of its descendants. /// /// Transactions are placed back in the queue. + #[instrument(target = COMPONENT, skip_all, fields(batch))] pub fn batch_failed(&mut self, batch: BatchId) { // Batch may already have been removed as part of a parent batches failure. if !self.batches.contains(&batch) { @@ -272,6 +276,7 @@ impl Mempool { } /// Marks a batch as proven if it exists. + #[instrument(target = COMPONENT, skip_all, fields(batch=%batch.id()))] pub fn batch_proved(&mut self, batch: TransactionBatch) { // Batch may have been removed as part of a parent batches failure. if !self.batches.contains(&batch.id()) { @@ -290,6 +295,7 @@ impl Mempool { /// # Panics /// /// Panics if there is already a block in flight. + #[instrument(target = COMPONENT, skip_all)] pub fn select_block(&mut self) -> (BlockNumber, Vec) { assert!(self.block_in_progress.is_none(), "Cannot have two blocks inflight."); @@ -304,6 +310,7 @@ impl Mempool { /// # Panics /// /// Panics if blocks are completed out-of-order or if there is no block in flight. + #[instrument(target = COMPONENT, skip_all, fields(block_number))] pub fn block_committed(&mut self, block_number: BlockNumber) { assert_eq!(block_number, self.chain_tip.next(), "Blocks must be submitted sequentially"); @@ -326,6 +333,7 @@ impl Mempool { /// /// Panics if there is no block in flight or if the block number does not match the current /// inflight block. + #[instrument(target = COMPONENT, skip_all, fields(block_number))] pub fn block_failed(&mut self, block_number: BlockNumber) { assert_eq!(block_number, self.chain_tip.next(), "Blocks must be submitted sequentially"); diff --git a/crates/block-producer/src/server/api.rs b/crates/block-producer/src/server/api.rs index 22405feda..cb800e791 100644 --- a/crates/block-producer/src/server/api.rs +++ b/crates/block-producer/src/server/api.rs @@ -35,7 +35,7 @@ where BB: BatchBuilder, { #[instrument( - target = "miden-block-producer", + target = COMPONENT, name = "block_producer:submit_proven_transaction", skip_all, err diff --git a/crates/block-producer/src/server/mod.rs b/crates/block-producer/src/server/mod.rs index c2cf192e8..ffaf87190 100644 --- a/crates/block-producer/src/server/mod.rs +++ b/crates/block-producer/src/server/mod.rs @@ -201,7 +201,7 @@ impl BlockProducerRpcServer { } #[instrument( - target = "miden-block-producer", + target = COMPONENT, name = "block_producer:submit_proven_transaction", skip_all, err diff --git a/crates/block-producer/src/state_view/mod.rs b/crates/block-producer/src/state_view/mod.rs index 7fb69faff..15710eadf 100644 --- a/crates/block-producer/src/state_view/mod.rs +++ b/crates/block-producer/src/state_view/mod.rs @@ -138,7 +138,7 @@ impl ApplyBlock for DefaultStateView where S: Store, { - #[instrument(target = "miden-block-producer", skip_all, err)] + #[instrument(target = COMPONENT, skip_all, err)] async fn apply_block(&self, block: &Block) -> Result<(), ApplyBlockError> { self.store.apply_block(block).await?; @@ -179,7 +179,7 @@ where /// - all notes in `tx_notes_not_in_store` are currently in flight /// /// The account state is not verified as it is performed by [InflightAccountStates]. -#[instrument(target = "miden-block-producer", skip_all, err)] +#[instrument(target = COMPONENT, skip_all, err)] fn ensure_in_flight_constraints( candidate_tx: &ProvenTransaction, accounts_in_flight: &InflightAccountStates, @@ -225,7 +225,7 @@ fn ensure_in_flight_constraints( /// - input notes must not be already consumed /// /// Returns a list of unauthenticated input notes that were not found in the store. -#[instrument(target = "miden-block-producer", skip_all, err)] +#[instrument(target = COMPONENT, skip_all, err)] fn ensure_tx_inputs_constraints( candidate_tx: &ProvenTransaction, tx_inputs: TransactionInputs, diff --git a/crates/block-producer/src/store/mod.rs b/crates/block-producer/src/store/mod.rs index 3b9ec7aa7..c69760f9f 100644 --- a/crates/block-producer/src/store/mod.rs +++ b/crates/block-producer/src/store/mod.rs @@ -130,7 +130,7 @@ impl StoreClient { } /// Returns the latest block's header from the store. - #[instrument(target = "miden-block-producer", skip_all, err)] + #[instrument(target = COMPONENT, skip_all, err)] pub async fn latest_header(&self) -> Result { let response = self .inner @@ -146,7 +146,7 @@ impl StoreClient { BlockHeader::try_from(response).map_err(Into::into) } - #[instrument(target = "miden-block-producer", skip_all, err)] + #[instrument(target = COMPONENT, skip_all, err)] pub async fn get_tx_inputs( &self, proven_tx: &ProvenTransaction, @@ -183,7 +183,7 @@ impl StoreClient { Ok(tx_inputs) } - #[instrument(target = "miden-block-producer", skip_all, err)] + #[instrument(target = COMPONENT, skip_all, err)] pub async fn get_block_inputs( &self, updated_accounts: impl Iterator + Send, @@ -201,7 +201,7 @@ impl StoreClient { store_response.try_into().map_err(Into::into) } - #[instrument(target = "miden-block-producer", skip_all, err)] + #[instrument(target = COMPONENT, skip_all, err)] pub async fn apply_block(&self, block: &Block) -> Result<(), StoreError> { let request = tonic::Request::new(ApplyBlockRequest { block: block.to_bytes() }); diff --git a/crates/rpc/src/server/api.rs b/crates/rpc/src/server/api.rs index 56f97c788..e815781e4 100644 --- a/crates/rpc/src/server/api.rs +++ b/crates/rpc/src/server/api.rs @@ -59,7 +59,7 @@ impl RpcApi { #[tonic::async_trait] impl api_server::Api for RpcApi { #[instrument( - target = "miden-rpc", + target = COMPONENT, name = "rpc:check_nullifiers", skip_all, ret(level = "debug"), @@ -82,7 +82,7 @@ impl api_server::Api for RpcApi { } #[instrument( - target = "miden-rpc", + target = COMPONENT, name = "rpc:check_nullifiers_by_prefix", skip_all, ret(level = "debug"), @@ -98,7 +98,7 @@ impl api_server::Api for RpcApi { } #[instrument( - target = "miden-rpc", + target = COMPONENT, name = "rpc:get_block_header_by_number", skip_all, ret(level = "debug"), @@ -114,7 +114,7 @@ impl api_server::Api for RpcApi { } #[instrument( - target = "miden-rpc", + target = COMPONENT, name = "rpc:sync_state", skip_all, ret(level = "debug"), @@ -130,7 +130,7 @@ impl api_server::Api for RpcApi { } #[instrument( - target = "miden-rpc", + target = COMPONENT, name = "rpc:sync_notes", skip_all, ret(level = "debug"), @@ -146,7 +146,7 @@ impl api_server::Api for RpcApi { } #[instrument( - target = "miden-rpc", + target = COMPONENT, name = "rpc:get_notes_by_id", skip_all, ret(level = "debug"), @@ -167,7 +167,7 @@ impl api_server::Api for RpcApi { self.store.clone().get_notes_by_id(request).await } - #[instrument(target = "miden-rpc", name = "rpc:submit_proven_transaction", skip_all, err)] + #[instrument(target = COMPONENT, name = "rpc:submit_proven_transaction", skip_all, err)] async fn submit_proven_transaction( &self, request: Request, @@ -190,7 +190,7 @@ impl api_server::Api for RpcApi { /// Returns details for public (public) account by id. #[instrument( - target = "miden-rpc", + target = COMPONENT, name = "rpc:get_account_details", skip_all, ret(level = "debug"), @@ -214,7 +214,7 @@ impl api_server::Api for RpcApi { } #[instrument( - target = "miden-rpc", + target = COMPONENT, name = "rpc:get_block_by_number", skip_all, ret(level = "debug"), @@ -232,7 +232,7 @@ impl api_server::Api for RpcApi { } #[instrument( - target = "miden-rpc", + target = COMPONENT, name = "rpc:get_account_state_delta", skip_all, ret(level = "debug"), @@ -250,7 +250,7 @@ impl api_server::Api for RpcApi { } #[instrument( - target = "miden-rpc", + target = COMPONENT, name = "rpc:get_account_proofs", skip_all, ret(level = "debug"), diff --git a/crates/store/src/db/migrations.rs b/crates/store/src/db/migrations.rs index 0336685de..ef7547ee2 100644 --- a/crates/store/src/db/migrations.rs +++ b/crates/store/src/db/migrations.rs @@ -24,7 +24,7 @@ fn up(s: &'static str) -> M<'static> { const DB_MIGRATION_HASH_FIELD: &str = "db-migration-hash"; const DB_SCHEMA_VERSION_FIELD: &str = "db-schema-version"; -#[instrument(target = "miden-store", skip_all, err)] +#[instrument(target = COMPONENT, skip_all, err)] pub fn apply_migrations(conn: &mut Connection) -> super::Result<()> { let version_before = MIGRATIONS.current_version(conn)?; diff --git a/crates/store/src/db/mod.rs b/crates/store/src/db/mod.rs index 554ecf794..ac757b095 100644 --- a/crates/store/src/db/mod.rs +++ b/crates/store/src/db/mod.rs @@ -132,7 +132,7 @@ impl Db { /// Open a connection to the DB, apply any pending migrations, and ensure that the genesis block /// is as expected and present in the database. // TODO: This span is logged in a root span, we should connect it to the parent one. - #[instrument(target = "miden-store", skip_all)] + #[instrument(target = COMPONENT, skip_all)] pub async fn setup( config: StoreConfig, block_store: Arc, @@ -199,7 +199,7 @@ impl Db { } /// Loads all the nullifiers from the DB. - #[instrument(target = "miden-store", skip_all, ret(level = "debug"), err)] + #[instrument(target = COMPONENT, skip_all, ret(level = "debug"), err)] pub async fn select_all_nullifiers(&self) -> Result> { self.pool .get() @@ -212,7 +212,7 @@ impl Db { } /// Loads the nullifiers that match the prefixes from the DB. - #[instrument(target = "miden-store", skip_all, ret(level = "debug"), err)] + #[instrument(target = COMPONENT, skip_all, ret(level = "debug"), err)] pub async fn select_nullifiers_by_prefix( &self, prefix_len: u32, @@ -233,7 +233,7 @@ impl Db { } /// Loads all the notes from the DB. - #[instrument(target = "miden-store", skip_all, ret(level = "debug"), err)] + #[instrument(target = COMPONENT, skip_all, ret(level = "debug"), err)] pub async fn select_all_notes(&self) -> Result> { self.pool.get().await?.interact(sql::select_all_notes).await.map_err(|err| { DatabaseError::InteractError(format!("Select notes task failed: {err}")) @@ -241,7 +241,7 @@ impl Db { } /// Loads all the accounts from the DB. - #[instrument(target = "miden-store", skip_all, ret(level = "debug"), err)] + #[instrument(target = COMPONENT, skip_all, ret(level = "debug"), err)] pub async fn select_all_accounts(&self) -> Result> { self.pool.get().await?.interact(sql::select_all_accounts).await.map_err(|err| { DatabaseError::InteractError(format!("Select accounts task failed: {err}")) @@ -251,7 +251,7 @@ impl Db { /// Search for a [BlockHeader] from the database by its `block_num`. /// /// When `block_number` is [None], the latest block header is returned. - #[instrument(target = "miden-store", skip_all, ret(level = "debug"), err)] + #[instrument(target = COMPONENT, skip_all, ret(level = "debug"), err)] pub async fn select_block_header_by_block_num( &self, block_number: Option, @@ -267,7 +267,7 @@ impl Db { } /// Loads multiple block headers from the DB. - #[instrument(target = "miden-store", skip_all, ret(level = "debug"), err)] + #[instrument(target = COMPONENT, skip_all, ret(level = "debug"), err)] pub async fn select_block_headers(&self, blocks: Vec) -> Result> { self.pool .get() @@ -282,7 +282,7 @@ impl Db { } /// Loads all the block headers from the DB. - #[instrument(target = "miden-store", skip_all, ret(level = "debug"), err)] + #[instrument(target = COMPONENT, skip_all, ret(level = "debug"), err)] pub async fn select_all_block_headers(&self) -> Result> { self.pool .get() @@ -295,7 +295,7 @@ impl Db { } /// Loads all the account hashes from the DB. - #[instrument(target = "miden-store", skip_all, ret(level = "debug"), err)] + #[instrument(target = COMPONENT, skip_all, ret(level = "debug"), err)] pub async fn select_all_account_hashes(&self) -> Result> { self.pool .get() @@ -308,7 +308,7 @@ impl Db { } /// Loads public account details from the DB. - #[instrument(target = "miden-store", skip_all, ret(level = "debug"), err)] + #[instrument(target = COMPONENT, skip_all, ret(level = "debug"), err)] pub async fn select_account(&self, id: AccountId) -> Result { self.pool .get() @@ -321,7 +321,7 @@ impl Db { } /// Loads public accounts details from the DB. - #[instrument(target = "miden-store", skip_all, ret(level = "debug"), err)] + #[instrument(target = COMPONENT, skip_all, ret(level = "debug"), err)] pub async fn select_accounts_by_ids( &self, account_ids: Vec, @@ -336,7 +336,7 @@ impl Db { })? } - #[instrument(target = "miden-store", skip_all, ret(level = "debug"), err)] + #[instrument(target = COMPONENT, skip_all, ret(level = "debug"), err)] pub async fn get_state_sync( &self, block_num: BlockNumber, @@ -357,7 +357,7 @@ impl Db { })? } - #[instrument(target = "miden-store", skip_all, ret(level = "debug"), err)] + #[instrument(target = COMPONENT, skip_all, ret(level = "debug"), err)] pub async fn get_note_sync( &self, block_num: BlockNumber, @@ -375,7 +375,7 @@ impl Db { } /// Loads all the Note's matching a certain NoteId from the database. - #[instrument(target = "miden-store", skip_all, ret(level = "debug"), err)] + #[instrument(target = COMPONENT, skip_all, ret(level = "debug"), err)] pub async fn select_notes_by_id(&self, note_ids: Vec) -> Result> { self.pool .get() @@ -388,7 +388,7 @@ impl Db { } /// Loads inclusion proofs for notes matching the given IDs. - #[instrument(target = "miden-store", skip_all, ret(level = "debug"), err)] + #[instrument(target = COMPONENT, skip_all, ret(level = "debug"), err)] pub async fn select_note_inclusion_proofs( &self, note_ids: BTreeSet, @@ -406,7 +406,7 @@ impl Db { } /// Loads all note IDs matching a certain NoteId from the database. - #[instrument(target = "miden-store", skip_all, ret(level = "debug"), err)] + #[instrument(target = COMPONENT, skip_all, ret(level = "debug"), err)] pub async fn select_note_ids(&self, note_ids: Vec) -> Result> { self.select_notes_by_id(note_ids) .await @@ -418,7 +418,7 @@ impl Db { /// `allow_acquire` and `acquire_done` are used to synchronize writes to the DB with writes to /// the in-memory trees. Further details available on [super::state::State::apply_block]. // TODO: This span is logged in a root span, we should connect it to the parent one. - #[instrument(target = "miden-store", skip_all, err)] + #[instrument(target = COMPONENT, skip_all, err)] pub async fn apply_block( &self, allow_acquire: oneshot::Sender<()>, @@ -485,7 +485,7 @@ impl Db { /// If the database is empty, generates and stores the genesis block. Otherwise, it ensures that /// the genesis block in the database is consistent with the genesis block data in the /// genesis JSON file. - #[instrument(target = "miden-store", skip_all, err)] + #[instrument(target = COMPONENT, skip_all, err)] async fn ensure_genesis_block( &self, genesis_filepath: &str, diff --git a/crates/store/src/server/api.rs b/crates/store/src/server/api.rs index 4b9903046..8eeadcf8c 100644 --- a/crates/store/src/server/api.rs +++ b/crates/store/src/server/api.rs @@ -59,7 +59,7 @@ impl api_server::Api for StoreApi { /// /// If the block number is not provided, block header for the latest block is returned. #[instrument( - target = "miden-store", + target = COMPONENT, name = "store:get_block_header_by_number", skip_all, ret(level = "debug"), @@ -91,7 +91,7 @@ impl api_server::Api for StoreApi { /// This endpoint also returns Merkle authentication path for each requested nullifier which can /// be verified against the latest root of the nullifier database. #[instrument( - target = "miden-store", + target = COMPONENT, name = "store:check_nullifiers", skip_all, ret(level = "debug"), @@ -115,7 +115,7 @@ impl api_server::Api for StoreApi { /// /// Currently the only supported prefix length is 16 bits. #[instrument( - target = "miden-store", + target = COMPONENT, name = "store:check_nullifiers_by_prefix", skip_all, ret(level = "debug"), @@ -148,7 +148,7 @@ impl api_server::Api for StoreApi { /// Returns info which can be used by the client to sync up to the latest state of the chain /// for the objects the client is interested in. #[instrument( - target = "miden-store", + target = COMPONENT, name = "store:sync_state", skip_all, ret(level = "debug"), @@ -212,7 +212,7 @@ impl api_server::Api for StoreApi { /// Returns info which can be used by the client to sync note state. #[instrument( - target = "miden-store", + target = COMPONENT, name = "store:sync_notes", skip_all, ret(level = "debug"), @@ -244,7 +244,7 @@ impl api_server::Api for StoreApi { /// /// If the list is empty or no Note matched the requested NoteId and empty list is returned. #[instrument( - target = "miden-store", + target = COMPONENT, name = "store:get_notes_by_id", skip_all, ret(level = "debug"), @@ -276,7 +276,7 @@ impl api_server::Api for StoreApi { /// Returns a list of Note inclusion proofs for the specified NoteId's. #[instrument( - target = "miden-store", + target = COMPONENT, name = "store:get_note_inclusion_proofs", skip_all, ret(level = "debug"), @@ -312,7 +312,7 @@ impl api_server::Api for StoreApi { /// Returns details for public (public) account by id. #[instrument( - target = "miden-store", + target = COMPONENT, name = "store:get_account_details", skip_all, ret(level = "debug"), @@ -340,7 +340,7 @@ impl api_server::Api for StoreApi { /// Updates the local DB by inserting a new block header and the related data. #[instrument( - target = "miden-store", + target = COMPONENT, name = "store:apply_block", skip_all, ret(level = "debug"), @@ -376,7 +376,7 @@ impl api_server::Api for StoreApi { /// Returns data needed by the block producer to construct and prove the next block. #[instrument( - target = "miden-store", + target = COMPONENT, name = "store:get_block_inputs", skip_all, ret(level = "debug"), @@ -402,7 +402,7 @@ impl api_server::Api for StoreApi { } #[instrument( - target = "miden-store", + target = COMPONENT, name = "store:get_transaction_inputs", skip_all, ret(level = "debug"), @@ -450,7 +450,7 @@ impl api_server::Api for StoreApi { } #[instrument( - target = "miden-store", + target = COMPONENT, name = "store:get_block_by_number", skip_all, ret(level = "debug"), @@ -470,7 +470,7 @@ impl api_server::Api for StoreApi { } #[instrument( - target = "miden-store", + target = COMPONENT, name = "store:get_account_proofs", skip_all, ret(level = "debug"), @@ -508,7 +508,7 @@ impl api_server::Api for StoreApi { } #[instrument( - target = "miden-store", + target = COMPONENT, name = "store:get_account_state_delta", skip_all, ret(level = "debug"), @@ -540,7 +540,7 @@ impl api_server::Api for StoreApi { /// Returns a list of all nullifiers #[instrument( - target = "miden-store", + target = COMPONENT, name = "store:list_nullifiers", skip_all, ret(level = "debug"), @@ -563,7 +563,7 @@ impl api_server::Api for StoreApi { /// Returns a list of all notes #[instrument( - target = "miden-store", + target = COMPONENT, name = "store:list_notes", skip_all, ret(level = "debug"), @@ -579,7 +579,7 @@ impl api_server::Api for StoreApi { /// Returns a list of all accounts #[instrument( - target = "miden-store", + target = COMPONENT, name = "store:list_accounts", skip_all, ret(level = "debug"), @@ -607,7 +607,7 @@ fn invalid_argument(err: E) -> Status { Status::invalid_argument(err.to_string()) } -#[instrument(target = "miden-store", skip_all, err)] +#[instrument(target = COMPONENT, skip_all, err)] fn validate_nullifiers(nullifiers: &[generated::digest::Digest]) -> Result, Status> { nullifiers .iter() @@ -617,7 +617,7 @@ fn validate_nullifiers(nullifiers: &[generated::digest::Digest]) -> Result Result, Status> { notes .iter() diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index caaf0b5c0..fb00628c1 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -126,7 +126,7 @@ pub struct State { impl State { /// Loads the state from the `db`. - #[instrument(target = "miden-store", skip_all)] + #[instrument(target = COMPONENT, skip_all)] pub async fn load( mut db: Db, block_store: Arc, @@ -167,7 +167,7 @@ impl State { /// - the in-memory structures are updated, including the latest block pointer and the lock is /// released. // TODO: This span is logged in a root span, we should connect it to the parent span. - #[instrument(target = "miden-store", skip_all, err)] + #[instrument(target = COMPONENT, skip_all, err)] pub async fn apply_block(&self, block: Block) -> Result<(), ApplyBlockError> { let _lock = self.writer.try_lock().map_err(|_| ApplyBlockError::ConcurrentWrite)?; @@ -377,7 +377,7 @@ impl State { /// /// If [None] is given as the value of `block_num`, the data for the latest [BlockHeader] is /// returned. - #[instrument(target = "miden-store", skip_all, ret(level = "debug"), err)] + #[instrument(target = COMPONENT, skip_all, ret(level = "debug"), err)] pub async fn get_block_header( &self, block_num: Option, @@ -410,7 +410,7 @@ impl State { /// tree. /// /// Note: these proofs are invalidated once the nullifier tree is modified, i.e. on a new block. - #[instrument(target = "miden-store", skip_all, ret(level = "debug"))] + #[instrument(target = COMPONENT, skip_all, ret(level = "debug"))] pub async fn check_nullifiers(&self, nullifiers: &[Nullifier]) -> Vec { let inner = self.inner.read().await; nullifiers.iter().map(|n| inner.nullifier_tree.open(n)).collect() @@ -502,7 +502,7 @@ impl State { /// with any matches tags. /// - `nullifier_prefixes`: Only the 16 high bits of the nullifiers the client is interested in, /// results will include nullifiers matching prefixes produced in the given block range. - #[instrument(target = "miden-store", skip_all, ret(level = "debug"), err)] + #[instrument(target = COMPONENT, skip_all, ret(level = "debug"), err)] pub async fn sync_state( &self, block_num: BlockNumber, @@ -552,7 +552,7 @@ impl State { /// - `block_num`: The last block *known* by the client, updates start from the next block. /// - `note_tags`: The tags the client is interested in, resulting notes are restricted to the /// first block containing a matching note. - #[instrument(target = "miden-store", skip_all, ret(level = "debug"), err)] + #[instrument(target = COMPONENT, skip_all, ret(level = "debug"), err)] pub async fn sync_notes( &self, block_num: BlockNumber, @@ -635,7 +635,7 @@ impl State { } /// Returns data needed by the block producer to verify transactions validity. - #[instrument(target = "miden-store", skip_all, ret)] + #[instrument(target = COMPONENT, skip_all, ret)] pub async fn get_transaction_inputs( &self, account_id: AccountId, @@ -792,7 +792,7 @@ impl State { // UTILITIES // ================================================================================================ -#[instrument(target = "miden-store", skip_all)] +#[instrument(target = COMPONENT, skip_all)] async fn load_nullifier_tree(db: &mut Db) -> Result { let nullifiers = db.select_all_nullifiers().await?; let len = nullifiers.len(); @@ -811,7 +811,7 @@ async fn load_nullifier_tree(db: &mut Db) -> Result Result { let block_hashes: Vec = db.select_all_block_headers().await?.iter().map(BlockHeader::hash).collect(); @@ -819,7 +819,7 @@ async fn load_mmr(db: &mut Db) -> Result { Ok(block_hashes.into()) } -#[instrument(target = "miden-store", skip_all)] +#[instrument(target = COMPONENT, skip_all)] async fn load_accounts( db: &mut Db, ) -> Result, StateInitializationError> { From b3263de86999d0044674c5c2874b9dbef3784b08 Mon Sep 17 00:00:00 2001 From: Mirko <48352201+Mirko-von-Leipzig@users.noreply.github.com> Date: Thu, 19 Dec 2024 17:24:02 +0200 Subject: [PATCH 31/50] refactor(rpc): invert tx inputs missing notes to found (#590) --- CHANGELOG.md | 4 ++++ .../block-producer/src/domain/transaction.rs | 14 ++------------ crates/block-producer/src/store/mod.rs | 18 ++++++++++-------- crates/block-producer/src/test_utils/store.rs | 7 +++---- crates/proto/src/generated/responses.rs | 2 +- crates/rpc-proto/proto/responses.proto | 2 +- crates/store/src/server/api.rs | 4 ++-- crates/store/src/state.rs | 10 ++-------- proto/responses.proto | 2 +- 9 files changed, 26 insertions(+), 37 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 85647670d..3ae26b06e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ - Support Https in endpoint configuration (#556). - Upgrade `block-producer` from FIFO queue to mempool dependency graph (#562). +### Changes + +- [BREAKING] Inverted `TransactionInputs.missing_unauthenticated_notes` to `found_missing_notes` (#509). + ## v0.6.0 (2024-11-05) ### Enhancements diff --git a/crates/block-producer/src/domain/transaction.rs b/crates/block-producer/src/domain/transaction.rs index 738fccf36..6215bb6ea 100644 --- a/crates/block-producer/src/domain/transaction.rs +++ b/crates/block-producer/src/domain/transaction.rs @@ -56,16 +56,9 @@ impl AuthenticatedTransaction { return Err(VerifyTxError::InputNotesAlreadyConsumed(nullifiers_already_spent)); } - // Invert the missing notes; i.e. we now know the rest were actually found. - let authenticated_notes = tx - .get_unauthenticated_notes() - .map(|header| header.id()) - .filter(|note_id| !inputs.missing_unauthenticated_notes.contains(note_id)) - .collect(); - Ok(AuthenticatedTransaction { inner: Arc::new(tx), - notes_authenticated_by_store: authenticated_notes, + notes_authenticated_by_store: inputs.found_unauthenticated_notes, authentication_height: BlockNumber::new(inputs.current_block_height), store_account_state: inputs.account_hash, }) @@ -137,10 +130,7 @@ impl AuthenticatedTransaction { account_id: inner.account_id(), account_hash: store_account_state, nullifiers: inner.get_nullifiers().map(|nullifier| (nullifier, None)).collect(), - missing_unauthenticated_notes: inner - .get_unauthenticated_notes() - .map(|header| header.id()) - .collect(), + found_unauthenticated_notes: Default::default(), current_block_height: Default::default(), }; // SAFETY: nullifiers were set to None aka are definitely unspent. diff --git a/crates/block-producer/src/store/mod.rs b/crates/block-producer/src/store/mod.rs index c69760f9f..76210dd7a 100644 --- a/crates/block-producer/src/store/mod.rs +++ b/crates/block-producer/src/store/mod.rs @@ -1,5 +1,5 @@ use std::{ - collections::BTreeMap, + collections::{BTreeMap, BTreeSet}, fmt::{Display, Formatter}, num::NonZeroU32, }; @@ -44,9 +44,11 @@ pub struct TransactionInputs { /// /// We use NonZeroU32 as the wire format uses 0 to encode none. pub nullifiers: BTreeMap>, - /// List of unauthenticated notes that were not found in the store - pub missing_unauthenticated_notes: Vec, - /// The current block height + /// Unauthenticated notes which are present in the store. + /// + /// These are notes which were committed _after_ the transaction was created. + pub found_unauthenticated_notes: BTreeSet, + /// The current block height. pub current_block_height: u32, } @@ -94,11 +96,11 @@ impl TryFrom for TransactionInputs { nullifiers.insert(nullifier, NonZeroU32::new(nullifier_record.block_num)); } - let missing_unauthenticated_notes = response - .missing_unauthenticated_notes + let found_unauthenticated_notes = response + .found_unauthenticated_notes .into_iter() .map(|digest| Ok(RpoDigest::try_from(digest)?.into())) - .collect::, ConversionError>>()?; + .collect::>()?; let current_block_height = response.block_height; @@ -106,8 +108,8 @@ impl TryFrom for TransactionInputs { account_id, account_hash, nullifiers, - missing_unauthenticated_notes, current_block_height, + found_unauthenticated_notes, }) } } diff --git a/crates/block-producer/src/test_utils/store.rs b/crates/block-producer/src/test_utils/store.rs index 4b3b40a95..246841c5a 100644 --- a/crates/block-producer/src/test_utils/store.rs +++ b/crates/block-producer/src/test_utils/store.rs @@ -1,7 +1,6 @@ use std::{ collections::{BTreeMap, BTreeSet}, num::NonZeroU32, - ops::Not, }; use miden_node_proto::domain::{blocks::BlockInclusionProof, notes::NoteAuthenticationInfo}; @@ -265,11 +264,11 @@ impl MockStoreSuccess { .collect(); let locked_notes = self.notes.read().await; - let missing_unauthenticated_notes = proven_tx + let found_unauthenticated_notes = proven_tx .get_unauthenticated_notes() .filter_map(|header| { let id = header.id(); - locked_notes.contains_key(&id).not().then_some(id) + locked_notes.contains_key(&id).then_some(id) }) .collect(); @@ -277,7 +276,7 @@ impl MockStoreSuccess { account_id: proven_tx.account_id(), account_hash, nullifiers, - missing_unauthenticated_notes, + found_unauthenticated_notes, current_block_height: 0, }) } diff --git a/crates/proto/src/generated/responses.rs b/crates/proto/src/generated/responses.rs index b0d695bb8..e3bc58652 100644 --- a/crates/proto/src/generated/responses.rs +++ b/crates/proto/src/generated/responses.rs @@ -138,7 +138,7 @@ pub struct GetTransactionInputsResponse { #[prost(message, repeated, tag = "2")] pub nullifiers: ::prost::alloc::vec::Vec, #[prost(message, repeated, tag = "3")] - pub missing_unauthenticated_notes: ::prost::alloc::vec::Vec, + pub found_unauthenticated_notes: ::prost::alloc::vec::Vec, #[prost(fixed32, tag = "4")] pub block_height: u32, } diff --git a/crates/rpc-proto/proto/responses.proto b/crates/rpc-proto/proto/responses.proto index b586458b1..4f1c017b8 100644 --- a/crates/rpc-proto/proto/responses.proto +++ b/crates/rpc-proto/proto/responses.proto @@ -126,7 +126,7 @@ message NullifierTransactionInputRecord { message GetTransactionInputsResponse { AccountTransactionInputRecord account_state = 1; repeated NullifierTransactionInputRecord nullifiers = 2; - repeated digest.Digest missing_unauthenticated_notes = 3; + repeated digest.Digest found_unauthenticated_notes = 3; fixed32 block_height = 4; } diff --git a/crates/store/src/server/api.rs b/crates/store/src/server/api.rs index 8eeadcf8c..09fd45bc4 100644 --- a/crates/store/src/server/api.rs +++ b/crates/store/src/server/api.rs @@ -440,8 +440,8 @@ impl api_server::Api for StoreApi { block_num: nullifier.block_num, }) .collect(), - missing_unauthenticated_notes: tx_inputs - .missing_unauthenticated_notes + found_unauthenticated_notes: tx_inputs + .found_unauthenticated_notes .into_iter() .map(Into::into) .collect(), diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index fb00628c1..3e048715f 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -86,7 +86,7 @@ impl From for GetBlockInputsResponse { pub struct TransactionInputs { pub account_hash: RpoDigest, pub nullifiers: Vec, - pub missing_unauthenticated_notes: Vec, + pub found_unauthenticated_notes: BTreeSet, } /// Container for state that needs to be updated atomically. @@ -659,16 +659,10 @@ impl State { let found_unauthenticated_notes = self.db.select_note_ids(unauthenticated_notes.clone()).await?; - let missing_unauthenticated_notes = unauthenticated_notes - .iter() - .filter(|note_id| !found_unauthenticated_notes.contains(note_id)) - .copied() - .collect(); - Ok(TransactionInputs { account_hash, nullifiers, - missing_unauthenticated_notes, + found_unauthenticated_notes, }) } diff --git a/proto/responses.proto b/proto/responses.proto index b586458b1..4f1c017b8 100644 --- a/proto/responses.proto +++ b/proto/responses.proto @@ -126,7 +126,7 @@ message NullifierTransactionInputRecord { message GetTransactionInputsResponse { AccountTransactionInputRecord account_state = 1; repeated NullifierTransactionInputRecord nullifiers = 2; - repeated digest.Digest missing_unauthenticated_notes = 3; + repeated digest.Digest found_unauthenticated_notes = 3; fixed32 block_height = 4; } From 3de6e219a36adf01e5e3c9e9dfb401fd87a21d89 Mon Sep 17 00:00:00 2001 From: Mirko <48352201+Mirko-von-Leipzig@users.noreply.github.com> Date: Thu, 19 Dec 2024 17:31:15 +0200 Subject: [PATCH 32/50] feat(block-producer): catch up missed blocks instead of skipping (#588) --- crates/block-producer/src/block_builder/mod.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/block-producer/src/block_builder/mod.rs b/crates/block-producer/src/block_builder/mod.rs index d007c34ac..45be364a3 100644 --- a/crates/block-producer/src/block_builder/mod.rs +++ b/crates/block-producer/src/block_builder/mod.rs @@ -63,7 +63,10 @@ impl BlockBuilder { ); let mut interval = tokio::time::interval(self.block_interval); - interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + // We set the inverval's missed tick behaviour to burst. This means we'll catch up missed + // blocks as fast as possible. In other words, we try our best to keep the desired block + // interval on average. The other options would result in at least one skipped block. + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Burst); loop { interval.tick().await; From ccd3519ab3d462dce517aed42ffca7a1a66857e4 Mon Sep 17 00:00:00 2001 From: Mirko <48352201+Mirko-von-Leipzig@users.noreply.github.com> Date: Thu, 19 Dec 2024 17:40:16 +0200 Subject: [PATCH 33/50] feat(block-producer): re-add batch inputs (#589) --- .../block-producer/src/batch_builder/mod.rs | 25 +++++++++++++----- crates/block-producer/src/errors.rs | 3 +++ crates/block-producer/src/server/mod.rs | 3 ++- crates/block-producer/src/store/mod.rs | 26 ++++++++++++++++++- 4 files changed, 48 insertions(+), 9 deletions(-) diff --git a/crates/block-producer/src/batch_builder/mod.rs b/crates/block-producer/src/batch_builder/mod.rs index 2bfc7b151..875c48db8 100644 --- a/crates/block-producer/src/batch_builder/mod.rs +++ b/crates/block-producer/src/batch_builder/mod.rs @@ -1,13 +1,14 @@ use std::{num::NonZeroUsize, ops::Range, time::Duration}; use batch::BatchId; +use miden_node_proto::domain::notes::NoteAuthenticationInfo; use rand::Rng; use tokio::{task::JoinSet, time}; use tracing::{debug, info, instrument, Span}; use crate::{ - domain::transaction::AuthenticatedTransaction, mempool::SharedMempool, COMPONENT, - SERVER_BUILD_BATCH_FREQUENCY, + domain::transaction::AuthenticatedTransaction, mempool::SharedMempool, store::StoreClient, + COMPONENT, SERVER_BUILD_BATCH_FREQUENCY, }; pub mod batch; @@ -54,7 +55,7 @@ impl BatchBuilder { /// A pool of batch-proving workers is spawned, which are fed new batch jobs periodically. /// A batch is skipped if there are no available workers, or if there are no transactions /// available to batch. - pub async fn run(self, mempool: SharedMempool) { + pub async fn run(self, mempool: SharedMempool, store: StoreClient) { assert!( self.failure_rate < 1.0 && self.failure_rate.is_sign_positive(), "Failure rate must be a percentage" @@ -64,7 +65,7 @@ impl BatchBuilder { interval.set_missed_tick_behavior(time::MissedTickBehavior::Delay); let mut worker_pool = - WorkerPool::new(self.workers, self.simulated_proof_time, self.failure_rate); + WorkerPool::new(self.workers, self.simulated_proof_time, self.failure_rate, store); loop { tokio::select! { @@ -122,6 +123,7 @@ struct WorkerPool { /// implement [Ord]. Given that the expected capacity is relatively low, this has no real /// impact beyond ergonomics. task_map: Vec<(tokio::task::Id, BatchId)>, + store: StoreClient, } impl WorkerPool { @@ -129,11 +131,13 @@ impl WorkerPool { capacity: NonZeroUsize, simulated_proof_time: Range, failure_rate: f32, + store: StoreClient, ) -> Self { Self { simulated_proof_time, failure_rate, capacity, + store, in_progress: JoinSet::default(), task_map: Default::default(), } @@ -210,11 +214,18 @@ impl WorkerPool { // // Note: Rng::gen rolls between [0, 1.0) for f32, so this works as expected. let failed = rand::thread_rng().gen::() < self.failure_rate; + let store = self.store.clone(); async move { tracing::debug!("Begin proving batch."); - let batch = Self::build_batch(transactions).map_err(|err| (id, err))?; + let inputs = store + .get_batch_inputs( + transactions.iter().flat_map(|tx| tx.unauthenticated_notes()), + ) + .await + .map_err(|err| (id, err.into()))?; + let batch = Self::build_batch(transactions, inputs).map_err(|err| (id, err))?; tokio::time::sleep(simulated_proof_time).await; if failed { @@ -237,6 +248,7 @@ impl WorkerPool { #[instrument(target = COMPONENT, skip_all, err, fields(batch_id))] fn build_batch( txs: Vec, + inputs: NoteAuthenticationInfo, ) -> Result { let num_txs = txs.len(); @@ -244,8 +256,7 @@ impl WorkerPool { debug!(target: COMPONENT, txs = %format_array(txs.iter().map(|tx| tx.id().to_hex()))); let txs = txs.iter().map(AuthenticatedTransaction::raw_proven_transaction); - // TODO: Found unauthenticated notes are no longer required.. potentially? - let batch = TransactionBatch::new(txs, Default::default())?; + let batch = TransactionBatch::new(txs, inputs)?; Span::current().record("batch_id", batch.id().to_string()); info!(target: COMPONENT, "Transaction batch built"); diff --git a/crates/block-producer/src/errors.rs b/crates/block-producer/src/errors.rs index 27f7b0564..dba60eff3 100644 --- a/crates/block-producer/src/errors.rs +++ b/crates/block-producer/src/errors.rs @@ -142,6 +142,9 @@ pub enum BuildBatchError { #[error("Batch proving task panic'd")] JoinError(#[from] tokio::task::JoinError), + + #[error("Fetching inputs from store failed")] + StoreError(#[from] StoreError), } // Block prover errors diff --git a/crates/block-producer/src/server/mod.rs b/crates/block-producer/src/server/mod.rs index ffaf87190..1a33ad14a 100644 --- a/crates/block-producer/src/server/mod.rs +++ b/crates/block-producer/src/server/mod.rs @@ -109,8 +109,9 @@ impl BlockProducer { let batch_builder_id = tasks .spawn({ let mempool = mempool.clone(); + let store = store.clone(); async { - batch_builder.run(mempool).await; + batch_builder.run(mempool, store).await; Ok(()) } }) diff --git a/crates/block-producer/src/store/mod.rs b/crates/block-producer/src/store/mod.rs index 76210dd7a..5d362a0f9 100644 --- a/crates/block-producer/src/store/mod.rs +++ b/crates/block-producer/src/store/mod.rs @@ -6,10 +6,14 @@ use std::{ use itertools::Itertools; use miden_node_proto::{ + domain::notes::NoteAuthenticationInfo, errors::{ConversionError, MissingFieldHelper}, generated::{ digest, - requests::{ApplyBlockRequest, GetBlockInputsRequest, GetTransactionInputsRequest}, + requests::{ + ApplyBlockRequest, GetBlockInputsRequest, GetNoteAuthenticationInfoRequest, + GetTransactionInputsRequest, + }, responses::{GetTransactionInputsResponse, NullifierTransactionInputRecord}, store::api_client as store_client, }, @@ -203,6 +207,26 @@ impl StoreClient { store_response.try_into().map_err(Into::into) } + #[instrument(target = COMPONENT, skip_all, err)] + pub async fn get_batch_inputs( + &self, + notes: impl Iterator + Send, + ) -> Result { + let request = tonic::Request::new(GetNoteAuthenticationInfoRequest { + note_ids: notes.map(digest::Digest::from).collect(), + }); + + let store_response = + self.inner.clone().get_note_authentication_info(request).await?.into_inner(); + + let note_authentication_info = store_response + .proofs + .ok_or(GetTransactionInputsResponse::missing_field("proofs"))? + .try_into()?; + + Ok(note_authentication_info) + } + #[instrument(target = COMPONENT, skip_all, err)] pub async fn apply_block(&self, block: &Block) -> Result<(), StoreError> { let request = tonic::Request::new(ApplyBlockRequest { block: block.to_bytes() }); From 96397552fed55109f711674a09f4428c28198f84 Mon Sep 17 00:00:00 2001 From: Bobbin Threadbare Date: Wed, 25 Dec 2024 10:29:06 -0800 Subject: [PATCH 34/50] chore: fix typos --- README.md | 2 +- crates/block-producer/src/domain/transaction.rs | 4 ++-- crates/rpc/README.md | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 614e749d4..8e080b997 100644 --- a/README.md +++ b/README.md @@ -78,7 +78,7 @@ This will install the latest official version of the node. You can install a spe cargo install miden-node --locked --version x.y.z ``` -You can also use `cargo` to compile the node from the source code if for some reason you need a specific git revision. Note that since these aren't official releases we cannot provide much support for any issues you run into, so consider this for advanced users only. The incantation is a little different as you'll be targetting this repo instead: +You can also use `cargo` to compile the node from the source code if for some reason you need a specific git revision. Note that since these aren't official releases we cannot provide much support for any issues you run into, so consider this for advanced users only. The incantation is a little different as you'll be targeting this repo instead: ```sh # Install from a specific branch diff --git a/crates/block-producer/src/domain/transaction.rs b/crates/block-producer/src/domain/transaction.rs index 6215bb6ea..01c74d371 100644 --- a/crates/block-producer/src/domain/transaction.rs +++ b/crates/block-producer/src/domain/transaction.rs @@ -25,7 +25,7 @@ pub struct AuthenticatedTransaction { /// This does not necessarily have to match the transaction's initial state /// as this may still be modified by inflight transactions. store_account_state: Option, - /// Unauthenticates notes that have now been authenticated by the store + /// Unauthenticated notes that have now been authenticated by the store /// [inputs](TransactionInputs). /// /// In other words, notes which were unauthenticated at the time the transaction was proven, @@ -38,7 +38,7 @@ pub struct AuthenticatedTransaction { impl AuthenticatedTransaction { /// Verifies the transaction against the inputs, enforcing that all nullifiers are unspent. /// - /// __No__ proof verification is peformed. The caller takes responsibility for ensuring + /// __No__ proof verification is performed. The caller takes responsibility for ensuring /// that the proof is valid. /// /// # Errors diff --git a/crates/rpc/README.md b/crates/rpc/README.md index 1e4f3d50b..d069f4f57 100644 --- a/crates/rpc/README.md +++ b/crates/rpc/README.md @@ -22,7 +22,7 @@ Here is a brief description of supported methods. ### CheckNullifiers -Gets a list of proofs for given nullifier hashes, each proof as a sparse Merkle Trees +Gets a list of proofs for given nullifier hashes, each proof as a sparse Merkle Tree. **Parameters:** From b240c2cec89b2200ef59d8294ba0b0056cffee3f Mon Sep 17 00:00:00 2001 From: Tomas Rodriguez Dala <43424983+tomyrd@users.noreply.github.com> Date: Wed, 25 Dec 2024 17:04:19 -0300 Subject: [PATCH 35/50] fix: support new two `Felt` account ID in node (#591) --- CHANGELOG.md | 3 +- Cargo.lock | 435 +++++++++--------- bin/faucet/src/main.rs | 14 +- bin/node/src/commands/genesis/mod.rs | 9 +- .../src/block_builder/prover/block_witness.rs | 4 +- .../src/block_builder/prover/tests.rs | 115 +++-- .../block-producer/src/test_utils/account.rs | 14 +- crates/block-producer/src/test_utils/store.rs | 5 +- crates/proto/src/domain/accounts.rs | 47 +- crates/proto/src/generated/account.rs | 13 +- crates/proto/src/generated/note.rs | 2 +- crates/proto/src/generated/requests.rs | 4 +- crates/proto/src/generated/responses.rs | 2 +- crates/proto/src/generated/transaction.rs | 2 +- crates/rpc-proto/proto/account.proto | 7 +- crates/rpc/src/server/api.rs | 1 + crates/store/src/db/migrations/001-init.sql | 18 +- crates/store/src/db/mod.rs | 4 +- crates/store/src/db/sql/mod.rs | 115 ++--- crates/store/src/db/sql/utils.rs | 29 +- crates/store/src/db/tests.rs | 77 ++-- crates/store/src/errors.rs | 3 +- crates/store/src/genesis.rs | 9 +- crates/store/src/server/api.rs | 46 +- crates/store/src/state.rs | 31 +- crates/store/src/types.rs | 1 - proto/account.proto | 7 +- 27 files changed, 568 insertions(+), 449 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ae26b06e..90e461c89 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,8 @@ ### Changes -- [BREAKING] Inverted `TransactionInputs.missing_unauthenticated_notes` to `found_missing_notes` (#509). +- [BREAKING] Added support for new two `Felt` account ID (#591). +- [BREAKING] Inverted `TransactionInputs.missing_unauthenticated_notes` to `found_missing_notes` (#509). ## v0.6.0 (2024-11-05) diff --git a/Cargo.lock b/Cargo.lock index 2d91e4f56..816b54c29 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -104,9 +104,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.92" +version = "1.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74f37166d7d48a0284b99dd824694c26119c700b53bf0d1540cdb147dbdaaf13" +checksum = "34ac096ce696dc2fcabef30516bb13c0a68a11d30131d3df6f04711467681b04" [[package]] name = "arrayref" @@ -191,9 +191,9 @@ checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "axum" -version = "0.7.7" +version = "0.7.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "504e3947307ac8326a5437504c517c4b56716c9d98fac0028c2acc7ca47d70ae" +checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" dependencies = [ "async-trait", "axum-core", @@ -215,9 +215,9 @@ dependencies = [ "serde_json", "serde_path_to_error", "serde_urlencoded", - "sync_wrapper 1.0.1", + "sync_wrapper", "tokio", - "tower 0.5.1", + "tower 0.5.2", "tower-layer", "tower-service", "tracing", @@ -238,7 +238,7 @@ dependencies = [ "mime", "pin-project-lite", "rustversion", - "sync_wrapper 1.0.1", + "sync_wrapper", "tower-layer", "tower-service", "tracing", @@ -306,7 +306,16 @@ version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1" dependencies = [ - "bit-vec", + "bit-vec 0.6.3", +] + +[[package]] +name = "bit-set" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" +dependencies = [ + "bit-vec 0.8.0", ] [[package]] @@ -315,6 +324,12 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" +[[package]] +name = "bit-vec" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" + [[package]] name = "bitflags" version = "2.6.0" @@ -323,9 +338,9 @@ checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" [[package]] name = "blake3" -version = "1.5.4" +version = "1.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d82033247fd8e890df8f740e407ad4d038debb9eb1f40533fffb32e7d17dc6f7" +checksum = "b8ee0c1824c4dea5b5f81736aff91bae041d2c07ee1192bec91054e10e3e601e" dependencies = [ "arrayref", "arrayvec", @@ -351,9 +366,9 @@ checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" [[package]] name = "bytemuck" -version = "1.19.0" +version = "1.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8334215b81e418a0a7bdb8ef0849474f40bb10c8b71f1c4ed315cff49f32494d" +checksum = "ef657dfab802224e671f5818e9a4935f9b1957ed18e58292690cc39e7a4092a3" [[package]] name = "byteorder" @@ -363,9 +378,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.8.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ac0150caa2ae65ca5bd83f25c7de183dea78d4d366469f148435e2acfbad0da" +checksum = "325918d6fe32f23b19878fe4b34794ae41fc19ddbe53b10571a4874d44ffd39b" [[package]] name = "camino" @@ -378,32 +393,32 @@ dependencies = [ [[package]] name = "cargo-platform" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24b1f0365a6c6bb4020cd05806fd0d33c44d38046b8bd7f0e40814b9763cabfc" +checksum = "e35af189006b9c0f00a064685c727031e3ed2d8020f7ba284d78cc2671bd36ea" dependencies = [ "serde", ] [[package]] name = "cargo_metadata" -version = "0.18.1" +version = "0.19.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d886547e41f740c616ae73108f6eb70afe6d940c7bc697cb30f13daec073037" +checksum = "8769706aad5d996120af43197bf46ef6ad0fda35216b4505f926a365a232d924" dependencies = [ "camino", "cargo-platform", - "semver 1.0.23", + "semver 1.0.24", "serde", "serde_json", - "thiserror 1.0.68", + "thiserror 2.0.9", ] [[package]] name = "cc" -version = "1.1.35" +version = "1.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f57c4b4da2a9d619dd035f27316d7a426305b75be93d09e92f2b9229c34feaf" +checksum = "c31a0499c1dc64f458ad13872de75c0eb7e3fdb0e67964610c914b034fc5956e" dependencies = [ "jobserver", "libc", @@ -437,9 +452,9 @@ dependencies = [ [[package]] name = "chrono" -version = "0.4.38" +version = "0.4.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" +checksum = "7e36cc9d416881d2e24f9a963be5fb1cd90966419ac844274161d10488b3e825" dependencies = [ "android-tzdata", "iana-time-zone", @@ -462,9 +477,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.20" +version = "4.5.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b97f376d85a664d5837dbae44bf546e6477a679ff6610010f17276f686d867e8" +checksum = "3135e7ec2ef7b10c6ed8950f0f792ed96ee093fa088608f1c76e569722700c84" dependencies = [ "clap_builder", "clap_derive", @@ -472,9 +487,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.20" +version = "4.5.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19bc80abd44e4bed93ca373a0704ccbd1b710dc5749406201bb018272808dc54" +checksum = "30582fc632330df2bd26877bde0c1f4470d57c582bbc070376afcd04d8cb4838" dependencies = [ "anstream", "anstyle", @@ -496,9 +511,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.7.2" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97" +checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" [[package]] name = "colorchoice" @@ -520,18 +535,18 @@ checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "cpufeatures" -version = "0.2.14" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "608697df725056feaccfa42cffdaeeec3fccc4ffc38358ecd19b243e716a78e0" +checksum = "16b80225097f2e5ae4e7179dd2266824648f3e2f49d9134d584b76389d31c4c3" dependencies = [ "libc", ] [[package]] name = "crossbeam-deque" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" dependencies = [ "crossbeam-epoch", "crossbeam-utils", @@ -548,9 +563,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.20" +version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" [[package]] name = "crunchy" @@ -749,12 +764,12 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.3.9" +version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" +checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -771,9 +786,9 @@ checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" [[package]] name = "fastrand" -version = "2.1.1" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8c02a5121d4ea3eb16a80748c74f5549a5665e4c21333c6098f283870fbdea6" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" [[package]] name = "figment" @@ -875,9 +890,9 @@ dependencies = [ [[package]] name = "generator" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbb949699c3e4df3a183b1d2142cb24277057055ed23c68ed58894f76c517223" +checksum = "cc6bd114ceda131d3b1d665eba35788690ad37f5916457286b32ab6fd3c438dd" dependencies = [ "cfg-if", "libc", @@ -923,9 +938,9 @@ checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" [[package]] name = "h2" -version = "0.4.6" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "524e8ac6999421f49a846c2d4411f337e53497d8ec55d67753beffa43c5d9205" +checksum = "ccae279728d634d083c00f6099cb58f01cc99c145b84b8be2f6c74618d79922e" dependencies = [ "atomic-waker", "bytes", @@ -933,7 +948,7 @@ dependencies = [ "futures-core", "futures-sink", "http", - "indexmap 2.6.0", + "indexmap 2.7.0", "slab", "tokio", "tokio-util", @@ -957,9 +972,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.15.1" +version = "0.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a9bfc1af68b1726ea47d3d5109de126281def866b33970e10fbab11b5dafab3" +checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" [[package]] name = "hashlink" @@ -990,9 +1005,9 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "http" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" +checksum = "f16ca2af56261c99fba8bac40a10251ce8188205a4c448fbb745a2e4daa76fea" dependencies = [ "bytes", "fnv", @@ -1036,9 +1051,9 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "hyper" -version = "1.5.0" +version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbbff0a806a4728c99295b254c8838933b5b082d75e3cb70c8dab21fdfbcfa9a" +checksum = "256fb8d4bd6413123cc9d91832d78325c48ff41677595be797d90f42969beae0" dependencies = [ "bytes", "futures-channel", @@ -1134,12 +1149,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.6.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" +checksum = "62f822373a4fe84d4bb149bf54e584a7f4abec90e072ed49cda0edea5b95471f" dependencies = [ "equivalent", - "hashbrown 0.15.1", + "hashbrown 0.15.2", ] [[package]] @@ -1189,9 +1204,9 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.11" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" +checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" [[package]] name = "jobserver" @@ -1204,10 +1219,11 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.72" +version = "0.3.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a88f1bda2bd75b0452a14784937d796722fdebfe50df998aeb3f0b7603019a9" +checksum = "6717b6b5b077764fb5966237269cb3c64edddde4b14ce42647430a78ced9e7b7" dependencies = [ + "once_cell", "wasm-bindgen", ] @@ -1227,7 +1243,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55cb077ad656299f160924eb2912aa147d7339ea7d69e1b5517326fdcec3c1ca" dependencies = [ "ascii-canvas", - "bit-set", + "bit-set 0.5.3", "ena", "itertools 0.11.0", "lalrpop-util", @@ -1261,15 +1277,15 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" -version = "0.2.161" +version = "0.2.169" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e9489c2807c139ffd9c1794f4af0ebe86a828db53ecdc7fea2111d0fed085d1" +checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" [[package]] name = "libloading" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4979f22fdb869068da03c9f7528f8297c6fd2606bc3a4affe42e6a823fdb8da4" +checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34" dependencies = [ "cfg-if", "windows-targets 0.52.6", @@ -1327,18 +1343,18 @@ checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" [[package]] name = "logos" -version = "0.14.2" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c6b6e02facda28ca5fb8dbe4b152496ba3b1bd5a4b40bb2b1b2d8ad74e0f39b" +checksum = "7251356ef8cb7aec833ddf598c6cb24d17b689d20b993f9d11a3d764e34e6458" dependencies = [ "logos-derive", ] [[package]] name = "logos-codegen" -version = "0.14.2" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b32eb6b5f26efacd015b000bfc562186472cd9b34bdba3f6b264e2a052676d10" +checksum = "59f80069600c0d66734f5ff52cc42f2dabd6b29d205f333d61fd7832e9e9963f" dependencies = [ "beef", "fnv", @@ -1351,9 +1367,9 @@ dependencies = [ [[package]] name = "logos-derive" -version = "0.14.2" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e5d0c5463c911ef55624739fc353238b4e310f0144be1f875dc42fec6bfd5ec" +checksum = "24fb722b06a9dc12adb0963ed585f19fc61dc5413e6a9be9422ef92c091e731d" dependencies = [ "logos-codegen", ] @@ -1481,12 +1497,12 @@ dependencies = [ "rand_chacha", "serde", "static-files", - "thiserror 2.0.3", + "thiserror 2.0.9", "tokio", "toml", "tonic", - "tower 0.5.1", - "tower-http 0.6.1", + "tower 0.5.2", + "tower-http 0.6.2", "tracing", ] @@ -1502,12 +1518,13 @@ dependencies = [ [[package]] name = "miden-lib" version = "0.7.0" -source = "git+https://github.com/0xPolygonMiden/miden-base.git?branch=next#bc91c604862e2c6013b21eb989a3640d30cce25b" +source = "git+https://github.com/0xPolygonMiden/miden-base.git?branch=next#64d7a2e5fa3fb1c65269369bf0b152057e45befd" dependencies = [ "miden-assembly", "miden-objects", "miden-stdlib", "regex", + "thiserror 2.0.9", "walkdir", ] @@ -1595,7 +1612,7 @@ dependencies = [ "rand", "rand_chacha", "serde", - "thiserror 2.0.3", + "thiserror 2.0.9", "tokio", "tokio-stream", "tonic", @@ -1615,7 +1632,7 @@ dependencies = [ "prost", "prost-build", "protox", - "thiserror 2.0.3", + "thiserror 2.0.9", "tonic", "tonic-build", ] @@ -1650,7 +1667,7 @@ dependencies = [ "rusqlite", "rusqlite_migration", "serde", - "thiserror 2.0.3", + "thiserror 2.0.9", "tokio", "tokio-stream", "tonic", @@ -1675,7 +1692,7 @@ dependencies = [ "miden-objects", "rand", "serde", - "thiserror 2.0.3", + "thiserror 2.0.9", "tonic", "tracing", "tracing-forest", @@ -1687,7 +1704,7 @@ dependencies = [ [[package]] name = "miden-objects" version = "0.7.0" -source = "git+https://github.com/0xPolygonMiden/miden-base.git?branch=next#bc91c604862e2c6013b21eb989a3640d30cce25b" +source = "git+https://github.com/0xPolygonMiden/miden-base.git?branch=next#64d7a2e5fa3fb1c65269369bf0b152057e45befd" dependencies = [ "getrandom", "miden-assembly", @@ -1696,7 +1713,7 @@ dependencies = [ "miden-processor", "miden-verifier", "rand", - "thiserror 2.0.3", + "thiserror 2.0.9", "winter-rand-utils", ] @@ -1761,7 +1778,7 @@ dependencies = [ [[package]] name = "miden-tx" version = "0.7.0" -source = "git+https://github.com/0xPolygonMiden/miden-base.git?branch=next#bc91c604862e2c6013b21eb989a3640d30cce25b" +source = "git+https://github.com/0xPolygonMiden/miden-base.git?branch=next#64d7a2e5fa3fb1c65269369bf0b152057e45befd" dependencies = [ "async-trait", "miden-lib", @@ -1771,6 +1788,7 @@ dependencies = [ "miden-verifier", "rand", "rand_chacha", + "thiserror 2.0.9", "winter-maybe-async", ] @@ -1788,21 +1806,21 @@ dependencies = [ [[package]] name = "miette" -version = "7.2.0" +version = "7.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4edc8853320c2a0dab800fbda86253c8938f6ea88510dc92c5f1ed20e794afc1" +checksum = "317f146e2eb7021892722af37cf1b971f0a70c8406f487e24952667616192c64" dependencies = [ "cfg-if", "miette-derive", - "thiserror 1.0.68", + "thiserror 1.0.69", "unicode-width 0.1.14", ] [[package]] name = "miette-derive" -version = "7.2.0" +version = "7.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcf09caffaac8068c346b6df2a7fc27a177fd20b39421a39ce0a211bde679a6c" +checksum = "23c9b935fbe1d6cbd1dac857b54a688145e2d93f48db36010514d0f612d0ad67" dependencies = [ "proc-macro2", "quote", @@ -1833,20 +1851,19 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.8.0" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" +checksum = "4ffbe83022cedc1d264172192511ae958937694cd57ce297164951b8b3568394" dependencies = [ "adler2", ] [[package]] name = "mio" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80e04d1dcff3aae0704555fe5fee3bcfaf3d1fdf8a7e521d5b9d2b42acb52cec" +checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd" dependencies = [ - "hermit-abi", "libc", "wasi", "windows-sys 0.52.0", @@ -1996,9 +2013,9 @@ dependencies = [ [[package]] name = "object" -version = "0.36.5" +version = "0.36.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aedf0a2d09c573ed1d8d85b30c119153926a2b36dce0ab28322c09a117a4683e" +checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" dependencies = [ "memchr", ] @@ -2095,7 +2112,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" dependencies = [ "fixedbitset", - "indexmap 2.6.0", + "indexmap 2.7.0", ] [[package]] @@ -2188,9 +2205,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.89" +version = "1.0.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f139b0662de085916d1fb67d2b4169d1addddda1919e696f3252b740b629986e" +checksum = "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0" dependencies = [ "unicode-ident", ] @@ -2210,12 +2227,12 @@ dependencies = [ [[package]] name = "proptest" -version = "1.5.0" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4c2511913b88df1637da85cc8d96ec8e43a3f8bb8ccb71ee1ac240d6f3df58d" +checksum = "14cae93065090804185d3b75f0bf93b8eeda30c7a9b4a33d3bdb3988d6229e50" dependencies = [ - "bit-set", - "bit-vec", + "bit-set 0.8.0", + "bit-vec 0.8.0", "bitflags", "lazy_static", "num-traits", @@ -2230,9 +2247,9 @@ dependencies = [ [[package]] name = "prost" -version = "0.13.3" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b0487d90e047de87f984913713b85c601c05609aad5b0df4b4573fbf69aa13f" +checksum = "2c0fef6c4230e4ccf618a35c59d7ede15dea37de8427500f50aff708806e42ec" dependencies = [ "bytes", "prost-derive", @@ -2240,11 +2257,10 @@ dependencies = [ [[package]] name = "prost-build" -version = "0.13.3" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c1318b19085f08681016926435853bbf7858f9c082d0999b80550ff5d9abe15" +checksum = "d0f3e5beed80eb580c68e2c600937ac2c4eedabdfd5ef1e5b7ea4f3fba84497b" dependencies = [ - "bytes", "heck", "itertools 0.13.0", "log", @@ -2261,9 +2277,9 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.13.3" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9552f850d5f0964a4e4d0bf306459ac29323ddfbae05e35a7c0d35cb0803cc5" +checksum = "157c5a9d7ea5c2ed2d9fb8f495b64759f7816c7eaea54ba3978f0d63000162e3" dependencies = [ "anyhow", "itertools 0.13.0", @@ -2274,9 +2290,9 @@ dependencies = [ [[package]] name = "prost-reflect" -version = "0.14.2" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b7535b02f0e5efe3e1dbfcb428be152226ed0c66cad9541f2274c8ba8d4cd40" +checksum = "20ae544fca2892fd4b7e9ff26cba1090cedf1d4d95c2aded1af15d2f93f270b8" dependencies = [ "logos", "miette", @@ -2287,9 +2303,9 @@ dependencies = [ [[package]] name = "prost-types" -version = "0.13.3" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4759aa0d3a6232fb8dbdb97b61de2c20047c68aca932c7ed76da9d788508d670" +checksum = "cc2f1e56baa61e93533aebc21af4d2134b70f66275e0fcdf3cbe43d77ff7e8fc" dependencies = [ "prost", ] @@ -2306,7 +2322,7 @@ dependencies = [ "prost-reflect", "prost-types", "protox-parse", - "thiserror 1.0.68", + "thiserror 1.0.69", ] [[package]] @@ -2318,7 +2334,7 @@ dependencies = [ "logos", "miette", "prost-types", - "thiserror 1.0.68", + "thiserror 1.0.69", ] [[package]] @@ -2397,9 +2413,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.7" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b6dfecf2c74bce2466cabf93f6664d6998a69eb21e39f4207930065b27b771f" +checksum = "03a862b389f93e68874fbf580b9de08dd02facb9a788ebadaf4a3fd33cf58834" dependencies = [ "bitflags", ] @@ -2412,7 +2428,7 @@ checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" dependencies = [ "getrandom", "libredox", - "thiserror 1.0.68", + "thiserror 1.0.69", ] [[package]] @@ -2423,7 +2439,7 @@ checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.4.8", + "regex-automata 0.4.9", "regex-syntax 0.8.5", ] @@ -2438,9 +2454,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.8" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "368758f23274712b504848e9d5a6f010445cc8b87a7cdb4d7cbee666c1288da3" +checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" dependencies = [ "aho-corasick", "memchr", @@ -2510,20 +2526,20 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" dependencies = [ - "semver 1.0.23", + "semver 1.0.24", ] [[package]] name = "rustix" -version = "0.38.39" +version = "0.38.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "375116bee2be9ed569afe2154ea6a99dfdffd257f533f187498c2a8f5feaf4ee" +checksum = "f93dc38ecbab2eb790ff964bb77fa94faf256fd3e73285fd7ba0903b76bedb85" dependencies = [ "bitflags", "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -2582,9 +2598,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.23" +version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" +checksum = "3cb6eb87a131f756572d7fb904f6e7b68633f09cca868c5df1c4b8d1a694bbba" dependencies = [ "serde", ] @@ -2597,18 +2613,18 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" [[package]] name = "serde" -version = "1.0.214" +version = "1.0.216" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f55c3193aca71c12ad7890f1785d2b73e1b9f63a0bbc353c08ef26fe03fc56b5" +checksum = "0b9781016e935a97e8beecf0c933758c97a5520d32930e460142b4cd80c6338e" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.214" +version = "1.0.216" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de523f781f095e28fa605cdce0f8307e451cc0fd14e2eb4cd2e98a355b147766" +checksum = "46f859dbbf73865c6627ed570e78961cd3ac92407a2d117204c49232485da55e" dependencies = [ "proc-macro2", "quote", @@ -2617,9 +2633,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.132" +version = "1.0.134" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d726bfaff4b320266d395898905d0eba0345aae23b54aee3a737e260fd46db03" +checksum = "d00f4175c42ee48b15416f6193a959ba3a0d67fc699a0db9ad12df9f83991c7d" dependencies = [ "itoa", "memchr", @@ -2712,9 +2728,9 @@ checksum = "b7c388c1b5e93756d0c740965c41e8822f866621d41acbdf6336a6a168f8840c" [[package]] name = "socket2" -version = "0.5.7" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce305eb0b4296696835b71df73eb912e0f1ffd2556a501fcede6e0c50349191c" +checksum = "c970269d99b64e60ec3bd6ad27270092a5394c4e309314b18ae3fe575695fbe8" dependencies = [ "libc", "windows-sys 0.52.0", @@ -2767,18 +2783,18 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] name = "supports-color" -version = "3.0.1" +version = "3.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8775305acf21c96926c900ad056abeef436701108518cf890020387236ac5a77" +checksum = "c64fc7232dd8d2e4ac5ce4ef302b1d81e0b80d055b9d77c7c4f51f6aa4c867d6" dependencies = [ "is_ci", ] [[package]] name = "supports-hyperlinks" -version = "3.0.0" +version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c0a1e5168041f5f3ff68ff7d95dcb9c8749df29f6e7e89ada40dd4c9de404ee" +checksum = "804f44ed3c63152de6a9f90acbea1a110441de43006ea51bcce8f436196a288b" [[package]] name = "supports-unicode" @@ -2788,9 +2804,9 @@ checksum = "b7401a30af6cb5818bb64852270bb722533397edcfc7344954a38f420819ece2" [[package]] name = "syn" -version = "2.0.87" +version = "2.0.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25aa4ce346d03a6dcd68dd8b4010bcb74e54e62c90c573f394c46eae99aba32d" +checksum = "d53cbcb5a243bd33b7858b1d7f4aca2153490815872d86d955d6ea29f743c035" dependencies = [ "proc-macro2", "quote", @@ -2799,15 +2815,9 @@ dependencies = [ [[package]] name = "sync_wrapper" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" - -[[package]] -name = "sync_wrapper" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" [[package]] name = "target-triple" @@ -2817,9 +2827,9 @@ checksum = "42a4d50cdb458045afc8131fd91b64904da29548bcb63c7236e0844936c13078" [[package]] name = "tempfile" -version = "3.13.0" +version = "3.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0f2c9fc62d0beef6951ccffd757e241266a2c833136efbe35af6cd2567dca5b" +checksum = "28cce251fcbc87fac86a866eeb0d6c2d536fc16d06f184bb61aeae11aa4cee0c" dependencies = [ "cfg-if", "fastrand", @@ -2871,27 +2881,27 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.68" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02dd99dc800bbb97186339685293e1cc5d9df1f8fae2d0aecd9ff1c77efea892" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" dependencies = [ - "thiserror-impl 1.0.68", + "thiserror-impl 1.0.69", ] [[package]] name = "thiserror" -version = "2.0.3" +version = "2.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c006c85c7651b3cf2ada4584faa36773bd07bac24acfb39f3c431b36d7e667aa" +checksum = "f072643fd0190df67a8bab670c20ef5d8737177d6ac6b2e9a236cb096206b2cc" dependencies = [ - "thiserror-impl 2.0.3", + "thiserror-impl 2.0.9", ] [[package]] name = "thiserror-impl" -version = "1.0.68" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7c61ec9a6f64d2793d8a45faba21efbe3ced62a886d44c36a009b2b519b4c7e" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", @@ -2900,9 +2910,9 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.3" +version = "2.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f077553d607adc1caf65430528a576c757a71ed73944b66ebb58ef2bbd243568" +checksum = "7b50fa271071aae2e6ee85f842e2e28ba8cd2c5fb67f11fcb1fd70b276f9e7d4" dependencies = [ "proc-macro2", "quote", @@ -2921,9 +2931,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.36" +version = "0.3.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" +checksum = "35e7868883861bd0e56d9ac6efcaaca0d6d5d82a2a7ec8209ff492c07cf37b21" dependencies = [ "deranged", "itoa", @@ -2944,9 +2954,9 @@ checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.2.18" +version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" +checksum = "2834e6017e3e5e4b9834939793b282bc03b37a3336245fa820e35e233e2a85de" dependencies = [ "num-conv", "time-core", @@ -2963,9 +2973,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.41.0" +version = "1.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "145f3413504347a2be84393cc8a7d2fb4d863b375909ea59f2158261aa258bbb" +checksum = "5cec9b21b0450273377fc97bd4c33a8acffc8c996c987a7c5b319a0083707551" dependencies = [ "backtrace", "bytes", @@ -2990,9 +3000,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.16" +version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f4e6ce100d0eb49a2734f8c0812bcd324cf357d21810932c5df6b96ef2b86f1" +checksum = "eca58d7bba4a75707817a2c44174253f9236b2d5fbd055602e9d5c07c139a047" dependencies = [ "futures-core", "pin-project-lite", @@ -3001,9 +3011,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.12" +version = "0.7.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61e7c3654c13bcd040d4a03abee2c75b1d14a37b423cf5a813ceae1cc903ec6a" +checksum = "d7fcaa8d55a2bdd6b83ace262b016eca0d79ee02818c5c1bcdf0305114081078" dependencies = [ "bytes", "futures-core", @@ -3039,7 +3049,7 @@ version = "0.22.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" dependencies = [ - "indexmap 2.6.0", + "indexmap 2.7.0", "serde", "serde_spanned", "toml_datetime", @@ -3132,14 +3142,14 @@ dependencies = [ [[package]] name = "tower" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2873938d487c3cfb9aed7546dc9f2711d867c9f90c46b889989a2cb84eba6b4f" +checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" dependencies = [ "futures-core", "futures-util", "pin-project-lite", - "sync_wrapper 0.1.2", + "sync_wrapper", "tokio", "tower-layer", "tower-service", @@ -3164,9 +3174,9 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.6.1" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8437150ab6bbc8c5f0f519e3d5ed4aa883a83dd4cdd3d1b21f9482936046cb97" +checksum = "403fa3b783d4b626a8ad51d766ab03cb6d2dbfc46b1c5d4448395e6628dc9697" dependencies = [ "bitflags", "bytes", @@ -3231,7 +3241,7 @@ checksum = "ee40835db14ddd1e3ba414292272eddde9dad04d3d4b65509656414d1c42592f" dependencies = [ "chrono", "smallvec", - "thiserror 1.0.68", + "thiserror 1.0.69", "tracing", "tracing-subscriber", ] @@ -3249,9 +3259,9 @@ dependencies = [ [[package]] name = "tracing-serde" -version = "0.1.3" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc6b213177105856957181934e4920de57730fc69bf42c37ee5bb664d406d9e1" +checksum = "704b1aeb7be0d0a84fc9828cae51dab5970fee5088f83d1dd7ee6f6246fc6ff1" dependencies = [ "serde", "tracing-core", @@ -3259,9 +3269,9 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.18" +version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" +checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008" dependencies = [ "matchers", "nu-ansi-term", @@ -3323,15 +3333,15 @@ dependencies = [ [[package]] name = "unicase" -version = "2.8.0" +version = "2.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e51b68083f157f853b6379db119d1c1be0e6e4dec98101079dec41f6f5cf6df" +checksum = "75b844d17643ee918803943289730bec8aac480150456169e647ed0b576ba539" [[package]] name = "unicode-ident" -version = "1.0.13" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" +checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83" [[package]] name = "unicode-linebreak" @@ -3377,9 +3387,9 @@ checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" [[package]] name = "vergen" -version = "9.0.1" +version = "9.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "349ed9e45296a581f455bc18039878f409992999bc1d5da12a6800eb18c8752f" +checksum = "31f25fc8f8f05df455c7941e87f093ad22522a9ff33d7a027774815acf6f0639" dependencies = [ "anyhow", "cargo_metadata", @@ -3392,9 +3402,9 @@ dependencies = [ [[package]] name = "vergen-gitcl" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a3a7f91caabecefc3c249fd864b11d4abe315c166fbdb568964421bccfd2b7a" +checksum = "0227006d09f98ab00ea69e9a5e055e676a813cfbed4232986176c86a6080b997" dependencies = [ "anyhow", "derive_builder", @@ -3406,9 +3416,9 @@ dependencies = [ [[package]] name = "vergen-lib" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "229eaddb0050920816cf051e619affaf18caa3dd512de8de5839ccbc8e53abb0" +checksum = "c0c767e6751c09fc85cde58722cf2f1007e80e4c8d5a4321fc90d83dc54ca147" dependencies = [ "anyhow", "derive_builder", @@ -3477,9 +3487,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.95" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "128d1e363af62632b8eb57219c8fd7877144af57558fb2ef0368d0087bddeb2e" +checksum = "a474f6281d1d70c17ae7aa6a613c87fce69a127e2624002df63dcb39d6cf6396" dependencies = [ "cfg-if", "once_cell", @@ -3488,13 +3498,12 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.95" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb6dd4d3ca0ddffd1dd1c9c04f94b868c37ff5fac97c30b97cff2d74fce3a358" +checksum = "5f89bb38646b4f81674e8f5c3fb81b562be1fd936d84320f3264486418519c79" dependencies = [ "bumpalo", "log", - "once_cell", "proc-macro2", "quote", "syn", @@ -3503,9 +3512,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.95" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e79384be7f8f5a9dd5d7167216f022090cf1f9ec128e6e6a482a2cb5c5422c56" +checksum = "2cc6181fd9a7492eef6fef1f33961e3695e4579b9872a6f7c83aee556666d4fe" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3513,9 +3522,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.95" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26c6ab57572f7a24a4985830b120de1594465e5d500f24afe89e16b4e833ef68" +checksum = "30d7a95b763d3c45903ed6c81f156801839e5ee968bb07e534c44df0fcd330c2" dependencies = [ "proc-macro2", "quote", @@ -3526,9 +3535,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.95" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65fc09f10666a9f147042251e0dda9c18f166ff7de300607007e96bdebc1068d" +checksum = "943aab3fdaaa029a6e0271b35ea10b72b943135afe9bffca82384098ad0e06a6" [[package]] name = "winapi" @@ -3793,9 +3802,9 @@ dependencies = [ [[package]] name = "winter-air" -version = "0.10.1" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29bec0b06b741543f43e3a6677b95b200d4cad2daab76e6721e14345345bfd0e" +checksum = "7c92f0d9a736cb744b0a3e267fafd50d27d6625be3681f55a2b7650ddbf3a2ce" dependencies = [ "libm", "winter-crypto", @@ -3806,9 +3815,9 @@ dependencies = [ [[package]] name = "winter-crypto" -version = "0.10.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "163da45f1d4d65cac361b8df4835a6daa95b3399154e16eb0305c178c6f6c1f4" +checksum = "3fcae1ada055aa10554910ecffc106cb116a19dba11ac91390ef982f94adb9c5" dependencies = [ "blake3", "sha3", @@ -3818,9 +3827,9 @@ dependencies = [ [[package]] name = "winter-fri" -version = "0.10.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b7b394670d68979a4cc21a37a95ef8ef350cf84be9256c53effe3052df50d26" +checksum = "ff88657560100f34fb83882a0adf33fb7caee235deb83193d0d251ddb28ed9c9" dependencies = [ "winter-crypto", "winter-math", @@ -3829,9 +3838,9 @@ dependencies = [ [[package]] name = "winter-math" -version = "0.10.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a8ba832121679e79b004b0003018c85873956d742a39c348c247f680fe15e00" +checksum = "82479f94efc0b5374a93e2074ba46ef404384fb1ea6e35a847febec53096509b" dependencies = [ "winter-utils", ] @@ -3848,9 +3857,9 @@ dependencies = [ [[package]] name = "winter-prover" -version = "0.10.1" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f55f0153d26691caaf969066a13a824bcf3c98719d71b0f569bf8dc40a06fb9" +checksum = "cab5e02d53d5df7903ebf3e1f4ba44b918267876bfd37eade046d9a669f4e9fb" dependencies = [ "tracing", "winter-air", @@ -3882,9 +3891,9 @@ dependencies = [ [[package]] name = "winter-verifier" -version = "0.10.1" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ae1648768f96f5e6321a48a5bff5cc3101d2e51b23a6a095c6c9c9e133ecb61" +checksum = "517c31712cceaafc3c7dcc9311f7d5d306b34e208bc72664c691056fd863f7b8" dependencies = [ "winter-air", "winter-crypto", diff --git a/bin/faucet/src/main.rs b/bin/faucet/src/main.rs index a172440ad..b489d8e1f 100644 --- a/bin/faucet/src/main.rs +++ b/bin/faucet/src/main.rs @@ -13,6 +13,7 @@ use axum::{ Router, }; use clap::{Parser, Subcommand}; +use client::initialize_faucet_client; use http::HeaderValue; use miden_lib::{accounts::faucets::create_basic_fungible_faucet, AuthScheme}; use miden_node_utils::{config::load_config, crypto::get_rpo_random_coin, version::LongVersion}; @@ -22,8 +23,8 @@ use miden_objects::{ crypto::dsa::rpo_falcon512::SecretKey, Felt, }; -use rand::Rng; -use rand_chacha::{rand_core::SeedableRng, ChaCha20Rng}; +use rand::{Rng, SeedableRng}; +use rand_chacha::ChaCha20Rng; use state::FaucetState; use tokio::net::TcpListener; use tower::ServiceBuilder; @@ -61,6 +62,8 @@ pub enum Command { /// Create a new public faucet account and save to the specified file CreateFaucetAccount { + #[arg(short, long, value_name = "FILE", default_value = FAUCET_CONFIG_FILE_PATH)] + config_path: PathBuf, #[arg(short, long, value_name = "FILE", default_value = DEFAULT_FAUCET_ACCOUNT_PATH)] output_path: PathBuf, #[arg(short, long)] @@ -128,6 +131,7 @@ async fn main() -> anyhow::Result<()> { }, Command::CreateFaucetAccount { + config_path, output_path, token_symbol, decimals, @@ -135,6 +139,11 @@ async fn main() -> anyhow::Result<()> { } => { println!("Generating new faucet account. This may take a few minutes..."); + let config: FaucetConfig = + load_config(config_path).context("Failed to load configuration file")?; + + let (_, root_block_header, _) = initialize_faucet_client(&config).await?; + let current_dir = std::env::current_dir().context("Failed to open current directory")?; @@ -144,6 +153,7 @@ async fn main() -> anyhow::Result<()> { let (account, account_seed) = create_basic_fungible_faucet( rng.gen(), + (&root_block_header).try_into().context("Failed to create anchor block")?, TokenSymbol::try_from(token_symbol.as_str()) .context("Failed to parse token symbol")?, *decimals, diff --git a/bin/node/src/commands/genesis/mod.rs b/bin/node/src/commands/genesis/mod.rs index 5c285b837..d3c00148f 100644 --- a/bin/node/src/commands/genesis/mod.rs +++ b/bin/node/src/commands/genesis/mod.rs @@ -9,13 +9,13 @@ use miden_lib::{accounts::faucets::create_basic_fungible_faucet, AuthScheme}; use miden_node_store::genesis::GenesisState; use miden_node_utils::{config::load_config, crypto::get_rpo_random_coin}; use miden_objects::{ - accounts::{Account, AccountData, AuthSecretKey}, + accounts::{Account, AccountData, AccountIdAnchor, AuthSecretKey}, assets::TokenSymbol, crypto::{dsa::rpo_falcon512::SecretKey, utils::Serializable}, Felt, ONE, }; -use rand::Rng; -use rand_chacha::{rand_core::SeedableRng, ChaCha20Rng}; +use rand::{Rng, SeedableRng}; +use rand_chacha::ChaCha20Rng; use tracing::info; mod inputs; @@ -116,7 +116,7 @@ fn create_accounts( let mut rng = ChaCha20Rng::from_seed(rand::random()); for account in accounts { - // build offchain account data from account inputs + // build account data from account inputs let (mut account_data, name) = match account { AccountInput::BasicFungibleFaucet(inputs) => { info!("Creating fungible faucet account..."); @@ -125,6 +125,7 @@ fn create_accounts( let storage_mode = inputs.storage_mode.as_str().try_into()?; let (account, account_seed) = create_basic_fungible_faucet( rng.gen(), + AccountIdAnchor::PRE_GENESIS, TokenSymbol::try_from(inputs.token_symbol.as_str())?, inputs.decimals, Felt::try_from(inputs.max_supply) diff --git a/crates/block-producer/src/block_builder/prover/block_witness.rs b/crates/block-producer/src/block_builder/prover/block_witness.rs index 10e37c00a..91d380c5b 100644 --- a/crates/block-producer/src/block_builder/prover/block_witness.rs +++ b/crates/block-producer/src/block_builder/prover/block_witness.rs @@ -188,7 +188,7 @@ impl BlockWitness { for (idx, (account_id, account_update)) in self.updated_accounts.iter().enumerate() { account_data.extend(account_update.final_state_hash); - account_data.push((*account_id).into()); + account_data.push(account_id.first_felt()); let idx = u64::try_from(idx).expect("can't be more than 2^64 - 1 accounts"); num_accounts_updated = idx + 1; @@ -265,7 +265,7 @@ impl BlockWitness { merkle_store .add_merkle_paths(self.updated_accounts.into_iter().map( |(account_id, AccountUpdateWitness { initial_state_hash, proof, .. })| { - (u64::from(account_id), initial_state_hash, proof) + (account_id.prefix().into(), initial_state_hash, proof) }, )) .map_err(BlockProverError::InvalidMerklePaths)?; diff --git a/crates/block-producer/src/block_builder/prover/tests.rs b/crates/block-producer/src/block_builder/prover/tests.rs index e8463f3c3..ca3047c13 100644 --- a/crates/block-producer/src/block_builder/prover/tests.rs +++ b/crates/block-producer/src/block_builder/prover/tests.rs @@ -2,18 +2,15 @@ use std::{collections::BTreeMap, iter}; use assert_matches::assert_matches; use miden_objects::{ - accounts::{ - account_id::testing::{ - ACCOUNT_ID_OFF_CHAIN_SENDER, ACCOUNT_ID_REGULAR_ACCOUNT_UPDATABLE_CODE_OFF_CHAIN, - }, - delta::AccountUpdateDetails, - AccountId, - }, + accounts::{delta::AccountUpdateDetails, AccountId, AccountStorageMode, AccountType}, block::{BlockAccountUpdate, BlockNoteIndex, BlockNoteTree}, crypto::merkle::{ EmptySubtreeRoots, LeafIndex, MerklePath, Mmr, MmrPeaks, Smt, SmtLeaf, SmtProof, SMT_DEPTH, }, notes::{NoteExecutionHint, NoteHeader, NoteMetadata, NoteTag, NoteType, Nullifier}, + testing::account_id::{ + ACCOUNT_ID_OFF_CHAIN_SENDER, ACCOUNT_ID_REGULAR_ACCOUNT_UPDATABLE_CODE_OFF_CHAIN, + }, transaction::{OutputNote, ProvenTransaction}, Felt, BATCH_NOTE_TREE_DEPTH, BLOCK_NOTE_TREE_DEPTH, ONE, ZERO, }; @@ -38,9 +35,21 @@ use crate::{ /// The store will contain accounts 1 & 2, while the transaction batches will contain 2 & 3. #[test] fn block_witness_validation_inconsistent_account_ids() { - let account_id_1 = AccountId::new_unchecked(Felt::new(ACCOUNT_ID_OFF_CHAIN_SENDER)); - let account_id_2 = AccountId::new_unchecked(Felt::new(ACCOUNT_ID_OFF_CHAIN_SENDER + 1)); - let account_id_3 = AccountId::new_unchecked(Felt::new(ACCOUNT_ID_OFF_CHAIN_SENDER + 2)); + let account_id_1 = AccountId::new_dummy( + [0; 15], + AccountType::RegularAccountImmutableCode, + miden_objects::accounts::AccountStorageMode::Private, + ); + let account_id_2 = AccountId::new_dummy( + [1; 15], + AccountType::RegularAccountImmutableCode, + miden_objects::accounts::AccountStorageMode::Private, + ); + let account_id_3 = AccountId::new_dummy( + [2; 15], + AccountType::RegularAccountImmutableCode, + miden_objects::accounts::AccountStorageMode::Private, + ); let block_inputs_from_store: BlockInputs = { let block_header = BlockHeader::mock(0, None, None, &[], Default::default()); @@ -98,8 +107,8 @@ fn block_witness_validation_inconsistent_account_ids() { #[test] fn block_witness_validation_inconsistent_account_hashes() { let account_id_1 = - AccountId::new_unchecked(Felt::new(ACCOUNT_ID_REGULAR_ACCOUNT_UPDATABLE_CODE_OFF_CHAIN)); - let account_id_2 = AccountId::new_unchecked(Felt::new(ACCOUNT_ID_OFF_CHAIN_SENDER)); + AccountId::try_from(ACCOUNT_ID_REGULAR_ACCOUNT_UPDATABLE_CODE_OFF_CHAIN).unwrap(); + let account_id_2 = AccountId::try_from(ACCOUNT_ID_OFF_CHAIN_SENDER).unwrap(); let account_1_hash_store = Digest::new([Felt::new(1u64), Felt::new(2u64), Felt::new(3u64), Felt::new(4u64)]); @@ -179,8 +188,8 @@ fn block_witness_validation_inconsistent_account_hashes() { #[test] fn block_witness_multiple_batches_per_account() { let x_account_id = - AccountId::new_unchecked(Felt::new(ACCOUNT_ID_REGULAR_ACCOUNT_UPDATABLE_CODE_OFF_CHAIN)); - let y_account_id = AccountId::new_unchecked(Felt::new(ACCOUNT_ID_OFF_CHAIN_SENDER)); + AccountId::try_from(ACCOUNT_ID_REGULAR_ACCOUNT_UPDATABLE_CODE_OFF_CHAIN).unwrap(); + let y_account_id = AccountId::try_from(ACCOUNT_ID_OFF_CHAIN_SENDER).unwrap(); let x_hashes = [ Digest::new((0..4).map(Felt::new).collect::>().try_into().unwrap()), @@ -273,11 +282,31 @@ async fn compute_account_root_success() { // Set up account states // --------------------------------------------------------------------------------------------- let account_ids = [ - AccountId::new_unchecked(Felt::new(ACCOUNT_ID_OFF_CHAIN_SENDER)), - AccountId::new_unchecked(Felt::new(ACCOUNT_ID_OFF_CHAIN_SENDER + 1)), - AccountId::new_unchecked(Felt::new(ACCOUNT_ID_OFF_CHAIN_SENDER + 2)), - AccountId::new_unchecked(Felt::new(ACCOUNT_ID_OFF_CHAIN_SENDER + 3)), - AccountId::new_unchecked(Felt::new(ACCOUNT_ID_OFF_CHAIN_SENDER + 4)), + AccountId::new_dummy( + [0; 15], + AccountType::RegularAccountImmutableCode, + miden_objects::accounts::AccountStorageMode::Private, + ), + AccountId::new_dummy( + [1; 15], + AccountType::RegularAccountImmutableCode, + miden_objects::accounts::AccountStorageMode::Private, + ), + AccountId::new_dummy( + [2; 15], + AccountType::RegularAccountImmutableCode, + miden_objects::accounts::AccountStorageMode::Private, + ), + AccountId::new_dummy( + [3; 15], + AccountType::RegularAccountImmutableCode, + miden_objects::accounts::AccountStorageMode::Private, + ), + AccountId::new_dummy( + [4; 15], + AccountType::RegularAccountImmutableCode, + miden_objects::accounts::AccountStorageMode::Private, + ), ]; let account_initial_states = [ @@ -373,11 +402,31 @@ async fn compute_account_root_empty_batches() { // Set up account states // --------------------------------------------------------------------------------------------- let account_ids = [ - AccountId::new_unchecked(Felt::new(0b0000_0000_0000_0000u64)), - AccountId::new_unchecked(Felt::new(0b1111_0000_0000_0000u64)), - AccountId::new_unchecked(Felt::new(0b1111_1111_0000_0000u64)), - AccountId::new_unchecked(Felt::new(0b1111_1111_1111_0000u64)), - AccountId::new_unchecked(Felt::new(0b1111_1111_1111_1111u64)), + AccountId::new_dummy( + [0; 15], + AccountType::RegularAccountImmutableCode, + AccountStorageMode::Private, + ), + AccountId::new_dummy( + [1; 15], + AccountType::RegularAccountImmutableCode, + AccountStorageMode::Private, + ), + AccountId::new_dummy( + [2; 15], + AccountType::RegularAccountImmutableCode, + AccountStorageMode::Private, + ), + AccountId::new_dummy( + [3; 15], + AccountType::RegularAccountImmutableCode, + AccountStorageMode::Private, + ), + AccountId::new_dummy( + [4; 15], + AccountType::RegularAccountImmutableCode, + AccountStorageMode::Private, + ), ]; let account_initial_states = [ @@ -495,9 +544,21 @@ async fn compute_note_root_empty_notes_success() { #[miden_node_test_macro::enable_logging] async fn compute_note_root_success() { let account_ids = [ - AccountId::new_unchecked(Felt::new(ACCOUNT_ID_OFF_CHAIN_SENDER)), - AccountId::new_unchecked(Felt::new(ACCOUNT_ID_OFF_CHAIN_SENDER + 1)), - AccountId::new_unchecked(Felt::new(ACCOUNT_ID_OFF_CHAIN_SENDER + 2)), + AccountId::new_dummy( + [0; 15], + AccountType::RegularAccountImmutableCode, + miden_objects::accounts::AccountStorageMode::Private, + ), + AccountId::new_dummy( + [1; 15], + AccountType::RegularAccountImmutableCode, + miden_objects::accounts::AccountStorageMode::Private, + ), + AccountId::new_dummy( + [2; 15], + AccountType::RegularAccountImmutableCode, + miden_objects::accounts::AccountStorageMode::Private, + ), ]; let notes_created: Vec = [ diff --git a/crates/block-producer/src/test_utils/account.rs b/crates/block-producer/src/test_utils/account.rs index af0d614c0..4c0930fa3 100644 --- a/crates/block-producer/src/test_utils/account.rs +++ b/crates/block-producer/src/test_utils/account.rs @@ -1,7 +1,7 @@ use std::{collections::HashMap, ops::Not, sync::LazyLock}; use miden_objects::{ - accounts::{get_account_seed, AccountStorageMode, AccountType}, + accounts::{AccountIdAnchor, AccountIdVersion, AccountStorageMode, AccountType}, Hasher, }; @@ -34,17 +34,25 @@ impl MockPrivateAccount { } fn generate(init_seed: [u8; 32], new_account: bool) -> Self { - let account_seed = get_account_seed( + let account_seed = AccountId::compute_account_seed( init_seed, AccountType::RegularAccountUpdatableCode, AccountStorageMode::Private, + AccountIdVersion::VERSION_0, + Digest::default(), Digest::default(), Digest::default(), ) .unwrap(); Self::new( - AccountId::new(account_seed, Digest::default(), Digest::default()).unwrap(), + AccountId::new( + account_seed, + AccountIdAnchor::PRE_GENESIS, + Digest::default(), + Digest::default(), + ) + .unwrap(), new_account.not().then(|| Hasher::hash(&init_seed)).unwrap_or_default(), ) } diff --git a/crates/block-producer/src/test_utils/store.rs b/crates/block-producer/src/test_utils/store.rs index 246841c5a..2314ad16b 100644 --- a/crates/block-producer/src/test_utils/store.rs +++ b/crates/block-producer/src/test_utils/store.rs @@ -42,7 +42,7 @@ impl MockStoreSuccessBuilder { let accounts = batches_iter .clone() .flat_map(TransactionBatch::account_initial_states) - .map(|(account_id, hash)| (account_id.into(), hash.into())); + .map(|(account_id, hash)| (account_id.prefix().into(), hash.into())); SimpleSmt::::with_leaves(accounts).unwrap() }; @@ -57,7 +57,8 @@ impl MockStoreSuccessBuilder { pub fn from_accounts(accounts: impl Iterator) -> Self { let accounts_smt = { - let accounts = accounts.map(|(account_id, hash)| (account_id.into(), hash.into())); + let accounts = + accounts.map(|(account_id, hash)| (account_id.prefix().into(), hash.into())); SimpleSmt::::with_leaves(accounts).unwrap() }; diff --git a/crates/proto/src/domain/accounts.rs b/crates/proto/src/domain/accounts.rs index 3a7de73d1..c19dbf56f 100644 --- a/crates/proto/src/domain/accounts.rs +++ b/crates/proto/src/domain/accounts.rs @@ -4,17 +4,14 @@ use miden_node_utils::formatting::format_opt; use miden_objects::{ accounts::{Account, AccountHeader, AccountId}, crypto::{hash::rpo::RpoDigest, merkle::MerklePath}, - utils::Serializable, + utils::{Deserializable, Serializable}, Digest, }; use crate::{ errors::{ConversionError, MissingFieldHelper}, generated::{ - account::{ - AccountHeader as AccountHeaderPb, AccountId as AccountIdPb, - AccountInfo as AccountInfoPb, AccountSummary as AccountSummaryPb, - }, + account as proto, responses::{AccountBlockInputRecord, AccountTransactionInputRecord}, }, }; @@ -22,13 +19,17 @@ use crate::{ // ACCOUNT ID // ================================================================================================ -impl Display for AccountIdPb { +impl Display for proto::AccountId { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - f.write_fmt(format_args!("0x{:x}", self.id)) + write!(f, "0x")?; + for byte in &self.id { + write!(f, "{:02x}", byte)?; + } + Ok(()) } } -impl Debug for AccountIdPb { +impl Debug for proto::AccountId { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { Display::fmt(self, f) } @@ -37,38 +38,26 @@ impl Debug for AccountIdPb { // INTO PROTO ACCOUNT ID // ------------------------------------------------------------------------------------------------ -impl From for AccountIdPb { - fn from(value: u64) -> Self { - AccountIdPb { id: value } - } -} - -impl From<&AccountId> for AccountIdPb { +impl From<&AccountId> for proto::AccountId { fn from(account_id: &AccountId) -> Self { (*account_id).into() } } -impl From for AccountIdPb { +impl From for proto::AccountId { fn from(account_id: AccountId) -> Self { - Self { id: account_id.into() } + Self { id: account_id.to_bytes() } } } // FROM PROTO ACCOUNT ID // ------------------------------------------------------------------------------------------------ -impl From for u64 { - fn from(value: AccountIdPb) -> Self { - value.id - } -} - -impl TryFrom for AccountId { +impl TryFrom for AccountId { type Error = ConversionError; - fn try_from(account_id: AccountIdPb) -> Result { - account_id.id.try_into().map_err(|_| ConversionError::NotAValidFelt) + fn try_from(account_id: proto::AccountId) -> Result { + AccountId::read_from_bytes(&account_id.id).map_err(|_| ConversionError::NotAValidFelt) } } @@ -82,7 +71,7 @@ pub struct AccountSummary { pub block_num: u32, } -impl From<&AccountSummary> for AccountSummaryPb { +impl From<&AccountSummary> for proto::AccountSummary { fn from(update: &AccountSummary) -> Self { Self { account_id: Some(update.account_id.into()), @@ -98,7 +87,7 @@ pub struct AccountInfo { pub details: Option, } -impl From<&AccountInfo> for AccountInfoPb { +impl From<&AccountInfo> for proto::AccountInfo { fn from(AccountInfo { summary, details }: &AccountInfo) -> Self { Self { summary: Some(summary.into()), @@ -180,7 +169,7 @@ impl From for AccountTransactionInputRecord { } } -impl From for AccountHeaderPb { +impl From for proto::AccountHeader { fn from(from: AccountHeader) -> Self { Self { vault_root: Some(from.vault_root().into()), diff --git a/crates/proto/src/generated/account.rs b/crates/proto/src/generated/account.rs index fc59f1fe3..85760632c 100644 --- a/crates/proto/src/generated/account.rs +++ b/crates/proto/src/generated/account.rs @@ -1,14 +1,13 @@ // This file is @generated by prost-build. -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, ::prost::Message)] #[prost(skip_debug)] pub struct AccountId { - /// A miden account is defined with a little bit of proof-of-work, the id itself is defined as - /// the first word of a hash digest. For this reason account ids can be considered as random - /// values, because of that the encoding below uses fixed 64 bits, instead of zig-zag encoding. - #[prost(fixed64, tag = "1")] - pub id: u64, + /// A Miden account ID is a 120-bit value derived from the commitments to account code and + /// storage, and a random user-provided seed. + #[prost(bytes = "vec", tag = "1")] + pub id: ::prost::alloc::vec::Vec, } -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct AccountSummary { #[prost(message, optional, tag = "1")] pub account_id: ::core::option::Option, diff --git a/crates/proto/src/generated/note.rs b/crates/proto/src/generated/note.rs index 563532b7d..3768961d0 100644 --- a/crates/proto/src/generated/note.rs +++ b/crates/proto/src/generated/note.rs @@ -1,5 +1,5 @@ // This file is @generated by prost-build. -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct NoteMetadata { #[prost(message, optional, tag = "1")] pub sender: ::core::option::Option, diff --git a/crates/proto/src/generated/requests.rs b/crates/proto/src/generated/requests.rs index db4d238a5..b58105d61 100644 --- a/crates/proto/src/generated/requests.rs +++ b/crates/proto/src/generated/requests.rs @@ -122,7 +122,7 @@ pub struct ListAccountsRequest {} #[derive(Clone, Copy, PartialEq, ::prost::Message)] pub struct ListNotesRequest {} /// Returns the latest state of an account with the specified ID. -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct GetAccountDetailsRequest { /// Account ID to get details. #[prost(message, optional, tag = "1")] @@ -136,7 +136,7 @@ pub struct GetBlockByNumberRequest { } /// Returns delta of the account states in the range from `from_block_num` (exclusive) to /// `to_block_num` (inclusive). -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct GetAccountStateDeltaRequest { /// ID of the account for which the delta is requested. #[prost(message, optional, tag = "1")] diff --git a/crates/proto/src/generated/responses.rs b/crates/proto/src/generated/responses.rs index e3bc58652..b635c10ed 100644 --- a/crates/proto/src/generated/responses.rs +++ b/crates/proto/src/generated/responses.rs @@ -114,7 +114,7 @@ pub struct GetBlockInputsResponse { >, } /// An account returned as a response to the GetTransactionInputs -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct AccountTransactionInputRecord { #[prost(message, optional, tag = "1")] pub account_id: ::core::option::Option, diff --git a/crates/proto/src/generated/transaction.rs b/crates/proto/src/generated/transaction.rs index ca77a09b3..7dca33a33 100644 --- a/crates/proto/src/generated/transaction.rs +++ b/crates/proto/src/generated/transaction.rs @@ -4,7 +4,7 @@ pub struct TransactionId { #[prost(message, optional, tag = "1")] pub id: ::core::option::Option, } -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct TransactionSummary { #[prost(message, optional, tag = "1")] pub transaction_id: ::core::option::Option, diff --git a/crates/rpc-proto/proto/account.proto b/crates/rpc-proto/proto/account.proto index 8f7becff9..6614b7bc7 100644 --- a/crates/rpc-proto/proto/account.proto +++ b/crates/rpc-proto/proto/account.proto @@ -4,10 +4,9 @@ package account; import "digest.proto"; message AccountId { - // A miden account is defined with a little bit of proof-of-work, the id itself is defined as - // the first word of a hash digest. For this reason account ids can be considered as random - // values, because of that the encoding below uses fixed 64 bits, instead of zig-zag encoding. - fixed64 id = 1; + // A Miden account ID is a 120-bit value derived from the commitments to account code and + // storage, and a random user-provided seed. + bytes id = 1; } message AccountSummary { diff --git a/crates/rpc/src/server/api.rs b/crates/rpc/src/server/api.rs index e815781e4..241fe9a8a 100644 --- a/crates/rpc/src/server/api.rs +++ b/crates/rpc/src/server/api.rs @@ -206,6 +206,7 @@ impl api_server::Api for RpcApi { let _account_id: AccountId = request .get_ref() .account_id + .clone() .ok_or(Status::invalid_argument("account_id is missing"))? .try_into() .map_err(|err| Status::invalid_argument(format!("Invalid account id: {err}")))?; diff --git a/crates/store/src/db/migrations/001-init.sql b/crates/store/src/db/migrations/001-init.sql index 293b9ca4d..287d9b478 100644 --- a/crates/store/src/db/migrations/001-init.sql +++ b/crates/store/src/db/migrations/001-init.sql @@ -28,7 +28,7 @@ CREATE TABLE note_index INTEGER NOT NULL, -- Index of note in batch, starting from 0 note_id BLOB NOT NULL, note_type INTEGER NOT NULL, -- 1-Public (0b01), 2-Private (0b10), 3-Encrypted (0b11) - sender INTEGER NOT NULL, + sender BLOB NOT NULL, tag INTEGER NOT NULL, aux INTEGER NOT NULL, execution_hint INTEGER NOT NULL, @@ -46,7 +46,7 @@ CREATE TABLE CREATE TABLE accounts ( - account_id INTEGER NOT NULL, + account_id BLOB NOT NULL, account_hash BLOB NOT NULL, block_num INTEGER NOT NULL, details BLOB, @@ -59,7 +59,7 @@ CREATE TABLE CREATE TABLE account_deltas ( - account_id INTEGER NOT NULL, + account_id BLOB NOT NULL, block_num INTEGER NOT NULL, nonce INTEGER NOT NULL, @@ -71,7 +71,7 @@ CREATE TABLE CREATE TABLE account_storage_slot_updates ( - account_id INTEGER NOT NULL, + account_id BLOB NOT NULL, block_num INTEGER NOT NULL, slot INTEGER NOT NULL, value BLOB NOT NULL, @@ -83,7 +83,7 @@ CREATE TABLE CREATE TABLE account_storage_map_updates ( - account_id INTEGER NOT NULL, + account_id BLOB NOT NULL, block_num INTEGER NOT NULL, slot INTEGER NOT NULL, key BLOB NOT NULL, @@ -96,9 +96,9 @@ CREATE TABLE CREATE TABLE account_fungible_asset_deltas ( - account_id INTEGER NOT NULL, + account_id BLOB NOT NULL, block_num INTEGER NOT NULL, - faucet_id INTEGER NOT NULL, + faucet_id BLOB NOT NULL, delta INTEGER NOT NULL, PRIMARY KEY (account_id, block_num, faucet_id), @@ -108,7 +108,7 @@ CREATE TABLE CREATE TABLE account_non_fungible_asset_updates ( - account_id INTEGER NOT NULL, + account_id BLOB NOT NULL, block_num INTEGER NOT NULL, vault_key BLOB NOT NULL, is_remove INTEGER NOT NULL, -- 0 - add, 1 - remove @@ -135,7 +135,7 @@ CREATE TABLE transactions ( transaction_id BLOB NOT NULL, - account_id INTEGER NOT NULL, + account_id BLOB NOT NULL, block_num INTEGER NOT NULL, PRIMARY KEY (transaction_id), diff --git a/crates/store/src/db/mod.rs b/crates/store/src/db/mod.rs index ac757b095..cc8a12be0 100644 --- a/crates/store/src/db/mod.rs +++ b/crates/store/src/db/mod.rs @@ -10,7 +10,7 @@ use miden_node_proto::{ generated::note::{Note as NotePb, NoteSyncRecord as NoteSyncRecordPb}, }; use miden_objects::{ - accounts::AccountDelta, + accounts::{AccountDelta, AccountId}, block::{Block, BlockNoteIndex}, crypto::{hash::rpo::RpoDigest, merkle::MerklePath, utils::Deserializable}, notes::{NoteId, NoteInclusionProof, NoteMetadata, Nullifier}, @@ -28,7 +28,7 @@ use crate::{ db::migrations::apply_migrations, errors::{DatabaseError, DatabaseSetupError, GenesisError, NoteSyncError, StateSyncError}, genesis::GenesisState, - types::{AccountId, BlockNumber}, + types::BlockNumber, COMPONENT, SQL_STATEMENT_CACHE_CAPACITY, }; diff --git a/crates/store/src/db/sql/mod.rs b/crates/store/src/db/sql/mod.rs index 5161ac386..359f4f79a 100644 --- a/crates/store/src/db/sql/mod.rs +++ b/crates/store/src/db/sql/mod.rs @@ -12,8 +12,9 @@ use std::{ use miden_node_proto::domain::accounts::{AccountInfo, AccountSummary}; use miden_objects::{ accounts::{ - delta::AccountUpdateDetails, AccountDelta, AccountStorageDelta, AccountVaultDelta, - FungibleAssetDelta, NonFungibleAssetDelta, NonFungibleDeltaAction, StorageMapDelta, + delta::AccountUpdateDetails, AccountDelta, AccountId, AccountStorageDelta, + AccountVaultDelta, FungibleAssetDelta, NonFungibleAssetDelta, NonFungibleDeltaAction, + StorageMapDelta, }, assets::NonFungibleAsset, block::{BlockAccountUpdate, BlockNoteIndex}, @@ -24,6 +25,7 @@ use miden_objects::{ BlockHeader, Digest, Word, }; use rusqlite::{params, types::Value, Connection, Transaction}; +use utils::read_from_blob_column; use super::{ NoteRecord, NoteSyncRecord, NoteSyncUpdate, NullifierInfo, Result, StateSyncUpdate, @@ -35,7 +37,7 @@ use crate::{ get_nullifier_prefix, u64_to_value, }, errors::{DatabaseError, NoteSyncError, StateSyncError}, - types::{AccountId, BlockNumber}, + types::BlockNumber, }; // ACCOUNT QUERIES // ================================================================================================ @@ -80,7 +82,7 @@ pub fn select_all_account_hashes(conn: &mut Connection) -> Result = account_ids.iter().copied().map(u64_to_value).collect(); + let account_ids: Vec = account_ids + .iter() + .copied() + .map(|account_id| account_id.to_bytes().into()) + .collect(); let mut rows = stmt.query(params![block_start, block_end, Rc::new(account_ids)])?; let mut result = Vec::new(); @@ -150,7 +156,7 @@ pub fn select_account(conn: &mut Connection, account_id: AccountId) -> Result = account_ids.iter().copied().map(u64_to_value).collect(); + let account_ids: Vec = account_ids + .iter() + .copied() + .map(|account_id| account_id.to_bytes().into()) + .collect(); let mut rows = stmt.query(params![Rc::new(account_ids)])?; let mut result = Vec::new(); @@ -295,7 +305,7 @@ pub fn select_account_delta( ", )?; - let account_id = u64_to_value(account_id); + let account_id = account_id.to_bytes(); let nonce = match select_nonce_stmt .query_row(params![account_id, block_start, block_end], |row| row.get::<_, u64>(0)) { @@ -337,9 +347,9 @@ pub fn select_account_delta( let mut rows = select_fungible_asset_deltas_stmt.query(params![account_id, block_start, block_end])?; while let Some(row) = rows.next()? { - let faucet_id: u64 = row.get(0)?; + let faucet_id: AccountId = read_from_blob_column(row, 0)?; let value = row.get(1)?; - fungible.insert(faucet_id.try_into()?, value); + fungible.insert(faucet_id, value); } let mut non_fungible_delta = NonFungibleAssetDelta::default(); @@ -350,8 +360,7 @@ pub fn select_account_delta( ])?; while let Some(row) = rows.next()? { let vault_key_data = row.get_ref(1)?.as_blob()?; - let vault_key = Word::read_from_bytes(vault_key_data)?; - let asset = NonFungibleAsset::try_from(vault_key) + let asset = NonFungibleAsset::read_from_bytes(vault_key_data) .map_err(|err| DatabaseError::DataCorrupted(err.to_string()))?; let action: usize = row.get(2)?; match action { @@ -394,11 +403,11 @@ pub fn upsert_accounts( let mut count = 0; for update in accounts.iter() { - let account_id = update.account_id().into(); + let account_id = update.account_id(); let (full_account, insert_delta) = match update.details() { AccountUpdateDetails::Private => (None, None), AccountUpdateDetails::New(account) => { - debug_assert_eq!(account_id, u64::from(account.id())); + debug_assert_eq!(account_id, account.id()); if account.hash() != update.new_state_hash() { return Err(DatabaseError::AccountHashesMismatch { @@ -412,7 +421,7 @@ pub fn upsert_accounts( (Some(Cow::Borrowed(account)), Some(Cow::Owned(insert_delta))) }, AccountUpdateDetails::Delta(delta) => { - let mut rows = select_details_stmt.query(params![u64_to_value(account_id)])?; + let mut rows = select_details_stmt.query(params![account_id.to_bytes()])?; let Some(row) = rows.next()? else { return Err(DatabaseError::AccountNotFoundInDb(account_id)); }; @@ -425,7 +434,7 @@ pub fn upsert_accounts( }; let inserted = upsert_stmt.execute(params![ - u64_to_value(account_id), + account_id.to_bytes(), update.new_state_hash().to_bytes(), block_num, full_account.as_ref().map(|account| account.to_bytes()), @@ -487,14 +496,14 @@ fn insert_account_delta( }))?; insert_acc_delta_stmt.execute(params![ - u64_to_value(account_id), + account_id.to_bytes(), block_number, delta.nonce().map(Into::::into).unwrap_or_default() ])?; for (&slot, value) in delta.storage().values() { insert_slot_update_stmt.execute(params![ - u64_to_value(account_id), + account_id.to_bytes(), block_number, slot, value.to_bytes() @@ -504,7 +513,7 @@ fn insert_account_delta( for (&slot, map_delta) in delta.storage().maps() { for (key, value) in map_delta.leaves() { insert_storage_map_update_stmt.execute(params![ - u64_to_value(account_id), + account_id.to_bytes(), block_number, slot, key.to_bytes(), @@ -515,9 +524,9 @@ fn insert_account_delta( for (&faucet_id, &delta) in delta.vault().fungible().iter() { insert_fungible_asset_delta_stmt.execute(params![ - u64_to_value(account_id), + account_id.to_bytes(), block_number, - u64_to_value(faucet_id.into()), + faucet_id.to_bytes(), delta, ])?; } @@ -528,9 +537,9 @@ fn insert_account_delta( NonFungibleDeltaAction::Remove => 1, }; insert_non_fungible_asset_update_stmt.execute(params![ - u64_to_value(account_id), + account_id.to_bytes(), block_number, - asset.vault_key().to_bytes(), + asset.to_bytes(), is_remove, ])?; } @@ -726,19 +735,14 @@ pub fn select_all_notes(conn: &mut Connection) -> Result> { let details = details_data.map(>::read_from_bytes).transpose()?; let note_type = row.get::<_, u8>(4)?.try_into()?; - let sender = column_value_as_u64(row, 5)?; + let sender = AccountId::read_from_bytes(row.get_ref(5)?.as_blob()?)?; let tag: u32 = row.get(6)?; let aux: u64 = row.get(7)?; let aux = aux.try_into().map_err(DatabaseError::InvalidFelt)?; let execution_hint = column_value_as_u64(row, 8)?; - let metadata = NoteMetadata::new( - sender.try_into()?, - note_type, - tag.into(), - execution_hint.try_into()?, - aux, - )?; + let metadata = + NoteMetadata::new(sender, note_type, tag.into(), execution_hint.try_into()?, aux)?; notes.push(NoteRecord { block_num: row.get(0)?, @@ -786,7 +790,7 @@ pub fn insert_notes(transaction: &Transaction, notes: &[NoteRecord]) -> Result::into(note.metadata.execution_hint()), @@ -820,26 +824,28 @@ pub fn select_notes_since_block_by_tag_and_sender( .prepare_cached(include_str!("queries/select_notes_since_block_by_tag_and_sender.sql"))?; let tags: Vec = tags.iter().copied().map(Into::into).collect(); - let account_ids: Vec = account_ids.iter().copied().map(u64_to_value).collect(); + let account_ids: Vec = account_ids + .iter() + .copied() + .map(|account_id| account_id.to_bytes().into()) + .collect(); let mut rows = stmt.query(params![Rc::new(tags), Rc::new(account_ids), block_num])?; let mut res = Vec::new(); while let Some(row) = rows.next()? { let block_num = row.get(0)?; let note_index = BlockNoteIndex::new(row.get(1)?, row.get(2)?)?; - let note_id_data = row.get_ref(3)?.as_blob()?; - let note_id = RpoDigest::read_from_bytes(note_id_data)?; + let note_id = read_from_blob_column(row, 3)?; let note_type = row.get::<_, u8>(4)?; - let sender = column_value_as_u64(row, 5)?; + let sender = read_from_blob_column(row, 5)?; let tag: u32 = row.get(6)?; let aux: u64 = row.get(7)?; let aux = aux.try_into().map_err(DatabaseError::InvalidFelt)?; let execution_hint = column_value_as_u64(row, 8)?; - let merkle_path_data = row.get_ref(9)?.as_blob()?; - let merkle_path = MerklePath::read_from_bytes(merkle_path_data)?; + let merkle_path = read_from_blob_column(row, 9)?; let metadata = NoteMetadata::new( - sender.try_into()?, + sender, NoteType::try_from(note_type)?, tag.into(), execution_hint.try_into()?, @@ -894,26 +900,20 @@ pub fn select_notes_by_id(conn: &mut Connection, note_ids: &[NoteId]) -> Result< let note_id_data = row.get_ref(3)?.as_blob()?; let note_id = NoteId::read_from_bytes(note_id_data)?; - let merkle_path_data = row.get_ref(9)?.as_blob()?; - let merkle_path = MerklePath::read_from_bytes(merkle_path_data)?; + let merkle_path = read_from_blob_column(row, 9)?; let details_data = row.get_ref(10)?.as_blob_or_null()?; let details = details_data.map(>::read_from_bytes).transpose()?; let note_type = row.get::<_, u8>(4)?.try_into()?; - let sender = column_value_as_u64(row, 5)?; + let sender = read_from_blob_column(row, 5)?; let tag: u32 = row.get(6)?; let aux: u64 = row.get(7)?; let aux = aux.try_into().map_err(DatabaseError::InvalidFelt)?; let execution_hint = column_value_as_u64(row, 8)?; - let metadata = NoteMetadata::new( - sender.try_into()?, - note_type, - tag.into(), - execution_hint.try_into()?, - aux, - )?; + let metadata = + NoteMetadata::new(sender, note_type, tag.into(), execution_hint.try_into()?, aux)?; notes.push(NoteRecord { block_num: row.get(0)?, @@ -1104,13 +1104,10 @@ pub fn insert_transactions( )?; let mut count = 0; for update in accounts { - let account_id = update.account_id().into(); + let account_id = update.account_id(); for transaction_id in update.transactions() { - count += stmt.execute(params![ - transaction_id.to_bytes(), - u64_to_value(account_id), - block_num - ])? + count += + stmt.execute(params![transaction_id.to_bytes(), account_id.to_bytes(), block_num])? } } Ok(count) @@ -1128,7 +1125,11 @@ pub fn select_transactions_by_accounts_and_block_range( block_end: BlockNumber, account_ids: &[AccountId], ) -> Result> { - let account_ids: Vec = account_ids.iter().copied().map(u64_to_value).collect(); + let account_ids: Vec = account_ids + .iter() + .copied() + .map(|account_id| account_id.to_bytes().into()) + .collect(); let mut stmt = conn.prepare_cached( " @@ -1151,7 +1152,7 @@ pub fn select_transactions_by_accounts_and_block_range( let mut result = vec![]; while let Some(row) = rows.next()? { - let account_id = column_value_as_u64(row, 0)?; + let account_id = read_from_blob_column(row, 0)?; let block_num = row.get(1)?; let transaction_id_data = row.get_ref(2)?.as_blob()?; let transaction_id = TransactionId::read_from_bytes(transaction_id_data)?; diff --git a/crates/store/src/db/sql/utils.rs b/crates/store/src/db/sql/utils.rs index 79bd1b198..5862a2f11 100644 --- a/crates/store/src/db/sql/utils.rs +++ b/crates/store/src/db/sql/utils.rs @@ -1,6 +1,6 @@ use miden_node_proto::domain::accounts::{AccountInfo, AccountSummary}; use miden_objects::{ - accounts::{Account, AccountDelta}, + accounts::{Account, AccountDelta, AccountId}, crypto::hash::rpo::RpoDigest, notes::Nullifier, utils::Deserializable, @@ -86,20 +86,33 @@ pub fn column_value_as_u64( Ok(value as u64) } +/// Gets a blob value from the database and tries to deserialize it into the necessary type. +pub fn read_from_blob_column(row: &rusqlite::Row<'_>, index: I) -> rusqlite::Result +where + I: rusqlite::RowIndex + Copy + Into, + T: Deserializable, +{ + let value = row.get_ref(index)?.as_blob()?; + + T::read_from_bytes(value).map_err(|err| { + rusqlite::Error::FromSqlConversionFailure( + index.into(), + rusqlite::types::Type::Blob, + Box::new(err), + ) + }) +} + /// Constructs `AccountSummary` from the row of `accounts` table. /// /// Note: field ordering must be the same, as in `accounts` table! pub fn account_summary_from_row(row: &rusqlite::Row<'_>) -> crate::db::Result { - let account_id = column_value_as_u64(row, 0)?; + let account_id = read_from_blob_column(row, 0)?; let account_hash_data = row.get_ref(1)?.as_blob()?; let account_hash = RpoDigest::read_from_bytes(account_hash_data)?; let block_num = row.get(2)?; - Ok(AccountSummary { - account_id: account_id.try_into()?, - account_hash, - block_num, - }) + Ok(AccountSummary { account_id, account_hash, block_num }) } /// Constructs `AccountInfo` from the row of `accounts` table. @@ -116,7 +129,7 @@ pub fn account_info_from_row(row: &rusqlite::Row<'_>) -> crate::db::Result, delta: &AccountDelta, final_state_hash: &RpoDigest, diff --git a/crates/store/src/db/tests.rs b/crates/store/src/db/tests.rs index f7260f761..614a91376 100644 --- a/crates/store/src/db/tests.rs +++ b/crates/store/src/db/tests.rs @@ -2,18 +2,18 @@ use miden_lib::transaction::TransactionKernel; use miden_node_proto::domain::accounts::AccountSummary; use miden_objects::{ accounts::{ - account_id::testing::{ - ACCOUNT_ID_FUNGIBLE_FAUCET_ON_CHAIN, ACCOUNT_ID_NON_FUNGIBLE_FAUCET_ON_CHAIN, - ACCOUNT_ID_OFF_CHAIN_SENDER, ACCOUNT_ID_REGULAR_ACCOUNT_UPDATABLE_CODE_OFF_CHAIN, - }, - delta::AccountUpdateDetails, - Account, AccountBuilder, AccountComponent, AccountDelta, AccountId, AccountStorageDelta, - AccountStorageMode, AccountType, AccountVaultDelta, StorageSlot, + delta::AccountUpdateDetails, Account, AccountBuilder, AccountComponent, AccountDelta, + AccountId, AccountStorageDelta, AccountStorageMode, AccountType, AccountVaultDelta, + StorageSlot, }, assets::{Asset, FungibleAsset, NonFungibleAsset, NonFungibleAssetDetails}, block::{BlockAccountUpdate, BlockNoteIndex, BlockNoteTree}, crypto::{hash::rpo::RpoDigest, merkle::MerklePath}, notes::{NoteExecutionHint, NoteId, NoteMetadata, NoteType, Nullifier}, + testing::account_id::{ + ACCOUNT_ID_FUNGIBLE_FAUCET_ON_CHAIN, ACCOUNT_ID_NON_FUNGIBLE_FAUCET_ON_CHAIN, + ACCOUNT_ID_OFF_CHAIN_SENDER, ACCOUNT_ID_REGULAR_ACCOUNT_UPDATABLE_CODE_OFF_CHAIN, + }, BlockHeader, Felt, FieldElement, Word, ZERO, }; use rusqlite::{vtab::array, Connection}; @@ -106,7 +106,13 @@ fn sql_insert_transactions() { #[test] fn sql_select_transactions() { fn query_transactions(conn: &mut Connection) -> Vec { - sql::select_transactions_by_accounts_and_block_range(conn, 0, 2, &[1]).unwrap() + sql::select_transactions_by_accounts_and_block_range( + conn, + 0, + 2, + &[AccountId::try_from(ACCOUNT_ID_OFF_CHAIN_SENDER).unwrap()], + ) + .unwrap() } let mut conn = create_db(); @@ -260,7 +266,7 @@ fn sql_select_notes_different_execution_hints() { ACCOUNT_ID_OFF_CHAIN_SENDER.try_into().unwrap(), NoteType::Public, 2.into(), - NoteExecutionHint::after_block(12), + NoteExecutionHint::after_block(12).unwrap(), Default::default(), ) .unwrap(), @@ -274,7 +280,7 @@ fn sql_select_notes_different_execution_hints() { assert_eq!(res.unwrap(), 1, "One element must have been inserted"); transaction.commit().unwrap(); let note = &sql::select_notes_by_id(&mut conn, &[num_to_rpo_digest(2).into()]).unwrap()[0]; - assert_eq!(note.metadata.execution_hint(), NoteExecutionHint::after_block(12)); + assert_eq!(note.metadata.execution_hint(), NoteExecutionHint::after_block(12).unwrap()); } #[test] @@ -289,16 +295,15 @@ fn sql_select_accounts() { assert!(accounts.is_empty()); // test multiple entries let mut state = vec![]; - for i in 0..10 { - let account_id = - ACCOUNT_ID_REGULAR_ACCOUNT_UPDATABLE_CODE_OFF_CHAIN + (i << 32) + 0b1111100000; - let account_hash = num_to_rpo_digest(i); + for i in 0..10u8 { + let account_id = AccountId::new_dummy( + [i; 15], + AccountType::RegularAccountImmutableCode, + miden_objects::accounts::AccountStorageMode::Private, + ); + let account_hash = num_to_rpo_digest(i as u64); state.push(AccountInfo { - summary: AccountSummary { - account_id: account_id.try_into().unwrap(), - account_hash, - block_num, - }, + summary: AccountSummary { account_id, account_hash, block_num }, details: None, }); @@ -306,7 +311,7 @@ fn sql_select_accounts() { let res = sql::upsert_accounts( &transaction, &[BlockAccountUpdate::new( - account_id.try_into().unwrap(), + account_id, account_hash, AccountUpdateDetails::Private, vec![], @@ -332,7 +337,7 @@ fn sql_public_account_details() { let nft1 = Asset::NonFungible( NonFungibleAsset::new( - &NonFungibleAssetDetails::new(non_fungible_faucet_id, vec![1, 2, 3]).unwrap(), + &NonFungibleAssetDetails::new(non_fungible_faucet_id.prefix(), vec![1, 2, 3]).unwrap(), ) .unwrap(), ); @@ -373,7 +378,7 @@ fn sql_public_account_details() { create_block(&mut conn, 2); - let read_delta = sql::select_account_delta(&mut conn, account.id().into(), 1, 2).unwrap(); + let read_delta = sql::select_account_delta(&mut conn, account.id(), 1, 2).unwrap(); assert_eq!(read_delta, None); @@ -382,7 +387,7 @@ fn sql_public_account_details() { let nft2 = Asset::NonFungible( NonFungibleAsset::new( - &NonFungibleAssetDetails::new(non_fungible_faucet_id, vec![4, 5, 6]).unwrap(), + &NonFungibleAssetDetails::new(non_fungible_faucet_id.prefix(), vec![4, 5, 6]).unwrap(), ) .unwrap(), ); @@ -421,7 +426,7 @@ fn sql_public_account_details() { assert_eq!(account_read.nonce(), account.nonce()); assert_eq!(account_read.storage(), account.storage()); - let read_delta = sql::select_account_delta(&mut conn, account.id().into(), 1, 2).unwrap(); + let read_delta = sql::select_account_delta(&mut conn, account.id(), 1, 2).unwrap(); assert_eq!(read_delta.as_ref(), Some(&delta2)); create_block(&mut conn, 3); @@ -464,7 +469,7 @@ fn sql_public_account_details() { assert_eq!(account_read.vault(), account.vault()); assert_eq!(account_read.nonce(), account.nonce()); - let read_delta = sql::select_account_delta(&mut conn, account.id().into(), 1, 3).unwrap(); + let read_delta = sql::select_account_delta(&mut conn, account.id(), 1, 3).unwrap(); delta2.merge(delta3).unwrap(); @@ -783,7 +788,11 @@ fn db_account() { create_block(&mut conn, block_num); // test empty table - let account_ids = vec![ACCOUNT_ID_REGULAR_ACCOUNT_UPDATABLE_CODE_OFF_CHAIN, 1, 2, 3, 4, 5]; + let account_ids: Vec = + [ACCOUNT_ID_REGULAR_ACCOUNT_UPDATABLE_CODE_OFF_CHAIN, 1, 2, 3, 4, 5] + .iter() + .map(|acc_id| (*acc_id).try_into().unwrap()) + .collect(); let res = sql::select_accounts_by_block_range(&mut conn, 0, u32::MAX, &account_ids).unwrap(); assert!(res.is_empty()); @@ -824,8 +833,13 @@ fn db_account() { assert!(res.is_empty()); // test query with unknown accounts - let res = sql::select_accounts_by_block_range(&mut conn, block_num + 1, u32::MAX, &[6, 7, 8]) - .unwrap(); + let res = sql::select_accounts_by_block_range( + &mut conn, + block_num + 1, + u32::MAX, + &[6.try_into().unwrap(), 7.try_into().unwrap(), 8.try_into().unwrap()], + ) + .unwrap(); assert!(res.is_empty()); } @@ -848,7 +862,7 @@ fn notes() { let note_index = BlockNoteIndex::new(0, 2).unwrap(); let note_id = num_to_rpo_digest(3); let tag = 5u32; - let sender = AccountId::new_unchecked(Felt::new(ACCOUNT_ID_OFF_CHAIN_SENDER)); + let sender = AccountId::try_from(ACCOUNT_ID_OFF_CHAIN_SENDER).unwrap(); let note_metadata = NoteMetadata::new(sender, NoteType::Public, tag.into(), NoteExecutionHint::none(), ZERO) .unwrap(); @@ -967,7 +981,10 @@ fn insert_transactions(conn: &mut Connection) -> usize { let count = sql::insert_transactions( &transaction, block_num, - &[mock_block_account_update(AccountId::new_unchecked(Felt::ONE), 1)], + &[mock_block_account_update( + AccountId::try_from(ACCOUNT_ID_OFF_CHAIN_SENDER).unwrap(), + 1, + )], ) .unwrap(); transaction.commit().unwrap(); diff --git a/crates/store/src/errors.rs b/crates/store/src/errors.rs index 2465da52f..87a139d76 100644 --- a/crates/store/src/errors.rs +++ b/crates/store/src/errors.rs @@ -2,6 +2,7 @@ use std::io; use deadpool_sqlite::PoolError; use miden_objects::{ + accounts::AccountId, crypto::{ hash::rpo::RpoDigest, merkle::{MerkleError, MmrError}, @@ -16,7 +17,7 @@ use thiserror::Error; use tokio::sync::oneshot::error::RecvError; use tonic::Status; -use crate::types::{AccountId, BlockNumber}; +use crate::types::BlockNumber; // INTERNAL ERRORS // ================================================================================================= diff --git a/crates/store/src/genesis.rs b/crates/store/src/genesis.rs index a19c6c882..98bf5379f 100644 --- a/crates/store/src/genesis.rs +++ b/crates/store/src/genesis.rs @@ -46,11 +46,10 @@ impl GenesisState { }) .collect(); - let account_smt: SimpleSmt = SimpleSmt::with_leaves( - accounts - .iter() - .map(|update| (update.account_id().into(), update.new_state_hash().into())), - )?; + let account_smt: SimpleSmt = + SimpleSmt::with_leaves(accounts.iter().map(|update| { + (update.account_id().prefix().into(), update.new_state_hash().into()) + }))?; let header = BlockHeader::new( self.version, diff --git a/crates/store/src/server/api.rs b/crates/store/src/server/api.rs index 09fd45bc4..fe1808003 100644 --- a/crates/store/src/server/api.rs +++ b/crates/store/src/server/api.rs @@ -2,7 +2,7 @@ use std::{collections::BTreeSet, sync::Arc}; use miden_node_proto::{ convert, - domain::notes::NoteAuthenticationInfo, + domain::{accounts::AccountInfo, notes::NoteAuthenticationInfo}, errors::ConversionError, generated::{ self, @@ -32,6 +32,7 @@ use miden_node_proto::{ try_convert, }; use miden_objects::{ + accounts::AccountId, block::Block, crypto::hash::rpo::RpoDigest, notes::{NoteId, Nullifier}, @@ -41,7 +42,7 @@ use miden_objects::{ use tonic::{Request, Response, Status}; use tracing::{debug, info, instrument}; -use crate::{state::State, types::AccountId, COMPONENT}; +use crate::{state::State, COMPONENT}; // STORE API // ================================================================================================ @@ -160,7 +161,7 @@ impl api_server::Api for StoreApi { ) -> Result, Status> { let request = request.into_inner(); - let account_ids: Vec = request.account_ids.iter().map(|e| e.id).collect(); + let account_ids: Vec = read_account_ids(&request.account_ids)?; let (state, delta) = self .state @@ -323,12 +324,8 @@ impl api_server::Api for StoreApi { request: Request, ) -> Result, Status> { let request = request.into_inner(); - let account_info = self - .state - .get_account_details( - request.account_id.ok_or(invalid_argument("Account missing id"))?.into(), - ) - .await?; + let account_id = read_account_id(request.account_id)?; + let account_info: AccountInfo = self.state.get_account_details(account_id).await?; Ok(Response::new(GetAccountDetailsResponse { details: Some((&account_info).into()), @@ -389,7 +386,7 @@ impl api_server::Api for StoreApi { let request = request.into_inner(); let nullifiers = validate_nullifiers(&request.nullifiers)?; - let account_ids: Vec = request.account_ids.iter().map(|e| e.id).collect(); + let account_ids = read_account_ids(&request.account_ids)?; let unauthenticated_notes = validate_notes(&request.unauthenticated_notes)?; let unauthenticated_notes = unauthenticated_notes.into_iter().collect(); @@ -416,7 +413,7 @@ impl api_server::Api for StoreApi { debug!(target: COMPONENT, ?request); - let account_id = request.account_id.ok_or(invalid_argument("`account_id` missing"))?.id; + let account_id = read_account_id(request.account_id)?; let nullifiers = validate_nullifiers(&request.nullifiers)?; let unauthenticated_notes = validate_notes(&request.unauthenticated_notes)?; @@ -490,7 +487,7 @@ impl api_server::Api for StoreApi { debug!(target: COMPONENT, ?request); let include_headers = request.include_headers.unwrap_or_default(); - let account_ids: Vec = convert(request.account_ids); + let account_ids: Vec = read_account_ids(&request.account_ids)?; let request_code_commitments: BTreeSet = try_convert(request.code_commitments) .map_err(|err| { Status::invalid_argument(format!("Invalid code commitment: {}", err)) @@ -522,13 +519,10 @@ impl api_server::Api for StoreApi { debug!(target: COMPONENT, ?request); + let account_id = read_account_id(request.account_id)?; let delta = self .state - .get_account_state_delta( - request.account_id.ok_or(invalid_argument("account_id is missing"))?.id, - request.from_block_num, - request.to_block_num, - ) + .get_account_state_delta(account_id, request.from_block_num, request.to_block_num) .await? .map(|delta| delta.to_bytes()); @@ -607,6 +601,24 @@ fn invalid_argument(err: E) -> Status { Status::invalid_argument(err.to_string()) } +fn read_account_id(id: Option) -> Result { + id.ok_or(invalid_argument("missing account ID"))? + .try_into() + .map_err(|err| invalid_argument(format!("invalid account ID: {}", err))) +} + +#[instrument(target = COMPONENT, skip_all, err)] +fn read_account_ids( + account_ids: &[generated::account::AccountId], +) -> Result, Status> { + account_ids + .iter() + .cloned() + .map(AccountId::try_from) + .collect::>() + .map_err(|_| invalid_argument("Byte array is not a valid AccountId")) +} + #[instrument(target = COMPONENT, skip_all, err)] fn validate_nullifiers(nullifiers: &[generated::digest::Digest]) -> Result, Status> { nullifiers diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index 3e048715f..87c624a11 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -15,9 +15,9 @@ use miden_node_proto::{ generated::responses::{AccountProofsResponse, AccountStateHeader, GetBlockInputsResponse}, AccountInputRecord, NullifierWitness, }; -use miden_node_utils::formatting::{format_account_id, format_array}; +use miden_node_utils::formatting::format_array; use miden_objects::{ - accounts::{AccountDelta, AccountHeader}, + accounts::{AccountDelta, AccountHeader, AccountId}, block::Block, crypto::{ hash::rpo::RpoDigest, @@ -45,7 +45,7 @@ use crate::{ StateSyncError, }, nullifier_tree::NullifierTree, - types::{AccountId, BlockNumber}, + types::BlockNumber, COMPONENT, }; // STRUCTURES @@ -254,7 +254,7 @@ impl State { let account_tree_update = inner.account_tree.compute_mutations( block.updated_accounts().iter().map(|update| { ( - LeafIndex::new_max_depth(update.account_id().into()), + LeafIndex::new_max_depth(update.account_id().prefix().into()), update.new_state_hash().into(), ) }), @@ -604,12 +604,8 @@ impl State { .cloned() .map(|account_id| { let ValuePath { value: account_hash, path: proof } = - inner.account_tree.open(&LeafIndex::new_max_depth(account_id)); - Ok(AccountInputRecord { - account_id: account_id.try_into()?, - account_hash, - proof, - }) + inner.account_tree.open(&LeafIndex::new_max_depth(account_id.prefix().into())); + Ok(AccountInputRecord { account_id, account_hash, proof }) }) .collect::>()?; @@ -642,11 +638,14 @@ impl State { nullifiers: &[Nullifier], unauthenticated_notes: Vec, ) -> Result { - info!(target: COMPONENT, account_id = %format_account_id(account_id), nullifiers = %format_array(nullifiers)); + info!(target: COMPONENT, account_id = %account_id.to_string(), nullifiers = %format_array(nullifiers)); let inner = self.inner.read().await; - let account_hash = inner.account_tree.open(&LeafIndex::new_max_depth(account_id)).value; + let account_hash = inner + .account_tree + .open(&LeafIndex::new_max_depth(account_id.prefix().into())) + .value; let nullifiers = nullifiers .iter() @@ -705,7 +704,7 @@ impl State { let infos = self.db.select_accounts_by_ids(account_ids.clone()).await?; if account_ids.len() > infos.len() { - let found_ids = infos.iter().map(|info| info.summary.account_id.into()).collect(); + let found_ids = infos.iter().map(|info| info.summary.account_id).collect(); return Err(DatabaseError::AccountsNotFoundInDb( BTreeSet::from_iter(account_ids).difference(&found_ids).copied().collect(), )); @@ -716,7 +715,7 @@ impl State { .filter_map(|info| { info.details.map(|details| { ( - info.summary.account_id.into(), + info.summary.account_id, AccountStateHeader { header: Some(AccountHeader::from(&details).into()), storage_header: details.storage().get_header().to_bytes(), @@ -736,7 +735,7 @@ impl State { let responses = account_ids .into_iter() .map(|account_id| { - let acc_leaf_idx = LeafIndex::new_max_depth(account_id); + let acc_leaf_idx = LeafIndex::new_max_depth(account_id.prefix().into()); let opening = inner_state.account_tree.open(&acc_leaf_idx); let state_header = state_headers.get(&account_id).cloned(); @@ -821,7 +820,7 @@ async fn load_accounts( .select_all_account_hashes() .await? .into_iter() - .map(|(id, account_hash)| (id, account_hash.into())) + .map(|(id, account_hash)| (id.prefix().into(), account_hash.into())) .collect(); SimpleSmt::with_leaves(account_data) diff --git a/crates/store/src/types.rs b/crates/store/src/types.rs index 44702b127..f75d6ec60 100644 --- a/crates/store/src/types.rs +++ b/crates/store/src/types.rs @@ -1,2 +1 @@ pub type BlockNumber = u32; -pub type AccountId = u64; diff --git a/proto/account.proto b/proto/account.proto index 8f7becff9..6614b7bc7 100644 --- a/proto/account.proto +++ b/proto/account.proto @@ -4,10 +4,9 @@ package account; import "digest.proto"; message AccountId { - // A miden account is defined with a little bit of proof-of-work, the id itself is defined as - // the first word of a hash digest. For this reason account ids can be considered as random - // values, because of that the encoding below uses fixed 64 bits, instead of zig-zag encoding. - fixed64 id = 1; + // A Miden account ID is a 120-bit value derived from the commitments to account code and + // storage, and a random user-provided seed. + bytes id = 1; } message AccountSummary { From 78ca36c6a3f574ff6131db8e8f51c460d2751d96 Mon Sep 17 00:00:00 2001 From: Mirko <48352201+Mirko-von-Leipzig@users.noreply.github.com> Date: Wed, 25 Dec 2024 22:42:18 +0200 Subject: [PATCH 36/50] feat: support transaction expiration (#582) --- CHANGELOG.md | 1 + .../block-producer/src/domain/transaction.rs | 4 + crates/block-producer/src/errors.rs | 7 + crates/block-producer/src/lib.rs | 8 + .../block-producer/src/mempool/batch_graph.rs | 26 ++ .../src/mempool/inflight_state/mod.rs | 86 +++++- crates/block-producer/src/mempool/mod.rs | 190 +++++++------- crates/block-producer/src/mempool/tests.rs | 245 ++++++++++++++++++ .../src/mempool/transaction_expiration.rs | 75 ++++++ .../src/{server/mod.rs => server.rs} | 13 +- crates/block-producer/src/server/api.rs | 74 ------ 11 files changed, 551 insertions(+), 178 deletions(-) create mode 100644 crates/block-producer/src/mempool/tests.rs create mode 100644 crates/block-producer/src/mempool/transaction_expiration.rs rename crates/block-producer/src/{server/mod.rs => server.rs} (95%) delete mode 100644 crates/block-producer/src/server/api.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 90e461c89..63d12a651 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ - Support Https in endpoint configuration (#556). - Upgrade `block-producer` from FIFO queue to mempool dependency graph (#562). +- Support transaction expiration (#582). ### Changes diff --git a/crates/block-producer/src/domain/transaction.rs b/crates/block-producer/src/domain/transaction.rs index 01c74d371..c36302c5f 100644 --- a/crates/block-producer/src/domain/transaction.rs +++ b/crates/block-producer/src/domain/transaction.rs @@ -113,6 +113,10 @@ impl AuthenticatedTransaction { pub fn raw_proven_transaction(&self) -> &ProvenTransaction { &self.inner } + + pub fn expires_at(&self) -> BlockNumber { + BlockNumber::new(self.inner.expiration_block_num()) + } } #[cfg(test)] diff --git a/crates/block-producer/src/errors.rs b/crates/block-producer/src/errors.rs index dba60eff3..021c0b90a 100644 --- a/crates/block-producer/src/errors.rs +++ b/crates/block-producer/src/errors.rs @@ -91,6 +91,12 @@ pub enum AddTransactionError { #[error("Deserialization failed: {0}")] DeserializationError(String), + + #[error("Transaction expired at {expired_at} but the limit was {limit}")] + Expired { + expired_at: BlockNumber, + limit: BlockNumber, + }, } impl From for tonic::Status { @@ -102,6 +108,7 @@ impl From for tonic::Status { | VerificationFailed(VerifyTxError::OutputNotesAlreadyExist(_)) | VerificationFailed(VerifyTxError::IncorrectAccountInitialHash { .. }) | VerificationFailed(VerifyTxError::InvalidTransactionProof(_)) + | Expired { .. } | DeserializationError(_) => Self::invalid_argument(value.to_string()), // Internal errors which should not be communicated to the user. diff --git a/crates/block-producer/src/lib.rs b/crates/block-producer/src/lib.rs index 16efd97b2..809d8c0ca 100644 --- a/crates/block-producer/src/lib.rs +++ b/crates/block-producer/src/lib.rs @@ -1,5 +1,7 @@ use std::time::Duration; +use mempool::BlockNumber; + #[cfg(test)] pub mod test_utils; @@ -38,6 +40,12 @@ const SERVER_MAX_BATCHES_PER_BLOCK: usize = 4; /// the store and verification in the mempool. const SERVER_MEMPOOL_STATE_RETENTION: usize = 5; +/// Transactions are rejected by the mempool if there is less than this amount of blocks between the +/// chain tip and the transaction's expiration block. +/// +/// This rejects transactions which would likely expire before making it into a block. +const SERVER_MEMPOOL_EXPIRATION_SLACK: BlockNumber = BlockNumber::new(2); + const _: () = assert!( SERVER_MAX_BATCHES_PER_BLOCK <= miden_objects::MAX_BATCHES_PER_BLOCK, "Server constraint cannot exceed the protocol's constraint" diff --git a/crates/block-producer/src/mempool/batch_graph.rs b/crates/block-producer/src/mempool/batch_graph.rs index 14dae83da..ef7822150 100644 --- a/crates/block-producer/src/mempool/batch_graph.rs +++ b/crates/block-producer/src/mempool/batch_graph.rs @@ -167,6 +167,26 @@ impl BatchGraph { Ok(batches) } + /// Remove all batches associated with the given transactions (and their descendents). + /// + /// Transactions not currently associated with a batch are allowed, but have no impact. + /// + /// # Returns + /// + /// Returns all removed batches and their transactions. + /// + /// Unlike [remove_batches](Self::remove_batches), this has no error condition as batches are + /// derived internally. + pub fn remove_batches_with_transactions<'a>( + &mut self, + txs: impl Iterator, + ) -> BTreeMap> { + let batches = txs.filter_map(|tx| self.transactions.get(tx)).copied().collect(); + + self.remove_batches(batches) + .expect("batches must exist as they have been taken from internal map") + } + /// Removes the set of committed batches from the graph. /// /// The batches _must_ have been previously selected for inclusion in a block using @@ -250,6 +270,12 @@ impl BatchGraph { pub fn contains(&self, id: &BatchId) -> bool { self.batches.contains_key(id) } + + /// Returns the transactions associated with the given batch, otherwise returns `None` if the + /// batch does not exist. + pub fn get_transactions(&self, id: &BatchId) -> Option<&[TransactionId]> { + self.batches.get(id).map(Vec::as_slice) + } } #[cfg(any(test, doctest))] diff --git a/crates/block-producer/src/mempool/inflight_state/mod.rs b/crates/block-producer/src/mempool/inflight_state/mod.rs index 2f4836621..a68d4283b 100644 --- a/crates/block-producer/src/mempool/inflight_state/mod.rs +++ b/crates/block-producer/src/mempool/inflight_state/mod.rs @@ -57,6 +57,12 @@ pub struct InflightState { /// The latest committed block height. chain_tip: BlockNumber, + + /// Number of blocks to allow between chain tip and a transaction's expiration block height + /// before rejecting it. + /// + /// A new transaction is rejected if its expiration block is this close to the chain tip. + expiration_slack: BlockNumber, } /// A summary of a transaction's impact on the state. @@ -83,10 +89,15 @@ impl Delta { impl InflightState { /// Creates an [InflightState] which will retain committed state for the given /// amount of blocks before pruning them. - pub fn new(chain_tip: BlockNumber, num_retained_blocks: usize) -> Self { + pub fn new( + chain_tip: BlockNumber, + num_retained_blocks: usize, + expiration_slack: BlockNumber, + ) -> Self { Self { num_retained_blocks, chain_tip, + expiration_slack, accounts: Default::default(), nullifiers: Default::default(), output_notes: Default::default(), @@ -123,6 +134,14 @@ impl InflightState { } fn verify_transaction(&self, tx: &AuthenticatedTransaction) -> Result<(), AddTransactionError> { + // Check that the transaction hasn't already expired. + if tx.expires_at() <= self.chain_tip + self.expiration_slack { + return Err(AddTransactionError::Expired { + expired_at: tx.expires_at(), + limit: self.chain_tip + self.expiration_slack, + }); + } + // The mempool retains recently committed blocks, in addition to the state that is currently // inflight. This overlap with the committed state allows us to verify incoming // transactions against the current state (committed + inflight). Transactions are @@ -362,6 +381,47 @@ mod tests { MockProvenTxBuilder, }; + #[test] + fn rejects_expired_transaction() { + let chain_tip = BlockNumber::new(123); + let mut uut = InflightState::new(chain_tip, 5, BlockNumber::new(0)); + + let expired = MockProvenTxBuilder::with_account_index(0) + .expiration_block_num(chain_tip.into_inner()) + .build(); + let expired = AuthenticatedTransaction::from_inner(expired) + .with_authentication_height(chain_tip.into_inner()); + + let err = uut.add_transaction(&expired).unwrap_err(); + assert_matches!(err, AddTransactionError::Expired { .. }); + } + + /// Ensures that the specified expiration slack is adhered to. + #[test] + fn expiration_slack_is_respected() { + let slack = BlockNumber::new(3); + let chain_tip = BlockNumber::new(123); + let expiration_limit = BlockNumber::new(chain_tip.into_inner() + slack.into_inner()); + let mut uut = InflightState::new(chain_tip, 5, slack); + + let unexpired = MockProvenTxBuilder::with_account_index(0) + .expiration_block_num(expiration_limit.next().into_inner()) + .build(); + let unexpired = AuthenticatedTransaction::from_inner(unexpired) + .with_authentication_height(chain_tip.into_inner()); + + uut.add_transaction(&unexpired).unwrap(); + + let expired = MockProvenTxBuilder::with_account_index(1) + .expiration_block_num(expiration_limit.into_inner()) + .build(); + let expired = AuthenticatedTransaction::from_inner(expired) + .with_authentication_height(chain_tip.into_inner()); + + let err = uut.add_transaction(&expired).unwrap_err(); + assert_matches!(err, AddTransactionError::Expired { .. }); + } + #[test] fn rejects_duplicate_nullifiers() { let account = mock_account_id(1); @@ -379,7 +439,7 @@ mod tests { .unauthenticated_notes(vec![mock_note(note_seed)]) .build(); - let mut uut = InflightState::new(BlockNumber::default(), 1); + let mut uut = InflightState::new(BlockNumber::default(), 1, BlockNumber::new(0)); uut.add_transaction(&AuthenticatedTransaction::from_inner(tx0)).unwrap(); uut.add_transaction(&AuthenticatedTransaction::from_inner(tx1)).unwrap(); @@ -406,7 +466,7 @@ mod tests { .output_notes(vec![note.clone()]) .build(); - let mut uut = InflightState::new(BlockNumber::default(), 1); + let mut uut = InflightState::new(BlockNumber::default(), 1, BlockNumber::new(0)); uut.add_transaction(&AuthenticatedTransaction::from_inner(tx0)).unwrap(); let err = uut.add_transaction(&AuthenticatedTransaction::from_inner(tx1)).unwrap_err(); @@ -426,7 +486,7 @@ mod tests { let tx = MockProvenTxBuilder::with_account(account, states[0], states[1]).build(); - let mut uut = InflightState::new(BlockNumber::default(), 1); + let mut uut = InflightState::new(BlockNumber::default(), 1, BlockNumber::new(0)); let err = uut .add_transaction(&AuthenticatedTransaction::from_inner(tx).with_store_state(states[2])) .unwrap_err(); @@ -448,7 +508,7 @@ mod tests { let tx0 = MockProvenTxBuilder::with_account(account, states[0], states[1]).build(); let tx1 = MockProvenTxBuilder::with_account(account, states[1], states[2]).build(); - let mut uut = InflightState::new(BlockNumber::default(), 1); + let mut uut = InflightState::new(BlockNumber::default(), 1, BlockNumber::new(0)); uut.add_transaction(&AuthenticatedTransaction::from_inner(tx0)).unwrap(); uut.add_transaction(&AuthenticatedTransaction::from_inner(tx1).with_empty_store_state()) .unwrap(); @@ -465,7 +525,7 @@ mod tests { ) .build(); - let mut uut = InflightState::new(BlockNumber::default(), 1); + let mut uut = InflightState::new(BlockNumber::default(), 1, BlockNumber::new(0)); uut.add_transaction(&AuthenticatedTransaction::from_inner(tx).with_empty_store_state()) .unwrap(); } @@ -478,7 +538,7 @@ mod tests { let tx0 = MockProvenTxBuilder::with_account(account, states[0], states[1]).build(); let tx1 = MockProvenTxBuilder::with_account(account, states[1], states[2]).build(); - let mut uut = InflightState::new(BlockNumber::default(), 1); + let mut uut = InflightState::new(BlockNumber::default(), 1, BlockNumber::new(0)); uut.add_transaction(&AuthenticatedTransaction::from_inner(tx0)).unwrap(); // Feed in an old state via input. This should be ignored, and the previous tx's final @@ -504,7 +564,7 @@ mod tests { .unauthenticated_notes(vec![mock_note(note_seed)]) .build(); - let mut uut = InflightState::new(BlockNumber::default(), 1); + let mut uut = InflightState::new(BlockNumber::default(), 1, BlockNumber::new(0)); uut.add_transaction(&AuthenticatedTransaction::from_inner(tx0.clone())).unwrap(); uut.add_transaction(&AuthenticatedTransaction::from_inner(tx1.clone())).unwrap(); @@ -535,7 +595,7 @@ mod tests { .unauthenticated_notes(vec![mock_note(note_seed)]) .build(); - let mut uut = InflightState::new(BlockNumber::default(), 1); + let mut uut = InflightState::new(BlockNumber::default(), 1, BlockNumber::new(0)); uut.add_transaction(&tx0.clone()).unwrap(); uut.add_transaction(&tx1.clone()).unwrap(); @@ -577,7 +637,7 @@ mod tests { for i in 0..states.len() { // Insert all txs and then revert the last `i` of them. // This should match only inserting the first `N-i` of them. - let mut reverted = InflightState::new(BlockNumber::default(), 1); + let mut reverted = InflightState::new(BlockNumber::default(), 1, BlockNumber::new(0)); for (idx, tx) in txs.iter().enumerate() { reverted.add_transaction(tx).unwrap_or_else(|err| { panic!("Inserting tx #{idx} in iteration {i} should succeed: {err}") @@ -587,7 +647,7 @@ mod tests { txs.iter().rev().take(i).rev().map(AuthenticatedTransaction::id).collect(), ); - let mut inserted = InflightState::new(BlockNumber::default(), 1); + let mut inserted = InflightState::new(BlockNumber::default(), 1, BlockNumber::new(0)); for (idx, tx) in txs.iter().rev().skip(i).rev().enumerate() { inserted.add_transaction(tx).unwrap_or_else(|err| { panic!("Inserting tx #{idx} in iteration {i} should succeed: {err}") @@ -631,7 +691,7 @@ mod tests { // This should match only inserting the final `N-i` transactions. // Note: we force all committed state to immedietely be pruned by setting // it to zero. - let mut committed = InflightState::new(BlockNumber::default(), 0); + let mut committed = InflightState::new(BlockNumber::default(), 0, BlockNumber::new(0)); for (idx, tx) in txs.iter().enumerate() { committed.add_transaction(tx).unwrap_or_else(|err| { panic!("Inserting tx #{idx} in iteration {i} should succeed: {err}") @@ -639,7 +699,7 @@ mod tests { } committed.commit_block(txs.iter().take(i).map(AuthenticatedTransaction::id)); - let mut inserted = InflightState::new(BlockNumber::new(1), 0); + let mut inserted = InflightState::new(BlockNumber::new(1), 0, BlockNumber::new(0)); for (idx, tx) in txs.iter().skip(i).enumerate() { // We need to adjust the height since we are effectively at block "1" now. let tx = tx.clone().with_authentication_height(1); diff --git a/crates/block-producer/src/mempool/mod.rs b/crates/block-producer/src/mempool/mod.rs index 8b1d756b1..b16efcd31 100644 --- a/crates/block-producer/src/mempool/mod.rs +++ b/crates/block-producer/src/mempool/mod.rs @@ -1,12 +1,15 @@ use std::{collections::BTreeSet, fmt::Display, sync::Arc}; use batch_graph::BatchGraph; +use graph::GraphError; use inflight_state::InflightState; use miden_objects::{ - MAX_ACCOUNTS_PER_BATCH, MAX_INPUT_NOTES_PER_BATCH, MAX_OUTPUT_NOTES_PER_BATCH, + transaction::TransactionId, MAX_ACCOUNTS_PER_BATCH, MAX_INPUT_NOTES_PER_BATCH, + MAX_OUTPUT_NOTES_PER_BATCH, }; use tokio::sync::Mutex; use tracing::instrument; +use transaction_expiration::TransactionExpirations; use transaction_graph::TransactionGraph; use crate::{ @@ -19,8 +22,12 @@ use crate::{ mod batch_graph; mod graph; mod inflight_state; +mod transaction_expiration; mod transaction_graph; +#[cfg(test)] +mod tests; + #[derive(Debug, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] pub struct BlockNumber(u32); @@ -30,8 +37,16 @@ impl Display for BlockNumber { } } +impl std::ops::Add for BlockNumber { + type Output = Self; + + fn add(self, rhs: Self) -> Self::Output { + BlockNumber::new(self.0 + rhs.0) + } +} + impl BlockNumber { - pub fn new(x: u32) -> Self { + pub const fn new(x: u32) -> Self { Self(x) } @@ -53,6 +68,10 @@ impl BlockNumber { pub fn checked_sub(&self, rhs: Self) -> Option { self.0.checked_sub(rhs.0).map(Self) } + + pub fn into_inner(self) -> u32 { + self.0 + } } // MEMPOOL BUDGET @@ -166,6 +185,13 @@ pub struct Mempool { /// Inflight transactions. transactions: TransactionGraph, + /// Tracks inflight transaction expirations. + /// + /// This is used to identify inflight transactions that have become invalid once their + /// expiration block constraint has been violated. This occurs naturally as blocks get + /// committed and the chain grows. + expirations: TransactionExpirations, + /// Inflight batches. batches: BatchGraph, @@ -186,8 +212,15 @@ impl Mempool { batch_budget: BatchBudget, block_budget: BlockBudget, state_retention: usize, + expiration_slack: BlockNumber, ) -> SharedMempool { - Arc::new(Mutex::new(Self::new(chain_tip, batch_budget, block_budget, state_retention))) + Arc::new(Mutex::new(Self::new( + chain_tip, + batch_budget, + block_budget, + state_retention, + expiration_slack, + ))) } fn new( @@ -195,15 +228,17 @@ impl Mempool { batch_budget: BatchBudget, block_budget: BlockBudget, state_retention: usize, + expiration_slack: BlockNumber, ) -> Mempool { Self { chain_tip, batch_budget, block_budget, - state: InflightState::new(chain_tip, state_retention), + state: InflightState::new(chain_tip, state_retention, expiration_slack), block_in_progress: Default::default(), transactions: Default::default(), batches: Default::default(), + expirations: Default::default(), } } @@ -224,6 +259,8 @@ impl Mempool { // Add transaction to inflight state. let parents = self.state.add_transaction(&transaction)?; + self.expirations.insert(transaction.id(), transaction.expires_at()); + self.transactions .insert(transaction, parents) .expect("Transaction should insert after passing inflight state"); @@ -321,10 +358,17 @@ impl Mempool { .commit_transactions(&transactions) .expect("Transaction graph malformed"); + // Remove the committed transactions from expiration tracking. + self.expirations.remove(transactions.iter()); + // Inform inflight state about committed data. self.state.commit_block(transactions); - self.chain_tip.increment(); + + // Revert expired transactions and their descendents. + let expired = self.expirations.get(block_number); + self.revert_transactions(expired.into_iter().collect()) + .expect("expired transactions must be part of the mempool"); } /// Block and all of its contents and dependents are purged from the mempool. @@ -339,93 +383,61 @@ impl Mempool { let batches = self.block_in_progress.take().expect("No block in progress to be failed"); - // Remove all transactions from the graphs. - let purged = self.batches.remove_batches(batches).expect("Batch should be removed"); - let transactions = purged.into_values().flatten().collect(); - - let transactions = self - .transactions - .remove_transactions(transactions) - .expect("Failed transactions should be removed"); - - // Rollback state. - self.state.revert_transactions(transactions); + // Revert all transactions. This is the nuclear (but simplest) solution. + // + // We currently don't have a way of determining why this block failed so take the safe route + // and just nuke all associated transactions. + // + // TODO: improve this strategy, e.g. count txn failures (as well as in e.g. batch failures), + // and only revert upon exceeding some threshold. + let txs = batches + .into_iter() + .flat_map(|batch_id| { + self.batches + .get_transactions(&batch_id) + .expect("batch from a block must be in the mempool") + }) + .copied() + .collect(); + self.revert_transactions(txs) + .expect("transactions from a block must be part of the mempool"); } -} -#[cfg(test)] -mod tests { - use pretty_assertions::assert_eq; - - use super::*; - use crate::test_utils::MockProvenTxBuilder; - - impl Mempool { - fn for_tests() -> Self { - Self::new(BlockNumber::new(0), Default::default(), Default::default(), 5) - } - } - - // BATCH REVERSION TESTS - // ================================================================================================ - - #[test] - fn children_of_reverted_batches_are_ignored() { - // Batches are proved concurrently. This makes it possible for a child job to complete after - // the parent has been reverted (and therefore reverting the child job). Such a child job - // should be ignored. - let txs = MockProvenTxBuilder::sequential(); - - let mut uut = Mempool::for_tests(); - uut.add_transaction(txs[0].clone()).unwrap(); - let (parent_batch, batch_txs) = uut.select_batch().unwrap(); - assert_eq!(batch_txs, vec![txs[0].clone()]); - - uut.add_transaction(txs[1].clone()).unwrap(); - let (child_batch_a, batch_txs) = uut.select_batch().unwrap(); - assert_eq!(batch_txs, vec![txs[1].clone()]); - - uut.add_transaction(txs[2].clone()).unwrap(); - let (_, batch_txs) = uut.select_batch().unwrap(); - assert_eq!(batch_txs, vec![txs[2].clone()]); - - // Child batch jobs are now dangling. - uut.batch_failed(parent_batch); - let reference = uut.clone(); - - // Success or failure of the child job should effectively do nothing. - uut.batch_failed(child_batch_a); - assert_eq!(uut, reference); - - let proof = - TransactionBatch::new([txs[2].raw_proven_transaction()], Default::default()).unwrap(); - uut.batch_proved(proof); - assert_eq!(uut, reference); - } - - #[test] - fn reverted_batch_transactions_are_requeued() { - let txs = MockProvenTxBuilder::sequential(); - - let mut uut = Mempool::for_tests(); - uut.add_transaction(txs[0].clone()).unwrap(); - uut.select_batch().unwrap(); - - uut.add_transaction(txs[1].clone()).unwrap(); - let (failed_batch, _) = uut.select_batch().unwrap(); - - uut.add_transaction(txs[2].clone()).unwrap(); - uut.select_batch().unwrap(); - - // Middle batch failed, so it and its child transaction should be re-entered into the queue. - uut.batch_failed(failed_batch); + /// Reverts the given transactions and their descendents from the mempool. + /// + /// This includes removing them from the transaction and batch graphs, as well as cleaning up + /// their inflight state and expiration mappings. + /// + /// Transactions that were in reverted batches but that are disjoint from the reverted + /// transactions (i.e. not descendents) are requeued and _not_ reverted. + /// + /// # Errors + /// + /// Returns an error if any transaction was not in the transaction graph i.e. if the transaction + /// is unknown. + fn revert_transactions( + &mut self, + txs: Vec, + ) -> Result<(), GraphError> { + // Revert all transactions and their descendents, and their associated batches. + let reverted = self.transactions.remove_transactions(txs)?; + let batches_reverted = self.batches.remove_batches_with_transactions(reverted.iter()); + + // Requeue transactions that are disjoint from the reverted set, but were part of the + // reverted batches. + let to_requeue = batches_reverted + .into_values() + .flatten() + .filter(|tx| !reverted.contains(tx)) + .collect(); + self.transactions + .requeue_transactions(to_requeue) + .expect("transactions from batches must be requeueable"); - let mut reference = Mempool::for_tests(); - reference.add_transaction(txs[0].clone()).unwrap(); - reference.select_batch().unwrap(); - reference.add_transaction(txs[1].clone()).unwrap(); - reference.add_transaction(txs[2].clone()).unwrap(); + // Cleanup state. + self.expirations.remove(reverted.iter()); + self.state.revert_transactions(reverted); - assert_eq!(uut, reference); + Ok(()) } } diff --git a/crates/block-producer/src/mempool/tests.rs b/crates/block-producer/src/mempool/tests.rs new file mode 100644 index 000000000..2646706a3 --- /dev/null +++ b/crates/block-producer/src/mempool/tests.rs @@ -0,0 +1,245 @@ +use pretty_assertions::assert_eq; + +use super::*; +use crate::test_utils::MockProvenTxBuilder; + +impl Mempool { + fn for_tests() -> Self { + Self::new( + BlockNumber::new(0), + Default::default(), + Default::default(), + 5, + Default::default(), + ) + } +} + +// BATCH FAILED TESTS +// ================================================================================================ + +#[test] +fn children_of_failed_batches_are_ignored() { + // Batches are proved concurrently. This makes it possible for a child job to complete after + // the parent has been reverted (and therefore reverting the child job). Such a child job + // should be ignored. + let txs = MockProvenTxBuilder::sequential(); + + let mut uut = Mempool::for_tests(); + uut.add_transaction(txs[0].clone()).unwrap(); + let (parent_batch, batch_txs) = uut.select_batch().unwrap(); + assert_eq!(batch_txs, vec![txs[0].clone()]); + + uut.add_transaction(txs[1].clone()).unwrap(); + let (child_batch_a, batch_txs) = uut.select_batch().unwrap(); + assert_eq!(batch_txs, vec![txs[1].clone()]); + + uut.add_transaction(txs[2].clone()).unwrap(); + let (_, batch_txs) = uut.select_batch().unwrap(); + assert_eq!(batch_txs, vec![txs[2].clone()]); + + // Child batch jobs are now dangling. + uut.batch_failed(parent_batch); + let reference = uut.clone(); + + // Success or failure of the child job should effectively do nothing. + uut.batch_failed(child_batch_a); + assert_eq!(uut, reference); + + let proof = + TransactionBatch::new([txs[2].raw_proven_transaction()], Default::default()).unwrap(); + uut.batch_proved(proof); + assert_eq!(uut, reference); +} + +#[test] +fn failed_batch_transactions_are_requeued() { + let txs = MockProvenTxBuilder::sequential(); + + let mut uut = Mempool::for_tests(); + uut.add_transaction(txs[0].clone()).unwrap(); + uut.select_batch().unwrap(); + + uut.add_transaction(txs[1].clone()).unwrap(); + let (failed_batch, _) = uut.select_batch().unwrap(); + + uut.add_transaction(txs[2].clone()).unwrap(); + uut.select_batch().unwrap(); + + // Middle batch failed, so it and its child transaction should be re-entered into the queue. + uut.batch_failed(failed_batch); + + let mut reference = Mempool::for_tests(); + reference.add_transaction(txs[0].clone()).unwrap(); + reference.select_batch().unwrap(); + reference.add_transaction(txs[1].clone()).unwrap(); + reference.add_transaction(txs[2].clone()).unwrap(); + + assert_eq!(uut, reference); +} + +// BLOCK COMMITTED TESTS +// ================================================================================================ + +/// Expired transactions should be reverted once their expiration block is committed. +#[test] +fn block_commit_reverts_expired_txns() { + let mut uut = Mempool::for_tests(); + + let tx_to_commit = MockProvenTxBuilder::with_account_index(0).build(); + let tx_to_commit = AuthenticatedTransaction::from_inner(tx_to_commit); + + // Force the tx into a pending block. + uut.add_transaction(tx_to_commit.clone()).unwrap(); + uut.select_batch().unwrap(); + uut.batch_proved( + TransactionBatch::new([tx_to_commit.raw_proven_transaction()], Default::default()).unwrap(), + ); + let (block, _) = uut.select_block(); + // A reverted transaction behaves as if it never existed, the current state is the expected + // outcome, plus an extra committed block at the end. + let mut reference = uut.clone(); + + // Add a new transaction which will expire when the pending block is committed. + let tx_to_revert = MockProvenTxBuilder::with_account_index(1) + .expiration_block_num(block.into_inner()) + .build(); + let tx_to_revert = AuthenticatedTransaction::from_inner(tx_to_revert); + uut.add_transaction(tx_to_revert).unwrap(); + + // Commit the pending block which should revert the above tx. + uut.block_committed(block); + reference.block_committed(block); + + assert_eq!(uut, reference); +} + +#[test] +fn empty_block_commitment() { + let mut uut = Mempool::for_tests(); + + for _ in 0..3 { + let (block, _) = uut.select_block(); + uut.block_committed(block); + } +} + +#[test] +#[should_panic] +fn blocks_must_be_committed_sequentially() { + let mut uut = Mempool::for_tests(); + + let (block, _) = uut.select_block(); + uut.block_committed(block.next()); +} + +#[test] +#[should_panic] +fn block_commitment_is_rejected_if_no_block_is_in_flight() { + Mempool::for_tests().block_committed(BlockNumber::new(1)); +} + +#[test] +#[should_panic] +fn cannot_have_multple_inflight_blocks() { + let mut uut = Mempool::for_tests(); + + uut.select_block(); + uut.select_block(); +} + +// BLOCK FAILED TESTS +// ================================================================================================ + +/// A failed block should have all of its transactions reverted. +#[test] +fn block_failure_reverts_its_transactions() { + let mut uut = Mempool::for_tests(); + // We will revert everything so the reference should be the empty mempool. + let reference = uut.clone(); + + let reverted_txs = MockProvenTxBuilder::sequential(); + + uut.add_transaction(reverted_txs[0].clone()).unwrap(); + uut.select_batch().unwrap(); + uut.batch_proved( + TransactionBatch::new([reverted_txs[0].raw_proven_transaction()], Default::default()) + .unwrap(), + ); + + // Block 1 will contain just the first batch. + let (block_number, _) = uut.select_block(); + + // Create another dependent batch. + uut.add_transaction(reverted_txs[1].clone()).unwrap(); + uut.select_batch(); + // Create another dependent transaction. + uut.add_transaction(reverted_txs[2].clone()).unwrap(); + + // Fail the block which should result in everything reverting. + uut.block_failed(block_number); + + assert_eq!(uut, reference); +} + +// TRANSACTION REVERSION TESTS +// ================================================================================================ + +/// Ensures that reverting transactions is equivalent to them never being inserted at all. +/// +/// This checks that there are no forgotten links to them exist anywhere in the mempool by +/// comparing to a reference mempool that never had them inserted. +#[test] +fn reverted_transactions_and_descendents_are_non_existent() { + let mut uut = Mempool::for_tests(); + + let reverted_txs = MockProvenTxBuilder::sequential(); + + uut.add_transaction(reverted_txs[0].clone()).unwrap(); + uut.select_batch().unwrap(); + + uut.add_transaction(reverted_txs[1].clone()).unwrap(); + uut.select_batch().unwrap(); + + uut.add_transaction(reverted_txs[2].clone()).unwrap(); + uut.revert_transactions(vec![reverted_txs[1].id()]).unwrap(); + + // We expect the second batch and the latter reverted txns to be non-existent. + let mut reference = Mempool::for_tests(); + reference.add_transaction(reverted_txs[0].clone()).unwrap(); + reference.select_batch().unwrap(); + + assert_eq!(uut, reference); +} + +/// Reverting transactions causes their batches to also revert. These batches in turn contain +/// non-reverted transactions which should be requeued (and not reverted). +#[test] +fn reverted_transaction_batches_are_requeued() { + let mut uut = Mempool::for_tests(); + + let unrelated_txs = MockProvenTxBuilder::sequential(); + let reverted_txs = MockProvenTxBuilder::sequential(); + + uut.add_transaction(reverted_txs[0].clone()).unwrap(); + uut.add_transaction(unrelated_txs[0].clone()).unwrap(); + uut.select_batch().unwrap(); + + uut.add_transaction(reverted_txs[1].clone()).unwrap(); + uut.add_transaction(unrelated_txs[1].clone()).unwrap(); + uut.select_batch().unwrap(); + + uut.add_transaction(reverted_txs[2].clone()).unwrap(); + uut.add_transaction(unrelated_txs[2].clone()).unwrap(); + uut.revert_transactions(vec![reverted_txs[1].id()]).unwrap(); + + // We expect the second batch and the latter reverted txns to be non-existent. + let mut reference = Mempool::for_tests(); + reference.add_transaction(reverted_txs[0].clone()).unwrap(); + reference.add_transaction(unrelated_txs[0].clone()).unwrap(); + reference.select_batch().unwrap(); + reference.add_transaction(unrelated_txs[1].clone()).unwrap(); + reference.add_transaction(unrelated_txs[2].clone()).unwrap(); + + assert_eq!(uut, reference); +} diff --git a/crates/block-producer/src/mempool/transaction_expiration.rs b/crates/block-producer/src/mempool/transaction_expiration.rs new file mode 100644 index 000000000..103e665f5 --- /dev/null +++ b/crates/block-producer/src/mempool/transaction_expiration.rs @@ -0,0 +1,75 @@ +use std::collections::{btree_map::Entry, BTreeMap, BTreeSet}; + +use miden_objects::transaction::TransactionId; + +use super::BlockNumber; + +/// Tracks transactions and their expiration block heights. +/// +/// Implemented as a bi-directional map internally to allow for efficient lookups via both +/// transaction ID and block number. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct TransactionExpirations { + /// Transaction lookup index. + by_tx: BTreeMap, + /// Block number lookup index. + by_block: BTreeMap>, +} + +impl TransactionExpirations { + /// Add the transaction to the tracker. + pub fn insert(&mut self, tx: TransactionId, block: BlockNumber) { + self.by_tx.insert(tx, block); + self.by_block.entry(block).or_default().insert(tx); + } + + /// Returns all transactions that are expiring at the given block number. + pub fn get(&mut self, block: BlockNumber) -> BTreeSet { + self.by_block.get(&block).cloned().unwrap_or_default() + } + + /// Removes the transactions from the tracker. + /// + /// Unknown transactions are ignored. + pub fn remove<'a>(&mut self, txs: impl Iterator) { + for tx in txs { + if let Some(block) = self.by_tx.remove(tx) { + let Entry::Occupied(entry) = self.by_block.entry(block).and_modify(|x| { + x.remove(tx); + }) else { + panic!("block entry must exist as this is a bidirectional mapping"); + }; + + // Prune the entire block's entry if no transactions are tracked. + if entry.get().is_empty() { + entry.remove(); + } + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_utils::Random; + + /// Removing a transaction may result in a block's mapping being empty. This test ensures that + /// such maps are pruned to prevent endless growth. + #[test] + fn remove_prunes_empty_block_maps() { + let tx = Random::with_random_seed().draw_tx_id(); + let block = BlockNumber::new(123); + + let mut uut = TransactionExpirations::default(); + uut.insert(tx, block); + uut.remove(std::iter::once(&tx)); + + assert_eq!(uut, Default::default()); + } + + #[test] + fn get_empty() { + assert!(TransactionExpirations::default().get(BlockNumber(123)).is_empty()); + } +} diff --git a/crates/block-producer/src/server/mod.rs b/crates/block-producer/src/server.rs similarity index 95% rename from crates/block-producer/src/server/mod.rs rename to crates/block-producer/src/server.rs index 1a33ad14a..a3ee6043c 100644 --- a/crates/block-producer/src/server/mod.rs +++ b/crates/block-producer/src/server.rs @@ -22,7 +22,7 @@ use crate::{ errors::{AddTransactionError, BlockProducerError, VerifyTxError}, mempool::{BatchBudget, BlockBudget, BlockNumber, Mempool, SharedMempool}, store::StoreClient, - COMPONENT, SERVER_MEMPOOL_STATE_RETENTION, + COMPONENT, SERVER_MEMPOOL_EXPIRATION_SLACK, SERVER_MEMPOOL_STATE_RETENTION, }; /// Represents an initialized block-producer component where the RPC connection is open, @@ -37,6 +37,7 @@ pub struct BlockProducer { batch_budget: BatchBudget, block_budget: BlockBudget, state_retention: usize, + expiration_slack: BlockNumber, rpc_listener: TcpListener, store: StoreClient, chain_tip: BlockNumber, @@ -78,6 +79,7 @@ impl BlockProducer { batch_budget: Default::default(), block_budget: Default::default(), state_retention: SERVER_MEMPOOL_STATE_RETENTION, + expiration_slack: SERVER_MEMPOOL_EXPIRATION_SLACK, store, rpc_listener, chain_tip, @@ -94,9 +96,16 @@ impl BlockProducer { rpc_listener, store, chain_tip, + expiration_slack, } = self; - let mempool = Mempool::shared(chain_tip, batch_budget, block_budget, state_retention); + let mempool = Mempool::shared( + chain_tip, + batch_budget, + block_budget, + state_retention, + expiration_slack, + ); // Spawn rpc server and batch and block provers. // diff --git a/crates/block-producer/src/server/api.rs b/crates/block-producer/src/server/api.rs deleted file mode 100644 index cb800e791..000000000 --- a/crates/block-producer/src/server/api.rs +++ /dev/null @@ -1,74 +0,0 @@ -use std::sync::Arc; - -use miden_node_proto::generated::{ - block_producer::api_server, requests::SubmitProvenTransactionRequest, - responses::SubmitProvenTransactionResponse, -}; -use miden_node_utils::formatting::{format_input_notes, format_output_notes}; -use miden_objects::{transaction::ProvenTransaction, utils::serde::Deserializable}; -use tonic::Status; -use tracing::{debug, info, instrument}; - -use crate::{ - batch_builder::BatchBuilder, - txqueue::{TransactionQueue, TransactionValidator}, - COMPONENT, -}; - -// BLOCK PRODUCER -// ================================================================================================ - -pub struct BlockProducerApi { - queue: Arc>, -} - -impl BlockProducerApi { - pub fn new(queue: Arc>) -> Self { - Self { queue } - } -} - -#[tonic::async_trait] -impl api_server::Api for BlockProducerApi -where - TV: TransactionValidator, - BB: BatchBuilder, -{ - #[instrument( - target = COMPONENT, - name = "block_producer:submit_proven_transaction", - skip_all, - err - )] - async fn submit_proven_transaction( - &self, - request: tonic::Request, - ) -> Result, Status> { - let request = request.into_inner(); - debug!(target: COMPONENT, ?request); - - let tx = ProvenTransaction::read_from_bytes(&request.transaction) - .map_err(|_| Status::invalid_argument("Invalid transaction"))?; - - info!( - target: COMPONENT, - tx_id = %tx.id().to_hex(), - account_id = %tx.account_id().to_hex(), - initial_account_hash = %tx.account_update().init_state_hash(), - final_account_hash = %tx.account_update().final_state_hash(), - input_notes = %format_input_notes(tx.input_notes()), - output_notes = %format_output_notes(tx.output_notes()), - block_ref = %tx.block_ref(), - "Deserialized transaction" - ); - debug!(target: COMPONENT, proof = ?tx.proof()); - - let block_height = self - .queue - .add_transaction(tx) - .await - .map_err(|err| Status::invalid_argument(format!("{:?}", err)))?; - - Ok(tonic::Response::new(SubmitProvenTransactionResponse { block_height })) - } -} From ecacf1c410881149c6812abe2735c2a25b813b4e Mon Sep 17 00:00:00 2001 From: Tomas Date: Mon, 13 Jan 2025 04:10:21 -0300 Subject: [PATCH 37/50] style: standardize proto aliases (#610) --- CHANGELOG.md | 1 + crates/proto/src/domain/accounts.rs | 57 +++++++++++++---------- crates/proto/src/domain/digest.rs | 14 +++--- crates/proto/src/domain/merkle.rs | 62 ++++++++++++------------- crates/proto/src/domain/notes.rs | 33 +++++++------ crates/proto/src/domain/nullifiers.rs | 26 +++++++---- crates/proto/src/domain/transactions.rs | 23 ++++----- crates/store/src/db/mod.rs | 6 +-- 8 files changed, 117 insertions(+), 105 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 63d12a651..43d54f049 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ ### Changes +- Standardized protobuf type aliases (#609). - [BREAKING] Added support for new two `Felt` account ID (#591). - [BREAKING] Inverted `TransactionInputs.missing_unauthenticated_notes` to `found_missing_notes` (#509). diff --git a/crates/proto/src/domain/accounts.rs b/crates/proto/src/domain/accounts.rs index c19dbf56f..08c1174e1 100644 --- a/crates/proto/src/domain/accounts.rs +++ b/crates/proto/src/domain/accounts.rs @@ -10,16 +10,13 @@ use miden_objects::{ use crate::{ errors::{ConversionError, MissingFieldHelper}, - generated::{ - account as proto, - responses::{AccountBlockInputRecord, AccountTransactionInputRecord}, - }, + generated as proto, }; // ACCOUNT ID // ================================================================================================ -impl Display for proto::AccountId { +impl Display for proto::account::AccountId { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { write!(f, "0x")?; for byte in &self.id { @@ -29,7 +26,7 @@ impl Display for proto::AccountId { } } -impl Debug for proto::AccountId { +impl Debug for proto::account::AccountId { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { Display::fmt(self, f) } @@ -38,13 +35,13 @@ impl Debug for proto::AccountId { // INTO PROTO ACCOUNT ID // ------------------------------------------------------------------------------------------------ -impl From<&AccountId> for proto::AccountId { +impl From<&AccountId> for proto::account::AccountId { fn from(account_id: &AccountId) -> Self { (*account_id).into() } } -impl From for proto::AccountId { +impl From for proto::account::AccountId { fn from(account_id: AccountId) -> Self { Self { id: account_id.to_bytes() } } @@ -53,10 +50,10 @@ impl From for proto::AccountId { // FROM PROTO ACCOUNT ID // ------------------------------------------------------------------------------------------------ -impl TryFrom for AccountId { +impl TryFrom for AccountId { type Error = ConversionError; - fn try_from(account_id: proto::AccountId) -> Result { + fn try_from(account_id: proto::account::AccountId) -> Result { AccountId::read_from_bytes(&account_id.id).map_err(|_| ConversionError::NotAValidFelt) } } @@ -71,7 +68,7 @@ pub struct AccountSummary { pub block_num: u32, } -impl From<&AccountSummary> for proto::AccountSummary { +impl From<&AccountSummary> for proto::account::AccountSummary { fn from(update: &AccountSummary) -> Self { Self { account_id: Some(update.account_id.into()), @@ -87,7 +84,7 @@ pub struct AccountInfo { pub details: Option, } -impl From<&AccountInfo> for proto::AccountInfo { +impl From<&AccountInfo> for proto::account::AccountInfo { fn from(AccountInfo { summary, details }: &AccountInfo) -> Self { Self { summary: Some(summary.into()), @@ -106,7 +103,7 @@ pub struct AccountInputRecord { pub proof: MerklePath, } -impl From for AccountBlockInputRecord { +impl From for proto::responses::AccountBlockInputRecord { fn from(from: AccountInputRecord) -> Self { Self { account_id: Some(from.account_id.into()), @@ -116,23 +113,29 @@ impl From for AccountBlockInputRecord { } } -impl TryFrom for AccountInputRecord { +impl TryFrom for AccountInputRecord { type Error = ConversionError; - fn try_from(account_input_record: AccountBlockInputRecord) -> Result { + fn try_from( + account_input_record: proto::responses::AccountBlockInputRecord, + ) -> Result { Ok(Self { account_id: account_input_record .account_id - .ok_or(AccountBlockInputRecord::missing_field(stringify!(account_id)))? + .ok_or(proto::responses::AccountBlockInputRecord::missing_field(stringify!( + account_id + )))? .try_into()?, account_hash: account_input_record .account_hash - .ok_or(AccountBlockInputRecord::missing_field(stringify!(account_hash)))? + .ok_or(proto::responses::AccountBlockInputRecord::missing_field(stringify!( + account_hash + )))? .try_into()?, proof: account_input_record .proof .as_ref() - .ok_or(AccountBlockInputRecord::missing_field(stringify!(proof)))? + .ok_or(proto::responses::AccountBlockInputRecord::missing_field(stringify!(proof)))? .try_into()?, }) } @@ -160,7 +163,7 @@ impl Display for AccountState { } } -impl From for AccountTransactionInputRecord { +impl From for proto::responses::AccountTransactionInputRecord { fn from(from: AccountState) -> Self { Self { account_id: Some(from.account_id.into()), @@ -169,7 +172,7 @@ impl From for AccountTransactionInputRecord { } } -impl From for proto::AccountHeader { +impl From for proto::account::AccountHeader { fn from(from: AccountHeader) -> Self { Self { vault_root: Some(from.vault_root().into()), @@ -180,18 +183,24 @@ impl From for proto::AccountHeader { } } -impl TryFrom for AccountState { +impl TryFrom for AccountState { type Error = ConversionError; - fn try_from(from: AccountTransactionInputRecord) -> Result { + fn try_from( + from: proto::responses::AccountTransactionInputRecord, + ) -> Result { let account_id = from .account_id - .ok_or(AccountTransactionInputRecord::missing_field(stringify!(account_id)))? + .ok_or(proto::responses::AccountTransactionInputRecord::missing_field(stringify!( + account_id + )))? .try_into()?; let account_hash = from .account_hash - .ok_or(AccountTransactionInputRecord::missing_field(stringify!(account_hash)))? + .ok_or(proto::responses::AccountTransactionInputRecord::missing_field(stringify!( + account_hash + )))? .try_into()?; // If the hash is equal to `Digest::default()`, it signifies that this is a new account diff --git a/crates/proto/src/domain/digest.rs b/crates/proto/src/domain/digest.rs index fb396cb99..811ee872d 100644 --- a/crates/proto/src/domain/digest.rs +++ b/crates/proto/src/domain/digest.rs @@ -207,23 +207,23 @@ mod test { use hex::{FromHex, ToHex}; use proptest::prelude::*; - use crate::generated::digest::Digest; + use crate::generated::digest as proto; #[test] fn hex_digest() { - let digest = Digest { + let digest = proto::Digest { d0: 3488802789098113751, d1: 5271242459988994564, d2: 17816570245237064784, d3: 10910963388447438895, }; let encoded: String = ToHex::encode_hex(&digest); - let round_trip: Result = FromHex::from_hex::<&[u8]>(encoded.as_ref()); + let round_trip: Result = FromHex::from_hex::<&[u8]>(encoded.as_ref()); assert_eq!(digest, round_trip.unwrap()); - let digest = Digest { d0: 0, d1: 0, d2: 0, d3: 0 }; + let digest = proto::Digest { d0: 0, d1: 0, d2: 0, d3: 0 }; let encoded: String = ToHex::encode_hex(&digest); - let round_trip: Result = FromHex::from_hex::<&[u8]>(encoded.as_ref()); + let round_trip: Result = FromHex::from_hex::<&[u8]>(encoded.as_ref()); assert_eq!(digest, round_trip.unwrap()); } @@ -235,9 +235,9 @@ mod test { d2: u64, d3: u64, ) { - let digest = Digest { d0, d1, d2, d3 }; + let digest = proto::Digest { d0, d1, d2, d3 }; let encoded: String = ToHex::encode_hex(&digest); - let round_trip: Result = FromHex::from_hex::<&[u8]>(encoded.as_ref()); + let round_trip: Result = FromHex::from_hex::<&[u8]>(encoded.as_ref()); assert_eq!(digest, round_trip.unwrap()); } } diff --git a/crates/proto/src/domain/merkle.rs b/crates/proto/src/domain/merkle.rs index f21bec087..2cd1adf74 100644 --- a/crates/proto/src/domain/merkle.rs +++ b/crates/proto/src/domain/merkle.rs @@ -6,29 +6,29 @@ use miden_objects::{ use super::{convert, try_convert}; use crate::{ errors::{ConversionError, MissingFieldHelper}, - generated, + generated as proto, }; // MERKLE PATH // ================================================================================================ -impl From<&MerklePath> for generated::merkle::MerklePath { +impl From<&MerklePath> for proto::merkle::MerklePath { fn from(value: &MerklePath) -> Self { - let siblings = value.nodes().iter().map(generated::digest::Digest::from).collect(); - generated::merkle::MerklePath { siblings } + let siblings = value.nodes().iter().map(proto::digest::Digest::from).collect(); + proto::merkle::MerklePath { siblings } } } -impl From for generated::merkle::MerklePath { +impl From for proto::merkle::MerklePath { fn from(value: MerklePath) -> Self { (&value).into() } } -impl TryFrom<&generated::merkle::MerklePath> for MerklePath { +impl TryFrom<&proto::merkle::MerklePath> for MerklePath { type Error = ConversionError; - fn try_from(merkle_path: &generated::merkle::MerklePath) -> Result { + fn try_from(merkle_path: &proto::merkle::MerklePath) -> Result { merkle_path.siblings.iter().map(Digest::try_from).collect() } } @@ -36,17 +36,17 @@ impl TryFrom<&generated::merkle::MerklePath> for MerklePath { // MMR DELTA // ================================================================================================ -impl From for generated::mmr::MmrDelta { +impl From for proto::mmr::MmrDelta { fn from(value: MmrDelta) -> Self { - let data = value.data.into_iter().map(generated::digest::Digest::from).collect(); - generated::mmr::MmrDelta { forest: value.forest as u64, data } + let data = value.data.into_iter().map(proto::digest::Digest::from).collect(); + proto::mmr::MmrDelta { forest: value.forest as u64, data } } } -impl TryFrom for MmrDelta { +impl TryFrom for MmrDelta { type Error = ConversionError; - fn try_from(value: generated::mmr::MmrDelta) -> Result { + fn try_from(value: proto::mmr::MmrDelta) -> Result { let data: Result, ConversionError> = value.data.into_iter().map(Digest::try_from).collect(); @@ -63,22 +63,22 @@ impl TryFrom for MmrDelta { // SMT LEAF // ------------------------------------------------------------------------------------------------ -impl TryFrom for SmtLeaf { +impl TryFrom for SmtLeaf { type Error = ConversionError; - fn try_from(value: generated::smt::SmtLeaf) -> Result { - let leaf = value.leaf.ok_or(generated::smt::SmtLeaf::missing_field(stringify!(leaf)))?; + fn try_from(value: proto::smt::SmtLeaf) -> Result { + let leaf = value.leaf.ok_or(proto::smt::SmtLeaf::missing_field(stringify!(leaf)))?; match leaf { - generated::smt::smt_leaf::Leaf::Empty(leaf_index) => { + proto::smt::smt_leaf::Leaf::Empty(leaf_index) => { Ok(Self::new_empty(LeafIndex::new_max_depth(leaf_index))) }, - generated::smt::smt_leaf::Leaf::Single(entry) => { + proto::smt::smt_leaf::Leaf::Single(entry) => { let (key, value): (Digest, Word) = entry.try_into()?; Ok(SmtLeaf::new_single(key, value)) }, - generated::smt::smt_leaf::Leaf::Multiple(entries) => { + proto::smt::smt_leaf::Leaf::Multiple(entries) => { let domain_entries: Vec<(Digest, Word)> = try_convert(entries.entries)?; Ok(SmtLeaf::new_multiple(domain_entries)?) @@ -87,15 +87,15 @@ impl TryFrom for SmtLeaf { } } -impl From for generated::smt::SmtLeaf { +impl From for proto::smt::SmtLeaf { fn from(smt_leaf: SmtLeaf) -> Self { - use generated::smt::smt_leaf::Leaf; + use proto::smt::smt_leaf::Leaf; let leaf = match smt_leaf { SmtLeaf::Empty(leaf_index) => Leaf::Empty(leaf_index.value()), SmtLeaf::Single(entry) => Leaf::Single(entry.into()), SmtLeaf::Multiple(entries) => { - Leaf::Multiple(generated::smt::SmtLeafEntries { entries: convert(entries) }) + Leaf::Multiple(proto::smt::SmtLeafEntries { entries: convert(entries) }) }, }; @@ -106,24 +106,24 @@ impl From for generated::smt::SmtLeaf { // SMT LEAF ENTRY // ------------------------------------------------------------------------------------------------ -impl TryFrom for (Digest, Word) { +impl TryFrom for (Digest, Word) { type Error = ConversionError; - fn try_from(entry: generated::smt::SmtLeafEntry) -> Result { + fn try_from(entry: proto::smt::SmtLeafEntry) -> Result { let key: Digest = entry .key - .ok_or(generated::smt::SmtLeafEntry::missing_field(stringify!(key)))? + .ok_or(proto::smt::SmtLeafEntry::missing_field(stringify!(key)))? .try_into()?; let value: Word = entry .value - .ok_or(generated::smt::SmtLeafEntry::missing_field(stringify!(value)))? + .ok_or(proto::smt::SmtLeafEntry::missing_field(stringify!(value)))? .try_into()?; Ok((key, value)) } } -impl From<(Digest, Word)> for generated::smt::SmtLeafEntry { +impl From<(Digest, Word)> for proto::smt::SmtLeafEntry { fn from((key, value): (Digest, Word)) -> Self { Self { key: Some(key.into()), @@ -135,25 +135,25 @@ impl From<(Digest, Word)> for generated::smt::SmtLeafEntry { // SMT PROOF // ------------------------------------------------------------------------------------------------ -impl TryFrom for SmtProof { +impl TryFrom for SmtProof { type Error = ConversionError; - fn try_from(opening: generated::smt::SmtOpening) -> Result { + fn try_from(opening: proto::smt::SmtOpening) -> Result { let path: MerklePath = opening .path .as_ref() - .ok_or(generated::smt::SmtOpening::missing_field(stringify!(path)))? + .ok_or(proto::smt::SmtOpening::missing_field(stringify!(path)))? .try_into()?; let leaf: SmtLeaf = opening .leaf - .ok_or(generated::smt::SmtOpening::missing_field(stringify!(leaf)))? + .ok_or(proto::smt::SmtOpening::missing_field(stringify!(leaf)))? .try_into()?; Ok(SmtProof::new(path, leaf)?) } } -impl From for generated::smt::SmtOpening { +impl From for proto::smt::SmtOpening { fn from(proof: SmtProof) -> Self { let (ref path, leaf) = proof.into_parts(); Self { diff --git a/crates/proto/src/domain/notes.rs b/crates/proto/src/domain/notes.rs index 10cb5db4f..4dafb028f 100644 --- a/crates/proto/src/domain/notes.rs +++ b/crates/proto/src/domain/notes.rs @@ -9,20 +9,17 @@ use crate::{ convert, domain::blocks::BlockInclusionProof, errors::{ConversionError, MissingFieldHelper}, - generated::note::{ - NoteAuthenticationInfo as NoteAuthenticationInfoProto, - NoteInclusionInBlockProof as NoteInclusionInBlockProofPb, NoteMetadata as NoteMetadataPb, - }, + generated::note as proto, try_convert, }; -impl TryFrom for NoteMetadata { +impl TryFrom for NoteMetadata { type Error = ConversionError; - fn try_from(value: NoteMetadataPb) -> Result { + fn try_from(value: proto::NoteMetadata) -> Result { let sender = value .sender - .ok_or_else(|| NoteMetadataPb::missing_field(stringify!(sender)))? + .ok_or_else(|| proto::NoteMetadata::missing_field(stringify!(sender)))? .try_into()?; let note_type = NoteType::try_from(value.note_type as u64)?; let tag = NoteTag::from(value.tag); @@ -35,7 +32,7 @@ impl TryFrom for NoteMetadata { } } -impl From for NoteMetadataPb { +impl From for proto::NoteMetadata { fn from(val: NoteMetadata) -> Self { let sender = Some(val.sender().into()); let note_type = val.note_type() as u32; @@ -43,7 +40,7 @@ impl From for NoteMetadataPb { let execution_hint: u64 = val.execution_hint().into(); let aux = val.aux().into(); - crate::generated::note::NoteMetadata { + proto::NoteMetadata { sender, note_type, tag, @@ -53,7 +50,7 @@ impl From for NoteMetadataPb { } } -impl From<(&NoteId, &NoteInclusionProof)> for NoteInclusionInBlockProofPb { +impl From<(&NoteId, &NoteInclusionProof)> for proto::NoteInclusionInBlockProof { fn from((note_id, proof): (&NoteId, &NoteInclusionProof)) -> Self { Self { note_id: Some(note_id.into()), @@ -64,18 +61,18 @@ impl From<(&NoteId, &NoteInclusionProof)> for NoteInclusionInBlockProofPb { } } -impl TryFrom<&NoteInclusionInBlockProofPb> for (NoteId, NoteInclusionProof) { +impl TryFrom<&proto::NoteInclusionInBlockProof> for (NoteId, NoteInclusionProof) { type Error = ConversionError; fn try_from( - proof: &NoteInclusionInBlockProofPb, + proof: &proto::NoteInclusionInBlockProof, ) -> Result<(NoteId, NoteInclusionProof), Self::Error> { Ok(( Digest::try_from( proof .note_id .as_ref() - .ok_or(NoteInclusionInBlockProofPb::missing_field(stringify!(note_id)))?, + .ok_or(proto::NoteInclusionInBlockProof::missing_field(stringify!(note_id)))?, )? .into(), NoteInclusionProof::new( @@ -84,7 +81,9 @@ impl TryFrom<&NoteInclusionInBlockProofPb> for (NoteId, NoteInclusionProof) { proof .merkle_path .as_ref() - .ok_or(NoteInclusionInBlockProofPb::missing_field(stringify!(merkle_path)))? + .ok_or(proto::NoteInclusionInBlockProof::missing_field(stringify!( + merkle_path + )))? .try_into()?, )?, )) @@ -107,7 +106,7 @@ impl NoteAuthenticationInfo { } } -impl From for NoteAuthenticationInfoProto { +impl From for proto::NoteAuthenticationInfo { fn from(value: NoteAuthenticationInfo) -> Self { Self { note_proofs: convert(&value.note_proofs), @@ -116,10 +115,10 @@ impl From for NoteAuthenticationInfoProto { } } -impl TryFrom for NoteAuthenticationInfo { +impl TryFrom for NoteAuthenticationInfo { type Error = ConversionError; - fn try_from(value: NoteAuthenticationInfoProto) -> Result { + fn try_from(value: proto::NoteAuthenticationInfo) -> Result { let result = Self { block_proofs: try_convert(value.block_proofs)?, note_proofs: try_convert(&value.note_proofs)?, diff --git a/crates/proto/src/domain/nullifiers.rs b/crates/proto/src/domain/nullifiers.rs index 0b3cc0d0c..3f7bebc03 100644 --- a/crates/proto/src/domain/nullifiers.rs +++ b/crates/proto/src/domain/nullifiers.rs @@ -5,19 +5,19 @@ use miden_objects::{ use crate::{ errors::{ConversionError, MissingFieldHelper}, - generated::{digest::Digest, responses::NullifierBlockInputRecord}, + generated as proto, }; // FROM NULLIFIER // ================================================================================================ -impl From<&Nullifier> for Digest { +impl From<&Nullifier> for proto::digest::Digest { fn from(value: &Nullifier) -> Self { (*value).inner().into() } } -impl From for Digest { +impl From for proto::digest::Digest { fn from(value: Nullifier) -> Self { value.inner().into() } @@ -26,10 +26,10 @@ impl From for Digest { // INTO NULLIFIER // ================================================================================================ -impl TryFrom for Nullifier { +impl TryFrom for Nullifier { type Error = ConversionError; - fn try_from(value: Digest) -> Result { + fn try_from(value: proto::digest::Digest) -> Result { let digest: RpoDigest = value.try_into()?; Ok(digest.into()) } @@ -44,24 +44,30 @@ pub struct NullifierWitness { pub proof: SmtProof, } -impl TryFrom for NullifierWitness { +impl TryFrom for NullifierWitness { type Error = ConversionError; - fn try_from(nullifier_input_record: NullifierBlockInputRecord) -> Result { + fn try_from( + nullifier_input_record: proto::responses::NullifierBlockInputRecord, + ) -> Result { Ok(Self { nullifier: nullifier_input_record .nullifier - .ok_or(NullifierBlockInputRecord::missing_field(stringify!(nullifier)))? + .ok_or(proto::responses::NullifierBlockInputRecord::missing_field(stringify!( + nullifier + )))? .try_into()?, proof: nullifier_input_record .opening - .ok_or(NullifierBlockInputRecord::missing_field(stringify!(opening)))? + .ok_or(proto::responses::NullifierBlockInputRecord::missing_field(stringify!( + opening + )))? .try_into()?, }) } } -impl From for NullifierBlockInputRecord { +impl From for proto::responses::NullifierBlockInputRecord { fn from(value: NullifierWitness) -> Self { Self { nullifier: Some(value.nullifier.into()), diff --git a/crates/proto/src/domain/transactions.rs b/crates/proto/src/domain/transactions.rs index 5f22cad37..d4f71c32f 100644 --- a/crates/proto/src/domain/transactions.rs +++ b/crates/proto/src/domain/transactions.rs @@ -1,32 +1,29 @@ use miden_objects::{crypto::hash::rpo::RpoDigest, transaction::TransactionId}; -use crate::{ - errors::ConversionError, - generated::{digest::Digest, transaction::TransactionId as TransactionIdPb}, -}; +use crate::{errors::ConversionError, generated as proto}; // FROM TRANSACTION ID // ================================================================================================ -impl From<&TransactionId> for Digest { +impl From<&TransactionId> for proto::digest::Digest { fn from(value: &TransactionId) -> Self { (*value).inner().into() } } -impl From for Digest { +impl From for proto::digest::Digest { fn from(value: TransactionId) -> Self { value.inner().into() } } -impl From<&TransactionId> for TransactionIdPb { +impl From<&TransactionId> for proto::transaction::TransactionId { fn from(value: &TransactionId) -> Self { - TransactionIdPb { id: Some(value.into()) } + proto::transaction::TransactionId { id: Some(value.into()) } } } -impl From for TransactionIdPb { +impl From for proto::transaction::TransactionId { fn from(value: TransactionId) -> Self { (&value).into() } @@ -35,19 +32,19 @@ impl From for TransactionIdPb { // INTO TRANSACTION ID // ================================================================================================ -impl TryFrom for TransactionId { +impl TryFrom for TransactionId { type Error = ConversionError; - fn try_from(value: Digest) -> Result { + fn try_from(value: proto::digest::Digest) -> Result { let digest: RpoDigest = value.try_into()?; Ok(digest.into()) } } -impl TryFrom for TransactionId { +impl TryFrom for TransactionId { type Error = ConversionError; - fn try_from(value: TransactionIdPb) -> Result { + fn try_from(value: proto::transaction::TransactionId) -> Result { value .id .ok_or(ConversionError::MissingFieldInProtobufRepresentation { diff --git a/crates/store/src/db/mod.rs b/crates/store/src/db/mod.rs index cc8a12be0..f78315375 100644 --- a/crates/store/src/db/mod.rs +++ b/crates/store/src/db/mod.rs @@ -7,7 +7,7 @@ use std::{ use deadpool_sqlite::{Config as SqliteConfig, Hook, HookError, Pool, Runtime}; use miden_node_proto::{ domain::accounts::{AccountInfo, AccountSummary}, - generated::note::{Note as NotePb, NoteSyncRecord as NoteSyncRecordPb}, + generated::note as proto, }; use miden_objects::{ accounts::{AccountDelta, AccountId}, @@ -68,7 +68,7 @@ pub struct NoteRecord { pub merkle_path: MerklePath, } -impl From for NotePb { +impl From for proto::Note { fn from(note: NoteRecord) -> Self { Self { block_num: note.block_num, @@ -105,7 +105,7 @@ pub struct NoteSyncRecord { pub merkle_path: MerklePath, } -impl From for NoteSyncRecordPb { +impl From for proto::NoteSyncRecord { fn from(note: NoteSyncRecord) -> Self { Self { note_index: note.note_index.leaf_index_value().into(), From ef07bbcc7a0c3dacba2aca28493ac0e85c6f397a Mon Sep 17 00:00:00 2001 From: Mirko <48352201+Mirko-von-Leipzig@users.noreply.github.com> Date: Mon, 13 Jan 2025 12:03:22 +0200 Subject: [PATCH 38/50] feat(store): remove testing endpoints (#608) --- CHANGELOG.md | 1 + crates/proto/src/generated/requests.rs | 6 - crates/proto/src/generated/responses.rs | 18 -- crates/proto/src/generated/store.rs | 230 ------------------------ crates/rpc-proto/proto/requests.proto | 6 - crates/rpc-proto/proto/responses.proto | 15 -- crates/rpc-proto/proto/store.proto | 3 - crates/store/README.md | 38 ---- crates/store/src/db/mod.rs | 16 -- crates/store/src/db/sql/mod.rs | 2 + crates/store/src/server/api.rs | 66 +------ crates/store/src/state.rs | 16 -- proto/requests.proto | 6 - proto/responses.proto | 15 -- proto/store.proto | 3 - 15 files changed, 5 insertions(+), 436 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 43d54f049..ff678a0b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ - Standardized protobuf type aliases (#609). - [BREAKING] Added support for new two `Felt` account ID (#591). - [BREAKING] Inverted `TransactionInputs.missing_unauthenticated_notes` to `found_missing_notes` (#509). +- [BREAKING] Remove store's `ListXXX` endpoints which were intended for test purposes (#608). ## v0.6.0 (2024-11-05) diff --git a/crates/proto/src/generated/requests.rs b/crates/proto/src/generated/requests.rs index b58105d61..5eccaedb9 100644 --- a/crates/proto/src/generated/requests.rs +++ b/crates/proto/src/generated/requests.rs @@ -115,12 +115,6 @@ pub struct GetNoteAuthenticationInfoRequest { #[prost(message, repeated, tag = "1")] pub note_ids: ::prost::alloc::vec::Vec, } -#[derive(Clone, Copy, PartialEq, ::prost::Message)] -pub struct ListNullifiersRequest {} -#[derive(Clone, Copy, PartialEq, ::prost::Message)] -pub struct ListAccountsRequest {} -#[derive(Clone, Copy, PartialEq, ::prost::Message)] -pub struct ListNotesRequest {} /// Returns the latest state of an account with the specified ID. #[derive(Clone, PartialEq, ::prost::Message)] pub struct GetAccountDetailsRequest { diff --git a/crates/proto/src/generated/responses.rs b/crates/proto/src/generated/responses.rs index b635c10ed..89690721b 100644 --- a/crates/proto/src/generated/responses.rs +++ b/crates/proto/src/generated/responses.rs @@ -160,24 +160,6 @@ pub struct GetNoteAuthenticationInfoResponse { pub proofs: ::core::option::Option, } #[derive(Clone, PartialEq, ::prost::Message)] -pub struct ListNullifiersResponse { - /// Lists all nullifiers of the current chain - #[prost(message, repeated, tag = "1")] - pub nullifiers: ::prost::alloc::vec::Vec, -} -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ListAccountsResponse { - /// Lists all accounts of the current chain - #[prost(message, repeated, tag = "1")] - pub accounts: ::prost::alloc::vec::Vec, -} -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ListNotesResponse { - /// Lists all notes of the current chain - #[prost(message, repeated, tag = "1")] - pub notes: ::prost::alloc::vec::Vec, -} -#[derive(Clone, PartialEq, ::prost::Message)] pub struct GetAccountDetailsResponse { /// Account info (with details for public accounts) #[prost(message, optional, tag = "1")] diff --git a/crates/proto/src/generated/store.rs b/crates/proto/src/generated/store.rs index 0a9055a9a..7d979f379 100644 --- a/crates/proto/src/generated/store.rs +++ b/crates/proto/src/generated/store.rs @@ -388,71 +388,6 @@ pub mod api_client { .insert(GrpcMethod::new("store.Api", "GetTransactionInputs")); self.inner.unary(req, path, codec).await } - pub async fn list_accounts( - &mut self, - request: impl tonic::IntoRequest, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - > { - self.inner - .ready() - .await - .map_err(|e| { - tonic::Status::unknown( - format!("Service was not ready: {}", e.into()), - ) - })?; - let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/store.Api/ListAccounts"); - let mut req = request.into_request(); - req.extensions_mut().insert(GrpcMethod::new("store.Api", "ListAccounts")); - self.inner.unary(req, path, codec).await - } - pub async fn list_notes( - &mut self, - request: impl tonic::IntoRequest, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - > { - self.inner - .ready() - .await - .map_err(|e| { - tonic::Status::unknown( - format!("Service was not ready: {}", e.into()), - ) - })?; - let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/store.Api/ListNotes"); - let mut req = request.into_request(); - req.extensions_mut().insert(GrpcMethod::new("store.Api", "ListNotes")); - self.inner.unary(req, path, codec).await - } - pub async fn list_nullifiers( - &mut self, - request: impl tonic::IntoRequest< - super::super::requests::ListNullifiersRequest, - >, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - > { - self.inner - .ready() - .await - .map_err(|e| { - tonic::Status::unknown( - format!("Service was not ready: {}", e.into()), - ) - })?; - let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/store.Api/ListNullifiers"); - let mut req = request.into_request(); - req.extensions_mut().insert(GrpcMethod::new("store.Api", "ListNullifiers")); - self.inner.unary(req, path, codec).await - } pub async fn sync_notes( &mut self, request: impl tonic::IntoRequest, @@ -600,27 +535,6 @@ pub mod api_server { tonic::Response, tonic::Status, >; - async fn list_accounts( - &self, - request: tonic::Request, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - >; - async fn list_notes( - &self, - request: tonic::Request, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - >; - async fn list_nullifiers( - &self, - request: tonic::Request, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - >; async fn sync_notes( &self, request: tonic::Request, @@ -1291,150 +1205,6 @@ pub mod api_server { }; Box::pin(fut) } - "/store.Api/ListAccounts" => { - #[allow(non_camel_case_types)] - struct ListAccountsSvc(pub Arc); - impl< - T: Api, - > tonic::server::UnaryService< - super::super::requests::ListAccountsRequest, - > for ListAccountsSvc { - type Response = super::super::responses::ListAccountsResponse; - type Future = BoxFuture< - tonic::Response, - tonic::Status, - >; - fn call( - &mut self, - request: tonic::Request< - super::super::requests::ListAccountsRequest, - >, - ) -> Self::Future { - let inner = Arc::clone(&self.0); - let fut = async move { - ::list_accounts(&inner, request).await - }; - Box::pin(fut) - } - } - let accept_compression_encodings = self.accept_compression_encodings; - let send_compression_encodings = self.send_compression_encodings; - let max_decoding_message_size = self.max_decoding_message_size; - let max_encoding_message_size = self.max_encoding_message_size; - let inner = self.inner.clone(); - let fut = async move { - let method = ListAccountsSvc(inner); - let codec = tonic::codec::ProstCodec::default(); - let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config( - accept_compression_encodings, - send_compression_encodings, - ) - .apply_max_message_size_config( - max_decoding_message_size, - max_encoding_message_size, - ); - let res = grpc.unary(method, req).await; - Ok(res) - }; - Box::pin(fut) - } - "/store.Api/ListNotes" => { - #[allow(non_camel_case_types)] - struct ListNotesSvc(pub Arc); - impl< - T: Api, - > tonic::server::UnaryService< - super::super::requests::ListNotesRequest, - > for ListNotesSvc { - type Response = super::super::responses::ListNotesResponse; - type Future = BoxFuture< - tonic::Response, - tonic::Status, - >; - fn call( - &mut self, - request: tonic::Request< - super::super::requests::ListNotesRequest, - >, - ) -> Self::Future { - let inner = Arc::clone(&self.0); - let fut = async move { - ::list_notes(&inner, request).await - }; - Box::pin(fut) - } - } - let accept_compression_encodings = self.accept_compression_encodings; - let send_compression_encodings = self.send_compression_encodings; - let max_decoding_message_size = self.max_decoding_message_size; - let max_encoding_message_size = self.max_encoding_message_size; - let inner = self.inner.clone(); - let fut = async move { - let method = ListNotesSvc(inner); - let codec = tonic::codec::ProstCodec::default(); - let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config( - accept_compression_encodings, - send_compression_encodings, - ) - .apply_max_message_size_config( - max_decoding_message_size, - max_encoding_message_size, - ); - let res = grpc.unary(method, req).await; - Ok(res) - }; - Box::pin(fut) - } - "/store.Api/ListNullifiers" => { - #[allow(non_camel_case_types)] - struct ListNullifiersSvc(pub Arc); - impl< - T: Api, - > tonic::server::UnaryService< - super::super::requests::ListNullifiersRequest, - > for ListNullifiersSvc { - type Response = super::super::responses::ListNullifiersResponse; - type Future = BoxFuture< - tonic::Response, - tonic::Status, - >; - fn call( - &mut self, - request: tonic::Request< - super::super::requests::ListNullifiersRequest, - >, - ) -> Self::Future { - let inner = Arc::clone(&self.0); - let fut = async move { - ::list_nullifiers(&inner, request).await - }; - Box::pin(fut) - } - } - let accept_compression_encodings = self.accept_compression_encodings; - let send_compression_encodings = self.send_compression_encodings; - let max_decoding_message_size = self.max_decoding_message_size; - let max_encoding_message_size = self.max_encoding_message_size; - let inner = self.inner.clone(); - let fut = async move { - let method = ListNullifiersSvc(inner); - let codec = tonic::codec::ProstCodec::default(); - let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config( - accept_compression_encodings, - send_compression_encodings, - ) - .apply_max_message_size_config( - max_decoding_message_size, - max_encoding_message_size, - ); - let res = grpc.unary(method, req).await; - Ok(res) - }; - Box::pin(fut) - } "/store.Api/SyncNotes" => { #[allow(non_camel_case_types)] struct SyncNotesSvc(pub Arc); diff --git a/crates/rpc-proto/proto/requests.proto b/crates/rpc-proto/proto/requests.proto index 31a0cb68e..899359488 100644 --- a/crates/rpc-proto/proto/requests.proto +++ b/crates/rpc-proto/proto/requests.proto @@ -104,12 +104,6 @@ message GetNoteAuthenticationInfoRequest { repeated digest.Digest note_ids = 1; } -message ListNullifiersRequest {} - -message ListAccountsRequest {} - -message ListNotesRequest {} - // Returns the latest state of an account with the specified ID. message GetAccountDetailsRequest { // Account ID to get details. diff --git a/crates/rpc-proto/proto/responses.proto b/crates/rpc-proto/proto/responses.proto index 4f1c017b8..233189a46 100644 --- a/crates/rpc-proto/proto/responses.proto +++ b/crates/rpc-proto/proto/responses.proto @@ -144,21 +144,6 @@ message GetNoteAuthenticationInfoResponse { note.NoteAuthenticationInfo proofs = 1; } -message ListNullifiersResponse { - // Lists all nullifiers of the current chain - repeated smt.SmtLeafEntry nullifiers = 1; -} - -message ListAccountsResponse { - // Lists all accounts of the current chain - repeated account.AccountInfo accounts = 1; -} - -message ListNotesResponse { - // Lists all notes of the current chain - repeated note.Note notes = 1; -} - message GetAccountDetailsResponse { // Account info (with details for public accounts) account.AccountInfo details = 1; diff --git a/crates/rpc-proto/proto/store.proto b/crates/rpc-proto/proto/store.proto index ec5a11270..b51198b96 100644 --- a/crates/rpc-proto/proto/store.proto +++ b/crates/rpc-proto/proto/store.proto @@ -20,9 +20,6 @@ service Api { rpc GetNoteAuthenticationInfo(requests.GetNoteAuthenticationInfoRequest) returns (responses.GetNoteAuthenticationInfoResponse) {} rpc GetNotesById(requests.GetNotesByIdRequest) returns (responses.GetNotesByIdResponse) {} rpc GetTransactionInputs(requests.GetTransactionInputsRequest) returns (responses.GetTransactionInputsResponse) {} - rpc ListAccounts(requests.ListAccountsRequest) returns (responses.ListAccountsResponse) {} - rpc ListNotes(requests.ListNotesRequest) returns (responses.ListNotesResponse) {} - rpc ListNullifiers(requests.ListNullifiersRequest) returns (responses.ListNullifiersResponse) {} rpc SyncNotes(requests.SyncNoteRequest) returns (responses.SyncNoteResponse) {} rpc SyncState(requests.SyncStateRequest) returns (responses.SyncStateResponse) {} } diff --git a/crates/store/README.md b/crates/store/README.md index 036d3f272..9dcaef115 100644 --- a/crates/store/README.md +++ b/crates/store/README.md @@ -157,44 +157,6 @@ contains excessive notes and nullifiers, client can make additional filtering of - `nullifiers`: `[NullifierUpdate]` – a list of nullifiers created between `request.block_num + 1` and `response.block_header.block_num`. - Each `NullifierUpdate` consists of the `nullifier` and `block_num` the block number in which the note corresponding to that nullifier was consumed. -## Methods for testing purposes - -### ListNullifiers - -Lists all nullifiers of the current chain. - -**Parameters** - -This request doesn't have any parameters. - -**Returns** - -- `nullifiers`: `[NullifierLeaf]` – lists of all nullifiers of the current chain. - -### ListAccounts - -Lists all accounts of the current chain. - -**Parameters** - -This request doesn't have any parameters. - -**Returns** - -- `accounts`: `[AccountInfo]` – list of all accounts of the current chain. - -### ListNotes - -Lists all notes of the current chain. - -**Parameters** - -This request doesn't have any parameters. - -**Returns** - -- `notes`: `[Note]` – list of all notes of the current chain. - ## License This project is [MIT licensed](../../LICENSE). diff --git a/crates/store/src/db/mod.rs b/crates/store/src/db/mod.rs index f78315375..abdd88d21 100644 --- a/crates/store/src/db/mod.rs +++ b/crates/store/src/db/mod.rs @@ -232,22 +232,6 @@ impl Db { })? } - /// Loads all the notes from the DB. - #[instrument(target = COMPONENT, skip_all, ret(level = "debug"), err)] - pub async fn select_all_notes(&self) -> Result> { - self.pool.get().await?.interact(sql::select_all_notes).await.map_err(|err| { - DatabaseError::InteractError(format!("Select notes task failed: {err}")) - })? - } - - /// Loads all the accounts from the DB. - #[instrument(target = COMPONENT, skip_all, ret(level = "debug"), err)] - pub async fn select_all_accounts(&self) -> Result> { - self.pool.get().await?.interact(sql::select_all_accounts).await.map_err(|err| { - DatabaseError::InteractError(format!("Select accounts task failed: {err}")) - })? - } - /// Search for a [BlockHeader] from the database by its `block_num`. /// /// When `block_number` is [None], the latest block header is returned. diff --git a/crates/store/src/db/sql/mod.rs b/crates/store/src/db/sql/mod.rs index 359f4f79a..fa93a26ec 100644 --- a/crates/store/src/db/sql/mod.rs +++ b/crates/store/src/db/sql/mod.rs @@ -47,6 +47,7 @@ use crate::{ /// # Returns /// /// A vector with accounts, or an error. +#[cfg(test)] pub fn select_all_accounts(conn: &mut Connection) -> Result> { let mut stmt = conn.prepare_cached( " @@ -700,6 +701,7 @@ pub fn select_nullifiers_by_prefix( /// # Returns /// /// A vector with notes, or an error. +#[cfg(test)] pub fn select_all_notes(conn: &mut Connection) -> Result> { let mut stmt = conn.prepare_cached( " diff --git a/crates/store/src/server/api.rs b/crates/store/src/server/api.rs index fe1808003..7ab399309 100644 --- a/crates/store/src/server/api.rs +++ b/crates/store/src/server/api.rs @@ -13,19 +13,16 @@ use miden_node_proto::{ GetAccountDetailsRequest, GetAccountProofsRequest, GetAccountStateDeltaRequest, GetBlockByNumberRequest, GetBlockHeaderByNumberRequest, GetBlockInputsRequest, GetNoteAuthenticationInfoRequest, GetNotesByIdRequest, GetTransactionInputsRequest, - ListAccountsRequest, ListNotesRequest, ListNullifiersRequest, SyncNoteRequest, - SyncStateRequest, + SyncNoteRequest, SyncStateRequest, }, responses::{ AccountTransactionInputRecord, ApplyBlockResponse, CheckNullifiersByPrefixResponse, CheckNullifiersResponse, GetAccountDetailsResponse, GetAccountProofsResponse, GetAccountStateDeltaResponse, GetBlockByNumberResponse, GetBlockHeaderByNumberResponse, GetBlockInputsResponse, GetNoteAuthenticationInfoResponse, GetNotesByIdResponse, - GetTransactionInputsResponse, ListAccountsResponse, ListNotesResponse, - ListNullifiersResponse, NullifierTransactionInputRecord, NullifierUpdate, + GetTransactionInputsResponse, NullifierTransactionInputRecord, NullifierUpdate, SyncNoteResponse, SyncStateResponse, }, - smt::SmtLeafEntry, store::api_server, transaction::TransactionSummary, }, @@ -37,7 +34,6 @@ use miden_objects::{ crypto::hash::rpo::RpoDigest, notes::{NoteId, Nullifier}, utils::{Deserializable, Serializable}, - Felt, ZERO, }; use tonic::{Request, Response, Status}; use tracing::{debug, info, instrument}; @@ -528,64 +524,6 @@ impl api_server::Api for StoreApi { Ok(Response::new(GetAccountStateDeltaResponse { delta })) } - - // TESTING ENDPOINTS - // -------------------------------------------------------------------------------------------- - - /// Returns a list of all nullifiers - #[instrument( - target = COMPONENT, - name = "store:list_nullifiers", - skip_all, - ret(level = "debug"), - err - )] - async fn list_nullifiers( - &self, - _request: Request, - ) -> Result, Status> { - let raw_nullifiers = self.state.list_nullifiers().await?; - let nullifiers = raw_nullifiers - .into_iter() - .map(|(key, block_num)| SmtLeafEntry { - key: Some(key.into()), - value: Some([Felt::from(block_num), ZERO, ZERO, ZERO].into()), - }) - .collect(); - Ok(Response::new(ListNullifiersResponse { nullifiers })) - } - - /// Returns a list of all notes - #[instrument( - target = COMPONENT, - name = "store:list_notes", - skip_all, - ret(level = "debug"), - err - )] - async fn list_notes( - &self, - _request: Request, - ) -> Result, Status> { - let notes = self.state.list_notes().await?.into_iter().map(Into::into).collect(); - Ok(Response::new(ListNotesResponse { notes })) - } - - /// Returns a list of all accounts - #[instrument( - target = COMPONENT, - name = "store:list_accounts", - skip_all, - ret(level = "debug"), - err - )] - async fn list_accounts( - &self, - _request: Request, - ) -> Result, Status> { - let accounts = self.state.list_accounts().await?.iter().map(Into::into).collect(); - Ok(Response::new(ListAccountsResponse { accounts })) - } } // UTILITIES diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index 87c624a11..a0b2bef69 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -665,22 +665,6 @@ impl State { }) } - /// Lists all known nullifiers with their inclusion blocks, intended for testing. - pub async fn list_nullifiers(&self) -> Result, DatabaseError> { - self.db.select_all_nullifiers().await - } - - /// Lists all known accounts, with their ids, latest state hash, and block at which the account - /// was last modified, intended for testing. - pub async fn list_accounts(&self) -> Result, DatabaseError> { - self.db.select_all_accounts().await - } - - /// Lists all known notes, intended for testing. - pub async fn list_notes(&self) -> Result, DatabaseError> { - self.db.select_all_notes().await - } - /// Returns details for public (on-chain) account. pub async fn get_account_details(&self, id: AccountId) -> Result { self.db.select_account(id).await diff --git a/proto/requests.proto b/proto/requests.proto index 31a0cb68e..899359488 100644 --- a/proto/requests.proto +++ b/proto/requests.proto @@ -104,12 +104,6 @@ message GetNoteAuthenticationInfoRequest { repeated digest.Digest note_ids = 1; } -message ListNullifiersRequest {} - -message ListAccountsRequest {} - -message ListNotesRequest {} - // Returns the latest state of an account with the specified ID. message GetAccountDetailsRequest { // Account ID to get details. diff --git a/proto/responses.proto b/proto/responses.proto index 4f1c017b8..233189a46 100644 --- a/proto/responses.proto +++ b/proto/responses.proto @@ -144,21 +144,6 @@ message GetNoteAuthenticationInfoResponse { note.NoteAuthenticationInfo proofs = 1; } -message ListNullifiersResponse { - // Lists all nullifiers of the current chain - repeated smt.SmtLeafEntry nullifiers = 1; -} - -message ListAccountsResponse { - // Lists all accounts of the current chain - repeated account.AccountInfo accounts = 1; -} - -message ListNotesResponse { - // Lists all notes of the current chain - repeated note.Note notes = 1; -} - message GetAccountDetailsResponse { // Account info (with details for public accounts) account.AccountInfo details = 1; diff --git a/proto/store.proto b/proto/store.proto index ec5a11270..b51198b96 100644 --- a/proto/store.proto +++ b/proto/store.proto @@ -20,9 +20,6 @@ service Api { rpc GetNoteAuthenticationInfo(requests.GetNoteAuthenticationInfoRequest) returns (responses.GetNoteAuthenticationInfoResponse) {} rpc GetNotesById(requests.GetNotesByIdRequest) returns (responses.GetNotesByIdResponse) {} rpc GetTransactionInputs(requests.GetTransactionInputsRequest) returns (responses.GetTransactionInputsResponse) {} - rpc ListAccounts(requests.ListAccountsRequest) returns (responses.ListAccountsResponse) {} - rpc ListNotes(requests.ListNotesRequest) returns (responses.ListNotesResponse) {} - rpc ListNullifiers(requests.ListNullifiersRequest) returns (responses.ListNullifiersResponse) {} rpc SyncNotes(requests.SyncNoteRequest) returns (responses.SyncNoteResponse) {} rpc SyncState(requests.SyncStateRequest) returns (responses.SyncStateResponse) {} } From f9da75e650975ef036d66e612e9c68b84ac98cae Mon Sep 17 00:00:00 2001 From: igamigo Date: Mon, 13 Jan 2025 16:46:01 -0300 Subject: [PATCH 39/50] feat: Enable storage map querying (#598) * feat: Enable storage map querying * reviews: Clarify doc comments; error messages; iterate over requests instead of retreived accounts; README suggestions --- CHANGELOG.md | 1 + crates/proto/src/domain/accounts.rs | 55 +++++++++++++++ crates/proto/src/generated/requests.rs | 34 +++++++++- crates/proto/src/generated/responses.rs | 17 ++++- crates/rpc-proto/proto/requests.proto | 29 +++++++- crates/rpc-proto/proto/responses.proto | 18 ++++- crates/rpc/src/server/api.rs | 10 ++- crates/store/README.md | 4 +- crates/store/src/server/api.rs | 33 ++++----- crates/store/src/state.rs | 90 ++++++++++++++++++------- proto/requests.proto | 29 +++++++- proto/responses.proto | 18 ++++- 12 files changed, 279 insertions(+), 59 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ff678a0b8..5ed082383 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ - [BREAKING] Added support for new two `Felt` account ID (#591). - [BREAKING] Inverted `TransactionInputs.missing_unauthenticated_notes` to `found_missing_notes` (#509). - [BREAKING] Remove store's `ListXXX` endpoints which were intended for test purposes (#608). +- [BREAKING] Added support for storage maps on `GetAccountProofs` endpoint (#598). ## v0.6.0 (2024-11-05) diff --git a/crates/proto/src/domain/accounts.rs b/crates/proto/src/domain/accounts.rs index 08c1174e1..cf358ab04 100644 --- a/crates/proto/src/domain/accounts.rs +++ b/crates/proto/src/domain/accounts.rs @@ -8,6 +8,7 @@ use miden_objects::{ Digest, }; +use super::try_convert; use crate::{ errors::{ConversionError, MissingFieldHelper}, generated as proto, @@ -93,6 +94,60 @@ impl From<&AccountInfo> for proto::account::AccountInfo { } } +// ACCOUNT STORAGE REQUEST +// ================================================================================================ + +/// Represents a request for an account proof alongside specific storage data. +pub struct AccountProofRequest { + pub account_id: AccountId, + pub storage_requests: Vec, +} + +impl TryInto for proto::requests::get_account_proofs_request::AccountRequest { + type Error = ConversionError; + + fn try_into(self) -> Result { + let proto::requests::get_account_proofs_request::AccountRequest { + account_id, + storage_requests, + } = self; + + Ok(AccountProofRequest { + account_id: account_id + .clone() + .ok_or(proto::requests::get_account_proofs_request::AccountRequest::missing_field( + stringify!(account_id), + ))? + .try_into()?, + storage_requests: try_convert(storage_requests)?, + }) + } +} + +/// Represents a request for an account's storage map values and its proof of existence. +pub struct StorageMapKeysProof { + /// Index of the storage map + pub storage_index: u8, + /// List of requested keys in the map + pub storage_keys: Vec, +} + +impl TryInto for proto::requests::get_account_proofs_request::StorageRequest { + type Error = ConversionError; + + fn try_into(self) -> Result { + let proto::requests::get_account_proofs_request::StorageRequest { + storage_slot_index, + map_keys, + } = self; + + Ok(StorageMapKeysProof { + storage_index: storage_slot_index.try_into()?, + storage_keys: try_convert(map_keys)?, + }) + } +} + // ACCOUNT INPUT RECORD // ================================================================================================ diff --git a/crates/proto/src/generated/requests.rs b/crates/proto/src/generated/requests.rs index 5eccaedb9..b28e0e3cf 100644 --- a/crates/proto/src/generated/requests.rs +++ b/crates/proto/src/generated/requests.rs @@ -142,12 +142,16 @@ pub struct GetAccountStateDeltaRequest { #[prost(fixed32, tag = "3")] pub to_block_num: u32, } +/// Request message to get account proofs. #[derive(Clone, PartialEq, ::prost::Message)] pub struct GetAccountProofsRequest { - /// List of account IDs to get states. + /// A list of account requests, including map keys + values. #[prost(message, repeated, tag = "1")] - pub account_ids: ::prost::alloc::vec::Vec, - /// Optional flag to include header and account code in the response. `false` by default. + pub account_requests: ::prost::alloc::vec::Vec< + get_account_proofs_request::AccountRequest, + >, + /// Optional flag to include account headers and account code in the response. If false, storage + /// requests are also ignored. False by default. #[prost(bool, optional, tag = "2")] pub include_headers: ::core::option::Option, /// Account code commitments corresponding to the last-known `AccountCode` for requested @@ -157,3 +161,27 @@ pub struct GetAccountProofsRequest { #[prost(message, repeated, tag = "3")] pub code_commitments: ::prost::alloc::vec::Vec, } +/// Nested message and enum types in `GetAccountProofsRequest`. +pub mod get_account_proofs_request { + /// Represents per-account requests where each account ID has its own list of + /// (storage_slot_index, map_keys) pairs. + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct AccountRequest { + /// The account ID for this request. + #[prost(message, optional, tag = "1")] + pub account_id: ::core::option::Option, + /// List of storage requests for this account. + #[prost(message, repeated, tag = "2")] + pub storage_requests: ::prost::alloc::vec::Vec, + } + /// Represents a storage slot index and the associated map keys. + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct StorageRequest { + /// Storage slot index (\[0..255\]) + #[prost(uint32, tag = "1")] + pub storage_slot_index: u32, + /// A list of map keys (Digests) associated with this storage slot. + #[prost(message, repeated, tag = "2")] + pub map_keys: ::prost::alloc::vec::Vec, + } +} diff --git a/crates/proto/src/generated/responses.rs b/crates/proto/src/generated/responses.rs index 89690721b..46ce0cd3d 100644 --- a/crates/proto/src/generated/responses.rs +++ b/crates/proto/src/generated/responses.rs @@ -209,8 +209,21 @@ pub struct AccountStateHeader { /// Values of all account storage slots (max 255). #[prost(bytes = "vec", tag = "2")] pub storage_header: ::prost::alloc::vec::Vec, - /// Account code, returned only when none of the request's code commitments match with the - /// current one. + /// Account code, returned only when none of the request's code commitments match + /// the current one. #[prost(bytes = "vec", optional, tag = "3")] pub account_code: ::core::option::Option<::prost::alloc::vec::Vec>, + /// Storage slots information for this account + #[prost(message, repeated, tag = "4")] + pub storage_maps: ::prost::alloc::vec::Vec, +} +/// Represents a single storage slot with the reuqested keys and their respective values. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct StorageSlotMapProof { + /// The storage slot index (\[0..255\]). + #[prost(uint32, tag = "1")] + pub storage_slot: u32, + /// Merkle proof of the map value + #[prost(bytes = "vec", tag = "2")] + pub smt_proof: ::prost::alloc::vec::Vec, } diff --git a/crates/rpc-proto/proto/requests.proto b/crates/rpc-proto/proto/requests.proto index 899359488..62ec750a1 100644 --- a/crates/rpc-proto/proto/requests.proto +++ b/crates/rpc-proto/proto/requests.proto @@ -126,11 +126,34 @@ message GetAccountStateDeltaRequest { fixed32 to_block_num = 3; } +// Request message to get account proofs. message GetAccountProofsRequest { - // List of account IDs to get states. - repeated account.AccountId account_ids = 1; - // Optional flag to include header and account code in the response. `false` by default. + // Represents per-account requests where each account ID has its own list of + // (storage_slot_index, map_keys) pairs. + message AccountRequest { + // The account ID for this request. + account.AccountId account_id = 1; + + // List of storage requests for this account. + repeated StorageRequest storage_requests = 2; + } + + // Represents a storage slot index and the associated map keys. + message StorageRequest { + // Storage slot index ([0..255]) + uint32 storage_slot_index = 1; + + // A list of map keys (Digests) associated with this storage slot. + repeated digest.Digest map_keys = 2; + } + + // A list of account requests, including map keys + values. + repeated AccountRequest account_requests = 1; + + // Optional flag to include account headers and account code in the response. If false, storage + // requests are also ignored. False by default. optional bool include_headers = 2; + // Account code commitments corresponding to the last-known `AccountCode` for requested // accounts. Responses will include only the ones that are not known to the caller. // These are not associated with a specific account but rather, they will be matched against diff --git a/crates/rpc-proto/proto/responses.proto b/crates/rpc-proto/proto/responses.proto index 233189a46..530a5dc18 100644 --- a/crates/rpc-proto/proto/responses.proto +++ b/crates/rpc-proto/proto/responses.proto @@ -180,9 +180,23 @@ message AccountProofsResponse { message AccountStateHeader { // Account header. account.AccountHeader header = 1; + // Values of all account storage slots (max 255). bytes storage_header = 2; - // Account code, returned only when none of the request's code commitments match with the - // current one. + + // Account code, returned only when none of the request's code commitments match + // the current one. optional bytes account_code = 3; + + // Storage slots information for this account + repeated StorageSlotMapProof storage_maps = 4; +} + +// Represents a single storage slot with the reuqested keys and their respective values. +message StorageSlotMapProof { + // The storage slot index ([0..255]). + uint32 storage_slot = 1; + + // Merkle proof of the map value + bytes smt_proof = 2; } diff --git a/crates/rpc/src/server/api.rs b/crates/rpc/src/server/api.rs index 241fe9a8a..96bca8096 100644 --- a/crates/rpc/src/server/api.rs +++ b/crates/rpc/src/server/api.rs @@ -265,13 +265,19 @@ impl api_server::Api for RpcApi { debug!(target: COMPONENT, ?request); - if request.account_ids.len() > MAX_NUM_FOREIGN_ACCOUNTS as usize { + if request.account_requests.len() > MAX_NUM_FOREIGN_ACCOUNTS as usize { return Err(Status::invalid_argument(format!( "Too many accounts requested: {}, limit: {MAX_NUM_FOREIGN_ACCOUNTS}", - request.account_ids.len() + request.account_requests.len() ))); } + if request.account_requests.len() < request.code_commitments.len() { + return Err(Status::invalid_argument( + "The number of code commitments should not exceed the number of requested accounts.", + )); + } + self.store.clone().get_account_proofs(request).await } } diff --git a/crates/store/README.md b/crates/store/README.md index 9dcaef115..a201956df 100644 --- a/crates/store/README.md +++ b/crates/store/README.md @@ -2,6 +2,8 @@ The **Store** maintains the state of the chain. It serves as the "source of truth" for the chain - i.e., if it is not in the store, the node does not consider it to be part of the chain. +Incoming requests to the store are trusted because they are validated in the RPC component. + **Store** is one of components of the [Miden node](..). ## Architecture @@ -16,7 +18,7 @@ The Store can be installed and run as part of [Miden node](../README.md#installi ## API -The **Store** serves connections using the [gRPC protocol](https://grpc.io) on a port, set in the previously mentioned configuration file. +The **Store** serves connections using the [gRPC protocol](https://grpc.io) on a port, set in the previously mentioned configuration file. Here is a brief description of supported methods. ### ApplyBlock diff --git a/crates/store/src/server/api.rs b/crates/store/src/server/api.rs index 7ab399309..1eb5f3627 100644 --- a/crates/store/src/server/api.rs +++ b/crates/store/src/server/api.rs @@ -2,7 +2,10 @@ use std::{collections::BTreeSet, sync::Arc}; use miden_node_proto::{ convert, - domain::{accounts::AccountInfo, notes::NoteAuthenticationInfo}, + domain::{ + accounts::{AccountInfo, AccountProofRequest}, + notes::NoteAuthenticationInfo, + }, errors::ConversionError, generated::{ self, @@ -473,25 +476,25 @@ impl api_server::Api for StoreApi { &self, request: Request, ) -> Result, Status> { - let request = request.into_inner(); - if request.account_ids.len() < request.code_commitments.len() { - return Err(Status::invalid_argument( - "The number of code commitments should not exceed the number of requested accounts.", - )); - } - debug!(target: COMPONENT, ?request); + let GetAccountProofsRequest { + account_requests, + include_headers, + code_commitments, + } = request.into_inner(); - let include_headers = request.include_headers.unwrap_or_default(); - let account_ids: Vec = read_account_ids(&request.account_ids)?; - let request_code_commitments: BTreeSet = try_convert(request.code_commitments) - .map_err(|err| { - Status::invalid_argument(format!("Invalid code commitment: {}", err)) - })?; + let include_headers = include_headers.unwrap_or_default(); + let request_code_commitments: BTreeSet = try_convert(code_commitments) + .map_err(|err| Status::invalid_argument(format!("Invalid code commitment: {}", err)))?; + + let account_requests: Vec = + try_convert(account_requests).map_err(|err| { + Status::invalid_argument(format!("Invalid account proofs request: {}", err)) + })?; let (block_num, infos) = self .state - .get_account_proofs(account_ids, request_code_commitments, include_headers) + .get_account_proofs(account_requests, request_code_commitments, include_headers) .await?; Ok(Response::new(GetAccountProofsResponse { diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index a0b2bef69..4da0cac78 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -11,13 +11,19 @@ use std::{ use miden_node_proto::{ convert, - domain::{accounts::AccountInfo, blocks::BlockInclusionProof, notes::NoteAuthenticationInfo}, - generated::responses::{AccountProofsResponse, AccountStateHeader, GetBlockInputsResponse}, + domain::{ + accounts::{AccountInfo, AccountProofRequest, StorageMapKeysProof}, + blocks::BlockInclusionProof, + notes::NoteAuthenticationInfo, + }, + generated::responses::{ + AccountProofsResponse, AccountStateHeader, GetBlockInputsResponse, StorageSlotMapProof, + }, AccountInputRecord, NullifierWitness, }; use miden_node_utils::formatting::format_array; use miden_objects::{ - accounts::{AccountDelta, AccountHeader, AccountId}, + accounts::{AccountDelta, AccountHeader, AccountId, StorageSlot}, block::Block, crypto::{ hash::rpo::RpoDigest, @@ -673,8 +679,8 @@ impl State { /// Returns account proofs with optional account and storage headers. pub async fn get_account_proofs( &self, - account_ids: Vec, - request_code_commitments: BTreeSet, + account_requests: Vec, + known_code_commitments: BTreeSet, include_headers: bool, ) -> Result<(BlockNumber, Vec), DatabaseError> { // Lock inner state for the whole operation. We need to hold this lock to prevent the @@ -682,11 +688,13 @@ impl State { // because changing one of them would lead to inconsistent state. let inner_state = self.inner.read().await; + let account_ids: Vec = + account_requests.iter().map(|req| req.account_id).collect(); + let state_headers = if !include_headers { BTreeMap::::default() } else { let infos = self.db.select_accounts_by_ids(account_ids.clone()).await?; - if account_ids.len() > infos.len() { let found_ids = infos.iter().map(|info| info.summary.account_id).collect(); return Err(DatabaseError::AccountsNotFoundInDb( @@ -694,26 +702,56 @@ impl State { )); } - infos - .into_iter() - .filter_map(|info| { - info.details.map(|details| { - ( - info.summary.account_id, - AccountStateHeader { - header: Some(AccountHeader::from(&details).into()), - storage_header: details.storage().get_header().to_bytes(), - // Only include account code if the request did not contain it - // (known by the caller) - account_code: request_code_commitments - .contains(&details.code().commitment()) - .not() - .then_some(details.code().to_bytes()), - }, - ) - }) - }) - .collect() + let mut headers_map = BTreeMap::new(); + + // Iterate and build state headers for public accounts + for request in account_requests { + let account_info = infos + .iter() + .find(|info| info.summary.account_id == request.account_id) + .expect("retrieved accounts were validated against request"); + + if let Some(details) = &account_info.details { + let mut storage_slot_map_keys = Vec::new(); + + for StorageMapKeysProof { storage_index, storage_keys } in + &request.storage_requests + { + if let Some(StorageSlot::Map(storage_map)) = + details.storage().slots().get(*storage_index as usize) + { + for map_key in storage_keys { + let proof = storage_map.open(map_key); + + let slot_map_key = StorageSlotMapProof { + storage_slot: *storage_index as u32, + smt_proof: proof.to_bytes(), + }; + storage_slot_map_keys.push(slot_map_key); + } + } else { + return Err(AccountError::StorageSlotNotMap(*storage_index).into()); + } + } + + // Only include unknown account codes + let account_code = known_code_commitments + .contains(&details.code().commitment()) + .not() + .then(|| details.code().to_bytes()); + + let state_header = AccountStateHeader { + header: Some(AccountHeader::from(details).into()), + storage_header: details.storage().get_header().to_bytes(), + account_code, + storage_maps: storage_slot_map_keys, + }; + + headers_map.insert(account_info.summary.account_id, state_header); + } + } + + headers_map }; let responses = account_ids diff --git a/proto/requests.proto b/proto/requests.proto index 899359488..62ec750a1 100644 --- a/proto/requests.proto +++ b/proto/requests.proto @@ -126,11 +126,34 @@ message GetAccountStateDeltaRequest { fixed32 to_block_num = 3; } +// Request message to get account proofs. message GetAccountProofsRequest { - // List of account IDs to get states. - repeated account.AccountId account_ids = 1; - // Optional flag to include header and account code in the response. `false` by default. + // Represents per-account requests where each account ID has its own list of + // (storage_slot_index, map_keys) pairs. + message AccountRequest { + // The account ID for this request. + account.AccountId account_id = 1; + + // List of storage requests for this account. + repeated StorageRequest storage_requests = 2; + } + + // Represents a storage slot index and the associated map keys. + message StorageRequest { + // Storage slot index ([0..255]) + uint32 storage_slot_index = 1; + + // A list of map keys (Digests) associated with this storage slot. + repeated digest.Digest map_keys = 2; + } + + // A list of account requests, including map keys + values. + repeated AccountRequest account_requests = 1; + + // Optional flag to include account headers and account code in the response. If false, storage + // requests are also ignored. False by default. optional bool include_headers = 2; + // Account code commitments corresponding to the last-known `AccountCode` for requested // accounts. Responses will include only the ones that are not known to the caller. // These are not associated with a specific account but rather, they will be matched against diff --git a/proto/responses.proto b/proto/responses.proto index 233189a46..530a5dc18 100644 --- a/proto/responses.proto +++ b/proto/responses.proto @@ -180,9 +180,23 @@ message AccountProofsResponse { message AccountStateHeader { // Account header. account.AccountHeader header = 1; + // Values of all account storage slots (max 255). bytes storage_header = 2; - // Account code, returned only when none of the request's code commitments match with the - // current one. + + // Account code, returned only when none of the request's code commitments match + // the current one. optional bytes account_code = 3; + + // Storage slots information for this account + repeated StorageSlotMapProof storage_maps = 4; +} + +// Represents a single storage slot with the reuqested keys and their respective values. +message StorageSlotMapProof { + // The storage slot index ([0..255]). + uint32 storage_slot = 1; + + // Merkle proof of the map value + bytes smt_proof = 2; } From 3f9152a6d75984a0df378ce2ad82776e9078d824 Mon Sep 17 00:00:00 2001 From: Mirko <48352201+Mirko-von-Leipzig@users.noreply.github.com> Date: Wed, 15 Jan 2025 08:20:46 +0200 Subject: [PATCH 40/50] feat: remove `testing` feature (#619) --- .github/workflows/amd_deb_packager.yml | 4 ++-- .github/workflows/arm_deb_packager.yml | 4 ++-- CHANGELOG.md | 1 + Makefile | 8 -------- README.md | 6 ------ bin/faucet/Cargo.toml | 5 ----- bin/faucet/README.md | 11 ++++------- bin/node/Cargo.toml | 3 --- bin/node/Dockerfile | 2 +- 9 files changed, 10 insertions(+), 34 deletions(-) diff --git a/.github/workflows/amd_deb_packager.yml b/.github/workflows/amd_deb_packager.yml index ea290aebe..2b17fe9ff 100644 --- a/.github/workflows/amd_deb_packager.yml +++ b/.github/workflows/amd_deb_packager.yml @@ -28,8 +28,8 @@ jobs: - name: Building for amd64 run: | - cargo build --release --locked --features testing --bin miden-node - cargo build --release --locked --features testing --bin miden-faucet + cargo build --release --locked --bin miden-node + cargo build --release --locked --bin miden-faucet - name: create package directories run: | diff --git a/.github/workflows/arm_deb_packager.yml b/.github/workflows/arm_deb_packager.yml index 9f38c4037..cb88d772b 100644 --- a/.github/workflows/arm_deb_packager.yml +++ b/.github/workflows/arm_deb_packager.yml @@ -48,8 +48,8 @@ jobs: - name: Building for arm64 run: | - cargo build --release --locked --features testing --bin miden-node - cargo build --release --locked --features testing --bin miden-faucet + cargo build --release --locked --bin miden-node + cargo build --release --locked --bin miden-faucet - name: create package directories run: | diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ed082383..fb2751afd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ - [BREAKING] Inverted `TransactionInputs.missing_unauthenticated_notes` to `found_missing_notes` (#509). - [BREAKING] Remove store's `ListXXX` endpoints which were intended for test purposes (#608). - [BREAKING] Added support for storage maps on `GetAccountProofs` endpoint (#598). +- [BREAKING] Removed the `testing` feature (#619). ## v0.6.0 (2024-11-05) diff --git a/Makefile b/Makefile index afa1a2da9..b8f6bc3c6 100644 --- a/Makefile +++ b/Makefile @@ -73,14 +73,6 @@ install-node: ## Installs node install-faucet: ## Installs faucet ${BUILD_PROTO} cargo install --path bin/faucet --locked -.PHONY: install-node-testing -install-node-testing: ## Installs node with testing feature enabled - ${BUILD_PROTO} cargo install --features testing --path bin/node --locked - -.PHONY: install-faucet-testing -install-faucet-testing: ## Installs faucet with testing feature enabled - ${BUILD_PROTO} cargo install --features testing --path bin/faucet --locked - # --- docker -------------------------------------------------------------------------------------- .PHONY: docker-build-node diff --git a/README.md b/README.md index 8e080b997..76874276f 100644 --- a/README.md +++ b/README.md @@ -93,12 +93,6 @@ cargo install --locked --git https://github.com/0xPolygonMiden/miden-node miden- More information on the various options can be found [here](https://doc.rust-lang.org/cargo/commands/cargo-install.html#install-options). -> [!TIP] -> Miden account generation uses a proof-of-work puzzle to prevent DoS attacks. These puzzles can be quite expensive, especially for test purposes. You can lower the difficulty of the puzzle by appending `--features testing` to the `cargo install ..` invocation. For example: -> ```sh -> cargo install miden-node --locked --features testing -> ``` - ### Verify installation You can verify the installation by checking the node's version: diff --git a/bin/faucet/Cargo.toml b/bin/faucet/Cargo.toml index 7d299e3e2..cc24efb54 100644 --- a/bin/faucet/Cargo.toml +++ b/bin/faucet/Cargo.toml @@ -11,11 +11,6 @@ authors.workspace = true homepage.workspace = true repository.workspace = true -[features] -# Makes `make-genesis` subcommand run faster. Is only suitable for testing. -# INFO: Make sure that all your components have matching features for them to function. -testing = ["miden-objects/testing", "miden-lib/testing"] - [dependencies] anyhow = "1.0" axum = { version = "0.7", features = ["tokio"] } diff --git a/bin/faucet/README.md b/bin/faucet/README.md index 2d7b15e55..f2a4d29a2 100644 --- a/bin/faucet/README.md +++ b/bin/faucet/README.md @@ -2,20 +2,17 @@ This crate contains a binary for running a Miden rollup faucet. -## Running the faucet in testing mode +## Running the faucet -> [!TIP] -> Miden account generation uses a proof-of-work puzzle to prevent DoS attacks. These puzzles can be quite expensive, especially for test purposes. You can lower the difficulty of the puzzle by appending `--features testing` to the `cargo install ..` invocation. - -1. Run a local node with the "testing" feature, for example using the docker image. From the "miden-node" repo root run the following commands: +1. Run a local node using the docker image. From the "miden-node" repo root run the following commands: ```bash make docker-build-node make docker-run-node ``` -2. Install the faucet (with the "testing" feature): +2. Install the faucet: ```bash -make install-faucet-testing +make install-faucet ``` 3. [Optional] Create faucet account (skip this step if you want to use an account from the genesis). This will generate authentication keypair and generate and write public faucet account data with its keypair into the file specified in `output-path`: diff --git a/bin/node/Cargo.toml b/bin/node/Cargo.toml index d2d9b5d91..7e441f9c0 100644 --- a/bin/node/Cargo.toml +++ b/bin/node/Cargo.toml @@ -12,9 +12,6 @@ homepage.workspace = true repository.workspace = true [features] -# Makes `make-genesis` subcommand run faster. Is only suitable for testing. -# INFO: Make sure that all your components have matching features for them to function. -testing = ["miden-lib/testing", "miden-objects/testing"] tracing-forest = ["miden-node-block-producer/tracing-forest"] [dependencies] diff --git a/bin/node/Dockerfile b/bin/node/Dockerfile index 577e5e61b..0195f0068 100644 --- a/bin/node/Dockerfile +++ b/bin/node/Dockerfile @@ -8,7 +8,7 @@ RUN apt-get update && \ WORKDIR /app COPY . . -RUN cargo install --features testing --path bin/node --locked +RUN cargo install --path bin/node --locked RUN miden-node make-genesis --inputs-path config/genesis.toml FROM debian:bookworm-slim From 0c17dbcb209dd7c48ed70efcd25698af66fc25c5 Mon Sep 17 00:00:00 2001 From: Mirko <48352201+Mirko-von-Leipzig@users.noreply.github.com> Date: Wed, 15 Jan 2025 11:07:29 +0200 Subject: [PATCH 41/50] refactor: standardize errors (#611) --- bin/faucet/src/client.rs | 8 +- bin/faucet/src/errors.rs | 38 ++-- bin/faucet/src/handlers.rs | 20 +- bin/faucet/src/main.rs | 3 +- .../block-producer/src/batch_builder/batch.rs | 8 +- .../block-producer/src/batch_builder/mod.rs | 2 +- .../block-producer/src/block_builder/mod.rs | 2 +- .../src/block_builder/prover/block_witness.rs | 4 +- crates/block-producer/src/errors.rs | 93 +++++---- .../block-producer/src/mempool/graph/mod.rs | 14 +- crates/block-producer/src/server.rs | 2 +- crates/proto/src/errors.rs | 20 +- crates/store/src/db/mod.rs | 6 +- crates/store/src/db/sql/utils.rs | 2 +- crates/store/src/errors.rs | 186 +++++++++--------- crates/store/src/nullifier_tree.rs | 4 +- crates/store/src/state.rs | 2 +- crates/utils/src/errors.rs | 12 +- crates/utils/src/logging.rs | 23 ++- 19 files changed, 216 insertions(+), 233 deletions(-) diff --git a/bin/faucet/src/client.rs b/bin/faucet/src/client.rs index ce76a37ed..8b05ddf4f 100644 --- a/bin/faucet/src/client.rs +++ b/bin/faucet/src/client.rs @@ -29,12 +29,7 @@ use rand::{random, rngs::StdRng}; use tonic::transport::Channel; use tracing::info; -use crate::{ - config::FaucetConfig, - errors::{ClientError, ImplError}, - store::FaucetDataStore, - COMPONENT, -}; +use crate::{config::FaucetConfig, errors::ClientError, store::FaucetDataStore, COMPONENT}; pub const DISTRIBUTE_FUNGIBLE_ASSET_SCRIPT: &str = include_str!("transaction_scripts/distribute_fungible_asset.masm"); @@ -255,7 +250,6 @@ async fn request_account_state( account_info.details.context("Account details field is empty")?; Account::read_from_bytes(&faucet_account_state_bytes) - .map_err(ImplError) .context("Failed to deserialize faucet account") .map_err(Into::into) } diff --git a/bin/faucet/src/errors.rs b/bin/faucet/src/errors.rs index c87054133..1ff978631 100644 --- a/bin/faucet/src/errors.rs +++ b/bin/faucet/src/errors.rs @@ -1,57 +1,51 @@ -use std::fmt::{Debug, Display}; +use std::fmt::Debug; use axum::{ http::{header, StatusCode}, response::{IntoResponse, Response}, }; +use miden_objects::AccountError; use thiserror::Error; -/// Wrapper for implementing `Error` trait for errors, which do not implement it, like -/// [miden_objects::crypto::utils::DeserializationError] and other error types from `miden-base`. -#[derive(Debug, Error)] -#[error("{0}")] -pub struct ImplError(pub E); - #[derive(Debug, Error)] pub enum ClientError { - #[error("Request error: {0}")] + #[error(transparent)] RequestError(#[from] tonic::Status), - #[error("Client error: {0:#}")] + #[error(transparent)] Other(#[from] anyhow::Error), } #[derive(Debug, Error)] pub enum HandlerError { - #[error("Node client error: {0}")] + #[error("client error")] ClientError(#[from] ClientError), - #[error("Server has encountered an internal error: {0:#}")] + #[error("internal error")] Internal(#[from] anyhow::Error), - #[error("Client has submitted a bad request: {0}")] - BadRequest(String), + #[error("account ID deserialization failed")] + AccountIdDeserializationError(#[source] AccountError), - #[error("Page not found: {0}")] - NotFound(String), + #[error("invalid asset amount {requested} requested, valid options are {options:?}")] + InvalidAssetAmount { requested: u64, options: Vec }, } impl HandlerError { fn status_code(&self) -> StatusCode { match *self { - Self::BadRequest(_) => StatusCode::BAD_REQUEST, - Self::NotFound(_) => StatusCode::NOT_FOUND, + Self::InvalidAssetAmount { .. } | Self::AccountIdDeserializationError(_) => { + StatusCode::BAD_REQUEST + }, Self::ClientError(_) | Self::Internal(_) => StatusCode::INTERNAL_SERVER_ERROR, } } - fn message(&self) -> String { + fn message(self) -> String { match self { - Self::BadRequest(msg) => msg, - Self::ClientError(_) | Self::Internal(_) => "Error processing request", - Self::NotFound(msg) => msg, + Self::ClientError(_) | Self::Internal(_) => "Internal error".to_string(), + other => format!("{:#}", anyhow::Error::new(other)), } - .to_string() } } diff --git a/bin/faucet/src/handlers.rs b/bin/faucet/src/handlers.rs index 764f997d6..e8d6e11bb 100644 --- a/bin/faucet/src/handlers.rs +++ b/bin/faucet/src/handlers.rs @@ -1,6 +1,6 @@ use anyhow::Context; use axum::{ - extract::{Path, State}, + extract::State, http::{Response, StatusCode}, response::IntoResponse, Json, @@ -56,14 +56,17 @@ pub async fn get_tokens( // Check that the amount is in the asset amount options if !state.config.asset_amount_options.contains(&req.asset_amount) { - return Err(HandlerError::BadRequest("Invalid asset amount".to_string())); + return Err(HandlerError::InvalidAssetAmount { + requested: req.asset_amount, + options: state.config.asset_amount_options.clone(), + }); } let mut client = state.client.lock().await; // Receive and hex user account id let target_account_id = AccountId::from_hex(req.account_id.as_str()) - .map_err(|err| HandlerError::BadRequest(err.to_string()))?; + .map_err(HandlerError::AccountIdDeserializationError)?; // Execute transaction info!(target: COMPONENT, "Executing mint transaction for account."); @@ -114,16 +117,9 @@ pub async fn get_tokens( } pub async fn get_index(state: State) -> Result { - get_static_file(state, Path("index.html".to_string())).await -} - -pub async fn get_static_file( - State(state): State, - Path(path): Path, -) -> Result { - info!(target: COMPONENT, path, "Serving static file"); + info!(target: COMPONENT, "Serving `index.html`"); - let static_file = state.static_files.get(path.as_str()).ok_or(HandlerError::NotFound(path))?; + let static_file = state.static_files.get("index.html").expect("index.html should be bundled"); Response::builder() .status(StatusCode::OK) diff --git a/bin/faucet/src/main.rs b/bin/faucet/src/main.rs index b489d8e1f..06a3a5fe9 100644 --- a/bin/faucet/src/main.rs +++ b/bin/faucet/src/main.rs @@ -33,7 +33,7 @@ use tracing::info; use crate::{ config::{FaucetConfig, DEFAULT_FAUCET_ACCOUNT_PATH}, - handlers::{get_index, get_metadata, get_static_file, get_tokens}, + handlers::{get_index, get_metadata, get_tokens}, }; // CONSTANTS @@ -105,7 +105,6 @@ async fn main() -> anyhow::Result<()> { .route("/", get(get_index)) .route("/get_metadata", get(get_metadata)) .route("/get_tokens", post(get_tokens)) - .route("/*path", get(get_static_file)) .layer( ServiceBuilder::new() .layer(TraceLayer::new_for_http()) diff --git a/crates/block-producer/src/batch_builder/batch.rs b/crates/block-producer/src/batch_builder/batch.rs index b6230b5ac..168463260 100644 --- a/crates/block-producer/src/batch_builder/batch.rs +++ b/crates/block-producer/src/batch_builder/batch.rs @@ -135,9 +135,11 @@ impl TransactionBatch { Entry::Vacant(vacant) => { vacant.insert(AccountUpdate::new(tx)); }, - Entry::Occupied(occupied) => occupied.into_mut().merge_tx(tx).map_err(|error| { - BuildBatchError::AccountUpdateError { account_id: tx.account_id(), error } - })?, + Entry::Occupied(occupied) => { + occupied.into_mut().merge_tx(tx).map_err(|source| { + BuildBatchError::AccountUpdateError { account_id: tx.account_id(), source } + })? + }, }; // Check unauthenticated input notes for duplicates: diff --git a/crates/block-producer/src/batch_builder/mod.rs b/crates/block-producer/src/batch_builder/mod.rs index 875c48db8..c504e4c7a 100644 --- a/crates/block-producer/src/batch_builder/mod.rs +++ b/crates/block-producer/src/batch_builder/mod.rs @@ -224,7 +224,7 @@ impl WorkerPool { transactions.iter().flat_map(|tx| tx.unauthenticated_notes()), ) .await - .map_err(|err| (id, err.into()))?; + .map_err(|err| (id, BuildBatchError::FetchBatchInputsFailed(err)))?; let batch = Self::build_batch(transactions, inputs).map_err(|err| (id, err))?; tokio::time::sleep(simulated_proof_time).await; diff --git a/crates/block-producer/src/block_builder/mod.rs b/crates/block-producer/src/block_builder/mod.rs index 45be364a3..ff79dd603 100644 --- a/crates/block-producer/src/block_builder/mod.rs +++ b/crates/block-producer/src/block_builder/mod.rs @@ -166,7 +166,7 @@ impl BlockBuilder { self.store .apply_block(&block) .await - .map_err(BuildBlockError::ApplyBlockFailed)?; + .map_err(BuildBlockError::StoreApplyBlockFailed)?; info!(target: COMPONENT, block_num, %block_hash, "block committed"); diff --git a/crates/block-producer/src/block_builder/prover/block_witness.rs b/crates/block-producer/src/block_builder/prover/block_witness.rs index 91d380c5b..cc9da466f 100644 --- a/crates/block-producer/src/block_builder/prover/block_witness.rs +++ b/crates/block-producer/src/block_builder/prover/block_witness.rs @@ -89,8 +89,8 @@ impl BlockWitness { details = Some(match details { None => update.details, - Some(details) => details.merge(update.details).map_err(|err| { - BuildBlockError::AccountUpdateError { account_id, error: err } + Some(details) => details.merge(update.details).map_err(|source| { + BuildBlockError::AccountUpdateError { account_id, source } })?, }); } diff --git a/crates/block-producer/src/errors.rs b/crates/block-producer/src/errors.rs index 021c0b90a..2dd8a4028 100644 --- a/crates/block-producer/src/errors.rs +++ b/crates/block-producer/src/errors.rs @@ -5,7 +5,7 @@ use miden_objects::{ crypto::merkle::MerkleError, notes::{NoteId, Nullifier}, transaction::TransactionId, - AccountDeltaError, Digest, TransactionInputError, + AccountDeltaError, Digest, }; use miden_processor::ExecutionError; use thiserror::Error; @@ -40,21 +40,23 @@ pub enum BlockProducerError { #[derive(Debug, Error)] pub enum VerifyTxError { /// Another transaction already consumed the notes with given nullifiers - #[error("Input notes with given nullifiers were already consumed by another transaction")] + #[error( + "input notes with given nullifiers were already consumed by another transaction: {0:?}" + )] InputNotesAlreadyConsumed(Vec), /// Unauthenticated transaction notes were not found in the store or in outputs of in-flight /// transactions #[error( - "Unauthenticated transaction notes were not found in the store or in outputs of in-flight transactions: {0:?}" + "unauthenticated transaction notes were not found in the store or in outputs of in-flight transactions: {0:?}" )] UnauthenticatedNotesNotFound(Vec), - #[error("Output note IDs already used: {0:?}")] + #[error("output note IDs already used: {0:?}")] OutputNotesAlreadyExist(Vec), /// The account's initial hash did not match the current account's hash - #[error("Incorrect account's initial hash ({tx_initial_account_hash}, current: {})", format_opt(.current_account_hash.as_ref()))] + #[error("incorrect account's initial hash ({tx_initial_account_hash}, current: {})", format_opt(.current_account_hash.as_ref()))] IncorrectAccountInitialHash { tx_initial_account_hash: Digest, current_account_hash: Option, @@ -64,14 +66,11 @@ pub enum VerifyTxError { /// /// TODO: Make this an "internal error". Q: Should we have a single `InternalError` enum for /// all internal errors that can occur across the system? - #[error("Failed to retrieve transaction inputs from the store: {0}")] + #[error("failed to retrieve transaction inputs from the store")] StoreConnectionFailed(#[from] StoreError), - #[error("Transaction input error: {0}")] - TransactionInputError(#[from] TransactionInputError), - /// Failed to verify the transaction execution proof - #[error("Invalid transaction proof error for transaction: {0}")] + #[error("invalid transaction proof error for transaction: {0}")] InvalidTransactionProof(TransactionId), } @@ -80,19 +79,21 @@ pub enum VerifyTxError { #[derive(Debug, Error)] pub enum AddTransactionError { - #[error("Transaction verification failed: {0}")] + #[error("transaction verification failed")] VerificationFailed(#[from] VerifyTxError), - #[error("Transaction input data is stale. Required data from {stale_limit} or newer, but inputs are from {input_block}.")] + #[error("transaction input data from block {input_block} is rejected as stale because it is older than the limit of {stale_limit}")] StaleInputs { input_block: BlockNumber, stale_limit: BlockNumber, }, - #[error("Deserialization failed: {0}")] - DeserializationError(String), + #[error("transaction deserialization failed")] + TransactionDeserializationFailed(#[source] miden_objects::utils::DeserializationError), - #[error("Transaction expired at {expired_at} but the limit was {limit}")] + #[error( + "transaction expired at block height {expired_at} but the block height limit was {limit}" + )] Expired { expired_at: BlockNumber, limit: BlockNumber, @@ -109,12 +110,12 @@ impl From for tonic::Status { | VerificationFailed(VerifyTxError::IncorrectAccountInitialHash { .. }) | VerificationFailed(VerifyTxError::InvalidTransactionProof(_)) | Expired { .. } - | DeserializationError(_) => Self::invalid_argument(value.to_string()), + | TransactionDeserializationFailed(_) => Self::invalid_argument(value.to_string()), // Internal errors which should not be communicated to the user. - VerificationFailed(VerifyTxError::TransactionInputError(_)) - | VerificationFailed(VerifyTxError::StoreConnectionFailed(_)) - | StaleInputs { .. } => Self::internal("Internal error"), + VerificationFailed(VerifyTxError::StoreConnectionFailed(_)) | StaleInputs { .. } => { + Self::internal("Internal error") + }, } } } @@ -125,33 +126,35 @@ impl From for tonic::Status { /// Error encountered while building a batch. #[derive(Debug, Error)] pub enum BuildBatchError { - #[error("Duplicated unauthenticated transaction input note ID in the batch: {0}")] + #[error("duplicated unauthenticated transaction input note ID in the batch: {0}")] DuplicateUnauthenticatedNote(NoteId), - #[error("Duplicated transaction output note ID in the batch: {0}")] + #[error("duplicated transaction output note ID in the batch: {0}")] DuplicateOutputNote(NoteId), - #[error("Note hashes mismatch for note {id}: (input: {input_hash}, output: {output_hash})")] + #[error("note hashes mismatch for note {id}: (input: {input_hash}, output: {output_hash})")] NoteHashesMismatch { id: NoteId, input_hash: Digest, output_hash: Digest, }, - #[error("Failed to merge transaction delta into account {account_id}: {error}")] + #[error("failed to merge transaction delta into account {account_id}")] AccountUpdateError { account_id: AccountId, - error: AccountDeltaError, + source: AccountDeltaError, }, - #[error("Nothing actually went wrong, failure was injected on purpose")] + /// We sometimes randomly inject errors into the batch building process to test our failure + /// responses. + #[error("nothing actually went wrong, failure was injected on purpose")] InjectedFailure, - #[error("Batch proving task panic'd")] + #[error("batch proving task panic'd")] JoinError(#[from] tokio::task::JoinError), - #[error("Fetching inputs from store failed")] - StoreError(#[from] StoreError), + #[error("failed to fetch batch inputs from store")] + FetchBatchInputsFailed(#[source] StoreError), } // Block prover errors @@ -159,11 +162,11 @@ pub enum BuildBatchError { #[derive(Debug, Error)] pub enum BlockProverError { - #[error("Received invalid merkle path")] - InvalidMerklePaths(MerkleError), - #[error("Program execution failed")] - ProgramExecutionFailed(ExecutionError), - #[error("Failed to retrieve {0} root from stack outputs")] + #[error("received invalid merkle path")] + InvalidMerklePaths(#[source] MerkleError), + #[error("program execution failed")] + ProgramExecutionFailed(#[source] ExecutionError), + #[error("failed to retrieve {0} root from stack outputs")] InvalidRootOutput(&'static str), } @@ -172,27 +175,31 @@ pub enum BlockProverError { #[derive(Debug, Error)] pub enum BuildBlockError { - #[error("failed to compute new block: {0}")] + #[error("failed to compute new block")] BlockProverFailed(#[from] BlockProverError), - #[error("failed to apply block: {0}")] - ApplyBlockFailed(#[source] StoreError), - #[error("failed to get block inputs from store: {0}")] + #[error("failed to apply block to store")] + StoreApplyBlockFailed(#[source] StoreError), + #[error("failed to get block inputs from store")] GetBlockInputsFailed(#[source] StoreError), - #[error("store did not produce data for account: {0}")] + #[error("block inputs from store did not contain data for account {0}")] MissingAccountInput(AccountId), - #[error("store produced extra account data. Offending accounts: {0:?}")] + #[error("block inputs from store contained extra data for accounts {0:?}")] ExtraStoreData(Vec), - #[error("no matching state transition found for account {0}. Current account state is {1}, remaining updates: {2:?}")] + #[error("account {0} with state {1} cannot transaction to remaining states {2:?}")] InconsistentAccountStateTransition(AccountId, Digest, Vec), - #[error("transaction batches and store don't produce the same nullifiers. Offending nullifiers: {0:?}")] + #[error( + "block inputs from store and transaction batches produced different nullifiers: {0:?}" + )] InconsistentNullifiers(Vec), #[error("unauthenticated transaction notes not found in the store or in outputs of other transactions in the block: {0:?}")] UnauthenticatedNotesNotFound(Vec), - #[error("failed to merge transaction delta into account {account_id}: {error}")] + #[error("failed to merge transaction delta into account {account_id}")] AccountUpdateError { account_id: AccountId, - error: AccountDeltaError, + source: AccountDeltaError, }, + /// We sometimes randomly inject errors into the batch building process to test our failure + /// responses. #[error("nothing actually went wrong, failure was injected on purpose")] InjectedFailure, } diff --git a/crates/block-producer/src/mempool/graph/mod.rs b/crates/block-producer/src/mempool/graph/mod.rs index d9199909d..5b0921d49 100644 --- a/crates/block-producer/src/mempool/graph/mod.rs +++ b/crates/block-producer/src/mempool/graph/mod.rs @@ -82,25 +82,25 @@ where #[derive(thiserror::Error, Debug, Clone, PartialEq, Eq)] pub enum GraphError { - #[error("Node {0} already exists")] + #[error("node {0} already exists")] DuplicateKey(K), - #[error("Parents not found: {0:?}")] + #[error("parents not found: {0:?}")] MissingParents(BTreeSet), - #[error("Nodes not found: {0:?}")] + #[error("nodes not found: {0:?}")] UnknownNodes(BTreeSet), - #[error("Nodes were not yet processed: {0:?}")] + #[error("nodes were not yet processed: {0:?}")] UnprocessedNodes(BTreeSet), - #[error("Nodes would be left dangling: {0:?}")] + #[error("nodes would be left dangling: {0:?}")] DanglingNodes(BTreeSet), - #[error("Node {0} is not a root node")] + #[error("node {0} is not a root node")] InvalidRootNode(K), - #[error("Node {0} is not a pending node")] + #[error("node {0} is not a pending node")] InvalidPendingNode(K), } diff --git a/crates/block-producer/src/server.rs b/crates/block-producer/src/server.rs index a3ee6043c..afd82c9e4 100644 --- a/crates/block-producer/src/server.rs +++ b/crates/block-producer/src/server.rs @@ -223,7 +223,7 @@ impl BlockProducerRpcServer { debug!(target: COMPONENT, ?request); let tx = ProvenTransaction::read_from_bytes(&request.transaction) - .map_err(|err| AddTransactionError::DeserializationError(err.to_string()))?; + .map_err(AddTransactionError::TransactionDeserializationFailed)?; let tx_id = tx.id(); diff --git a/crates/proto/src/errors.rs b/crates/proto/src/errors.rs index ae3acab58..8af3c59ea 100644 --- a/crates/proto/src/errors.rs +++ b/crates/proto/src/errors.rs @@ -5,28 +5,28 @@ use thiserror::Error; #[derive(Debug, Error)] pub enum ConversionError { - #[error("Hex error: {0}")] + #[error("hex error")] HexError(#[from] hex::FromHexError), - #[error("Note error: {0}")] + #[error("note error")] NoteError(#[from] miden_objects::NoteError), - #[error("SMT leaf error: {0}")] + #[error("SMT leaf error")] SmtLeafError(#[from] SmtLeafError), - #[error("SMT proof error: {0}")] + #[error("SMT proof error")] SmtProofError(#[from] SmtProofError), - #[error("Integer conversion error: {0}")] + #[error("integer conversion error: {0}")] TryFromIntError(#[from] TryFromIntError), - #[error("Too much data, expected {expected}, got {got}")] + #[error("too much data, expected {expected}, got {got}")] TooMuchData { expected: usize, got: usize }, - #[error("Not enough data, expected {expected}, got {got}")] + #[error("not enough data, expected {expected}, got {got}")] InsufficientData { expected: usize, got: usize }, - #[error("Value is not in the range 0..MODULUS")] + #[error("value is not in the range 0..MODULUS")] NotAValidFelt, - #[error("Field `{field_name}` required to be filled in protobuf representation of {entity}")] + #[error("field `{entity}::{field_name}` is missing")] MissingFieldInProtobufRepresentation { entity: &'static str, field_name: &'static str, }, - #[error("MMR error: {0}")] + #[error("MMR error")] MmrError(#[from] miden_objects::crypto::merkle::MmrError), } diff --git a/crates/store/src/db/mod.rs b/crates/store/src/db/mod.rs index abdd88d21..64e0ebdb0 100644 --- a/crates/store/src/db/mod.rs +++ b/crates/store/src/db/mod.rs @@ -476,10 +476,10 @@ impl Db { block_store: Arc, ) -> Result<(), GenesisError> { let genesis_block = { - let file_contents = fs::read(genesis_filepath).map_err(|error| { + let file_contents = fs::read(genesis_filepath).map_err(|source| { GenesisError::FailedToReadGenesisFile { genesis_filepath: genesis_filepath.to_string(), - error, + source, } })?; @@ -535,7 +535,7 @@ impl Db { Ok(()) }) .await - .map_err(|err| GenesisError::ApplyBlockFailed(err.to_string()))??; + .map_err(GenesisError::ApplyBlockFailed)??; }, } diff --git a/crates/store/src/db/sql/utils.rs b/crates/store/src/db/sql/utils.rs index 5862a2f11..03faf052c 100644 --- a/crates/store/src/db/sql/utils.rs +++ b/crates/store/src/db/sql/utils.rs @@ -138,7 +138,7 @@ pub fn apply_delta( let account = account.map(Account::read_from_bytes).transpose()?; let Some(mut account) = account else { - return Err(DatabaseError::AccountNotOnChain(account_id)); + return Err(DatabaseError::AccountNotPublic(account_id)); }; account.apply_delta(delta)?; diff --git a/crates/store/src/errors.rs b/crates/store/src/errors.rs index 87a139d76..6d3e14ed9 100644 --- a/crates/store/src/errors.rs +++ b/crates/store/src/errors.rs @@ -1,6 +1,6 @@ use std::io; -use deadpool_sqlite::PoolError; +use deadpool_sqlite::{InteractError, PoolError}; use miden_objects::{ accounts::AccountId, crypto::{ @@ -24,13 +24,11 @@ use crate::types::BlockNumber; #[derive(Debug, Error)] pub enum NullifierTreeError { - #[error("Merkle error: {0}")] - MerkleError(#[from] MerkleError), - #[error("Nullifier {nullifier} for block #{block_num} already exists in the nullifier tree")] - NullifierAlreadyExists { - nullifier: Nullifier, - block_num: BlockNumber, - }, + #[error("failed to create nullifier tree")] + CreationFailed(#[source] MerkleError), + + #[error("failed to mutate nullifier tree")] + MutationFailed(#[source] MerkleError), } // DATABASE ERRORS @@ -40,71 +38,65 @@ pub enum NullifierTreeError { pub enum DatabaseError { // ERRORS WITH AUTOMATIC CONVERSIONS FROM NESTED ERROR TYPES // --------------------------------------------------------------------------------------------- - #[error("Account error: {0}")] + #[error("account error")] AccountError(#[from] AccountError), - #[error("Account delta error: {0}")] + #[error("account delta error")] AccountDeltaError(#[from] AccountDeltaError), - #[error("Block error: {0}")] + #[error("block error")] BlockError(#[from] BlockError), - #[error("Closed channel: {0}")] + #[error("closed channel")] ClosedChannel(#[from] RecvError), - #[error("Deserialization of BLOB data from database failed: {0}")] - DeserializationError(DeserializationError), - #[error("Hex parsing error: {0}")] + #[error("deserialization failed")] + DeserializationError(#[from] DeserializationError), + #[error("hex parsing error")] FromHexError(#[from] hex::FromHexError), - #[error("SQLite error: {0}")] + #[error("SQLite deserialization error")] FromSqlError(#[from] FromSqlError), - #[error("I/O error: {0}")] + #[error("I/O error")] IoError(#[from] io::Error), - #[error("Migration error: {0}")] + #[error("migration failed")] MigrationError(#[from] rusqlite_migration::Error), - #[error("Missing database connection: {0}")] + #[error("missing database connection")] MissingDbConnection(#[from] PoolError), - #[error("Note error: {0}")] + #[error("note error")] NoteError(#[from] NoteError), - #[error("SQLite error: {0}")] + #[error("SQLite error")] SqliteError(#[from] rusqlite::Error), // OTHER ERRORS // --------------------------------------------------------------------------------------------- - #[error("Account hashes mismatch (expected {expected}, but calculated is {calculated})")] + #[error("account hash mismatch (expected {expected}, but calculated is {calculated})")] AccountHashesMismatch { expected: RpoDigest, calculated: RpoDigest, }, - #[error("Account {0} not found in the database")] + #[error("account {0} not found")] AccountNotFoundInDb(AccountId), - #[error("Accounts {0:?} not found in the database")] + #[error("accounts {0:?} not found")] AccountsNotFoundInDb(Vec), - #[error("Account {0} is not on the chain")] - AccountNotOnChain(AccountId), - #[error("Block {0} not found in the database")] + #[error("account {0} is not on the chain")] + AccountNotPublic(AccountId), + #[error("block {0} not found")] BlockNotFoundInDb(BlockNumber), - #[error("Data corrupted: {0}")] + #[error("data corrupted: {0}")] DataCorrupted(String), - #[error("SQLite pool interaction task failed: {0}")] + #[error("SQLite pool interaction failed: {0}")] InteractError(String), - #[error("Invalid Felt: {0}")] + #[error("invalid Felt: {0}")] InvalidFelt(String), #[error( - "Unsupported database version. There is no migration chain from/to this version. \ + "unsupported database version. There is no migration chain from/to this version. \ Remove all database files and try again." )] UnsupportedDatabaseVersion, } -impl From for DatabaseError { - fn from(value: DeserializationError) -> Self { - Self::DeserializationError(value) - } -} - impl From for Status { fn from(err: DatabaseError) -> Self { match err { DatabaseError::AccountNotFoundInDb(_) | DatabaseError::AccountsNotFoundInDb(_) - | DatabaseError::AccountNotOnChain(_) + | DatabaseError::AccountNotPublic(_) | DatabaseError::BlockNotFoundInDb(_) => Status::not_found(err.to_string()), _ => Status::internal(err.to_string()), @@ -117,25 +109,25 @@ impl From for Status { #[derive(Error, Debug)] pub enum StateInitializationError { - #[error("Database error: {0}")] + #[error("database error")] DatabaseError(#[from] DatabaseError), - #[error("Failed to create nullifier tree: {0}")] - FailedToCreateNullifierTree(NullifierTreeError), - #[error("Failed to create accounts tree: {0}")] - FailedToCreateAccountsTree(MerkleError), + #[error("failed to create nullifier tree")] + FailedToCreateNullifierTree(#[from] NullifierTreeError), + #[error("failed to create accounts tree")] + FailedToCreateAccountsTree(#[from] MerkleError), } #[derive(Debug, Error)] pub enum DatabaseSetupError { - #[error("I/O error: {0}")] + #[error("I/O error")] IoError(#[from] io::Error), - #[error("Database error: {0}")] + #[error("database error")] DatabaseError(#[from] DatabaseError), - #[error("Genesis block error: {0}")] + #[error("genesis block error")] GenesisBlockError(#[from] GenesisError), - #[error("Pool build error: {0}")] + #[error("pool build error")] PoolBuildError(#[from] deadpool_sqlite::BuildError), - #[error("SQLite migration error: {0}")] + #[error("SQLite migration error")] SqliteMigrationError(#[from] rusqlite_migration::Error), } @@ -143,54 +135,54 @@ pub enum DatabaseSetupError { pub enum GenesisError { // ERRORS WITH AUTOMATIC CONVERSIONS FROM NESTED ERROR TYPES // --------------------------------------------------------------------------------------------- - #[error("Database error: {0}")] + #[error("database error")] DatabaseError(#[from] DatabaseError), - #[error("Block error: {0}")] + #[error("block error")] BlockError(#[from] BlockError), - #[error("Merkle error: {0}")] + #[error("merkle error")] MerkleError(#[from] MerkleError), + #[error("failed to deserialize genesis file")] + GenesisFileDeserializationError(#[from] DeserializationError), + #[error("retrieving genesis block header failed")] + SelectBlockHeaderByBlockNumError(#[from] Box), // OTHER ERRORS // --------------------------------------------------------------------------------------------- - #[error("Apply block failed: {0}")] - ApplyBlockFailed(String), - #[error("Failed to read genesis file \"{genesis_filepath}\": {error}")] + #[error("apply block failed")] + ApplyBlockFailed(#[source] InteractError), + #[error("failed to read genesis file \"{genesis_filepath}\"")] FailedToReadGenesisFile { genesis_filepath: String, - error: io::Error, + source: io::Error, }, - #[error("Block header in store doesn't match block header in genesis file. Expected {expected_genesis_header:?}, but store contained {block_header_in_store:?}")] + #[error("block header in store doesn't match block header in genesis file. Expected {expected_genesis_header:?}, but store contained {block_header_in_store:?}")] GenesisBlockHeaderMismatch { expected_genesis_header: Box, block_header_in_store: Box, }, - #[error("Failed to deserialize genesis file: {0}")] - GenesisFileDeserializationError(DeserializationError), - #[error("Retrieving genesis block header failed: {0}")] - SelectBlockHeaderByBlockNumError(Box), } // ENDPOINT ERRORS // ================================================================================================= #[derive(Error, Debug)] pub enum InvalidBlockError { - #[error("Duplicated nullifiers {0:?}")] + #[error("duplicated nullifiers {0:?}")] DuplicatedNullifiers(Vec), - #[error("Invalid output note type: {0:?}")] + #[error("invalid output note type: {0:?}")] InvalidOutputNoteType(Box), - #[error("Invalid tx hash: expected {expected}, but got {actual}")] - InvalidTxHash { expected: RpoDigest, actual: RpoDigest }, - #[error("Received invalid account tree root")] + #[error("invalid block tx hash: expected {expected}, but got {actual}")] + InvalidBlockTxHash { expected: RpoDigest, actual: RpoDigest }, + #[error("received invalid account tree root")] NewBlockInvalidAccountRoot, - #[error("New block number must be 1 greater than the current block number")] + #[error("new block number must be 1 greater than the current block number")] NewBlockInvalidBlockNum, - #[error("New block chain root is not consistent with chain MMR")] + #[error("new block chain root is not consistent with chain MMR")] NewBlockInvalidChainRoot, - #[error("Received invalid note root")] + #[error("received invalid note root")] NewBlockInvalidNoteRoot, - #[error("Received invalid nullifier root")] + #[error("received invalid nullifier root")] NewBlockInvalidNullifierRoot, - #[error("New block `prev_hash` must match the chain's tip")] + #[error("new block `prev_hash` must match the chain's tip")] NewBlockInvalidPrevHash, } @@ -198,24 +190,24 @@ pub enum InvalidBlockError { pub enum ApplyBlockError { // ERRORS WITH AUTOMATIC CONVERSIONS FROM NESTED ERROR TYPES // --------------------------------------------------------------------------------------------- - #[error("Database error: {0}")] + #[error("database error")] DatabaseError(#[from] DatabaseError), - #[error("I/O error: {0}")] + #[error("I/O error")] IoError(#[from] io::Error), - #[error("Task join error: {0}")] + #[error("task join error")] TokioJoinError(#[from] tokio::task::JoinError), - #[error("Invalid block error: {0}")] + #[error("invalid block error")] InvalidBlockError(#[from] InvalidBlockError), // OTHER ERRORS // --------------------------------------------------------------------------------------------- - #[error("Block applying was cancelled because of closed channel on database side: {0}")] - ClosedChannel(RecvError), - #[error("Concurrent write detected")] + #[error("block applying was cancelled because of closed channel on database side")] + ClosedChannel(#[from] RecvError), + #[error("concurrent write detected")] ConcurrentWrite, - #[error("Database doesn't have any block header data")] + #[error("database doesn't have any block header data")] DbBlockHeaderEmpty, - #[error("Database update task failed: {0}")] + #[error("database update failed: {0}")] DbUpdateTaskFailed(String), } @@ -231,26 +223,26 @@ impl From for Status { #[derive(Error, Debug)] pub enum GetBlockHeaderError { - #[error("Database error: {0}")] + #[error("database error")] DatabaseError(#[from] DatabaseError), - #[error("Error retrieving the merkle proof for the block: {0}")] + #[error("error retrieving the merkle proof for the block")] MmrError(#[from] MmrError), } #[derive(Error, Debug)] pub enum GetBlockInputsError { - #[error("Account error: {0}")] + #[error("account error")] AccountError(#[from] AccountError), - #[error("Database error: {0}")] + #[error("database error")] DatabaseError(#[from] DatabaseError), - #[error("Database doesn't have any block header data")] + #[error("database doesn't have any block header data")] DbBlockHeaderEmpty, - #[error("Failed to get MMR peaks for forest ({forest}): {error}")] + #[error("failed to get MMR peaks for forest ({forest}): {error}")] FailedToGetMmrPeaksForForest { forest: usize, error: MmrError }, - #[error("Chain MMR forest expected to be 1 less than latest header's block num. Chain MMR forest: {forest}, block num: {block_num}")] + #[error("chain MMR forest expected to be 1 less than latest header's block num. Chain MMR forest: {forest}, block num: {block_num}")] IncorrectChainMmrForestNumber { forest: usize, block_num: u32 }, - #[error("Note inclusion proof MMR error: {0}")] - NoteInclusionMmr(MmrError), + #[error("note inclusion proof MMR error")] + NoteInclusionMmr(#[from] MmrError), } impl From for GetBlockInputsError { @@ -264,28 +256,28 @@ impl From for GetBlockInputsError { #[derive(Error, Debug)] pub enum StateSyncError { - #[error("Database error: {0}")] + #[error("database error")] DatabaseError(#[from] DatabaseError), - #[error("Block headers table is empty")] + #[error("block headers table is empty")] EmptyBlockHeadersTable, - #[error("Failed to build MMR delta: {0}")] - FailedToBuildMmrDelta(MmrError), + #[error("failed to build MMR delta")] + FailedToBuildMmrDelta(#[from] MmrError), } #[derive(Error, Debug)] pub enum NoteSyncError { - #[error("Database error: {0}")] + #[error("database error")] DatabaseError(#[from] DatabaseError), - #[error("Block headers table is empty")] + #[error("block headers table is empty")] EmptyBlockHeadersTable, - #[error("Error retrieving the merkle proof for the block: {0}")] + #[error("error retrieving the merkle proof for the block")] MmrError(#[from] MmrError), } #[derive(Error, Debug)] pub enum GetNoteInclusionProofError { - #[error("Database error: {0}")] + #[error("database error")] DatabaseError(#[from] DatabaseError), - #[error("Mmr error: {0}")] + #[error("Mmr error")] MmrError(#[from] MmrError), } diff --git a/crates/store/src/nullifier_tree.rs b/crates/store/src/nullifier_tree.rs index 0d6ed342c..282b27f44 100644 --- a/crates/store/src/nullifier_tree.rs +++ b/crates/store/src/nullifier_tree.rs @@ -22,7 +22,7 @@ impl NullifierTree { (nullifier.inner(), Self::block_num_to_leaf_value(block_num)) }); - let inner = Smt::with_entries(leaves)?; + let inner = Smt::with_entries(leaves).map_err(NullifierTreeError::CreationFailed)?; Ok(Self(inner)) } @@ -63,7 +63,7 @@ impl NullifierTree { &mut self, mutations: MutationSet, ) -> Result<(), NullifierTreeError> { - self.0.apply_mutations(mutations).map_err(Into::into) + self.0.apply_mutations(mutations).map_err(NullifierTreeError::MutationFailed) } // HELPER FUNCTIONS diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index 4da0cac78..ab1e0045b 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -181,7 +181,7 @@ impl State { let tx_hash = block.compute_tx_hash(); if header.tx_hash() != tx_hash { - return Err(InvalidBlockError::InvalidTxHash { + return Err(InvalidBlockError::InvalidBlockTxHash { expected: tx_hash, actual: header.tx_hash(), } diff --git a/crates/utils/src/errors.rs b/crates/utils/src/errors.rs index 6c70f3f06..283da7c5b 100644 --- a/crates/utils/src/errors.rs +++ b/crates/utils/src/errors.rs @@ -3,22 +3,22 @@ use tonic::transport::Error as TransportError; #[derive(Debug, Error)] pub enum ApiError { - #[error("An I/O error has occurred: {0}")] + #[error("an I/O error has occurred")] IoError(#[from] std::io::Error), #[error("initialisation of the Api has failed: {0}")] ApiInitialisationFailed(String), - #[error("Serving the Api server has failed.")] - ApiServeFailed(TransportError), + #[error("serving the Api server has failed")] + ApiServeFailed(#[from] TransportError), - #[error("Resolution of the server address has failed: {0}")] + #[error("resolution of the server address has failed: {0}")] AddressResolutionFailed(String), /// Converting the provided `Endpoint` into a socket address has failed - #[error("Converting the `Endpoint` into a socket address failed: {0}")] + #[error("converting the `Endpoint` into a socket address failed: {0}")] EndpointToSocketFailed(std::io::Error), - #[error("Connection to the database has failed: {0}")] + #[error("connection to the database has failed: {0}")] DatabaseConnectionFailed(String), } diff --git a/crates/utils/src/logging.rs b/crates/utils/src/logging.rs index a29e22ae7..dc58d7e88 100644 --- a/crates/utils/src/logging.rs +++ b/crates/utils/src/logging.rs @@ -1,8 +1,5 @@ use anyhow::Result; -use tracing::{ - level_filters::LevelFilter, - subscriber::{self, Subscriber}, -}; +use tracing::subscriber::{self, Subscriber}; use tracing_subscriber::EnvFilter; pub fn setup_logging() -> Result<()> { @@ -22,11 +19,11 @@ pub fn subscriber() -> impl Subscriber + core::fmt::Debug { .with_file(true) .with_line_number(true) .with_target(true) - .with_env_filter( - EnvFilter::builder() - .with_default_directive(LevelFilter::INFO.into()) - .from_env_lossy(), - ) + .with_env_filter(EnvFilter::try_from_default_env().unwrap_or_else(|_| { + // axum logs rejections from built-in extracts on the trace level, so we enable this + // manually. + "info,axum::rejection=trace".into() + })) .with_span_events(FmtSpan::NEW | FmtSpan::CLOSE) .finish() } @@ -37,8 +34,10 @@ pub fn subscriber() -> impl Subscriber + core::fmt::Debug { pub use tracing_subscriber::{layer::SubscriberExt, Registry}; Registry::default().with(ForestLayer::default()).with( - EnvFilter::builder() - .with_default_directive(LevelFilter::INFO.into()) - .from_env_lossy(), + EnvFilter::try_from_default_env().unwrap_or_else(|_| { + // axum logs rejections from built-in extracts on the trace level, so we enable this + // manually. + "info,axum::rejection=trace".into() + }), ) } From 16482775ec71f73147e529f92df2bd827d7fb5b4 Mon Sep 17 00:00:00 2001 From: igamigo Date: Wed, 15 Jan 2025 15:14:55 -0300 Subject: [PATCH 42/50] chore: Move to latest next and remove concurrent (#622) --- Cargo.lock | 226 +++++++++--------- bin/faucet/Cargo.toml | 4 +- bin/faucet/src/errors.rs | 4 +- bin/node/Cargo.toml | 2 +- .../src/block_builder/prover/block_witness.rs | 2 +- .../src/block_builder/prover/tests.rs | 52 ++-- .../block-producer/src/test_utils/account.rs | 3 +- crates/store/src/db/tests.rs | 10 +- 8 files changed, 165 insertions(+), 138 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 816b54c29..3856d74c2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -94,11 +94,12 @@ dependencies = [ [[package]] name = "anstyle-wincon" -version = "3.0.6" +version = "3.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2109dbce0e72be3ec00bed26e6a7479ca384ad226efdd66db8fa2e3a38c83125" +checksum = "ca3534e77181a9cc07539ad51f2141fe32f6c3ffd4df76db8ad92346b003ae4e" dependencies = [ "anstyle", + "once_cell", "windows-sys 0.59.0", ] @@ -159,9 +160,9 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.83" +version = "0.1.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" +checksum = "3f934833b4b7233644e5848f235df3f57ed8c80f1528a26c3dfa13d2147fa056" dependencies = [ "proc-macro2", "quote", @@ -332,9 +333,9 @@ checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" [[package]] name = "bitflags" -version = "2.6.0" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" +checksum = "8f68f53c83ab957f72c32642f3868eec03eb974d1fb82e453128456482613d36" [[package]] name = "blake3" @@ -411,14 +412,14 @@ dependencies = [ "semver 1.0.24", "serde", "serde_json", - "thiserror 2.0.9", + "thiserror 2.0.11", ] [[package]] name = "cc" -version = "1.2.5" +version = "1.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c31a0499c1dc64f458ad13872de75c0eb7e3fdb0e67964610c914b034fc5956e" +checksum = "c8293772165d9345bdaaa39b45b2109591e63fe5e6fbc23c6ff930a048aa310b" dependencies = [ "jobserver", "libc", @@ -477,9 +478,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.23" +version = "4.5.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3135e7ec2ef7b10c6ed8950f0f792ed96ee093fa088608f1c76e569722700c84" +checksum = "a8eb5e908ef3a6efbe1ed62520fb7287959888c88485abe072543190ecc66783" dependencies = [ "clap_builder", "clap_derive", @@ -487,9 +488,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.23" +version = "4.5.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30582fc632330df2bd26877bde0c1f4470d57c582bbc070376afcd04d8cb4838" +checksum = "96b01801b5fc6a0a232407abc821660c9c6d25a1cafc0d4f85f29fb8d9afc121" dependencies = [ "anstream", "anstyle", @@ -499,9 +500,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.18" +version = "4.5.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ac6a0c7b1a9e9a5186361f67dfa1b88213572f427fb9ab038efb2bd8c582dab" +checksum = "54b755194d6389280185988721fffba69495eed5ee9feeee9a599b53db80318c" dependencies = [ "heck", "proc-macro2", @@ -932,9 +933,9 @@ checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" [[package]] name = "glob" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" +checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" [[package]] name = "h2" @@ -1219,9 +1220,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.76" +version = "0.3.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6717b6b5b077764fb5966237269cb3c64edddde4b14ce42647430a78ced9e7b7" +checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" dependencies = [ "once_cell", "wasm-bindgen", @@ -1321,9 +1322,9 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.4.14" +version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" [[package]] name = "lock_api" @@ -1337,9 +1338,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.22" +version = "0.4.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" +checksum = "04cbf5b083de1c7e0222a7a51dbfdba1cbe1c6ab0b15e29fff3f6c077fd9cd9f" [[package]] name = "logos" @@ -1497,7 +1498,7 @@ dependencies = [ "rand_chacha", "serde", "static-files", - "thiserror 2.0.9", + "thiserror 2.0.11", "tokio", "toml", "tonic", @@ -1518,13 +1519,13 @@ dependencies = [ [[package]] name = "miden-lib" version = "0.7.0" -source = "git+https://github.com/0xPolygonMiden/miden-base.git?branch=next#64d7a2e5fa3fb1c65269369bf0b152057e45befd" +source = "git+https://github.com/0xPolygonMiden/miden-base.git?branch=next#d09a78e8674f983f30e38b5e4a2d21bed592bc95" dependencies = [ "miden-assembly", "miden-objects", "miden-stdlib", "regex", - "thiserror 2.0.9", + "thiserror 2.0.11", "walkdir", ] @@ -1612,7 +1613,7 @@ dependencies = [ "rand", "rand_chacha", "serde", - "thiserror 2.0.9", + "thiserror 2.0.11", "tokio", "tokio-stream", "tonic", @@ -1632,7 +1633,7 @@ dependencies = [ "prost", "prost-build", "protox", - "thiserror 2.0.9", + "thiserror 2.0.11", "tonic", "tonic-build", ] @@ -1667,7 +1668,7 @@ dependencies = [ "rusqlite", "rusqlite_migration", "serde", - "thiserror 2.0.9", + "thiserror 2.0.11", "tokio", "tokio-stream", "tonic", @@ -1692,7 +1693,7 @@ dependencies = [ "miden-objects", "rand", "serde", - "thiserror 2.0.9", + "thiserror 2.0.11", "tonic", "tracing", "tracing-forest", @@ -1704,7 +1705,7 @@ dependencies = [ [[package]] name = "miden-objects" version = "0.7.0" -source = "git+https://github.com/0xPolygonMiden/miden-base.git?branch=next#64d7a2e5fa3fb1c65269369bf0b152057e45befd" +source = "git+https://github.com/0xPolygonMiden/miden-base.git?branch=next#d09a78e8674f983f30e38b5e4a2d21bed592bc95" dependencies = [ "getrandom", "miden-assembly", @@ -1713,7 +1714,11 @@ dependencies = [ "miden-processor", "miden-verifier", "rand", - "thiserror 2.0.9", + "rand_xoshiro", + "semver 1.0.24", + "serde", + "thiserror 2.0.11", + "toml", "winter-rand-utils", ] @@ -1778,7 +1783,7 @@ dependencies = [ [[package]] name = "miden-tx" version = "0.7.0" -source = "git+https://github.com/0xPolygonMiden/miden-base.git?branch=next#64d7a2e5fa3fb1c65269369bf0b152057e45befd" +source = "git+https://github.com/0xPolygonMiden/miden-base.git?branch=next#d09a78e8674f983f30e38b5e4a2d21bed592bc95" dependencies = [ "async-trait", "miden-lib", @@ -1788,7 +1793,7 @@ dependencies = [ "miden-verifier", "rand", "rand_chacha", - "thiserror 2.0.9", + "thiserror 2.0.11", "winter-maybe-async", ] @@ -1851,9 +1856,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ffbe83022cedc1d264172192511ae958937694cd57ce297164951b8b3568394" +checksum = "b8402cab7aefae129c6977bb0ff1b8fd9a04eb5b51efc50a70bea51cda0c7924" dependencies = [ "adler2", ] @@ -2126,18 +2131,18 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.7" +version = "1.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be57f64e946e500c8ee36ef6331845d40a93055567ec57e8fae13efd33759b95" +checksum = "1e2ec53ad785f4d35dac0adea7f7dc6f1bb277ad84a680c7afefeae05d1f5916" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.7" +version = "1.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c0f5fad0874fc7abcd4d750e76917eaebbecaa2c20bde22e1dbeeba8beb758c" +checksum = "d56a66c0c55993aa927429d0f8a0abfd74f084e4d9c192cffed01e418d83eefb" dependencies = [ "proc-macro2", "quote", @@ -2146,9 +2151,9 @@ dependencies = [ [[package]] name = "pin-project-lite" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "915a1e146535de9163f3987b8944ed8cf49a18bb0056bcebcdcece385cece4ff" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" [[package]] name = "pin-utils" @@ -2195,9 +2200,9 @@ dependencies = [ [[package]] name = "prettyplease" -version = "0.2.25" +version = "0.2.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64d1ec885c64d0457d564db4ec299b2dae3f9c02808b8ad9c3a089c591b18033" +checksum = "6924ced06e1f7dfe3fa48d57b9f74f55d8915f5036121bef647ef4b204895fac" dependencies = [ "proc-macro2", "syn", @@ -2205,9 +2210,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.92" +version = "1.0.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0" +checksum = "60946a68e5f9d28b0dc1c21bb8a97ee7d018a8b322fa57838ba31cc878e22d99" dependencies = [ "unicode-ident", ] @@ -2290,9 +2295,9 @@ dependencies = [ [[package]] name = "prost-reflect" -version = "0.14.3" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20ae544fca2892fd4b7e9ff26cba1090cedf1d4d95c2aded1af15d2f93f270b8" +checksum = "bc9647f03b808b79abca8408b1609be9887ba90453c940d00332a60eeb6f5748" dependencies = [ "logos", "miette", @@ -2312,9 +2317,9 @@ dependencies = [ [[package]] name = "protox" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "873f359bdecdfe6e353752f97cb9ee69368df55b16363ed2216da85e03232a58" +checksum = "6f352af331bf637b8ecc720f7c87bf903d2571fa2e14a66e9b2558846864b54a" dependencies = [ "bytes", "miette", @@ -2345,9 +2350,9 @@ checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" [[package]] name = "quote" -version = "1.0.37" +version = "1.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" +checksum = "0e4dccaaaf89514f546c693ddc140f729f958c247918a13380cccc6078391acc" dependencies = [ "proc-macro2", ] @@ -2391,6 +2396,15 @@ dependencies = [ "rand_core", ] +[[package]] +name = "rand_xoshiro" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f97cdb2a36ed4183de61b2f824cc45c9f1037f28afe0a322e9fff4c108b5aaa" +dependencies = [ + "rand_core", +] + [[package]] name = "rayon" version = "1.10.0" @@ -2531,9 +2545,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.42" +version = "0.38.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f93dc38ecbab2eb790ff964bb77fa94faf256fd3e73285fd7ba0903b76bedb85" +checksum = "a78891ee6bf2340288408954ac787aa063d8e8817e9f53abb37c695c6d834ef6" dependencies = [ "bitflags", "errno", @@ -2544,9 +2558,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.18" +version = "1.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e819f2bc632f285be6d7cd36e25940d45b2391dd6d9b939e79de557f7014248" +checksum = "f7c45b9784283f1b2e7fb61b42047c2fd678ef0960d4f6f1eba131594cc369d4" [[package]] name = "rusty-fork" @@ -2613,18 +2627,18 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" [[package]] name = "serde" -version = "1.0.216" +version = "1.0.217" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b9781016e935a97e8beecf0c933758c97a5520d32930e460142b4cd80c6338e" +checksum = "02fc4265df13d6fa1d00ecff087228cc0a2b5f3c0e87e258d8b94a156e984c70" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.216" +version = "1.0.217" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46f859dbbf73865c6627ed570e78961cd3ac92407a2d117204c49232485da55e" +checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0" dependencies = [ "proc-macro2", "quote", @@ -2633,9 +2647,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.134" +version = "1.0.135" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d00f4175c42ee48b15416f6193a959ba3a0d67fc699a0db9ad12df9f83991c7d" +checksum = "2b0d7ba2887406110130a978386c4e1befb98c674b4fba677954e4db976630d9" dependencies = [ "itoa", "memchr", @@ -2768,9 +2782,9 @@ dependencies = [ [[package]] name = "strip-ansi-escapes" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55ff8ef943b384c414f54aefa961dd2bd853add74ec75e7ac74cf91dba62bcfa" +checksum = "2a8f8038e7e7969abb3f1b7c2a811225e9296da208539e0f79c5251d6cac0025" dependencies = [ "vte", ] @@ -2804,9 +2818,9 @@ checksum = "b7401a30af6cb5818bb64852270bb722533397edcfc7344954a38f420819ece2" [[package]] name = "syn" -version = "2.0.91" +version = "2.0.96" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d53cbcb5a243bd33b7858b1d7f4aca2153490815872d86d955d6ea29f743c035" +checksum = "d5d0adab1ae378d7f53bdebc67a39f1f151407ef230f0ce2883572f5d8985c80" dependencies = [ "proc-macro2", "quote", @@ -2827,12 +2841,13 @@ checksum = "42a4d50cdb458045afc8131fd91b64904da29548bcb63c7236e0844936c13078" [[package]] name = "tempfile" -version = "3.14.0" +version = "3.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28cce251fcbc87fac86a866eeb0d6c2d536fc16d06f184bb61aeae11aa4cee0c" +checksum = "9a8a559c81686f576e8cd0290cd2a24a2a9ad80c98b3478856500fcbd7acd704" dependencies = [ "cfg-if", "fastrand", + "getrandom", "once_cell", "rustix", "windows-sys 0.59.0", @@ -2890,11 +2905,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.9" +version = "2.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f072643fd0190df67a8bab670c20ef5d8737177d6ac6b2e9a236cb096206b2cc" +checksum = "d452f284b73e6d76dd36758a0c8684b1d5be31f92b89d07fd5822175732206fc" dependencies = [ - "thiserror-impl 2.0.9", + "thiserror-impl 2.0.11", ] [[package]] @@ -2910,9 +2925,9 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.9" +version = "2.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b50fa271071aae2e6ee85f842e2e28ba8cd2c5fb67f11fcb1fd70b276f9e7d4" +checksum = "26afc1baea8a989337eeb52b6e72a039780ce45c3edfcc9c5b9d112feeb173c2" dependencies = [ "proc-macro2", "quote", @@ -2973,9 +2988,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.42.0" +version = "1.43.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cec9b21b0450273377fc97bd4c33a8acffc8c996c987a7c5b319a0083707551" +checksum = "3d61fa4ffa3de412bfea335c6ecff681de2b609ba3c77ef3e00e521813a9ed9e" dependencies = [ "backtrace", "bytes", @@ -2989,9 +3004,9 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.4.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" +checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" dependencies = [ "proc-macro2", "quote", @@ -3387,9 +3402,9 @@ checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" [[package]] name = "vergen" -version = "9.0.2" +version = "9.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31f25fc8f8f05df455c7941e87f093ad22522a9ff33d7a027774815acf6f0639" +checksum = "e0d2f179f8075b805a43a2a21728a46f0cc2921b3c58695b28fa8817e103cd9a" dependencies = [ "anyhow", "cargo_metadata", @@ -3402,9 +3417,9 @@ dependencies = [ [[package]] name = "vergen-gitcl" -version = "1.0.2" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0227006d09f98ab00ea69e9a5e055e676a813cfbed4232986176c86a6080b997" +checksum = "b2f89d70a58a4506a6079cedf575c64cf51649ccbb4e02a63dac539b264b7711" dependencies = [ "anyhow", "derive_builder", @@ -3416,9 +3431,9 @@ dependencies = [ [[package]] name = "vergen-lib" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0c767e6751c09fc85cde58722cf2f1007e80e4c8d5a4321fc90d83dc54ca147" +checksum = "9b07e6010c0f3e59fcb164e0163834597da68d1f864e2b8ca49f74de01e9c166" dependencies = [ "anyhow", "derive_builder", @@ -3433,22 +3448,11 @@ checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" [[package]] name = "vte" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5022b5fbf9407086c180e9557be968742d839e68346af7792b8592489732197" -dependencies = [ - "utf8parse", - "vte_generate_state_changes", -] - -[[package]] -name = "vte_generate_state_changes" -version = "0.1.2" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e369bee1b05d510a7b4ed645f5faa90619e05437111783ea5848f28d97d3c2e" +checksum = "231fdcd7ef3037e8330d8e17e61011a2c244126acc0a982f4040ac3f9f0bc077" dependencies = [ - "proc-macro2", - "quote", + "memchr", ] [[package]] @@ -3487,20 +3491,21 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.99" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a474f6281d1d70c17ae7aa6a613c87fce69a127e2624002df63dcb39d6cf6396" +checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" dependencies = [ "cfg-if", "once_cell", + "rustversion", "wasm-bindgen-macro", ] [[package]] name = "wasm-bindgen-backend" -version = "0.2.99" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f89bb38646b4f81674e8f5c3fb81b562be1fd936d84320f3264486418519c79" +checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" dependencies = [ "bumpalo", "log", @@ -3512,9 +3517,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.99" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cc6181fd9a7492eef6fef1f33961e3695e4579b9872a6f7c83aee556666d4fe" +checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3522,9 +3527,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.99" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30d7a95b763d3c45903ed6c81f156801839e5ee968bb07e534c44df0fcd330c2" +checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" dependencies = [ "proc-macro2", "quote", @@ -3535,9 +3540,12 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.99" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "943aab3fdaaa029a6e0271b35ea10b72b943135afe9bffca82384098ad0e06a6" +checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" +dependencies = [ + "unicode-ident", +] [[package]] name = "winapi" @@ -3793,9 +3801,9 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winnow" -version = "0.6.20" +version = "0.6.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36c1fec1a2bb5866f07c25f68c26e565c4c200aebb96d7e55710c19d3e8ac49b" +checksum = "c8d71a593cc5c42ad7876e2c1fda56f314f3754c084128833e64f1345ff8a03a" dependencies = [ "memchr", ] diff --git a/bin/faucet/Cargo.toml b/bin/faucet/Cargo.toml index cc24efb54..5d9d8a148 100644 --- a/bin/faucet/Cargo.toml +++ b/bin/faucet/Cargo.toml @@ -18,10 +18,10 @@ clap = { version = "4.5", features = ["derive", "string"] } figment = { version = "0.10", features = ["toml", "env"] } http = "1.1" http-body-util = "0.1" -miden-lib = { workspace = true, features = ["concurrent"] } +miden-lib = { workspace = true } miden-node-proto = { workspace = true } miden-node-utils = { workspace = true } -miden-objects = { workspace = true , features = ["concurrent"] } +miden-objects = { workspace = true } miden-tx = { workspace = true, features = ["concurrent"] } mime = "0.3" rand = { workspace = true } diff --git a/bin/faucet/src/errors.rs b/bin/faucet/src/errors.rs index 1ff978631..476f066aa 100644 --- a/bin/faucet/src/errors.rs +++ b/bin/faucet/src/errors.rs @@ -4,7 +4,7 @@ use axum::{ http::{header, StatusCode}, response::{IntoResponse, Response}, }; -use miden_objects::AccountError; +use miden_objects::AccountIdError; use thiserror::Error; #[derive(Debug, Error)] @@ -25,7 +25,7 @@ pub enum HandlerError { Internal(#[from] anyhow::Error), #[error("account ID deserialization failed")] - AccountIdDeserializationError(#[source] AccountError), + AccountIdDeserializationError(#[source] AccountIdError), #[error("invalid asset amount {requested} requested, valid options are {options:?}")] InvalidAssetAmount { requested: u64, options: Vec }, diff --git a/bin/node/Cargo.toml b/bin/node/Cargo.toml index 7e441f9c0..9e25829b7 100644 --- a/bin/node/Cargo.toml +++ b/bin/node/Cargo.toml @@ -17,7 +17,7 @@ tracing-forest = ["miden-node-block-producer/tracing-forest"] [dependencies] anyhow = { version = "1.0" } clap = { version = "4.5", features = ["derive", "string"] } -miden-lib = { workspace = true, features = ["concurrent"] } +miden-lib = { workspace = true } miden-node-block-producer = { workspace = true } miden-node-rpc = { workspace = true } miden-node-store = { workspace = true } diff --git a/crates/block-producer/src/block_builder/prover/block_witness.rs b/crates/block-producer/src/block_builder/prover/block_witness.rs index cc9da466f..57205244c 100644 --- a/crates/block-producer/src/block_builder/prover/block_witness.rs +++ b/crates/block-producer/src/block_builder/prover/block_witness.rs @@ -188,7 +188,7 @@ impl BlockWitness { for (idx, (account_id, account_update)) in self.updated_accounts.iter().enumerate() { account_data.extend(account_update.final_state_hash); - account_data.push(account_id.first_felt()); + account_data.push(account_id.prefix().as_felt()); let idx = u64::try_from(idx).expect("can't be more than 2^64 - 1 accounts"); num_accounts_updated = idx + 1; diff --git a/crates/block-producer/src/block_builder/prover/tests.rs b/crates/block-producer/src/block_builder/prover/tests.rs index ca3047c13..927493975 100644 --- a/crates/block-producer/src/block_builder/prover/tests.rs +++ b/crates/block-producer/src/block_builder/prover/tests.rs @@ -2,7 +2,9 @@ use std::{collections::BTreeMap, iter}; use assert_matches::assert_matches; use miden_objects::{ - accounts::{delta::AccountUpdateDetails, AccountId, AccountStorageMode, AccountType}, + accounts::{ + delta::AccountUpdateDetails, AccountId, AccountIdVersion, AccountStorageMode, AccountType, + }, block::{BlockAccountUpdate, BlockNoteIndex, BlockNoteTree}, crypto::merkle::{ EmptySubtreeRoots, LeafIndex, MerklePath, Mmr, MmrPeaks, Smt, SmtLeaf, SmtProof, SMT_DEPTH, @@ -35,18 +37,21 @@ use crate::{ /// The store will contain accounts 1 & 2, while the transaction batches will contain 2 & 3. #[test] fn block_witness_validation_inconsistent_account_ids() { - let account_id_1 = AccountId::new_dummy( + let account_id_1 = AccountId::dummy( [0; 15], + AccountIdVersion::Version0, AccountType::RegularAccountImmutableCode, miden_objects::accounts::AccountStorageMode::Private, ); - let account_id_2 = AccountId::new_dummy( + let account_id_2 = AccountId::dummy( [1; 15], + AccountIdVersion::Version0, AccountType::RegularAccountImmutableCode, miden_objects::accounts::AccountStorageMode::Private, ); - let account_id_3 = AccountId::new_dummy( + let account_id_3 = AccountId::dummy( [2; 15], + AccountIdVersion::Version0, AccountType::RegularAccountImmutableCode, miden_objects::accounts::AccountStorageMode::Private, ); @@ -282,28 +287,33 @@ async fn compute_account_root_success() { // Set up account states // --------------------------------------------------------------------------------------------- let account_ids = [ - AccountId::new_dummy( + AccountId::dummy( [0; 15], + AccountIdVersion::Version0, AccountType::RegularAccountImmutableCode, miden_objects::accounts::AccountStorageMode::Private, ), - AccountId::new_dummy( + AccountId::dummy( [1; 15], + AccountIdVersion::Version0, AccountType::RegularAccountImmutableCode, miden_objects::accounts::AccountStorageMode::Private, ), - AccountId::new_dummy( + AccountId::dummy( [2; 15], + AccountIdVersion::Version0, AccountType::RegularAccountImmutableCode, miden_objects::accounts::AccountStorageMode::Private, ), - AccountId::new_dummy( + AccountId::dummy( [3; 15], + AccountIdVersion::Version0, AccountType::RegularAccountImmutableCode, miden_objects::accounts::AccountStorageMode::Private, ), - AccountId::new_dummy( + AccountId::dummy( [4; 15], + AccountIdVersion::Version0, AccountType::RegularAccountImmutableCode, miden_objects::accounts::AccountStorageMode::Private, ), @@ -402,28 +412,33 @@ async fn compute_account_root_empty_batches() { // Set up account states // --------------------------------------------------------------------------------------------- let account_ids = [ - AccountId::new_dummy( + AccountId::dummy( [0; 15], + AccountIdVersion::Version0, AccountType::RegularAccountImmutableCode, AccountStorageMode::Private, ), - AccountId::new_dummy( + AccountId::dummy( [1; 15], + AccountIdVersion::Version0, AccountType::RegularAccountImmutableCode, AccountStorageMode::Private, ), - AccountId::new_dummy( + AccountId::dummy( [2; 15], + AccountIdVersion::Version0, AccountType::RegularAccountImmutableCode, AccountStorageMode::Private, ), - AccountId::new_dummy( + AccountId::dummy( [3; 15], + AccountIdVersion::Version0, AccountType::RegularAccountImmutableCode, AccountStorageMode::Private, ), - AccountId::new_dummy( + AccountId::dummy( [4; 15], + AccountIdVersion::Version0, AccountType::RegularAccountImmutableCode, AccountStorageMode::Private, ), @@ -544,18 +559,21 @@ async fn compute_note_root_empty_notes_success() { #[miden_node_test_macro::enable_logging] async fn compute_note_root_success() { let account_ids = [ - AccountId::new_dummy( + AccountId::dummy( [0; 15], + AccountIdVersion::Version0, AccountType::RegularAccountImmutableCode, miden_objects::accounts::AccountStorageMode::Private, ), - AccountId::new_dummy( + AccountId::dummy( [1; 15], + AccountIdVersion::Version0, AccountType::RegularAccountImmutableCode, miden_objects::accounts::AccountStorageMode::Private, ), - AccountId::new_dummy( + AccountId::dummy( [2; 15], + AccountIdVersion::Version0, AccountType::RegularAccountImmutableCode, miden_objects::accounts::AccountStorageMode::Private, ), diff --git a/crates/block-producer/src/test_utils/account.rs b/crates/block-producer/src/test_utils/account.rs index 4c0930fa3..935ba11ba 100644 --- a/crates/block-producer/src/test_utils/account.rs +++ b/crates/block-producer/src/test_utils/account.rs @@ -38,7 +38,7 @@ impl MockPrivateAccount { init_seed, AccountType::RegularAccountUpdatableCode, AccountStorageMode::Private, - AccountIdVersion::VERSION_0, + AccountIdVersion::Version0, Digest::default(), Digest::default(), Digest::default(), @@ -49,6 +49,7 @@ impl MockPrivateAccount { AccountId::new( account_seed, AccountIdAnchor::PRE_GENESIS, + AccountIdVersion::Version0, Digest::default(), Digest::default(), ) diff --git a/crates/store/src/db/tests.rs b/crates/store/src/db/tests.rs index 614a91376..b931e850e 100644 --- a/crates/store/src/db/tests.rs +++ b/crates/store/src/db/tests.rs @@ -3,8 +3,8 @@ use miden_node_proto::domain::accounts::AccountSummary; use miden_objects::{ accounts::{ delta::AccountUpdateDetails, Account, AccountBuilder, AccountComponent, AccountDelta, - AccountId, AccountStorageDelta, AccountStorageMode, AccountType, AccountVaultDelta, - StorageSlot, + AccountId, AccountIdVersion, AccountStorageDelta, AccountStorageMode, AccountType, + AccountVaultDelta, StorageSlot, }, assets::{Asset, FungibleAsset, NonFungibleAsset, NonFungibleAssetDetails}, block::{BlockAccountUpdate, BlockNoteIndex, BlockNoteTree}, @@ -296,8 +296,9 @@ fn sql_select_accounts() { // test multiple entries let mut state = vec![]; for i in 0..10u8 { - let account_id = AccountId::new_dummy( + let account_id = AccountId::dummy( [i; 15], + AccountIdVersion::Version0, AccountType::RegularAccountImmutableCode, miden_objects::accounts::AccountStorageMode::Private, ); @@ -1021,8 +1022,7 @@ fn mock_account_code_and_storage( .unwrap() .with_supported_type(account_type); - AccountBuilder::new() - .init_seed([0; 32]) + AccountBuilder::new([0; 32]) .account_type(account_type) .storage_mode(storage_mode) .with_assets(assets) From c5da1e68a0db4b89afbd69aad7f5ca64e3d42d18 Mon Sep 17 00:00:00 2001 From: Mirko <48352201+Mirko-von-Leipzig@users.noreply.github.com> Date: Thu, 16 Jan 2025 21:31:42 +0200 Subject: [PATCH 43/50] ci: check for unused dependencies (#625) --- .github/workflows/lint.yml | 8 ++++++++ Cargo.lock | 2 -- bin/faucet/Cargo.toml | 1 - bin/node/Cargo.toml | 1 - 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 3b97b0d25..5c898fde8 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -68,6 +68,14 @@ jobs: - name: check rust versions run: ./scripts/check-rust-version.sh + unused_deps: + name: check for unused dependencies + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@main + - name: machete + uses: bnjbvr/cargo-machete@main + proto: name: proto check runs-on: ubuntu-latest diff --git a/Cargo.lock b/Cargo.lock index 3856d74c2..ebdb1852a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1485,7 +1485,6 @@ dependencies = [ "anyhow", "axum", "clap", - "figment", "http", "http-body-util", "miden-lib", @@ -1590,7 +1589,6 @@ dependencies = [ "tokio", "toml", "tracing", - "tracing-subscriber", ] [[package]] diff --git a/bin/faucet/Cargo.toml b/bin/faucet/Cargo.toml index 5d9d8a148..095613fab 100644 --- a/bin/faucet/Cargo.toml +++ b/bin/faucet/Cargo.toml @@ -15,7 +15,6 @@ repository.workspace = true anyhow = "1.0" axum = { version = "0.7", features = ["tokio"] } clap = { version = "4.5", features = ["derive", "string"] } -figment = { version = "0.10", features = ["toml", "env"] } http = "1.1" http-body-util = "0.1" miden-lib = { workspace = true } diff --git a/bin/node/Cargo.toml b/bin/node/Cargo.toml index 9e25829b7..eb4c1caa3 100644 --- a/bin/node/Cargo.toml +++ b/bin/node/Cargo.toml @@ -29,7 +29,6 @@ serde = { version = "1.0", features = ["derive"] } tokio = { workspace = true, features = ["rt-multi-thread", "net", "macros"] } toml = { version = "0.8" } tracing = { workspace = true } -tracing-subscriber = { workspace = true } [dev-dependencies] figment = { version = "0.10", features = ["toml", "env", "test"] } From 4a90936d7367a254e68c3e2f7abc19a05dbf5e9a Mon Sep 17 00:00:00 2001 From: polydez <155382956+polydez@users.noreply.github.com> Date: Fri, 17 Jan 2025 04:34:35 +0500 Subject: [PATCH 44/50] docs: Improved RPC documentation (#620) --- CHANGELOG.md | 1 + crates/block-producer/README.md | 2 +- crates/proto/src/generated/account.rs | 17 +++- crates/proto/src/generated/block.rs | 26 +++--- crates/proto/src/generated/block_producer.rs | 2 + crates/proto/src/generated/merkle.rs | 2 + crates/proto/src/generated/mmr.rs | 3 + crates/proto/src/generated/note.rs | 32 ++++++- crates/proto/src/generated/requests.rs | 34 +++++-- crates/proto/src/generated/responses.rs | 88 ++++++++++++------ crates/proto/src/generated/rpc.rs | 60 ++++++++++++ crates/proto/src/generated/smt.rs | 11 ++- crates/proto/src/generated/store.rs | 66 +++++++++++++ crates/proto/src/generated/transaction.rs | 6 ++ crates/rpc-proto/proto/account.proto | 23 ++++- crates/rpc-proto/proto/block.proto | 38 +++++--- crates/rpc-proto/proto/block_producer.proto | 1 + crates/rpc-proto/proto/merkle.proto | 2 + crates/rpc-proto/proto/mmr.proto | 4 + crates/rpc-proto/proto/note.proto | 48 +++++++++- crates/rpc-proto/proto/requests.proto | 39 +++++--- crates/rpc-proto/proto/responses.proto | 97 ++++++++++++++------ crates/rpc-proto/proto/rpc.proto | 40 ++++++++ crates/rpc-proto/proto/smt.proto | 17 +++- crates/rpc-proto/proto/store.proto | 46 ++++++++++ crates/rpc-proto/proto/transaction.proto | 8 ++ crates/rpc/README.md | 4 +- crates/store/README.md | 8 +- crates/store/src/server/api.rs | 2 +- proto/account.proto | 23 ++++- proto/block.proto | 38 +++++--- proto/block_producer.proto | 1 + proto/merkle.proto | 2 + proto/mmr.proto | 4 + proto/note.proto | 48 +++++++++- proto/requests.proto | 39 +++++--- proto/responses.proto | 97 ++++++++++++++------ proto/rpc.proto | 40 ++++++++ proto/smt.proto | 17 +++- proto/store.proto | 46 ++++++++++ proto/transaction.proto | 8 ++ 41 files changed, 918 insertions(+), 172 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fb2751afd..87319b3a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ - Support Https in endpoint configuration (#556). - Upgrade `block-producer` from FIFO queue to mempool dependency graph (#562). - Support transaction expiration (#582). +- Improved RPC endpoints doc comments (#620). ### Changes diff --git a/crates/block-producer/README.md b/crates/block-producer/README.md index 3de8f2085..cedb36b96 100644 --- a/crates/block-producer/README.md +++ b/crates/block-producer/README.md @@ -25,7 +25,7 @@ Submits proven transaction to the Miden network. **Parameters** -* `transaction`: `bytes` - transaction encoded using Miden's native format. +* `transaction`: `bytes` - transaction encoded using [winter_utils::Serializable](https://github.com/facebook/winterfell/blob/main/utils/core/src/serde/mod.rs#L26) implementation for [miden_objects::transaction::proven_tx::ProvenTransaction](https://github.com/0xPolygonMiden/miden-base/blob/main/objects/src/transaction/proven_tx.rs#L22). **Returns** diff --git a/crates/proto/src/generated/account.rs b/crates/proto/src/generated/account.rs index 85760632c..a2683eb0d 100644 --- a/crates/proto/src/generated/account.rs +++ b/crates/proto/src/generated/account.rs @@ -1,28 +1,41 @@ // This file is @generated by prost-build. +/// Uniquely identifies a specific account. +/// +/// A Miden account ID is a 120-bit value derived from the commitments to account code and storage, +/// and a random user-provided seed. #[derive(Clone, PartialEq, ::prost::Message)] #[prost(skip_debug)] pub struct AccountId { - /// A Miden account ID is a 120-bit value derived from the commitments to account code and - /// storage, and a random user-provided seed. + /// 15 bytes (120 bits) encoded using \[winter_utils::Serializable\] implementation for + /// \[miden_objects::accounts::account_id::AccountId\]. #[prost(bytes = "vec", tag = "1")] pub id: ::prost::alloc::vec::Vec, } +/// The state of an account at a specific block height. #[derive(Clone, PartialEq, ::prost::Message)] pub struct AccountSummary { + /// The account ID. #[prost(message, optional, tag = "1")] pub account_id: ::core::option::Option, + /// The current account hash or zero if the account does not exist. #[prost(message, optional, tag = "2")] pub account_hash: ::core::option::Option, + /// Block number at which the summary was made. #[prost(uint32, tag = "3")] pub block_num: u32, } +/// An account info. #[derive(Clone, PartialEq, ::prost::Message)] pub struct AccountInfo { + /// Account summary. #[prost(message, optional, tag = "1")] pub summary: ::core::option::Option, + /// Account details encoded using \[winter_utils::Serializable\] implementation for + /// \[miden_objects::accounts::Account\]. #[prost(bytes = "vec", optional, tag = "2")] pub details: ::core::option::Option<::prost::alloc::vec::Vec>, } +/// An account header. #[derive(Clone, Copy, PartialEq, ::prost::Message)] pub struct AccountHeader { /// Vault root hash. diff --git a/crates/proto/src/generated/block.rs b/crates/proto/src/generated/block.rs index 2c0898b1f..915dce7ec 100644 --- a/crates/proto/src/generated/block.rs +++ b/crates/proto/src/generated/block.rs @@ -1,44 +1,48 @@ // This file is @generated by prost-build. +/// Represents a block header. #[derive(Clone, Copy, PartialEq, ::prost::Message)] pub struct BlockHeader { - /// specifies the version of the protocol. + /// Specifies the version of the protocol. #[prost(uint32, tag = "1")] pub version: u32, - /// the hash of the previous blocks header. + /// The hash of the previous blocks header. #[prost(message, optional, tag = "2")] pub prev_hash: ::core::option::Option, - /// a unique sequential number of the current block. + /// A unique sequential number of the current block. #[prost(fixed32, tag = "3")] pub block_num: u32, - /// a commitment to an MMR of the entire chain where each block is a leaf. + /// A commitment to an MMR of the entire chain where each block is a leaf. #[prost(message, optional, tag = "4")] pub chain_root: ::core::option::Option, - /// a commitment to account database. + /// A commitment to account database. #[prost(message, optional, tag = "5")] pub account_root: ::core::option::Option, - /// a commitment to the nullifier database. + /// A commitment to the nullifier database. #[prost(message, optional, tag = "6")] pub nullifier_root: ::core::option::Option, - /// a commitment to all notes created in the current block. + /// A commitment to all notes created in the current block. #[prost(message, optional, tag = "7")] pub note_root: ::core::option::Option, - /// a commitment to a set of IDs of transactions which affected accounts in this block. + /// A commitment to a set of IDs of transactions which affected accounts in this block. #[prost(message, optional, tag = "8")] pub tx_hash: ::core::option::Option, - /// a hash of a STARK proof attesting to the correct state transition. + /// A hash of a STARK proof attesting to the correct state transition. #[prost(message, optional, tag = "9")] pub proof_hash: ::core::option::Option, - /// a commitment to all transaction kernels supported by this block. + /// A commitment to all transaction kernels supported by this block. #[prost(message, optional, tag = "10")] pub kernel_root: ::core::option::Option, - /// the time when the block was created. + /// The time when the block was created. #[prost(fixed32, tag = "11")] pub timestamp: u32, } +/// Represents a block inclusion proof. #[derive(Clone, PartialEq, ::prost::Message)] pub struct BlockInclusionProof { + /// Block header associated with the inclusion proof. #[prost(message, optional, tag = "1")] pub block_header: ::core::option::Option, + /// Merkle path associated with the inclusion proof. #[prost(message, optional, tag = "2")] pub mmr_path: ::core::option::Option, /// The chain length associated with `mmr_path`. diff --git a/crates/proto/src/generated/block_producer.rs b/crates/proto/src/generated/block_producer.rs index fefbd4351..c4e6ce35d 100644 --- a/crates/proto/src/generated/block_producer.rs +++ b/crates/proto/src/generated/block_producer.rs @@ -90,6 +90,7 @@ pub mod api_client { self.inner = self.inner.max_encoding_message_size(limit); self } + /// Submits proven transaction to the Miden network pub async fn submit_proven_transaction( &mut self, request: impl tonic::IntoRequest< @@ -133,6 +134,7 @@ pub mod api_server { /// Generated trait containing gRPC methods that should be implemented for use with ApiServer. #[async_trait] pub trait Api: std::marker::Send + std::marker::Sync + 'static { + /// Submits proven transaction to the Miden network async fn submit_proven_transaction( &self, request: tonic::Request< diff --git a/crates/proto/src/generated/merkle.rs b/crates/proto/src/generated/merkle.rs index 0aed4fbdc..e5c4a657a 100644 --- a/crates/proto/src/generated/merkle.rs +++ b/crates/proto/src/generated/merkle.rs @@ -1,6 +1,8 @@ // This file is @generated by prost-build. +/// Represents a Merkle path. #[derive(Clone, PartialEq, ::prost::Message)] pub struct MerklePath { + /// List of sibling node hashes, in order from the root to the leaf. #[prost(message, repeated, tag = "1")] pub siblings: ::prost::alloc::vec::Vec, } diff --git a/crates/proto/src/generated/mmr.rs b/crates/proto/src/generated/mmr.rs index f53ebe2d3..ca5591f71 100644 --- a/crates/proto/src/generated/mmr.rs +++ b/crates/proto/src/generated/mmr.rs @@ -1,8 +1,11 @@ // This file is @generated by prost-build. +/// Represents an MMR delta. #[derive(Clone, PartialEq, ::prost::Message)] pub struct MmrDelta { + /// The number of leaf nodes in the MMR. #[prost(uint64, tag = "1")] pub forest: u64, + /// New and changed MMR peaks. #[prost(message, repeated, tag = "2")] pub data: ::prost::alloc::vec::Vec, } diff --git a/crates/proto/src/generated/note.rs b/crates/proto/src/generated/note.rs index 3768961d0..b64565915 100644 --- a/crates/proto/src/generated/note.rs +++ b/crates/proto/src/generated/note.rs @@ -1,56 +1,84 @@ // This file is @generated by prost-build. +/// Represents a note's metadata. #[derive(Clone, PartialEq, ::prost::Message)] pub struct NoteMetadata { + /// The account which sent the note. #[prost(message, optional, tag = "1")] pub sender: ::core::option::Option, + /// The type of the note (0b01 = public, 0b10 = private, 0b11 = encrypted). #[prost(uint32, tag = "2")] pub note_type: u32, + /// A value which can be used by the recipient(s) to identify notes intended for them. + /// + /// See `miden_objects::notes::note_tag` for more info. #[prost(fixed32, tag = "3")] pub tag: u32, + /// Specifies when a note is ready to be consumed. + /// + /// See `miden_objects::notes::execution_hint` for more info. #[prost(fixed64, tag = "4")] pub execution_hint: u64, + /// An arbitrary user-defined value. #[prost(fixed64, tag = "5")] pub aux: u64, } +/// Represents a note. #[derive(Clone, PartialEq, ::prost::Message)] pub struct Note { + /// The block number in which the note was created. #[prost(fixed32, tag = "1")] pub block_num: u32, + /// The index of the note in the block. #[prost(uint32, tag = "2")] pub note_index: u32, + /// The ID of the note. #[prost(message, optional, tag = "3")] pub note_id: ::core::option::Option, + /// The note's metadata. #[prost(message, optional, tag = "4")] pub metadata: ::core::option::Option, + /// The note's inclusion proof in the block. #[prost(message, optional, tag = "5")] pub merkle_path: ::core::option::Option, - /// This field will be present when the note is public. - /// details contain the `Note` in a serialized format. + /// Serialized details of the public note (empty for private notes). #[prost(bytes = "vec", optional, tag = "6")] pub details: ::core::option::Option<::prost::alloc::vec::Vec>, } +/// Represents a proof of note's inclusion in a block. +/// +/// Does not include proof of the block's inclusion in the chain. #[derive(Clone, PartialEq, ::prost::Message)] pub struct NoteInclusionInBlockProof { + /// A unique identifier of the note which is a 32-byte commitment to the underlying note data. #[prost(message, optional, tag = "1")] pub note_id: ::core::option::Option, + /// The block number in which the note was created. #[prost(fixed32, tag = "2")] pub block_num: u32, + /// The index of the note in the block. #[prost(uint32, tag = "3")] pub note_index_in_block: u32, + /// The note's inclusion proof in the block. #[prost(message, optional, tag = "4")] pub merkle_path: ::core::option::Option, } +/// Represents proof of a note inclusion in the block. #[derive(Clone, PartialEq, ::prost::Message)] pub struct NoteSyncRecord { + /// The index of the note. #[prost(uint32, tag = "1")] pub note_index: u32, + /// A unique identifier of the note which is a 32-byte commitment to the underlying note data. #[prost(message, optional, tag = "2")] pub note_id: ::core::option::Option, + /// The note's metadata. #[prost(message, optional, tag = "3")] pub metadata: ::core::option::Option, + /// The note's inclusion proof in the block. #[prost(message, optional, tag = "4")] pub merkle_path: ::core::option::Option, } +/// Represents proof of notes inclusion in the block(s) and block(s) inclusion in the chain. #[derive(Clone, PartialEq, ::prost::Message)] pub struct NoteAuthenticationInfo { /// Proof of each note's inclusion in a block. diff --git a/crates/proto/src/generated/requests.rs b/crates/proto/src/generated/requests.rs index b28e0e3cf..6cdccab33 100644 --- a/crates/proto/src/generated/requests.rs +++ b/crates/proto/src/generated/requests.rs @@ -1,6 +1,9 @@ // This file is @generated by prost-build. +/// Applies changes of a new block to the DB and in-memory data structures. #[derive(Clone, PartialEq, ::prost::Message)] pub struct ApplyBlockRequest { + /// Block data encoded using \[winter_utils::Serializable\] implementation for + /// \[miden_objects::block::Block\]. #[prost(bytes = "vec", tag = "1")] pub block: ::prost::alloc::vec::Vec, } @@ -11,12 +14,14 @@ pub struct CheckNullifiersByPrefixRequest { #[prost(uint32, tag = "1")] pub prefix_len: u32, /// List of nullifiers to check. Each nullifier is specified by its prefix with length equal - /// to prefix_len + /// to `prefix_len`. #[prost(uint32, repeated, tag = "2")] pub nullifiers: ::prost::alloc::vec::Vec, } +/// Get a list of proofs for given nullifier hashes, each proof as a sparse Merkle Tree. #[derive(Clone, PartialEq, ::prost::Message)] pub struct CheckNullifiersRequest { + /// List of nullifiers to return proofs for. #[prost(message, repeated, tag = "1")] pub nullifiers: ::prost::alloc::vec::Vec, } @@ -26,9 +31,7 @@ pub struct CheckNullifiersRequest { /// The Merkle path is an MMR proof for the block's leaf, based on the current chain length. #[derive(Clone, Copy, PartialEq, ::prost::Message)] pub struct GetBlockHeaderByNumberRequest { - /// The block number of the target block. - /// - /// If not provided, means latest known block. + /// The target block height, defaults to latest if not provided. #[prost(uint32, optional, tag = "1")] pub block_num: ::core::option::Option, /// Whether or not to return authentication data for the block header. @@ -76,42 +79,54 @@ pub struct SyncNoteRequest { #[prost(fixed32, repeated, tag = "2")] pub note_tags: ::prost::alloc::vec::Vec, } +/// Returns data required to prove the next block. #[derive(Clone, PartialEq, ::prost::Message)] pub struct GetBlockInputsRequest { /// ID of the account against which a transaction is executed. #[prost(message, repeated, tag = "1")] pub account_ids: ::prost::alloc::vec::Vec, - /// Array of nullifiers for all notes consumed by a transaction. + /// Set of nullifiers consumed by this transaction. #[prost(message, repeated, tag = "2")] pub nullifiers: ::prost::alloc::vec::Vec, /// Array of note IDs to be checked for existence in the database. #[prost(message, repeated, tag = "3")] pub unauthenticated_notes: ::prost::alloc::vec::Vec, } +/// Returns data required to validate a new transaction. #[derive(Clone, PartialEq, ::prost::Message)] pub struct GetTransactionInputsRequest { + /// ID of the account against which a transaction is executed. #[prost(message, optional, tag = "1")] pub account_id: ::core::option::Option, + /// Set of nullifiers consumed by this transaction. #[prost(message, repeated, tag = "2")] pub nullifiers: ::prost::alloc::vec::Vec, + /// Set of unauthenticated notes to check for existence on-chain. + /// + /// These are notes which were not on-chain at the state the transaction was proven, + /// but could by now be present. #[prost(message, repeated, tag = "3")] pub unauthenticated_notes: ::prost::alloc::vec::Vec, } +/// Submits proven transaction to the Miden network. #[derive(Clone, PartialEq, ::prost::Message)] pub struct SubmitProvenTransactionRequest { - /// Transaction encoded using miden's native format + /// Transaction encoded using \[winter_utils::Serializable\] implementation for + /// \[miden_objects::transaction::proven_tx::ProvenTransaction\]. #[prost(bytes = "vec", tag = "1")] pub transaction: ::prost::alloc::vec::Vec, } +/// Returns a list of notes matching the provided note IDs. #[derive(Clone, PartialEq, ::prost::Message)] pub struct GetNotesByIdRequest { - /// List of NoteId's to be queried from the database + /// List of notes to be queried from the database. #[prost(message, repeated, tag = "1")] pub note_ids: ::prost::alloc::vec::Vec, } +/// Returns a list of Note inclusion proofs for the specified Note IDs. #[derive(Clone, PartialEq, ::prost::Message)] pub struct GetNoteAuthenticationInfoRequest { - /// List of NoteId's to be queried from the database + /// List of notes to be queried from the database. #[prost(message, repeated, tag = "1")] pub note_ids: ::prost::alloc::vec::Vec, } @@ -122,6 +137,7 @@ pub struct GetAccountDetailsRequest { #[prost(message, optional, tag = "1")] pub account_id: ::core::option::Option, } +/// Retrieves block data by given block number. #[derive(Clone, Copy, PartialEq, ::prost::Message)] pub struct GetBlockByNumberRequest { /// The block number of the target block. @@ -142,7 +158,7 @@ pub struct GetAccountStateDeltaRequest { #[prost(fixed32, tag = "3")] pub to_block_num: u32, } -/// Request message to get account proofs. +/// Returns the latest state proofs of the specified accounts. #[derive(Clone, PartialEq, ::prost::Message)] pub struct GetAccountProofsRequest { /// A list of account requests, including map keys + values. diff --git a/crates/proto/src/generated/responses.rs b/crates/proto/src/generated/responses.rs index 46ce0cd3d..ee222ce8e 100644 --- a/crates/proto/src/generated/responses.rs +++ b/crates/proto/src/generated/responses.rs @@ -1,68 +1,77 @@ // This file is @generated by prost-build. +/// Represents the result of applying a block. #[derive(Clone, Copy, PartialEq, ::prost::Message)] pub struct ApplyBlockResponse {} +/// Represents the result of checking nullifiers. #[derive(Clone, PartialEq, ::prost::Message)] pub struct CheckNullifiersResponse { /// Each requested nullifier has its corresponding nullifier proof at the same position. #[prost(message, repeated, tag = "1")] pub proofs: ::prost::alloc::vec::Vec, } +/// Represents the result of checking nullifiers by prefix. #[derive(Clone, PartialEq, ::prost::Message)] pub struct CheckNullifiersByPrefixResponse { /// List of nullifiers matching the prefixes specified in the request. #[prost(message, repeated, tag = "1")] pub nullifiers: ::prost::alloc::vec::Vec, } +/// Represents the result of getting a block header by block number. #[derive(Clone, PartialEq, ::prost::Message)] pub struct GetBlockHeaderByNumberResponse { - /// The requested block header + /// The requested block header. #[prost(message, optional, tag = "1")] pub block_header: ::core::option::Option, - /// Merkle path to verify the block's inclusion in the MMR at the returned `chain_length` + /// Merkle path to verify the block's inclusion in the MMR at the returned `chain_length`. #[prost(message, optional, tag = "2")] pub mmr_path: ::core::option::Option, - /// Current chain length + /// Current chain length. #[prost(fixed32, optional, tag = "3")] pub chain_length: ::core::option::Option, } +/// Represents a single nullifier update. #[derive(Clone, Copy, PartialEq, ::prost::Message)] pub struct NullifierUpdate { + /// Nullifier ID. #[prost(message, optional, tag = "1")] pub nullifier: ::core::option::Option, + /// Block number. #[prost(fixed32, tag = "2")] pub block_num: u32, } +/// Represents the result of syncing state request. #[derive(Clone, PartialEq, ::prost::Message)] pub struct SyncStateResponse { - /// Number of the latest block in the chain + /// Number of the latest block in the chain. #[prost(fixed32, tag = "1")] pub chain_tip: u32, - /// Block header of the block with the first note matching the specified criteria + /// Block header of the block with the first note matching the specified criteria. #[prost(message, optional, tag = "2")] pub block_header: ::core::option::Option, - /// Data needed to update the partial MMR from `request.block_num + 1` to `response.block_header.block_num` + /// Data needed to update the partial MMR from `request.block_num + 1` to `response.block_header.block_num`. #[prost(message, optional, tag = "3")] pub mmr_delta: ::core::option::Option, - /// List of account hashes updated after `request.block_num + 1` but not after `response.block_header.block_num` + /// List of account hashes updated after `request.block_num + 1` but not after `response.block_header.block_num`. #[prost(message, repeated, tag = "5")] pub accounts: ::prost::alloc::vec::Vec, /// List of transactions executed against requested accounts between `request.block_num + 1` and - /// `response.block_header.block_num` + /// `response.block_header.block_num`. #[prost(message, repeated, tag = "6")] pub transactions: ::prost::alloc::vec::Vec, - /// List of all notes together with the Merkle paths from `response.block_header.note_root` + /// List of all notes together with the Merkle paths from `response.block_header.note_root`. #[prost(message, repeated, tag = "7")] pub notes: ::prost::alloc::vec::Vec, - /// List of nullifiers created between `request.block_num + 1` and `response.block_header.block_num` + /// List of nullifiers created between `request.block_num + 1` and `response.block_header.block_num`. #[prost(message, repeated, tag = "8")] pub nullifiers: ::prost::alloc::vec::Vec, } +/// Represents the result of syncing notes request. #[derive(Clone, PartialEq, ::prost::Message)] pub struct SyncNoteResponse { - /// Number of the latest block in the chain + /// Number of the latest block in the chain. #[prost(fixed32, tag = "1")] pub chain_tip: u32, - /// Block header of the block with the first note matching the specified criteria + /// Block header of the block with the first note matching the specified criteria. #[prost(message, optional, tag = "2")] pub block_header: ::core::option::Option, /// Merkle path to verify the block's inclusion in the MMR at the returned `chain_tip`. @@ -71,112 +80,135 @@ pub struct SyncNoteResponse { /// an MMR of forest `chain_tip` with this path. #[prost(message, optional, tag = "3")] pub mmr_path: ::core::option::Option, - /// List of all notes together with the Merkle paths from `response.block_header.note_root` + /// List of all notes together with the Merkle paths from `response.block_header.note_root`. #[prost(message, repeated, tag = "4")] pub notes: ::prost::alloc::vec::Vec, } -/// An account returned as a response to the GetBlockInputs +/// An account returned as a response to the `GetBlockInputs`. #[derive(Clone, PartialEq, ::prost::Message)] pub struct AccountBlockInputRecord { + /// The account ID. #[prost(message, optional, tag = "1")] pub account_id: ::core::option::Option, + /// The latest account hash, zero hash if the account doesn't exist. #[prost(message, optional, tag = "2")] pub account_hash: ::core::option::Option, + /// Merkle path to verify the account's inclusion in the MMR. #[prost(message, optional, tag = "3")] pub proof: ::core::option::Option, } -/// A nullifier returned as a response to the GetBlockInputs +/// A nullifier returned as a response to the `GetBlockInputs`. #[derive(Clone, PartialEq, ::prost::Message)] pub struct NullifierBlockInputRecord { + /// The nullifier ID. #[prost(message, optional, tag = "1")] pub nullifier: ::core::option::Option, + /// Merkle path to verify the nullifier's inclusion in the MMR. #[prost(message, optional, tag = "2")] pub opening: ::core::option::Option, } +/// Represents the result of getting block inputs. #[derive(Clone, PartialEq, ::prost::Message)] pub struct GetBlockInputsResponse { - /// The latest block header + /// The latest block header. #[prost(message, optional, tag = "1")] pub block_header: ::core::option::Option, - /// Peaks of the above block's mmr, The `forest` value is equal to the block number + /// Peaks of the above block's mmr, The `forest` value is equal to the block number. #[prost(message, repeated, tag = "2")] pub mmr_peaks: ::prost::alloc::vec::Vec, - /// The hashes of the requested accounts and their authentication paths + /// The hashes of the requested accounts and their authentication paths. #[prost(message, repeated, tag = "3")] pub account_states: ::prost::alloc::vec::Vec, - /// The requested nullifiers and their authentication paths + /// The requested nullifiers and their authentication paths. #[prost(message, repeated, tag = "4")] pub nullifiers: ::prost::alloc::vec::Vec, - /// The list of requested notes which were found in the database + /// The list of requested notes which were found in the database. #[prost(message, optional, tag = "5")] pub found_unauthenticated_notes: ::core::option::Option< super::note::NoteAuthenticationInfo, >, } -/// An account returned as a response to the GetTransactionInputs +/// An account returned as a response to the `GetTransactionInputs`. #[derive(Clone, PartialEq, ::prost::Message)] pub struct AccountTransactionInputRecord { + /// The account ID. #[prost(message, optional, tag = "1")] pub account_id: ::core::option::Option, /// The latest account hash, zero hash if the account doesn't exist. #[prost(message, optional, tag = "2")] pub account_hash: ::core::option::Option, } -/// A nullifier returned as a response to the GetTransactionInputs +/// A nullifier returned as a response to the `GetTransactionInputs`. #[derive(Clone, Copy, PartialEq, ::prost::Message)] pub struct NullifierTransactionInputRecord { + /// The nullifier ID. #[prost(message, optional, tag = "1")] pub nullifier: ::core::option::Option, /// The block at which the nullifier has been consumed, zero if not consumed. #[prost(fixed32, tag = "2")] pub block_num: u32, } +/// Represents the result of getting transaction inputs. #[derive(Clone, PartialEq, ::prost::Message)] pub struct GetTransactionInputsResponse { + /// Account state proof. #[prost(message, optional, tag = "1")] pub account_state: ::core::option::Option, + /// List of nullifiers that have been consumed. #[prost(message, repeated, tag = "2")] pub nullifiers: ::prost::alloc::vec::Vec, + /// List of unauthenticated notes that were not found in the database. #[prost(message, repeated, tag = "3")] pub found_unauthenticated_notes: ::prost::alloc::vec::Vec, + /// The node's current block height. #[prost(fixed32, tag = "4")] pub block_height: u32, } +/// Represents the result of submitting proven transaction. #[derive(Clone, Copy, PartialEq, ::prost::Message)] pub struct SubmitProvenTransactionResponse { - /// The node's current block height + /// The node's current block height. #[prost(fixed32, tag = "1")] pub block_height: u32, } +/// Represents the result of getting notes by IDs. #[derive(Clone, PartialEq, ::prost::Message)] pub struct GetNotesByIdResponse { - /// Lists Note's returned by the database + /// Lists Note's returned by the database. #[prost(message, repeated, tag = "1")] pub notes: ::prost::alloc::vec::Vec, } +/// Represents the result of getting note authentication info. #[derive(Clone, PartialEq, ::prost::Message)] pub struct GetNoteAuthenticationInfoResponse { + /// Proofs of note inclusions in blocks and block inclusions in chain. #[prost(message, optional, tag = "1")] pub proofs: ::core::option::Option, } +/// Represents the result of getting account details. #[derive(Clone, PartialEq, ::prost::Message)] pub struct GetAccountDetailsResponse { - /// Account info (with details for public accounts) + /// Account info (with details for public accounts). #[prost(message, optional, tag = "1")] pub details: ::core::option::Option, } +/// Represents the result of getting block by number. #[derive(Clone, PartialEq, ::prost::Message)] pub struct GetBlockByNumberResponse { - /// The requested `Block` data encoded using miden native format + /// The requested block data encoded using \[winter_utils::Serializable\] implementation for + /// \[miden_objects::block::Block\]. #[prost(bytes = "vec", optional, tag = "1")] pub block: ::core::option::Option<::prost::alloc::vec::Vec>, } +/// Represents the result of getting account state delta. #[derive(Clone, PartialEq, ::prost::Message)] pub struct GetAccountStateDeltaResponse { - /// The calculated `AccountStateDelta` encoded using miden native format + /// The calculated account delta encoded using \[winter_utils::Serializable\] implementation + /// for \[miden_objects::accounts::delta::AccountDelta\]. #[prost(bytes = "vec", optional, tag = "1")] pub delta: ::core::option::Option<::prost::alloc::vec::Vec>, } +/// Represents the result of getting account proofs. #[derive(Clone, PartialEq, ::prost::Message)] pub struct GetAccountProofsResponse { /// Block number at which the state of the account was returned. @@ -186,6 +218,7 @@ pub struct GetAccountProofsResponse { #[prost(message, repeated, tag = "2")] pub account_proofs: ::prost::alloc::vec::Vec, } +/// A single account proof returned as a response to the `GetAccountProofs`. #[derive(Clone, PartialEq, ::prost::Message)] pub struct AccountProofsResponse { /// Account ID. @@ -201,6 +234,7 @@ pub struct AccountProofsResponse { #[prost(message, optional, tag = "4")] pub state_header: ::core::option::Option, } +/// State header for public accounts. #[derive(Clone, PartialEq, ::prost::Message)] pub struct AccountStateHeader { /// Account header. diff --git a/crates/proto/src/generated/rpc.rs b/crates/proto/src/generated/rpc.rs index c7181943c..f9af9ad7c 100644 --- a/crates/proto/src/generated/rpc.rs +++ b/crates/proto/src/generated/rpc.rs @@ -90,6 +90,7 @@ pub mod api_client { self.inner = self.inner.max_encoding_message_size(limit); self } + /// Gets a list of proofs for given nullifier hashes, each proof as a sparse Merkle Tree. pub async fn check_nullifiers( &mut self, request: impl tonic::IntoRequest< @@ -113,6 +114,7 @@ pub mod api_client { req.extensions_mut().insert(GrpcMethod::new("rpc.Api", "CheckNullifiers")); self.inner.unary(req, path, codec).await } + /// Returns a list of nullifiers that match the specified prefixes and are recorded in the node. pub async fn check_nullifiers_by_prefix( &mut self, request: impl tonic::IntoRequest< @@ -139,6 +141,7 @@ pub mod api_client { .insert(GrpcMethod::new("rpc.Api", "CheckNullifiersByPrefix")); self.inner.unary(req, path, codec).await } + /// Returns the latest state of an account with the specified ID. pub async fn get_account_details( &mut self, request: impl tonic::IntoRequest< @@ -164,6 +167,7 @@ pub mod api_client { req.extensions_mut().insert(GrpcMethod::new("rpc.Api", "GetAccountDetails")); self.inner.unary(req, path, codec).await } + /// Returns the latest state proofs of the specified accounts. pub async fn get_account_proofs( &mut self, request: impl tonic::IntoRequest< @@ -187,6 +191,8 @@ pub mod api_client { req.extensions_mut().insert(GrpcMethod::new("rpc.Api", "GetAccountProofs")); self.inner.unary(req, path, codec).await } + /// Returns delta of the account states in the range from `from_block_num` (exclusive) to + /// `to_block_num` (inclusive). pub async fn get_account_state_delta( &mut self, request: impl tonic::IntoRequest< @@ -213,6 +219,7 @@ pub mod api_client { .insert(GrpcMethod::new("rpc.Api", "GetAccountStateDelta")); self.inner.unary(req, path, codec).await } + /// Retrieves block data by given block number. pub async fn get_block_by_number( &mut self, request: impl tonic::IntoRequest< @@ -236,6 +243,8 @@ pub mod api_client { req.extensions_mut().insert(GrpcMethod::new("rpc.Api", "GetBlockByNumber")); self.inner.unary(req, path, codec).await } + /// Retrieves block header by given block number. Optionally, it also returns the MMR path + /// and current chain length to authenticate the block's inclusion. pub async fn get_block_header_by_number( &mut self, request: impl tonic::IntoRequest< @@ -262,6 +271,7 @@ pub mod api_client { .insert(GrpcMethod::new("rpc.Api", "GetBlockHeaderByNumber")); self.inner.unary(req, path, codec).await } + /// Returns a list of notes matching the provided note IDs. pub async fn get_notes_by_id( &mut self, request: impl tonic::IntoRequest, @@ -283,6 +293,7 @@ pub mod api_client { req.extensions_mut().insert(GrpcMethod::new("rpc.Api", "GetNotesById")); self.inner.unary(req, path, codec).await } + /// Submits proven transaction to the Miden network. pub async fn submit_proven_transaction( &mut self, request: impl tonic::IntoRequest< @@ -309,6 +320,10 @@ pub mod api_client { .insert(GrpcMethod::new("rpc.Api", "SubmitProvenTransaction")); self.inner.unary(req, path, codec).await } + /// Note synchronization request. + /// + /// Specifies note tags that client is interested in. The server will return the first block which + /// contains a note matching `note_tags` or the chain tip. pub async fn sync_notes( &mut self, request: impl tonic::IntoRequest, @@ -330,6 +345,21 @@ pub mod api_client { req.extensions_mut().insert(GrpcMethod::new("rpc.Api", "SyncNotes")); self.inner.unary(req, path, codec).await } + /// Returns info which can be used by the client to sync up to the latest state of the chain + /// for the objects (accounts, notes, nullifiers) the client is interested in. + /// + /// This request returns the next block containing requested data. It also returns `chain_tip` + /// which is the latest block number in the chain. Client is expected to repeat these requests + /// in a loop until `response.block_header.block_num == response.chain_tip`, at which point + /// the client is fully synchronized with the chain. + /// + /// Each request also returns info about new notes, nullifiers etc. created. It also returns + /// Chain MMR delta that can be used to update the state of Chain MMR. This includes both chain + /// MMR peaks and chain MMR nodes. + /// + /// For preserving some degree of privacy, note tags and nullifiers filters contain only high + /// part of hashes. Thus, returned data contains excessive notes and nullifiers, client can make + /// additional filtering of that data on its side. pub async fn sync_state( &mut self, request: impl tonic::IntoRequest, @@ -366,6 +396,7 @@ pub mod api_server { /// Generated trait containing gRPC methods that should be implemented for use with ApiServer. #[async_trait] pub trait Api: std::marker::Send + std::marker::Sync + 'static { + /// Gets a list of proofs for given nullifier hashes, each proof as a sparse Merkle Tree. async fn check_nullifiers( &self, request: tonic::Request, @@ -373,6 +404,7 @@ pub mod api_server { tonic::Response, tonic::Status, >; + /// Returns a list of nullifiers that match the specified prefixes and are recorded in the node. async fn check_nullifiers_by_prefix( &self, request: tonic::Request< @@ -382,6 +414,7 @@ pub mod api_server { tonic::Response, tonic::Status, >; + /// Returns the latest state of an account with the specified ID. async fn get_account_details( &self, request: tonic::Request, @@ -389,6 +422,7 @@ pub mod api_server { tonic::Response, tonic::Status, >; + /// Returns the latest state proofs of the specified accounts. async fn get_account_proofs( &self, request: tonic::Request, @@ -396,6 +430,8 @@ pub mod api_server { tonic::Response, tonic::Status, >; + /// Returns delta of the account states in the range from `from_block_num` (exclusive) to + /// `to_block_num` (inclusive). async fn get_account_state_delta( &self, request: tonic::Request, @@ -403,6 +439,7 @@ pub mod api_server { tonic::Response, tonic::Status, >; + /// Retrieves block data by given block number. async fn get_block_by_number( &self, request: tonic::Request, @@ -410,6 +447,8 @@ pub mod api_server { tonic::Response, tonic::Status, >; + /// Retrieves block header by given block number. Optionally, it also returns the MMR path + /// and current chain length to authenticate the block's inclusion. async fn get_block_header_by_number( &self, request: tonic::Request< @@ -419,6 +458,7 @@ pub mod api_server { tonic::Response, tonic::Status, >; + /// Returns a list of notes matching the provided note IDs. async fn get_notes_by_id( &self, request: tonic::Request, @@ -426,6 +466,7 @@ pub mod api_server { tonic::Response, tonic::Status, >; + /// Submits proven transaction to the Miden network. async fn submit_proven_transaction( &self, request: tonic::Request< @@ -435,6 +476,10 @@ pub mod api_server { tonic::Response, tonic::Status, >; + /// Note synchronization request. + /// + /// Specifies note tags that client is interested in. The server will return the first block which + /// contains a note matching `note_tags` or the chain tip. async fn sync_notes( &self, request: tonic::Request, @@ -442,6 +487,21 @@ pub mod api_server { tonic::Response, tonic::Status, >; + /// Returns info which can be used by the client to sync up to the latest state of the chain + /// for the objects (accounts, notes, nullifiers) the client is interested in. + /// + /// This request returns the next block containing requested data. It also returns `chain_tip` + /// which is the latest block number in the chain. Client is expected to repeat these requests + /// in a loop until `response.block_header.block_num == response.chain_tip`, at which point + /// the client is fully synchronized with the chain. + /// + /// Each request also returns info about new notes, nullifiers etc. created. It also returns + /// Chain MMR delta that can be used to update the state of Chain MMR. This includes both chain + /// MMR peaks and chain MMR nodes. + /// + /// For preserving some degree of privacy, note tags and nullifiers filters contain only high + /// part of hashes. Thus, returned data contains excessive notes and nullifiers, client can make + /// additional filtering of that data on its side. async fn sync_state( &self, request: tonic::Request, diff --git a/crates/proto/src/generated/smt.rs b/crates/proto/src/generated/smt.rs index c4aaf2d77..90e9394d3 100644 --- a/crates/proto/src/generated/smt.rs +++ b/crates/proto/src/generated/smt.rs @@ -1,14 +1,18 @@ // This file is @generated by prost-build. -/// An entry in a leaf. +/// Represents a single SMT leaf entry. #[derive(Clone, Copy, PartialEq, ::prost::Message)] pub struct SmtLeafEntry { + /// The key of the entry. #[prost(message, optional, tag = "1")] pub key: ::core::option::Option, + /// The value of the entry. #[prost(message, optional, tag = "2")] pub value: ::core::option::Option, } +/// Represents multiple leaf entries in an SMT. #[derive(Clone, PartialEq, ::prost::Message)] pub struct SmtLeafEntries { + /// The entries list. #[prost(message, repeated, tag = "1")] pub entries: ::prost::alloc::vec::Vec, } @@ -22,10 +26,13 @@ pub struct SmtLeaf { pub mod smt_leaf { #[derive(Clone, PartialEq, ::prost::Oneof)] pub enum Leaf { + /// An empty leaf. #[prost(uint64, tag = "1")] Empty(u64), + /// A single leaf entry. #[prost(message, tag = "2")] Single(super::SmtLeafEntry), + /// Multiple leaf entries. #[prost(message, tag = "3")] Multiple(super::SmtLeafEntries), } @@ -33,8 +40,10 @@ pub mod smt_leaf { /// The opening of a leaf in an SMT. #[derive(Clone, PartialEq, ::prost::Message)] pub struct SmtOpening { + /// The merkle path to the leaf. #[prost(message, optional, tag = "1")] pub path: ::core::option::Option, + /// The leaf itself. #[prost(message, optional, tag = "2")] pub leaf: ::core::option::Option, } diff --git a/crates/proto/src/generated/store.rs b/crates/proto/src/generated/store.rs index 7d979f379..6b1aa2cee 100644 --- a/crates/proto/src/generated/store.rs +++ b/crates/proto/src/generated/store.rs @@ -90,6 +90,7 @@ pub mod api_client { self.inner = self.inner.max_encoding_message_size(limit); self } + /// Applies changes of a new block to the DB and in-memory data structures. pub async fn apply_block( &mut self, request: impl tonic::IntoRequest, @@ -111,6 +112,7 @@ pub mod api_client { req.extensions_mut().insert(GrpcMethod::new("store.Api", "ApplyBlock")); self.inner.unary(req, path, codec).await } + /// Gets a list of proofs for given nullifier hashes, each proof as a sparse Merkle Tree. pub async fn check_nullifiers( &mut self, request: impl tonic::IntoRequest< @@ -136,6 +138,7 @@ pub mod api_client { req.extensions_mut().insert(GrpcMethod::new("store.Api", "CheckNullifiers")); self.inner.unary(req, path, codec).await } + /// Returns a list of nullifiers that match the specified prefixes and are recorded in the node. pub async fn check_nullifiers_by_prefix( &mut self, request: impl tonic::IntoRequest< @@ -162,6 +165,7 @@ pub mod api_client { .insert(GrpcMethod::new("store.Api", "CheckNullifiersByPrefix")); self.inner.unary(req, path, codec).await } + /// Returns the latest state of an account with the specified ID. pub async fn get_account_details( &mut self, request: impl tonic::IntoRequest< @@ -188,6 +192,7 @@ pub mod api_client { .insert(GrpcMethod::new("store.Api", "GetAccountDetails")); self.inner.unary(req, path, codec).await } + /// Returns the latest state proofs of the specified accounts. pub async fn get_account_proofs( &mut self, request: impl tonic::IntoRequest< @@ -214,6 +219,8 @@ pub mod api_client { .insert(GrpcMethod::new("store.Api", "GetAccountProofs")); self.inner.unary(req, path, codec).await } + /// Returns delta of the account states in the range from `from_block_num` (exclusive) to + /// `to_block_num` (inclusive). pub async fn get_account_state_delta( &mut self, request: impl tonic::IntoRequest< @@ -240,6 +247,7 @@ pub mod api_client { .insert(GrpcMethod::new("store.Api", "GetAccountStateDelta")); self.inner.unary(req, path, codec).await } + /// Retrieves block data by given block number. pub async fn get_block_by_number( &mut self, request: impl tonic::IntoRequest< @@ -266,6 +274,8 @@ pub mod api_client { .insert(GrpcMethod::new("store.Api", "GetBlockByNumber")); self.inner.unary(req, path, codec).await } + /// Retrieves block header by given block number. Optionally, it also returns the MMR path + /// and current chain length to authenticate the block's inclusion. pub async fn get_block_header_by_number( &mut self, request: impl tonic::IntoRequest< @@ -292,6 +302,7 @@ pub mod api_client { .insert(GrpcMethod::new("store.Api", "GetBlockHeaderByNumber")); self.inner.unary(req, path, codec).await } + /// Returns data required to prove the next block. pub async fn get_block_inputs( &mut self, request: impl tonic::IntoRequest< @@ -315,6 +326,7 @@ pub mod api_client { req.extensions_mut().insert(GrpcMethod::new("store.Api", "GetBlockInputs")); self.inner.unary(req, path, codec).await } + /// Returns a list of Note inclusion proofs for the specified Note IDs. pub async fn get_note_authentication_info( &mut self, request: impl tonic::IntoRequest< @@ -341,6 +353,7 @@ pub mod api_client { .insert(GrpcMethod::new("store.Api", "GetNoteAuthenticationInfo")); self.inner.unary(req, path, codec).await } + /// Returns a list of notes matching the provided note IDs. pub async fn get_notes_by_id( &mut self, request: impl tonic::IntoRequest, @@ -362,6 +375,7 @@ pub mod api_client { req.extensions_mut().insert(GrpcMethod::new("store.Api", "GetNotesById")); self.inner.unary(req, path, codec).await } + /// Returns data required to validate a new transaction. pub async fn get_transaction_inputs( &mut self, request: impl tonic::IntoRequest< @@ -388,6 +402,10 @@ pub mod api_client { .insert(GrpcMethod::new("store.Api", "GetTransactionInputs")); self.inner.unary(req, path, codec).await } + /// Note synchronization request. + /// + /// Specifies note tags that client is interested in. The server will return the first block which + /// contains a note matching `note_tags` or the chain tip. pub async fn sync_notes( &mut self, request: impl tonic::IntoRequest, @@ -409,6 +427,21 @@ pub mod api_client { req.extensions_mut().insert(GrpcMethod::new("store.Api", "SyncNotes")); self.inner.unary(req, path, codec).await } + /// Returns info which can be used by the client to sync up to the latest state of the chain + /// for the objects (accounts, notes, nullifiers) the client is interested in. + /// + /// This request returns the next block containing requested data. It also returns `chain_tip` + /// which is the latest block number in the chain. Client is expected to repeat these requests + /// in a loop until `response.block_header.block_num == response.chain_tip`, at which point + /// the client is fully synchronized with the chain. + /// + /// Each request also returns info about new notes, nullifiers etc. created. It also returns + /// Chain MMR delta that can be used to update the state of Chain MMR. This includes both chain + /// MMR peaks and chain MMR nodes. + /// + /// For preserving some degree of privacy, note tags and nullifiers filters contain only high + /// part of hashes. Thus, returned data contains excessive notes and nullifiers, client can make + /// additional filtering of that data on its side. pub async fn sync_state( &mut self, request: impl tonic::IntoRequest, @@ -445,6 +478,7 @@ pub mod api_server { /// Generated trait containing gRPC methods that should be implemented for use with ApiServer. #[async_trait] pub trait Api: std::marker::Send + std::marker::Sync + 'static { + /// Applies changes of a new block to the DB and in-memory data structures. async fn apply_block( &self, request: tonic::Request, @@ -452,6 +486,7 @@ pub mod api_server { tonic::Response, tonic::Status, >; + /// Gets a list of proofs for given nullifier hashes, each proof as a sparse Merkle Tree. async fn check_nullifiers( &self, request: tonic::Request, @@ -459,6 +494,7 @@ pub mod api_server { tonic::Response, tonic::Status, >; + /// Returns a list of nullifiers that match the specified prefixes and are recorded in the node. async fn check_nullifiers_by_prefix( &self, request: tonic::Request< @@ -468,6 +504,7 @@ pub mod api_server { tonic::Response, tonic::Status, >; + /// Returns the latest state of an account with the specified ID. async fn get_account_details( &self, request: tonic::Request, @@ -475,6 +512,7 @@ pub mod api_server { tonic::Response, tonic::Status, >; + /// Returns the latest state proofs of the specified accounts. async fn get_account_proofs( &self, request: tonic::Request, @@ -482,6 +520,8 @@ pub mod api_server { tonic::Response, tonic::Status, >; + /// Returns delta of the account states in the range from `from_block_num` (exclusive) to + /// `to_block_num` (inclusive). async fn get_account_state_delta( &self, request: tonic::Request, @@ -489,6 +529,7 @@ pub mod api_server { tonic::Response, tonic::Status, >; + /// Retrieves block data by given block number. async fn get_block_by_number( &self, request: tonic::Request, @@ -496,6 +537,8 @@ pub mod api_server { tonic::Response, tonic::Status, >; + /// Retrieves block header by given block number. Optionally, it also returns the MMR path + /// and current chain length to authenticate the block's inclusion. async fn get_block_header_by_number( &self, request: tonic::Request< @@ -505,6 +548,7 @@ pub mod api_server { tonic::Response, tonic::Status, >; + /// Returns data required to prove the next block. async fn get_block_inputs( &self, request: tonic::Request, @@ -512,6 +556,7 @@ pub mod api_server { tonic::Response, tonic::Status, >; + /// Returns a list of Note inclusion proofs for the specified Note IDs. async fn get_note_authentication_info( &self, request: tonic::Request< @@ -521,6 +566,7 @@ pub mod api_server { tonic::Response, tonic::Status, >; + /// Returns a list of notes matching the provided note IDs. async fn get_notes_by_id( &self, request: tonic::Request, @@ -528,6 +574,7 @@ pub mod api_server { tonic::Response, tonic::Status, >; + /// Returns data required to validate a new transaction. async fn get_transaction_inputs( &self, request: tonic::Request, @@ -535,6 +582,10 @@ pub mod api_server { tonic::Response, tonic::Status, >; + /// Note synchronization request. + /// + /// Specifies note tags that client is interested in. The server will return the first block which + /// contains a note matching `note_tags` or the chain tip. async fn sync_notes( &self, request: tonic::Request, @@ -542,6 +593,21 @@ pub mod api_server { tonic::Response, tonic::Status, >; + /// Returns info which can be used by the client to sync up to the latest state of the chain + /// for the objects (accounts, notes, nullifiers) the client is interested in. + /// + /// This request returns the next block containing requested data. It also returns `chain_tip` + /// which is the latest block number in the chain. Client is expected to repeat these requests + /// in a loop until `response.block_header.block_num == response.chain_tip`, at which point + /// the client is fully synchronized with the chain. + /// + /// Each request also returns info about new notes, nullifiers etc. created. It also returns + /// Chain MMR delta that can be used to update the state of Chain MMR. This includes both chain + /// MMR peaks and chain MMR nodes. + /// + /// For preserving some degree of privacy, note tags and nullifiers filters contain only high + /// part of hashes. Thus, returned data contains excessive notes and nullifiers, client can make + /// additional filtering of that data on its side. async fn sync_state( &self, request: tonic::Request, diff --git a/crates/proto/src/generated/transaction.rs b/crates/proto/src/generated/transaction.rs index 7dca33a33..3439e78ea 100644 --- a/crates/proto/src/generated/transaction.rs +++ b/crates/proto/src/generated/transaction.rs @@ -1,15 +1,21 @@ // This file is @generated by prost-build. +/// Represents a transaction ID. #[derive(Clone, Copy, PartialEq, ::prost::Message)] pub struct TransactionId { + /// The transaction ID. #[prost(message, optional, tag = "1")] pub id: ::core::option::Option, } +/// Represents a transaction summary. #[derive(Clone, PartialEq, ::prost::Message)] pub struct TransactionSummary { + /// A unique 32-byte identifier of a transaction. #[prost(message, optional, tag = "1")] pub transaction_id: ::core::option::Option, + /// The block number in which the transaction was executed. #[prost(fixed32, tag = "2")] pub block_num: u32, + /// The ID of the account affected by the transaction. #[prost(message, optional, tag = "3")] pub account_id: ::core::option::Option, } diff --git a/crates/rpc-proto/proto/account.proto b/crates/rpc-proto/proto/account.proto index 6614b7bc7..f2c1d6e50 100644 --- a/crates/rpc-proto/proto/account.proto +++ b/crates/rpc-proto/proto/account.proto @@ -3,30 +3,49 @@ package account; import "digest.proto"; +// Uniquely identifies a specific account. +// +// A Miden account ID is a 120-bit value derived from the commitments to account code and storage, +// and a random user-provided seed. message AccountId { - // A Miden account ID is a 120-bit value derived from the commitments to account code and - // storage, and a random user-provided seed. + // 15 bytes (120 bits) encoded using [winter_utils::Serializable] implementation for + // [miden_objects::accounts::account_id::AccountId]. bytes id = 1; } +// The state of an account at a specific block height. message AccountSummary { + // The account ID. AccountId account_id = 1; + + // The current account hash or zero if the account does not exist. digest.Digest account_hash = 2; + + // Block number at which the summary was made. uint32 block_num = 3; } +// An account info. message AccountInfo { + // Account summary. AccountSummary summary = 1; + + // Account details encoded using [winter_utils::Serializable] implementation for + // [miden_objects::accounts::Account]. optional bytes details = 2; } +// An account header. message AccountHeader { // Vault root hash. digest.Digest vault_root = 1; + // Storage root hash. digest.Digest storage_commitment = 2; + // Code root hash. digest.Digest code_commitment = 3; + // Account nonce. uint64 nonce = 4; } diff --git a/crates/rpc-proto/proto/block.proto b/crates/rpc-proto/proto/block.proto index 616f927b4..f9a41a99c 100644 --- a/crates/rpc-proto/proto/block.proto +++ b/crates/rpc-proto/proto/block.proto @@ -4,34 +4,50 @@ package block; import "digest.proto"; import "merkle.proto"; +// Represents a block header. message BlockHeader { - // specifies the version of the protocol. + // Specifies the version of the protocol. uint32 version = 1; - // the hash of the previous blocks header. + + // The hash of the previous blocks header. digest.Digest prev_hash = 2; - // a unique sequential number of the current block. + + // A unique sequential number of the current block. fixed32 block_num = 3; - // a commitment to an MMR of the entire chain where each block is a leaf. + + // A commitment to an MMR of the entire chain where each block is a leaf. digest.Digest chain_root = 4; - // a commitment to account database. + + // A commitment to account database. digest.Digest account_root = 5; - // a commitment to the nullifier database. + + // A commitment to the nullifier database. digest.Digest nullifier_root = 6; - // a commitment to all notes created in the current block. + + // A commitment to all notes created in the current block. digest.Digest note_root = 7; - // a commitment to a set of IDs of transactions which affected accounts in this block. + + // A commitment to a set of IDs of transactions which affected accounts in this block. digest.Digest tx_hash = 8; - // a hash of a STARK proof attesting to the correct state transition. + + // A hash of a STARK proof attesting to the correct state transition. digest.Digest proof_hash = 9; - // a commitment to all transaction kernels supported by this block. + + // A commitment to all transaction kernels supported by this block. digest.Digest kernel_root = 10; - // the time when the block was created. + + // The time when the block was created. fixed32 timestamp = 11; } +// Represents a block inclusion proof. message BlockInclusionProof { + // Block header associated with the inclusion proof. BlockHeader block_header = 1; + + // Merkle path associated with the inclusion proof. merkle.MerklePath mmr_path = 2; + // The chain length associated with `mmr_path`. fixed32 chain_length = 3; } diff --git a/crates/rpc-proto/proto/block_producer.proto b/crates/rpc-proto/proto/block_producer.proto index d4f2c0062..054c14912 100644 --- a/crates/rpc-proto/proto/block_producer.proto +++ b/crates/rpc-proto/proto/block_producer.proto @@ -6,6 +6,7 @@ import "requests.proto"; import "responses.proto"; service Api { + // Submits proven transaction to the Miden network rpc SubmitProvenTransaction(requests.SubmitProvenTransactionRequest) returns (responses.SubmitProvenTransactionResponse) {} } diff --git a/crates/rpc-proto/proto/merkle.proto b/crates/rpc-proto/proto/merkle.proto index abded7231..88336deda 100644 --- a/crates/rpc-proto/proto/merkle.proto +++ b/crates/rpc-proto/proto/merkle.proto @@ -3,6 +3,8 @@ package merkle; import "digest.proto"; +// Represents a Merkle path. message MerklePath { + // List of sibling node hashes, in order from the root to the leaf. repeated digest.Digest siblings = 1; } diff --git a/crates/rpc-proto/proto/mmr.proto b/crates/rpc-proto/proto/mmr.proto index baaced2c9..f5e60e570 100644 --- a/crates/rpc-proto/proto/mmr.proto +++ b/crates/rpc-proto/proto/mmr.proto @@ -3,7 +3,11 @@ package mmr; import "digest.proto"; +// Represents an MMR delta. message MmrDelta { + // The number of leaf nodes in the MMR. uint64 forest = 1; + + // New and changed MMR peaks. repeated digest.Digest data = 2; } diff --git a/crates/rpc-proto/proto/note.proto b/crates/rpc-proto/proto/note.proto index d424f696b..f65960d67 100644 --- a/crates/rpc-proto/proto/note.proto +++ b/crates/rpc-proto/proto/note.proto @@ -6,42 +6,86 @@ import "block.proto"; import "digest.proto"; import "merkle.proto"; +// Represents a note's metadata. message NoteMetadata { + // The account which sent the note. account.AccountId sender = 1; + + // The type of the note (0b01 = public, 0b10 = private, 0b11 = encrypted). uint32 note_type = 2; + + // A value which can be used by the recipient(s) to identify notes intended for them. + // + // See `miden_objects::notes::note_tag` for more info. fixed32 tag = 3; + + // Specifies when a note is ready to be consumed. + // + // See `miden_objects::notes::execution_hint` for more info. fixed64 execution_hint = 4; + + // An arbitrary user-defined value. fixed64 aux = 5; } +// Represents a note. message Note { + // The block number in which the note was created. fixed32 block_num = 1; + + // The index of the note in the block. uint32 note_index = 2; + + // The ID of the note. digest.Digest note_id = 3; + + // The note's metadata. NoteMetadata metadata = 4; + + // The note's inclusion proof in the block. merkle.MerklePath merkle_path = 5; - // This field will be present when the note is public. - // details contain the `Note` in a serialized format. + + // Serialized details of the public note (empty for private notes). optional bytes details = 6; } +// Represents a proof of note's inclusion in a block. +// +// Does not include proof of the block's inclusion in the chain. message NoteInclusionInBlockProof { + // A unique identifier of the note which is a 32-byte commitment to the underlying note data. digest.Digest note_id = 1; + + // The block number in which the note was created. fixed32 block_num = 2; + + // The index of the note in the block. uint32 note_index_in_block = 3; + + // The note's inclusion proof in the block. merkle.MerklePath merkle_path = 4; } +// Represents proof of a note inclusion in the block. message NoteSyncRecord { + // The index of the note. uint32 note_index = 1; + + // A unique identifier of the note which is a 32-byte commitment to the underlying note data. digest.Digest note_id = 2; + + // The note's metadata. NoteMetadata metadata = 3; + + // The note's inclusion proof in the block. merkle.MerklePath merkle_path = 4; } +// Represents proof of notes inclusion in the block(s) and block(s) inclusion in the chain. message NoteAuthenticationInfo { // Proof of each note's inclusion in a block. repeated note.NoteInclusionInBlockProof note_proofs = 1; + // Proof of each block's inclusion in the chain. repeated block.BlockInclusionProof block_proofs = 2; } diff --git a/crates/rpc-proto/proto/requests.proto b/crates/rpc-proto/proto/requests.proto index 62ec750a1..b8e2e0817 100644 --- a/crates/rpc-proto/proto/requests.proto +++ b/crates/rpc-proto/proto/requests.proto @@ -3,9 +3,11 @@ package requests; import "account.proto"; import "digest.proto"; -import "note.proto"; +// Applies changes of a new block to the DB and in-memory data structures. message ApplyBlockRequest { + // Block data encoded using [winter_utils::Serializable] implementation for + // [miden_objects::block::Block]. bytes block = 1; } @@ -14,11 +16,13 @@ message CheckNullifiersByPrefixRequest { // Number of bits used for nullifier prefix. Currently the only supported value is 16. uint32 prefix_len = 1; // List of nullifiers to check. Each nullifier is specified by its prefix with length equal - // to prefix_len + // to `prefix_len`. repeated uint32 nullifiers = 2; } +// Get a list of proofs for given nullifier hashes, each proof as a sparse Merkle Tree. message CheckNullifiersRequest { + // List of nullifiers to return proofs for. repeated digest.Digest nullifiers = 1; } @@ -27,9 +31,7 @@ message CheckNullifiersRequest { // // The Merkle path is an MMR proof for the block's leaf, based on the current chain length. message GetBlockHeaderByNumberRequest { - // The block number of the target block. - // - // If not provided, means latest known block. + // The target block height, defaults to latest if not provided. optional uint32 block_num = 1; // Whether or not to return authentication data for the block header. optional bool include_mmr_proof = 2; @@ -74,33 +76,45 @@ message SyncNoteRequest { repeated fixed32 note_tags = 2; } +// Returns data required to prove the next block. message GetBlockInputsRequest { // ID of the account against which a transaction is executed. repeated account.AccountId account_ids = 1; - // Array of nullifiers for all notes consumed by a transaction. + // Set of nullifiers consumed by this transaction. repeated digest.Digest nullifiers = 2; // Array of note IDs to be checked for existence in the database. repeated digest.Digest unauthenticated_notes = 3; } +// Returns data required to validate a new transaction. message GetTransactionInputsRequest { + // ID of the account against which a transaction is executed. account.AccountId account_id = 1; + // Set of nullifiers consumed by this transaction. repeated digest.Digest nullifiers = 2; + // Set of unauthenticated notes to check for existence on-chain. + // + // These are notes which were not on-chain at the state the transaction was proven, + // but could by now be present. repeated digest.Digest unauthenticated_notes = 3; } +// Submits proven transaction to the Miden network. message SubmitProvenTransactionRequest { - // Transaction encoded using miden's native format + // Transaction encoded using [winter_utils::Serializable] implementation for + // [miden_objects::transaction::proven_tx::ProvenTransaction]. bytes transaction = 1; } +// Returns a list of notes matching the provided note IDs. message GetNotesByIdRequest { - // List of NoteId's to be queried from the database + // List of notes to be queried from the database. repeated digest.Digest note_ids = 1; } +// Returns a list of Note inclusion proofs for the specified Note IDs. message GetNoteAuthenticationInfoRequest { - // List of NoteId's to be queried from the database + // List of notes to be queried from the database. repeated digest.Digest note_ids = 1; } @@ -110,6 +124,7 @@ message GetAccountDetailsRequest { account.AccountId account_id = 1; } +// Retrieves block data by given block number. message GetBlockByNumberRequest { // The block number of the target block. fixed32 block_num = 1; @@ -126,9 +141,9 @@ message GetAccountStateDeltaRequest { fixed32 to_block_num = 3; } -// Request message to get account proofs. +// Returns the latest state proofs of the specified accounts. message GetAccountProofsRequest { - // Represents per-account requests where each account ID has its own list of + // Represents per-account requests where each account ID has its own list of // (storage_slot_index, map_keys) pairs. message AccountRequest { // The account ID for this request. @@ -154,7 +169,7 @@ message GetAccountProofsRequest { // requests are also ignored. False by default. optional bool include_headers = 2; - // Account code commitments corresponding to the last-known `AccountCode` for requested + // Account code commitments corresponding to the last-known `AccountCode` for requested // accounts. Responses will include only the ones that are not known to the caller. // These are not associated with a specific account but rather, they will be matched against // all requested accounts. diff --git a/crates/rpc-proto/proto/responses.proto b/crates/rpc-proto/proto/responses.proto index 530a5dc18..860368e67 100644 --- a/crates/rpc-proto/proto/responses.proto +++ b/crates/rpc-proto/proto/responses.proto @@ -10,63 +10,73 @@ import "note.proto"; import "smt.proto"; import "transaction.proto"; +// Represents the result of applying a block. message ApplyBlockResponse {} +// Represents the result of checking nullifiers. message CheckNullifiersResponse { // Each requested nullifier has its corresponding nullifier proof at the same position. repeated smt.SmtOpening proofs = 1; } +// Represents the result of checking nullifiers by prefix. message CheckNullifiersByPrefixResponse { // List of nullifiers matching the prefixes specified in the request. repeated NullifierUpdate nullifiers = 1; } +// Represents the result of getting a block header by block number. message GetBlockHeaderByNumberResponse { - // The requested block header + // The requested block header. block.BlockHeader block_header = 1; - // Merkle path to verify the block's inclusion in the MMR at the returned `chain_length` + // Merkle path to verify the block's inclusion in the MMR at the returned `chain_length`. optional merkle.MerklePath mmr_path = 2; - // Current chain length + // Current chain length. optional fixed32 chain_length = 3; } +// Represents a single nullifier update. message NullifierUpdate { + // Nullifier ID. digest.Digest nullifier = 1; + + // Block number. fixed32 block_num = 2; } +// Represents the result of syncing state request. message SyncStateResponse { - // Number of the latest block in the chain + // Number of the latest block in the chain. fixed32 chain_tip = 1; - // Block header of the block with the first note matching the specified criteria + // Block header of the block with the first note matching the specified criteria. block.BlockHeader block_header = 2; - // Data needed to update the partial MMR from `request.block_num + 1` to `response.block_header.block_num` + // Data needed to update the partial MMR from `request.block_num + 1` to `response.block_header.block_num`. mmr.MmrDelta mmr_delta = 3; - // List of account hashes updated after `request.block_num + 1` but not after `response.block_header.block_num` + // List of account hashes updated after `request.block_num + 1` but not after `response.block_header.block_num`. repeated account.AccountSummary accounts = 5; // List of transactions executed against requested accounts between `request.block_num + 1` and - // `response.block_header.block_num` + // `response.block_header.block_num`. repeated transaction.TransactionSummary transactions = 6; - // List of all notes together with the Merkle paths from `response.block_header.note_root` + // List of all notes together with the Merkle paths from `response.block_header.note_root`. repeated note.NoteSyncRecord notes = 7; - // List of nullifiers created between `request.block_num + 1` and `response.block_header.block_num` + // List of nullifiers created between `request.block_num + 1` and `response.block_header.block_num`. repeated NullifierUpdate nullifiers = 8; } +// Represents the result of syncing notes request. message SyncNoteResponse { - // Number of the latest block in the chain + // Number of the latest block in the chain. fixed32 chain_tip = 1; - // Block header of the block with the first note matching the specified criteria + // Block header of the block with the first note matching the specified criteria. block.BlockHeader block_header = 2; // Merkle path to verify the block's inclusion in the MMR at the returned `chain_tip`. @@ -75,90 +85,121 @@ message SyncNoteResponse { // an MMR of forest `chain_tip` with this path. merkle.MerklePath mmr_path = 3; - // List of all notes together with the Merkle paths from `response.block_header.note_root` + // List of all notes together with the Merkle paths from `response.block_header.note_root`. repeated note.NoteSyncRecord notes = 4; } -// An account returned as a response to the GetBlockInputs +// An account returned as a response to the `GetBlockInputs`. message AccountBlockInputRecord { + // The account ID. account.AccountId account_id = 1; + + // The latest account hash, zero hash if the account doesn't exist. digest.Digest account_hash = 2; + + // Merkle path to verify the account's inclusion in the MMR. merkle.MerklePath proof = 3; } -// A nullifier returned as a response to the GetBlockInputs +// A nullifier returned as a response to the `GetBlockInputs`. message NullifierBlockInputRecord { + // The nullifier ID. digest.Digest nullifier = 1; + + // Merkle path to verify the nullifier's inclusion in the MMR. smt.SmtOpening opening = 2; } +// Represents the result of getting block inputs. message GetBlockInputsResponse { - // The latest block header + // The latest block header. block.BlockHeader block_header = 1; - // Peaks of the above block's mmr, The `forest` value is equal to the block number + // Peaks of the above block's mmr, The `forest` value is equal to the block number. repeated digest.Digest mmr_peaks = 2; - // The hashes of the requested accounts and their authentication paths + // The hashes of the requested accounts and their authentication paths. repeated AccountBlockInputRecord account_states = 3; - // The requested nullifiers and their authentication paths + // The requested nullifiers and their authentication paths. repeated NullifierBlockInputRecord nullifiers = 4; - // The list of requested notes which were found in the database + // The list of requested notes which were found in the database. note.NoteAuthenticationInfo found_unauthenticated_notes = 5; } -// An account returned as a response to the GetTransactionInputs +// An account returned as a response to the `GetTransactionInputs`. message AccountTransactionInputRecord { + // The account ID. account.AccountId account_id = 1; + // The latest account hash, zero hash if the account doesn't exist. digest.Digest account_hash = 2; } -// A nullifier returned as a response to the GetTransactionInputs +// A nullifier returned as a response to the `GetTransactionInputs`. message NullifierTransactionInputRecord { + // The nullifier ID. digest.Digest nullifier = 1; + // The block at which the nullifier has been consumed, zero if not consumed. fixed32 block_num = 2; } +// Represents the result of getting transaction inputs. message GetTransactionInputsResponse { + // Account state proof. AccountTransactionInputRecord account_state = 1; + + // List of nullifiers that have been consumed. repeated NullifierTransactionInputRecord nullifiers = 2; + + // List of unauthenticated notes that were not found in the database. repeated digest.Digest found_unauthenticated_notes = 3; + + // The node's current block height. fixed32 block_height = 4; } +// Represents the result of submitting proven transaction. message SubmitProvenTransactionResponse { - // The node's current block height + // The node's current block height. fixed32 block_height = 1; } +// Represents the result of getting notes by IDs. message GetNotesByIdResponse { - // Lists Note's returned by the database + // Lists Note's returned by the database. repeated note.Note notes = 1; } +// Represents the result of getting note authentication info. message GetNoteAuthenticationInfoResponse { + // Proofs of note inclusions in blocks and block inclusions in chain. note.NoteAuthenticationInfo proofs = 1; } +// Represents the result of getting account details. message GetAccountDetailsResponse { - // Account info (with details for public accounts) + // Account info (with details for public accounts). account.AccountInfo details = 1; } +// Represents the result of getting block by number. message GetBlockByNumberResponse { - // The requested `Block` data encoded using miden native format + // The requested block data encoded using [winter_utils::Serializable] implementation for + // [miden_objects::block::Block]. optional bytes block = 1; } +// Represents the result of getting account state delta. message GetAccountStateDeltaResponse { - // The calculated `AccountStateDelta` encoded using miden native format + // The calculated account delta encoded using [winter_utils::Serializable] implementation + // for [miden_objects::accounts::delta::AccountDelta]. optional bytes delta = 1; } +// Represents the result of getting account proofs. message GetAccountProofsResponse { // Block number at which the state of the account was returned. fixed32 block_num = 1; @@ -166,6 +207,7 @@ message GetAccountProofsResponse { repeated AccountProofsResponse account_proofs = 2; } +// A single account proof returned as a response to the `GetAccountProofs`. message AccountProofsResponse { // Account ID. account.AccountId account_id = 1; @@ -177,6 +219,7 @@ message AccountProofsResponse { optional AccountStateHeader state_header = 4; } +// State header for public accounts. message AccountStateHeader { // Account header. account.AccountHeader header = 1; diff --git a/crates/rpc-proto/proto/rpc.proto b/crates/rpc-proto/proto/rpc.proto index 13934b2c4..bd5d359ad 100644 --- a/crates/rpc-proto/proto/rpc.proto +++ b/crates/rpc-proto/proto/rpc.proto @@ -6,15 +6,55 @@ import "requests.proto"; import "responses.proto"; service Api { + // Gets a list of proofs for given nullifier hashes, each proof as a sparse Merkle Tree. rpc CheckNullifiers(requests.CheckNullifiersRequest) returns (responses.CheckNullifiersResponse) {} + + // Returns a list of nullifiers that match the specified prefixes and are recorded in the node. rpc CheckNullifiersByPrefix(requests.CheckNullifiersByPrefixRequest) returns (responses.CheckNullifiersByPrefixResponse) {} + + // Returns the latest state of an account with the specified ID. rpc GetAccountDetails(requests.GetAccountDetailsRequest) returns (responses.GetAccountDetailsResponse) {} + + // Returns the latest state proofs of the specified accounts. rpc GetAccountProofs(requests.GetAccountProofsRequest) returns (responses.GetAccountProofsResponse) {} + + // Returns delta of the account states in the range from `from_block_num` (exclusive) to + // `to_block_num` (inclusive). rpc GetAccountStateDelta(requests.GetAccountStateDeltaRequest) returns (responses.GetAccountStateDeltaResponse) {} + + // Retrieves block data by given block number. rpc GetBlockByNumber(requests.GetBlockByNumberRequest) returns (responses.GetBlockByNumberResponse) {} + + // Retrieves block header by given block number. Optionally, it also returns the MMR path + // and current chain length to authenticate the block's inclusion. rpc GetBlockHeaderByNumber(requests.GetBlockHeaderByNumberRequest) returns (responses.GetBlockHeaderByNumberResponse) {} + + // Returns a list of notes matching the provided note IDs. rpc GetNotesById(requests.GetNotesByIdRequest) returns (responses.GetNotesByIdResponse) {} + + // Submits proven transaction to the Miden network. rpc SubmitProvenTransaction(requests.SubmitProvenTransactionRequest) returns (responses.SubmitProvenTransactionResponse) {} + + // Note synchronization request. + // + // Specifies note tags that client is interested in. The server will return the first block which + // contains a note matching `note_tags` or the chain tip. rpc SyncNotes(requests.SyncNoteRequest) returns (responses.SyncNoteResponse) {} + + // Returns info which can be used by the client to sync up to the latest state of the chain + // for the objects (accounts, notes, nullifiers) the client is interested in. + // + // This request returns the next block containing requested data. It also returns `chain_tip` + // which is the latest block number in the chain. Client is expected to repeat these requests + // in a loop until `response.block_header.block_num == response.chain_tip`, at which point + // the client is fully synchronized with the chain. + // + // Each request also returns info about new notes, nullifiers etc. created. It also returns + // Chain MMR delta that can be used to update the state of Chain MMR. This includes both chain + // MMR peaks and chain MMR nodes. + // + // For preserving some degree of privacy, note tags and nullifiers filters contain only high + // part of hashes. Thus, returned data contains excessive notes and nullifiers, client can make + // additional filtering of that data on its side. rpc SyncState(requests.SyncStateRequest) returns (responses.SyncStateResponse) {} } diff --git a/crates/rpc-proto/proto/smt.proto b/crates/rpc-proto/proto/smt.proto index d62634dd4..484508b1b 100644 --- a/crates/rpc-proto/proto/smt.proto +++ b/crates/rpc-proto/proto/smt.proto @@ -1,4 +1,4 @@ -// Message definitions related to Sparse Merkle Trees (SMT). +// Message definitions related to Sparse Merkle Trees (SMT) syntax = "proto3"; package smt; @@ -6,27 +6,40 @@ package smt; import "digest.proto"; import "merkle.proto"; -// An entry in a leaf. +// Represents a single SMT leaf entry. message SmtLeafEntry { + // The key of the entry. digest.Digest key = 1; + + // The value of the entry. digest.Digest value = 2; } +// Represents multiple leaf entries in an SMT. message SmtLeafEntries { + // The entries list. repeated SmtLeafEntry entries = 1; } // A leaf in an SMT, sitting at depth 64. A leaf can contain 0, 1 or multiple leaf entries. message SmtLeaf { oneof leaf { + // An empty leaf. uint64 empty = 1; + + // A single leaf entry. SmtLeafEntry single = 2; + + // Multiple leaf entries. SmtLeafEntries multiple = 3; } } // The opening of a leaf in an SMT. message SmtOpening { + // The merkle path to the leaf. merkle.MerklePath path = 1; + + // The leaf itself. SmtLeaf leaf = 2; } diff --git a/crates/rpc-proto/proto/store.proto b/crates/rpc-proto/proto/store.proto index b51198b96..5ed980aba 100644 --- a/crates/rpc-proto/proto/store.proto +++ b/crates/rpc-proto/proto/store.proto @@ -8,18 +8,64 @@ import "requests.proto"; import "responses.proto"; service Api { + // Applies changes of a new block to the DB and in-memory data structures. rpc ApplyBlock(requests.ApplyBlockRequest) returns (responses.ApplyBlockResponse) {} + + // Gets a list of proofs for given nullifier hashes, each proof as a sparse Merkle Tree. rpc CheckNullifiers(requests.CheckNullifiersRequest) returns (responses.CheckNullifiersResponse) {} + + // Returns a list of nullifiers that match the specified prefixes and are recorded in the node. rpc CheckNullifiersByPrefix(requests.CheckNullifiersByPrefixRequest) returns (responses.CheckNullifiersByPrefixResponse) {} + + // Returns the latest state of an account with the specified ID. rpc GetAccountDetails(requests.GetAccountDetailsRequest) returns (responses.GetAccountDetailsResponse) {} + + // Returns the latest state proofs of the specified accounts. rpc GetAccountProofs(requests.GetAccountProofsRequest) returns (responses.GetAccountProofsResponse) {} + + // Returns delta of the account states in the range from `from_block_num` (exclusive) to + // `to_block_num` (inclusive). rpc GetAccountStateDelta(requests.GetAccountStateDeltaRequest) returns (responses.GetAccountStateDeltaResponse) {} + + // Retrieves block data by given block number. rpc GetBlockByNumber(requests.GetBlockByNumberRequest) returns (responses.GetBlockByNumberResponse) {} + + // Retrieves block header by given block number. Optionally, it also returns the MMR path + // and current chain length to authenticate the block's inclusion. rpc GetBlockHeaderByNumber(requests.GetBlockHeaderByNumberRequest) returns (responses.GetBlockHeaderByNumberResponse) {} + + // Returns data required to prove the next block. rpc GetBlockInputs(requests.GetBlockInputsRequest) returns (responses.GetBlockInputsResponse) {} + + // Returns a list of Note inclusion proofs for the specified Note IDs. rpc GetNoteAuthenticationInfo(requests.GetNoteAuthenticationInfoRequest) returns (responses.GetNoteAuthenticationInfoResponse) {} + + // Returns a list of notes matching the provided note IDs. rpc GetNotesById(requests.GetNotesByIdRequest) returns (responses.GetNotesByIdResponse) {} + + // Returns data required to validate a new transaction. rpc GetTransactionInputs(requests.GetTransactionInputsRequest) returns (responses.GetTransactionInputsResponse) {} + + // Note synchronization request. + // + // Specifies note tags that client is interested in. The server will return the first block which + // contains a note matching `note_tags` or the chain tip. rpc SyncNotes(requests.SyncNoteRequest) returns (responses.SyncNoteResponse) {} + + // Returns info which can be used by the client to sync up to the latest state of the chain + // for the objects (accounts, notes, nullifiers) the client is interested in. + // + // This request returns the next block containing requested data. It also returns `chain_tip` + // which is the latest block number in the chain. Client is expected to repeat these requests + // in a loop until `response.block_header.block_num == response.chain_tip`, at which point + // the client is fully synchronized with the chain. + // + // Each request also returns info about new notes, nullifiers etc. created. It also returns + // Chain MMR delta that can be used to update the state of Chain MMR. This includes both chain + // MMR peaks and chain MMR nodes. + // + // For preserving some degree of privacy, note tags and nullifiers filters contain only high + // part of hashes. Thus, returned data contains excessive notes and nullifiers, client can make + // additional filtering of that data on its side. rpc SyncState(requests.SyncStateRequest) returns (responses.SyncStateResponse) {} } diff --git a/crates/rpc-proto/proto/transaction.proto b/crates/rpc-proto/proto/transaction.proto index 996b15bc4..9a9520c61 100644 --- a/crates/rpc-proto/proto/transaction.proto +++ b/crates/rpc-proto/proto/transaction.proto @@ -4,13 +4,21 @@ package transaction; import "account.proto"; import "digest.proto"; +// Represents a transaction ID. message TransactionId { + // The transaction ID. digest.Digest id = 1; } +// Represents a transaction summary. message TransactionSummary { + // A unique 32-byte identifier of a transaction. TransactionId transaction_id = 1; + + // The block number in which the transaction was executed. fixed32 block_num = 2; + + // The ID of the account affected by the transaction. account.AccountId account_id = 3; } diff --git a/crates/rpc/README.md b/crates/rpc/README.md index d069f4f57..500e56ebd 100644 --- a/crates/rpc/README.md +++ b/crates/rpc/README.md @@ -54,7 +54,7 @@ Retrieves block data by given block number. **Returns:** -- `block`: `Block` – block data encoded in Miden native format. +- `block`: `Block` – block data encoded using [winter_utils::Serializable](https://github.com/facebook/winterfell/blob/main/utils/core/src/serde/mod.rs#L26) implementation for [miden_objects::block::Block](https://github.com/0xPolygonMiden/miden-base/blob/main/objects/src/block/mod.rs#L43). ### GetNotesById @@ -119,7 +119,7 @@ Submits proven transaction to the Miden network. **Parameters** -- `transaction`: `bytes` - transaction encoded using Miden's native format. +- `transaction`: `bytes` - transaction encoded using [winter_utils::Serializable](https://github.com/facebook/winterfell/blob/main/utils/core/src/serde/mod.rs#L26) implementation for [miden_objects::transaction::proven_tx::ProvenTransaction](https://github.com/0xPolygonMiden/miden-base/blob/main/objects/src/transaction/proven_tx.rs#L22). **Returns** diff --git a/crates/store/README.md b/crates/store/README.md index a201956df..2932726ca 100644 --- a/crates/store/README.md +++ b/crates/store/README.md @@ -70,11 +70,11 @@ Retrieves block data by given block number. **Returns:** -- `block`: `Block` – block data encoded in Miden native format. +- `block`: `Block` – block data encoded using [winter_utils::Serializable](https://github.com/facebook/winterfell/blob/main/utils/core/src/serde/mod.rs#L26) implementation for [miden_objects::block::Block](https://github.com/0xPolygonMiden/miden-base/blob/main/objects/src/block/mod.rs#L43). ### GetBlockInputs -Returns data needed by the block producer to construct and prove the next block. +Returns data required to prove the next block. **Parameters** @@ -90,12 +90,12 @@ Returns data needed by the block producer to construct and prove the next block. ### GetTransactionInputs -Returns the data needed by the block producer to check validity of an incoming transaction. +Returns data required to validate a new transaction. **Parameters** - `account_id`: `AccountId` – ID of the account against which a transaction is executed. -- `nullifiers`: `[Digest]` – array of nullifiers for all notes consumed by a transaction. +- `nullifiers`: `[Digest]` – set of nullifiers consumed by this transaction. **Returns** diff --git a/crates/store/src/server/api.rs b/crates/store/src/server/api.rs index 1eb5f3627..006561592 100644 --- a/crates/store/src/server/api.rs +++ b/crates/store/src/server/api.rs @@ -274,7 +274,7 @@ impl api_server::Api for StoreApi { Ok(Response::new(GetNotesByIdResponse { notes })) } - /// Returns a list of Note inclusion proofs for the specified NoteId's. + /// Returns the inclusion proofs of the specified notes. #[instrument( target = COMPONENT, name = "store:get_note_inclusion_proofs", diff --git a/proto/account.proto b/proto/account.proto index 6614b7bc7..f2c1d6e50 100644 --- a/proto/account.proto +++ b/proto/account.proto @@ -3,30 +3,49 @@ package account; import "digest.proto"; +// Uniquely identifies a specific account. +// +// A Miden account ID is a 120-bit value derived from the commitments to account code and storage, +// and a random user-provided seed. message AccountId { - // A Miden account ID is a 120-bit value derived from the commitments to account code and - // storage, and a random user-provided seed. + // 15 bytes (120 bits) encoded using [winter_utils::Serializable] implementation for + // [miden_objects::accounts::account_id::AccountId]. bytes id = 1; } +// The state of an account at a specific block height. message AccountSummary { + // The account ID. AccountId account_id = 1; + + // The current account hash or zero if the account does not exist. digest.Digest account_hash = 2; + + // Block number at which the summary was made. uint32 block_num = 3; } +// An account info. message AccountInfo { + // Account summary. AccountSummary summary = 1; + + // Account details encoded using [winter_utils::Serializable] implementation for + // [miden_objects::accounts::Account]. optional bytes details = 2; } +// An account header. message AccountHeader { // Vault root hash. digest.Digest vault_root = 1; + // Storage root hash. digest.Digest storage_commitment = 2; + // Code root hash. digest.Digest code_commitment = 3; + // Account nonce. uint64 nonce = 4; } diff --git a/proto/block.proto b/proto/block.proto index 616f927b4..f9a41a99c 100644 --- a/proto/block.proto +++ b/proto/block.proto @@ -4,34 +4,50 @@ package block; import "digest.proto"; import "merkle.proto"; +// Represents a block header. message BlockHeader { - // specifies the version of the protocol. + // Specifies the version of the protocol. uint32 version = 1; - // the hash of the previous blocks header. + + // The hash of the previous blocks header. digest.Digest prev_hash = 2; - // a unique sequential number of the current block. + + // A unique sequential number of the current block. fixed32 block_num = 3; - // a commitment to an MMR of the entire chain where each block is a leaf. + + // A commitment to an MMR of the entire chain where each block is a leaf. digest.Digest chain_root = 4; - // a commitment to account database. + + // A commitment to account database. digest.Digest account_root = 5; - // a commitment to the nullifier database. + + // A commitment to the nullifier database. digest.Digest nullifier_root = 6; - // a commitment to all notes created in the current block. + + // A commitment to all notes created in the current block. digest.Digest note_root = 7; - // a commitment to a set of IDs of transactions which affected accounts in this block. + + // A commitment to a set of IDs of transactions which affected accounts in this block. digest.Digest tx_hash = 8; - // a hash of a STARK proof attesting to the correct state transition. + + // A hash of a STARK proof attesting to the correct state transition. digest.Digest proof_hash = 9; - // a commitment to all transaction kernels supported by this block. + + // A commitment to all transaction kernels supported by this block. digest.Digest kernel_root = 10; - // the time when the block was created. + + // The time when the block was created. fixed32 timestamp = 11; } +// Represents a block inclusion proof. message BlockInclusionProof { + // Block header associated with the inclusion proof. BlockHeader block_header = 1; + + // Merkle path associated with the inclusion proof. merkle.MerklePath mmr_path = 2; + // The chain length associated with `mmr_path`. fixed32 chain_length = 3; } diff --git a/proto/block_producer.proto b/proto/block_producer.proto index d4f2c0062..054c14912 100644 --- a/proto/block_producer.proto +++ b/proto/block_producer.proto @@ -6,6 +6,7 @@ import "requests.proto"; import "responses.proto"; service Api { + // Submits proven transaction to the Miden network rpc SubmitProvenTransaction(requests.SubmitProvenTransactionRequest) returns (responses.SubmitProvenTransactionResponse) {} } diff --git a/proto/merkle.proto b/proto/merkle.proto index abded7231..88336deda 100644 --- a/proto/merkle.proto +++ b/proto/merkle.proto @@ -3,6 +3,8 @@ package merkle; import "digest.proto"; +// Represents a Merkle path. message MerklePath { + // List of sibling node hashes, in order from the root to the leaf. repeated digest.Digest siblings = 1; } diff --git a/proto/mmr.proto b/proto/mmr.proto index baaced2c9..f5e60e570 100644 --- a/proto/mmr.proto +++ b/proto/mmr.proto @@ -3,7 +3,11 @@ package mmr; import "digest.proto"; +// Represents an MMR delta. message MmrDelta { + // The number of leaf nodes in the MMR. uint64 forest = 1; + + // New and changed MMR peaks. repeated digest.Digest data = 2; } diff --git a/proto/note.proto b/proto/note.proto index d424f696b..f65960d67 100644 --- a/proto/note.proto +++ b/proto/note.proto @@ -6,42 +6,86 @@ import "block.proto"; import "digest.proto"; import "merkle.proto"; +// Represents a note's metadata. message NoteMetadata { + // The account which sent the note. account.AccountId sender = 1; + + // The type of the note (0b01 = public, 0b10 = private, 0b11 = encrypted). uint32 note_type = 2; + + // A value which can be used by the recipient(s) to identify notes intended for them. + // + // See `miden_objects::notes::note_tag` for more info. fixed32 tag = 3; + + // Specifies when a note is ready to be consumed. + // + // See `miden_objects::notes::execution_hint` for more info. fixed64 execution_hint = 4; + + // An arbitrary user-defined value. fixed64 aux = 5; } +// Represents a note. message Note { + // The block number in which the note was created. fixed32 block_num = 1; + + // The index of the note in the block. uint32 note_index = 2; + + // The ID of the note. digest.Digest note_id = 3; + + // The note's metadata. NoteMetadata metadata = 4; + + // The note's inclusion proof in the block. merkle.MerklePath merkle_path = 5; - // This field will be present when the note is public. - // details contain the `Note` in a serialized format. + + // Serialized details of the public note (empty for private notes). optional bytes details = 6; } +// Represents a proof of note's inclusion in a block. +// +// Does not include proof of the block's inclusion in the chain. message NoteInclusionInBlockProof { + // A unique identifier of the note which is a 32-byte commitment to the underlying note data. digest.Digest note_id = 1; + + // The block number in which the note was created. fixed32 block_num = 2; + + // The index of the note in the block. uint32 note_index_in_block = 3; + + // The note's inclusion proof in the block. merkle.MerklePath merkle_path = 4; } +// Represents proof of a note inclusion in the block. message NoteSyncRecord { + // The index of the note. uint32 note_index = 1; + + // A unique identifier of the note which is a 32-byte commitment to the underlying note data. digest.Digest note_id = 2; + + // The note's metadata. NoteMetadata metadata = 3; + + // The note's inclusion proof in the block. merkle.MerklePath merkle_path = 4; } +// Represents proof of notes inclusion in the block(s) and block(s) inclusion in the chain. message NoteAuthenticationInfo { // Proof of each note's inclusion in a block. repeated note.NoteInclusionInBlockProof note_proofs = 1; + // Proof of each block's inclusion in the chain. repeated block.BlockInclusionProof block_proofs = 2; } diff --git a/proto/requests.proto b/proto/requests.proto index 62ec750a1..b8e2e0817 100644 --- a/proto/requests.proto +++ b/proto/requests.proto @@ -3,9 +3,11 @@ package requests; import "account.proto"; import "digest.proto"; -import "note.proto"; +// Applies changes of a new block to the DB and in-memory data structures. message ApplyBlockRequest { + // Block data encoded using [winter_utils::Serializable] implementation for + // [miden_objects::block::Block]. bytes block = 1; } @@ -14,11 +16,13 @@ message CheckNullifiersByPrefixRequest { // Number of bits used for nullifier prefix. Currently the only supported value is 16. uint32 prefix_len = 1; // List of nullifiers to check. Each nullifier is specified by its prefix with length equal - // to prefix_len + // to `prefix_len`. repeated uint32 nullifiers = 2; } +// Get a list of proofs for given nullifier hashes, each proof as a sparse Merkle Tree. message CheckNullifiersRequest { + // List of nullifiers to return proofs for. repeated digest.Digest nullifiers = 1; } @@ -27,9 +31,7 @@ message CheckNullifiersRequest { // // The Merkle path is an MMR proof for the block's leaf, based on the current chain length. message GetBlockHeaderByNumberRequest { - // The block number of the target block. - // - // If not provided, means latest known block. + // The target block height, defaults to latest if not provided. optional uint32 block_num = 1; // Whether or not to return authentication data for the block header. optional bool include_mmr_proof = 2; @@ -74,33 +76,45 @@ message SyncNoteRequest { repeated fixed32 note_tags = 2; } +// Returns data required to prove the next block. message GetBlockInputsRequest { // ID of the account against which a transaction is executed. repeated account.AccountId account_ids = 1; - // Array of nullifiers for all notes consumed by a transaction. + // Set of nullifiers consumed by this transaction. repeated digest.Digest nullifiers = 2; // Array of note IDs to be checked for existence in the database. repeated digest.Digest unauthenticated_notes = 3; } +// Returns data required to validate a new transaction. message GetTransactionInputsRequest { + // ID of the account against which a transaction is executed. account.AccountId account_id = 1; + // Set of nullifiers consumed by this transaction. repeated digest.Digest nullifiers = 2; + // Set of unauthenticated notes to check for existence on-chain. + // + // These are notes which were not on-chain at the state the transaction was proven, + // but could by now be present. repeated digest.Digest unauthenticated_notes = 3; } +// Submits proven transaction to the Miden network. message SubmitProvenTransactionRequest { - // Transaction encoded using miden's native format + // Transaction encoded using [winter_utils::Serializable] implementation for + // [miden_objects::transaction::proven_tx::ProvenTransaction]. bytes transaction = 1; } +// Returns a list of notes matching the provided note IDs. message GetNotesByIdRequest { - // List of NoteId's to be queried from the database + // List of notes to be queried from the database. repeated digest.Digest note_ids = 1; } +// Returns a list of Note inclusion proofs for the specified Note IDs. message GetNoteAuthenticationInfoRequest { - // List of NoteId's to be queried from the database + // List of notes to be queried from the database. repeated digest.Digest note_ids = 1; } @@ -110,6 +124,7 @@ message GetAccountDetailsRequest { account.AccountId account_id = 1; } +// Retrieves block data by given block number. message GetBlockByNumberRequest { // The block number of the target block. fixed32 block_num = 1; @@ -126,9 +141,9 @@ message GetAccountStateDeltaRequest { fixed32 to_block_num = 3; } -// Request message to get account proofs. +// Returns the latest state proofs of the specified accounts. message GetAccountProofsRequest { - // Represents per-account requests where each account ID has its own list of + // Represents per-account requests where each account ID has its own list of // (storage_slot_index, map_keys) pairs. message AccountRequest { // The account ID for this request. @@ -154,7 +169,7 @@ message GetAccountProofsRequest { // requests are also ignored. False by default. optional bool include_headers = 2; - // Account code commitments corresponding to the last-known `AccountCode` for requested + // Account code commitments corresponding to the last-known `AccountCode` for requested // accounts. Responses will include only the ones that are not known to the caller. // These are not associated with a specific account but rather, they will be matched against // all requested accounts. diff --git a/proto/responses.proto b/proto/responses.proto index 530a5dc18..860368e67 100644 --- a/proto/responses.proto +++ b/proto/responses.proto @@ -10,63 +10,73 @@ import "note.proto"; import "smt.proto"; import "transaction.proto"; +// Represents the result of applying a block. message ApplyBlockResponse {} +// Represents the result of checking nullifiers. message CheckNullifiersResponse { // Each requested nullifier has its corresponding nullifier proof at the same position. repeated smt.SmtOpening proofs = 1; } +// Represents the result of checking nullifiers by prefix. message CheckNullifiersByPrefixResponse { // List of nullifiers matching the prefixes specified in the request. repeated NullifierUpdate nullifiers = 1; } +// Represents the result of getting a block header by block number. message GetBlockHeaderByNumberResponse { - // The requested block header + // The requested block header. block.BlockHeader block_header = 1; - // Merkle path to verify the block's inclusion in the MMR at the returned `chain_length` + // Merkle path to verify the block's inclusion in the MMR at the returned `chain_length`. optional merkle.MerklePath mmr_path = 2; - // Current chain length + // Current chain length. optional fixed32 chain_length = 3; } +// Represents a single nullifier update. message NullifierUpdate { + // Nullifier ID. digest.Digest nullifier = 1; + + // Block number. fixed32 block_num = 2; } +// Represents the result of syncing state request. message SyncStateResponse { - // Number of the latest block in the chain + // Number of the latest block in the chain. fixed32 chain_tip = 1; - // Block header of the block with the first note matching the specified criteria + // Block header of the block with the first note matching the specified criteria. block.BlockHeader block_header = 2; - // Data needed to update the partial MMR from `request.block_num + 1` to `response.block_header.block_num` + // Data needed to update the partial MMR from `request.block_num + 1` to `response.block_header.block_num`. mmr.MmrDelta mmr_delta = 3; - // List of account hashes updated after `request.block_num + 1` but not after `response.block_header.block_num` + // List of account hashes updated after `request.block_num + 1` but not after `response.block_header.block_num`. repeated account.AccountSummary accounts = 5; // List of transactions executed against requested accounts between `request.block_num + 1` and - // `response.block_header.block_num` + // `response.block_header.block_num`. repeated transaction.TransactionSummary transactions = 6; - // List of all notes together with the Merkle paths from `response.block_header.note_root` + // List of all notes together with the Merkle paths from `response.block_header.note_root`. repeated note.NoteSyncRecord notes = 7; - // List of nullifiers created between `request.block_num + 1` and `response.block_header.block_num` + // List of nullifiers created between `request.block_num + 1` and `response.block_header.block_num`. repeated NullifierUpdate nullifiers = 8; } +// Represents the result of syncing notes request. message SyncNoteResponse { - // Number of the latest block in the chain + // Number of the latest block in the chain. fixed32 chain_tip = 1; - // Block header of the block with the first note matching the specified criteria + // Block header of the block with the first note matching the specified criteria. block.BlockHeader block_header = 2; // Merkle path to verify the block's inclusion in the MMR at the returned `chain_tip`. @@ -75,90 +85,121 @@ message SyncNoteResponse { // an MMR of forest `chain_tip` with this path. merkle.MerklePath mmr_path = 3; - // List of all notes together with the Merkle paths from `response.block_header.note_root` + // List of all notes together with the Merkle paths from `response.block_header.note_root`. repeated note.NoteSyncRecord notes = 4; } -// An account returned as a response to the GetBlockInputs +// An account returned as a response to the `GetBlockInputs`. message AccountBlockInputRecord { + // The account ID. account.AccountId account_id = 1; + + // The latest account hash, zero hash if the account doesn't exist. digest.Digest account_hash = 2; + + // Merkle path to verify the account's inclusion in the MMR. merkle.MerklePath proof = 3; } -// A nullifier returned as a response to the GetBlockInputs +// A nullifier returned as a response to the `GetBlockInputs`. message NullifierBlockInputRecord { + // The nullifier ID. digest.Digest nullifier = 1; + + // Merkle path to verify the nullifier's inclusion in the MMR. smt.SmtOpening opening = 2; } +// Represents the result of getting block inputs. message GetBlockInputsResponse { - // The latest block header + // The latest block header. block.BlockHeader block_header = 1; - // Peaks of the above block's mmr, The `forest` value is equal to the block number + // Peaks of the above block's mmr, The `forest` value is equal to the block number. repeated digest.Digest mmr_peaks = 2; - // The hashes of the requested accounts and their authentication paths + // The hashes of the requested accounts and their authentication paths. repeated AccountBlockInputRecord account_states = 3; - // The requested nullifiers and their authentication paths + // The requested nullifiers and their authentication paths. repeated NullifierBlockInputRecord nullifiers = 4; - // The list of requested notes which were found in the database + // The list of requested notes which were found in the database. note.NoteAuthenticationInfo found_unauthenticated_notes = 5; } -// An account returned as a response to the GetTransactionInputs +// An account returned as a response to the `GetTransactionInputs`. message AccountTransactionInputRecord { + // The account ID. account.AccountId account_id = 1; + // The latest account hash, zero hash if the account doesn't exist. digest.Digest account_hash = 2; } -// A nullifier returned as a response to the GetTransactionInputs +// A nullifier returned as a response to the `GetTransactionInputs`. message NullifierTransactionInputRecord { + // The nullifier ID. digest.Digest nullifier = 1; + // The block at which the nullifier has been consumed, zero if not consumed. fixed32 block_num = 2; } +// Represents the result of getting transaction inputs. message GetTransactionInputsResponse { + // Account state proof. AccountTransactionInputRecord account_state = 1; + + // List of nullifiers that have been consumed. repeated NullifierTransactionInputRecord nullifiers = 2; + + // List of unauthenticated notes that were not found in the database. repeated digest.Digest found_unauthenticated_notes = 3; + + // The node's current block height. fixed32 block_height = 4; } +// Represents the result of submitting proven transaction. message SubmitProvenTransactionResponse { - // The node's current block height + // The node's current block height. fixed32 block_height = 1; } +// Represents the result of getting notes by IDs. message GetNotesByIdResponse { - // Lists Note's returned by the database + // Lists Note's returned by the database. repeated note.Note notes = 1; } +// Represents the result of getting note authentication info. message GetNoteAuthenticationInfoResponse { + // Proofs of note inclusions in blocks and block inclusions in chain. note.NoteAuthenticationInfo proofs = 1; } +// Represents the result of getting account details. message GetAccountDetailsResponse { - // Account info (with details for public accounts) + // Account info (with details for public accounts). account.AccountInfo details = 1; } +// Represents the result of getting block by number. message GetBlockByNumberResponse { - // The requested `Block` data encoded using miden native format + // The requested block data encoded using [winter_utils::Serializable] implementation for + // [miden_objects::block::Block]. optional bytes block = 1; } +// Represents the result of getting account state delta. message GetAccountStateDeltaResponse { - // The calculated `AccountStateDelta` encoded using miden native format + // The calculated account delta encoded using [winter_utils::Serializable] implementation + // for [miden_objects::accounts::delta::AccountDelta]. optional bytes delta = 1; } +// Represents the result of getting account proofs. message GetAccountProofsResponse { // Block number at which the state of the account was returned. fixed32 block_num = 1; @@ -166,6 +207,7 @@ message GetAccountProofsResponse { repeated AccountProofsResponse account_proofs = 2; } +// A single account proof returned as a response to the `GetAccountProofs`. message AccountProofsResponse { // Account ID. account.AccountId account_id = 1; @@ -177,6 +219,7 @@ message AccountProofsResponse { optional AccountStateHeader state_header = 4; } +// State header for public accounts. message AccountStateHeader { // Account header. account.AccountHeader header = 1; diff --git a/proto/rpc.proto b/proto/rpc.proto index 13934b2c4..bd5d359ad 100644 --- a/proto/rpc.proto +++ b/proto/rpc.proto @@ -6,15 +6,55 @@ import "requests.proto"; import "responses.proto"; service Api { + // Gets a list of proofs for given nullifier hashes, each proof as a sparse Merkle Tree. rpc CheckNullifiers(requests.CheckNullifiersRequest) returns (responses.CheckNullifiersResponse) {} + + // Returns a list of nullifiers that match the specified prefixes and are recorded in the node. rpc CheckNullifiersByPrefix(requests.CheckNullifiersByPrefixRequest) returns (responses.CheckNullifiersByPrefixResponse) {} + + // Returns the latest state of an account with the specified ID. rpc GetAccountDetails(requests.GetAccountDetailsRequest) returns (responses.GetAccountDetailsResponse) {} + + // Returns the latest state proofs of the specified accounts. rpc GetAccountProofs(requests.GetAccountProofsRequest) returns (responses.GetAccountProofsResponse) {} + + // Returns delta of the account states in the range from `from_block_num` (exclusive) to + // `to_block_num` (inclusive). rpc GetAccountStateDelta(requests.GetAccountStateDeltaRequest) returns (responses.GetAccountStateDeltaResponse) {} + + // Retrieves block data by given block number. rpc GetBlockByNumber(requests.GetBlockByNumberRequest) returns (responses.GetBlockByNumberResponse) {} + + // Retrieves block header by given block number. Optionally, it also returns the MMR path + // and current chain length to authenticate the block's inclusion. rpc GetBlockHeaderByNumber(requests.GetBlockHeaderByNumberRequest) returns (responses.GetBlockHeaderByNumberResponse) {} + + // Returns a list of notes matching the provided note IDs. rpc GetNotesById(requests.GetNotesByIdRequest) returns (responses.GetNotesByIdResponse) {} + + // Submits proven transaction to the Miden network. rpc SubmitProvenTransaction(requests.SubmitProvenTransactionRequest) returns (responses.SubmitProvenTransactionResponse) {} + + // Note synchronization request. + // + // Specifies note tags that client is interested in. The server will return the first block which + // contains a note matching `note_tags` or the chain tip. rpc SyncNotes(requests.SyncNoteRequest) returns (responses.SyncNoteResponse) {} + + // Returns info which can be used by the client to sync up to the latest state of the chain + // for the objects (accounts, notes, nullifiers) the client is interested in. + // + // This request returns the next block containing requested data. It also returns `chain_tip` + // which is the latest block number in the chain. Client is expected to repeat these requests + // in a loop until `response.block_header.block_num == response.chain_tip`, at which point + // the client is fully synchronized with the chain. + // + // Each request also returns info about new notes, nullifiers etc. created. It also returns + // Chain MMR delta that can be used to update the state of Chain MMR. This includes both chain + // MMR peaks and chain MMR nodes. + // + // For preserving some degree of privacy, note tags and nullifiers filters contain only high + // part of hashes. Thus, returned data contains excessive notes and nullifiers, client can make + // additional filtering of that data on its side. rpc SyncState(requests.SyncStateRequest) returns (responses.SyncStateResponse) {} } diff --git a/proto/smt.proto b/proto/smt.proto index d62634dd4..484508b1b 100644 --- a/proto/smt.proto +++ b/proto/smt.proto @@ -1,4 +1,4 @@ -// Message definitions related to Sparse Merkle Trees (SMT). +// Message definitions related to Sparse Merkle Trees (SMT) syntax = "proto3"; package smt; @@ -6,27 +6,40 @@ package smt; import "digest.proto"; import "merkle.proto"; -// An entry in a leaf. +// Represents a single SMT leaf entry. message SmtLeafEntry { + // The key of the entry. digest.Digest key = 1; + + // The value of the entry. digest.Digest value = 2; } +// Represents multiple leaf entries in an SMT. message SmtLeafEntries { + // The entries list. repeated SmtLeafEntry entries = 1; } // A leaf in an SMT, sitting at depth 64. A leaf can contain 0, 1 or multiple leaf entries. message SmtLeaf { oneof leaf { + // An empty leaf. uint64 empty = 1; + + // A single leaf entry. SmtLeafEntry single = 2; + + // Multiple leaf entries. SmtLeafEntries multiple = 3; } } // The opening of a leaf in an SMT. message SmtOpening { + // The merkle path to the leaf. merkle.MerklePath path = 1; + + // The leaf itself. SmtLeaf leaf = 2; } diff --git a/proto/store.proto b/proto/store.proto index b51198b96..5ed980aba 100644 --- a/proto/store.proto +++ b/proto/store.proto @@ -8,18 +8,64 @@ import "requests.proto"; import "responses.proto"; service Api { + // Applies changes of a new block to the DB and in-memory data structures. rpc ApplyBlock(requests.ApplyBlockRequest) returns (responses.ApplyBlockResponse) {} + + // Gets a list of proofs for given nullifier hashes, each proof as a sparse Merkle Tree. rpc CheckNullifiers(requests.CheckNullifiersRequest) returns (responses.CheckNullifiersResponse) {} + + // Returns a list of nullifiers that match the specified prefixes and are recorded in the node. rpc CheckNullifiersByPrefix(requests.CheckNullifiersByPrefixRequest) returns (responses.CheckNullifiersByPrefixResponse) {} + + // Returns the latest state of an account with the specified ID. rpc GetAccountDetails(requests.GetAccountDetailsRequest) returns (responses.GetAccountDetailsResponse) {} + + // Returns the latest state proofs of the specified accounts. rpc GetAccountProofs(requests.GetAccountProofsRequest) returns (responses.GetAccountProofsResponse) {} + + // Returns delta of the account states in the range from `from_block_num` (exclusive) to + // `to_block_num` (inclusive). rpc GetAccountStateDelta(requests.GetAccountStateDeltaRequest) returns (responses.GetAccountStateDeltaResponse) {} + + // Retrieves block data by given block number. rpc GetBlockByNumber(requests.GetBlockByNumberRequest) returns (responses.GetBlockByNumberResponse) {} + + // Retrieves block header by given block number. Optionally, it also returns the MMR path + // and current chain length to authenticate the block's inclusion. rpc GetBlockHeaderByNumber(requests.GetBlockHeaderByNumberRequest) returns (responses.GetBlockHeaderByNumberResponse) {} + + // Returns data required to prove the next block. rpc GetBlockInputs(requests.GetBlockInputsRequest) returns (responses.GetBlockInputsResponse) {} + + // Returns a list of Note inclusion proofs for the specified Note IDs. rpc GetNoteAuthenticationInfo(requests.GetNoteAuthenticationInfoRequest) returns (responses.GetNoteAuthenticationInfoResponse) {} + + // Returns a list of notes matching the provided note IDs. rpc GetNotesById(requests.GetNotesByIdRequest) returns (responses.GetNotesByIdResponse) {} + + // Returns data required to validate a new transaction. rpc GetTransactionInputs(requests.GetTransactionInputsRequest) returns (responses.GetTransactionInputsResponse) {} + + // Note synchronization request. + // + // Specifies note tags that client is interested in. The server will return the first block which + // contains a note matching `note_tags` or the chain tip. rpc SyncNotes(requests.SyncNoteRequest) returns (responses.SyncNoteResponse) {} + + // Returns info which can be used by the client to sync up to the latest state of the chain + // for the objects (accounts, notes, nullifiers) the client is interested in. + // + // This request returns the next block containing requested data. It also returns `chain_tip` + // which is the latest block number in the chain. Client is expected to repeat these requests + // in a loop until `response.block_header.block_num == response.chain_tip`, at which point + // the client is fully synchronized with the chain. + // + // Each request also returns info about new notes, nullifiers etc. created. It also returns + // Chain MMR delta that can be used to update the state of Chain MMR. This includes both chain + // MMR peaks and chain MMR nodes. + // + // For preserving some degree of privacy, note tags and nullifiers filters contain only high + // part of hashes. Thus, returned data contains excessive notes and nullifiers, client can make + // additional filtering of that data on its side. rpc SyncState(requests.SyncStateRequest) returns (responses.SyncStateResponse) {} } diff --git a/proto/transaction.proto b/proto/transaction.proto index 996b15bc4..9a9520c61 100644 --- a/proto/transaction.proto +++ b/proto/transaction.proto @@ -4,13 +4,21 @@ package transaction; import "account.proto"; import "digest.proto"; +// Represents a transaction ID. message TransactionId { + // The transaction ID. digest.Digest id = 1; } +// Represents a transaction summary. message TransactionSummary { + // A unique 32-byte identifier of a transaction. TransactionId transaction_id = 1; + + // The block number in which the transaction was executed. fixed32 block_num = 2; + + // The ID of the account affected by the transaction. account.AccountId account_id = 3; } From 61ea4e95709c83e18a0651ab1356c1b21847312a Mon Sep 17 00:00:00 2001 From: Mirko <48352201+Mirko-von-Leipzig@users.noreply.github.com> Date: Fri, 17 Jan 2025 09:52:50 +0200 Subject: [PATCH 45/50] chore: enable pedantic lints (#623) --- Cargo.toml | 14 ++++ bin/faucet/Cargo.toml | 3 + bin/faucet/src/client.rs | 4 +- bin/faucet/src/handlers.rs | 2 +- bin/faucet/src/main.rs | 2 +- bin/faucet/src/state.rs | 2 +- bin/faucet/src/store.rs | 6 +- bin/node/Cargo.toml | 3 + bin/node/src/commands/genesis/inputs.rs | 2 +- bin/node/src/commands/genesis/mod.rs | 26 +++---- bin/node/src/commands/init.rs | 14 ++-- bin/node/src/config.rs | 4 +- bin/node/src/main.rs | 6 +- crates/block-producer/Cargo.toml | 3 + .../block-producer/src/batch_builder/batch.rs | 22 +++--- .../block-producer/src/batch_builder/mod.rs | 16 ++-- .../block-producer/src/block_builder/mod.rs | 12 ++- .../src/block_builder/prover/block_witness.rs | 9 +-- .../src/block_builder/prover/tests.rs | 74 ++++++++++--------- .../block-producer/src/domain/transaction.rs | 10 +-- crates/block-producer/src/errors.rs | 26 ++++--- .../block-producer/src/mempool/batch_graph.rs | 18 ++--- .../block-producer/src/mempool/graph/mod.rs | 16 ++-- .../block-producer/src/mempool/graph/tests.rs | 4 +- .../mempool/inflight_state/account_state.rs | 26 +++---- .../src/mempool/inflight_state/mod.rs | 14 ++-- crates/block-producer/src/mempool/mod.rs | 34 ++++----- crates/block-producer/src/mempool/tests.rs | 23 ++++-- .../src/mempool/transaction_expiration.rs | 2 +- .../src/mempool/transaction_graph.rs | 16 ++-- crates/block-producer/src/server.rs | 6 +- crates/block-producer/src/store/mod.rs | 14 ++-- .../block-producer/src/test_utils/account.rs | 2 +- crates/block-producer/src/test_utils/batch.rs | 8 +- crates/block-producer/src/test_utils/block.rs | 3 + .../src/test_utils/proven_tx.rs | 6 ++ crates/block-producer/src/test_utils/store.rs | 6 +- crates/proto/Cargo.toml | 3 + crates/proto/build.rs | 12 +-- crates/proto/src/domain/accounts.rs | 4 +- crates/proto/src/domain/digest.rs | 28 +++---- crates/proto/src/domain/mod.rs | 4 +- crates/proto/src/domain/notes.rs | 2 +- crates/proto/src/generated/mod.rs | 2 +- crates/rpc-proto/Cargo.toml | 3 + crates/rpc-proto/build.rs | 6 +- crates/rpc-proto/src/lib.rs | 9 +-- crates/rpc/Cargo.toml | 3 + crates/rpc/src/server/api.rs | 4 +- crates/store/Cargo.toml | 3 + crates/store/src/config.rs | 2 +- crates/store/src/db/mod.rs | 2 +- crates/store/src/db/sql/mod.rs | 73 +++++++++--------- crates/store/src/db/sql/utils.rs | 10 ++- crates/store/src/db/tests.rs | 23 +++--- crates/store/src/genesis.rs | 2 +- crates/store/src/nullifier_tree.rs | 2 +- crates/store/src/server/api.rs | 12 +-- crates/store/src/state.rs | 18 ++--- crates/test-macro/Cargo.toml | 3 + crates/utils/Cargo.toml | 3 + crates/utils/src/formatting.rs | 6 +- crates/utils/src/version/mod.rs | 2 +- 63 files changed, 388 insertions(+), 311 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index eb13182a8..e3fa468fa 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -46,3 +46,17 @@ tokio-stream = { version = "0.1" } tonic = { version = "0.12" } tracing = { version = "0.1" } tracing-subscriber = { version = "0.3", features = ["fmt", "json", "env-filter"] } + +# Lints are set to warn for development, which are promoted to errors in CI. +[workspace.lints.clippy] +# Pedantic lints are set to a lower priority which allows lints in the group to be selectively enabled. +pedantic = { level = "warn", priority = -1 } +cast_possible_truncation = "allow" # Overly many instances especially regarding indices. +ignored_unit_patterns = "allow" # Stylistic choice. +large_types_passed_by_value = "allow" # Triggered by BlockHeader being Copy + 334 bytes. +missing_errors_doc = "allow" # TODO: fixup and enable this. +missing_panics_doc = "allow" # TODO: fixup and enable this. +module_name_repetitions = "allow" # Many triggers, and is a stylistic choice. +must_use_candidate = "allow" # This marks many fn's which isn't helpful. +should_panic_without_expect = "allow" # We don't care about the specific panic message. +# End of pedantic lints. diff --git a/bin/faucet/Cargo.toml b/bin/faucet/Cargo.toml index 095613fab..cce8844fd 100644 --- a/bin/faucet/Cargo.toml +++ b/bin/faucet/Cargo.toml @@ -11,6 +11,9 @@ authors.workspace = true homepage.workspace = true repository.workspace = true +[lints] +workspace = true + [dependencies] anyhow = "1.0" axum = { version = "0.7", features = ["tokio"] } diff --git a/bin/faucet/src/client.rs b/bin/faucet/src/client.rs index 8b05ddf4f..562ff483d 100644 --- a/bin/faucet/src/client.rs +++ b/bin/faucet/src/client.rs @@ -111,7 +111,7 @@ impl FaucetClient { let coin_seed: [u64; 4] = random(); let rng = RpoRandomCoin::new(coin_seed.map(Felt::new)); - Ok(Self { data_store, rpc_api, executor, id, rng }) + Ok(Self { rpc_api, executor, data_store, id, rng }) } /// Executes a mint transaction for the target account. @@ -137,7 +137,7 @@ impl FaucetClient { target_account_id, vec![asset.into()], note_type, - Default::default(), + Felt::default(), &mut self.rng, ) .context("Failed to create P2ID note")?; diff --git a/bin/faucet/src/handlers.rs b/bin/faucet/src/handlers.rs index e8d6e11bb..9037c2a24 100644 --- a/bin/faucet/src/handlers.rs +++ b/bin/faucet/src/handlers.rs @@ -86,7 +86,7 @@ pub async fn get_tokens( let block_height = client.prove_and_submit_transaction(executed_tx).await?; // Update data store with the new faucet state - client.data_store().update_faucet_state(faucet_account).await?; + client.data_store().update_faucet_state(faucet_account); let note_id: NoteId = created_note.id(); let note_details = diff --git a/bin/faucet/src/main.rs b/bin/faucet/src/main.rs index 06a3a5fe9..cb4053ead 100644 --- a/bin/faucet/src/main.rs +++ b/bin/faucet/src/main.rs @@ -203,7 +203,7 @@ mod static_resources { include!(concat!(env!("OUT_DIR"), "/generated.rs")); } -/// Generates [LongVersion] using the metadata generated by build.rs. +/// Generates [`LongVersion`] using the metadata generated by build.rs. fn long_version() -> LongVersion { // Use optional to allow for build script embedding failure. LongVersion { diff --git a/bin/faucet/src/state.rs b/bin/faucet/src/state.rs index fb6940812..6f70a0207 100644 --- a/bin/faucet/src/state.rs +++ b/bin/faucet/src/state.rs @@ -31,6 +31,6 @@ impl FaucetState { info!(target: COMPONENT, account_id = %id, "Faucet initialization successful"); - Ok(FaucetState { client, id, config, static_files }) + Ok(FaucetState { id, client, config, static_files }) } } diff --git a/bin/faucet/src/store.rs b/bin/faucet/src/store.rs index 02050765e..a4a3c831e 100644 --- a/bin/faucet/src/store.rs +++ b/bin/faucet/src/store.rs @@ -8,8 +8,6 @@ use miden_objects::{ }; use miden_tx::{DataStore, DataStoreError}; -use crate::errors::HandlerError; - pub struct FaucetDataStore { faucet_account: Mutex, /// Optional initial seed used for faucet account creation. @@ -42,10 +40,8 @@ impl FaucetDataStore { } /// Updates the stored faucet account with the new one. - pub async fn update_faucet_state(&self, new_faucet_state: Account) -> Result<(), HandlerError> { + pub fn update_faucet_state(&self, new_faucet_state: Account) { *self.faucet_account.lock().expect("Poisoned lock") = new_faucet_state; - - Ok(()) } } diff --git a/bin/node/Cargo.toml b/bin/node/Cargo.toml index eb4c1caa3..37eda9a12 100644 --- a/bin/node/Cargo.toml +++ b/bin/node/Cargo.toml @@ -11,6 +11,9 @@ authors.workspace = true homepage.workspace = true repository.workspace = true +[lints] +workspace = true + [features] tracing-forest = ["miden-node-block-producer/tracing-forest"] diff --git a/bin/node/src/commands/genesis/inputs.rs b/bin/node/src/commands/genesis/inputs.rs index 2f93d1d60..c0f5a765e 100644 --- a/bin/node/src/commands/genesis/inputs.rs +++ b/bin/node/src/commands/genesis/inputs.rs @@ -47,7 +47,7 @@ impl Default for GenesisInput { auth_scheme: AuthSchemeInput::RpoFalcon512, token_symbol: "POL".to_string(), decimals: 12, - max_supply: 1000000, + max_supply: 1_000_000, storage_mode: "public".to_string(), })]), } diff --git a/bin/node/src/commands/genesis/mod.rs b/bin/node/src/commands/genesis/mod.rs index d3c00148f..1875bb940 100644 --- a/bin/node/src/commands/genesis/mod.rs +++ b/bin/node/src/commands/genesis/mod.rs @@ -38,7 +38,7 @@ const DEFAULT_ACCOUNTS_DIR: &str = "accounts/"; /// This function returns a `Result` type. On successful creation of the genesis file, it returns /// `Ok(())`. If it fails at any point, due to issues like file existence checks or read/write /// operations, it returns an `Err` with a detailed error message. -pub fn make_genesis(inputs_path: &PathBuf, output_path: &PathBuf, force: &bool) -> Result<()> { +pub fn make_genesis(inputs_path: &PathBuf, output_path: &PathBuf, force: bool) -> Result<()> { let inputs_path = Path::new(inputs_path); let output_path = Path::new(output_path); @@ -63,14 +63,8 @@ pub fn make_genesis(inputs_path: &PathBuf, output_path: &PathBuf, force: &bool) return Err(anyhow!("Failed to open {} file.", inputs_path.display())); } - let parent_path = match output_path.parent() { - Some(path) => path, - None => { - return Err(anyhow!( - "There has been an error processing output_path: {}", - output_path.display() - )) - }, + let Some(parent_path) = output_path.parent() else { + anyhow::bail!("There has been an error processing output_path: {}", output_path.display()); }; let genesis_input: GenesisInput = load_config(inputs_path).map_err(|err| { @@ -97,7 +91,7 @@ pub fn make_genesis(inputs_path: &PathBuf, output_path: &PathBuf, force: &bool) fn create_accounts( accounts: &[AccountInput], accounts_path: impl AsRef, - force: &bool, + force: bool, ) -> Result> { if accounts_path.as_ref().try_exists()? { if !force { @@ -120,7 +114,7 @@ fn create_accounts( let (mut account_data, name) = match account { AccountInput::BasicFungibleFaucet(inputs) => { info!("Creating fungible faucet account..."); - let (auth_scheme, auth_secret_key) = gen_auth_keys(inputs.auth_scheme, &mut rng)?; + let (auth_scheme, auth_secret_key) = gen_auth_keys(inputs.auth_scheme, &mut rng); let storage_mode = inputs.storage_mode.as_str().try_into()?; let (account, account_seed) = create_basic_fungible_faucet( @@ -166,15 +160,15 @@ fn create_accounts( fn gen_auth_keys( auth_scheme_input: AuthSchemeInput, rng: &mut ChaCha20Rng, -) -> Result<(AuthScheme, AuthSecretKey)> { +) -> (AuthScheme, AuthSecretKey) { match auth_scheme_input { AuthSchemeInput::RpoFalcon512 => { let secret = SecretKey::with_rng(&mut get_rpo_random_coin(rng)); - Ok(( + ( AuthScheme::RpoFalcon512 { pub_key: secret.public_key() }, AuthSecretKey::RpoFalcon512(secret), - )) + ) }, } } @@ -217,7 +211,7 @@ mod tests { let genesis_dat_file_path = PathBuf::from(DEFAULT_GENESIS_FILE_PATH); // run make_genesis to generate genesis.dat and accounts folder and files - super::make_genesis(&genesis_inputs_file_path, &genesis_dat_file_path, &true).unwrap(); + super::make_genesis(&genesis_inputs_file_path, &genesis_dat_file_path, true).unwrap(); let a0_file_path = PathBuf::from("accounts/faucet.mac"); @@ -235,7 +229,7 @@ mod tests { let genesis_state = GenesisState::read_from_bytes(&genesis_file_contents).unwrap(); // build supposed genesis_state - let supposed_genesis_state = GenesisState::new(vec![a0.account], 1, 1672531200); + let supposed_genesis_state = GenesisState::new(vec![a0.account], 1, 1_672_531_200); // assert that both genesis_state(s) are eq assert_eq!(genesis_state, supposed_genesis_state); diff --git a/bin/node/src/commands/init.rs b/bin/node/src/commands/init.rs index 13ea35c45..7b36a70ff 100644 --- a/bin/node/src/commands/init.rs +++ b/bin/node/src/commands/init.rs @@ -1,4 +1,4 @@ -use std::{fs::File, io::Write, path::PathBuf}; +use std::{fs::File, io::Write, path::Path}; use anyhow::{anyhow, Result}; @@ -7,27 +7,27 @@ use crate::{commands::genesis::GenesisInput, config::NodeConfig}; // INIT // =================================================================================================== -pub fn init_config_files(config_file_path: PathBuf, genesis_file_path: PathBuf) -> Result<()> { +pub fn init_config_files(config_file_path: &Path, genesis_file_path: &Path) -> Result<()> { let config = NodeConfig::default(); let config_as_toml_string = toml::to_string(&config) .map_err(|err| anyhow!("Failed to serialize default config: {}", err))?; - write_string_in_file(config_as_toml_string, &config_file_path)?; + write_string_in_file(&config_as_toml_string, config_file_path)?; - println!("Config file successfully created at: {:?}", config_file_path); + println!("Config file successfully created at: {config_file_path:?}"); let genesis = GenesisInput::default(); let genesis_as_toml_string = toml::to_string(&genesis) .map_err(|err| anyhow!("Failed to serialize default config: {}", err))?; - write_string_in_file(genesis_as_toml_string, &genesis_file_path)?; + write_string_in_file(&genesis_as_toml_string, genesis_file_path)?; - println!("Genesis config file successfully created at: {:?}", genesis_file_path); + println!("Genesis config file successfully created at: {genesis_file_path:?}"); Ok(()) } -fn write_string_in_file(content: String, path: &PathBuf) -> Result<()> { +fn write_string_in_file(content: &str, path: &Path) -> Result<()> { let mut file_handle = File::options() .write(true) .create_new(true) diff --git a/bin/node/src/config.rs b/bin/node/src/config.rs index 8e44ad064..21a018920 100644 --- a/bin/node/src/config.rs +++ b/bin/node/src/config.rs @@ -13,14 +13,14 @@ pub struct NodeConfig { store: StoreConfig, } -/// A specialized variant of [RpcConfig] with redundant fields within [NodeConfig] removed. +/// A specialized variant of [`RpcConfig`] with redundant fields within [`NodeConfig`] removed. #[derive(Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Debug, Serialize, Deserialize)] #[serde(deny_unknown_fields)] struct NormalizedRpcConfig { endpoint: Endpoint, } -/// A specialized variant of [BlockProducerConfig] with redundant fields within [NodeConfig] +/// A specialized variant of [`BlockProducerConfig`] with redundant fields within [`NodeConfig`] /// removed. #[derive(Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Debug, Serialize, Deserialize)] #[serde(deny_unknown_fields)] diff --git a/bin/node/src/main.rs b/bin/node/src/main.rs index 1839d7484..998b54461 100644 --- a/bin/node/src/main.rs +++ b/bin/node/src/main.rs @@ -121,7 +121,7 @@ async fn main() -> anyhow::Result<()> { }, }, Command::MakeGenesis { output_path, force, inputs_path } => { - commands::make_genesis(inputs_path, output_path, force) + commands::make_genesis(inputs_path, output_path, *force) }, Command::Init { config_path, genesis_path } => { let current_dir = std::env::current_dir() @@ -130,12 +130,12 @@ async fn main() -> anyhow::Result<()> { let config = current_dir.join(config_path); let genesis = current_dir.join(genesis_path); - init_config_files(config, genesis) + init_config_files(&config, &genesis) }, } } -/// Generates [LongVersion] using the metadata generated by build.rs. +/// Generates [`LongVersion`] using the metadata generated by build.rs. fn long_version() -> LongVersion { LongVersion { version: env!("CARGO_PKG_VERSION"), diff --git a/crates/block-producer/Cargo.toml b/crates/block-producer/Cargo.toml index 52c898f8c..ee65ddd0f 100644 --- a/crates/block-producer/Cargo.toml +++ b/crates/block-producer/Cargo.toml @@ -11,6 +11,9 @@ authors.workspace = true homepage.workspace = true repository.workspace = true +[lints] +workspace = true + [features] tracing-forest = ["miden-node-utils/tracing-forest"] diff --git a/crates/block-producer/src/batch_builder/batch.rs b/crates/block-producer/src/batch_builder/batch.rs index 168463260..28cc3fa26 100644 --- a/crates/block-producer/src/batch_builder/batch.rs +++ b/crates/block-producer/src/batch_builder/batch.rs @@ -21,7 +21,7 @@ use crate::{errors::BuildBatchError, COMPONENT}; // BATCH ID // ================================================================================================ -/// Uniquely identifies a [TransactionBatch]. +/// Uniquely identifies a [`TransactionBatch`]. #[derive(Debug, Copy, Clone, Eq, Ord, PartialEq, PartialOrd)] pub struct BatchId(Blake3Digest<32>); @@ -138,7 +138,7 @@ impl TransactionBatch { Entry::Occupied(occupied) => { occupied.into_mut().merge_tx(tx).map_err(|source| { BuildBatchError::AccountUpdateError { account_id: tx.account_id(), source } - })? + })?; }, }; @@ -181,7 +181,7 @@ impl TransactionBatch { }, None => input_note.clone(), }; - input_notes.push(input_note) + input_notes.push(input_note); } } @@ -210,7 +210,7 @@ impl TransactionBatch { self.id } - /// Returns an iterator over (account_id, init_state_hash) tuples for accounts that were + /// Returns an iterator over (`account_id`, `init_state_hash`) tuples for accounts that were /// modified in this transaction batch. #[cfg(test)] pub fn account_initial_states(&self) -> impl Iterator + '_ { @@ -219,8 +219,8 @@ impl TransactionBatch { .map(|(&account_id, update)| (account_id, update.init_state)) } - /// Returns an iterator over (account_id, details, new_state_hash) tuples for accounts that were - /// modified in this transaction batch. + /// Returns an iterator over (`account_id`, details, `new_state_hash`) tuples for accounts that + /// were modified in this transaction batch. pub fn updated_accounts(&self) -> impl Iterator + '_ { self.updated_accounts.iter() } @@ -330,7 +330,7 @@ mod tests { match OutputNoteTracker::new(txs.iter()) { Err(BuildBatchError::DuplicateOutputNote(note_id)) => { - assert_eq!(note_id, duplicate_output_note.id()) + assert_eq!(note_id, duplicate_output_note.id()); }, res => panic!("Unexpected result: {res:?}"), } @@ -364,9 +364,9 @@ mod tests { let mut txs = mock_proven_txs(); let duplicate_note = mock_note(5); txs.push(mock_proven_tx(4, vec![duplicate_note.clone()], vec![mock_output_note(9)])); - match TransactionBatch::new(&txs, Default::default()) { + match TransactionBatch::new(&txs, NoteAuthenticationInfo::default()) { Err(BuildBatchError::DuplicateUnauthenticatedNote(note_id)) => { - assert_eq!(note_id, duplicate_note.id()) + assert_eq!(note_id, duplicate_note.id()); }, res => panic!("Unexpected result: {res:?}"), } @@ -382,7 +382,7 @@ mod tests { vec![mock_output_note(9), mock_output_note(10)], )); - let batch = TransactionBatch::new(&txs, Default::default()).unwrap(); + let batch = TransactionBatch::new(&txs, NoteAuthenticationInfo::default()).unwrap(); // One of the unauthenticated notes must be removed from the batch due to the consumption // of the corresponding output note @@ -424,7 +424,7 @@ mod tests { )]); let found_unauthenticated_notes = NoteAuthenticationInfo { note_proofs: found_unauthenticated_notes, - block_proofs: Default::default(), + block_proofs: Vec::default(), }; let batch = TransactionBatch::new(&txs, found_unauthenticated_notes).unwrap(); diff --git a/crates/block-producer/src/batch_builder/mod.rs b/crates/block-producer/src/batch_builder/mod.rs index c504e4c7a..58283d02b 100644 --- a/crates/block-producer/src/batch_builder/mod.rs +++ b/crates/block-producer/src/batch_builder/mod.rs @@ -20,7 +20,7 @@ use crate::errors::BuildBatchError; // BATCH BUILDER // ================================================================================================ -/// Builds [TransactionBatch] from sets of transactions. +/// Builds [`TransactionBatch`] from sets of transactions. /// /// Transaction sets are pulled from the [Mempool] at a configurable interval, and passed to a pool /// of provers for proof generation. Proving is currently unimplemented and is instead simulated via @@ -50,7 +50,7 @@ impl Default for BatchBuilder { } impl BatchBuilder { - /// Starts the [BatchBuilder], creating and proving batches at the configured interval. + /// Starts the [`BatchBuilder`], creating and proving batches at the configured interval. /// /// A pool of batch-proving workers is spawned, which are fed new batch jobs periodically. /// A batch is skipped if there are no available workers, or if there are no transactions @@ -109,7 +109,7 @@ type BatchResult = Result; /// Represents a pool of batch provers. /// -/// Effectively a wrapper around tokio's JoinSet that remains pending if the set is empty, +/// Effectively a wrapper around tokio's `JoinSet` that remains pending if the set is empty, /// instead of returning None. struct WorkerPool { in_progress: JoinSet, @@ -139,13 +139,13 @@ impl WorkerPool { capacity, store, in_progress: JoinSet::default(), - task_map: Default::default(), + task_map: Vec::default(), } } /// Returns the next batch proof result. /// - /// Will return pending if there are no jobs in progress (unlike tokio's [JoinSet::join_next] + /// Will return pending if there are no jobs in progress (unlike tokio's [`JoinSet::join_next`] /// which returns an option). async fn join_next(&mut self) -> BatchResult { if self.in_progress.is_empty() { @@ -193,7 +193,7 @@ impl WorkerPool { /// # Errors /// /// Returns an error if no workers are available which can be checked using - /// [has_capacity](Self::has_capacity). + /// [`has_capacity`](Self::has_capacity). fn spawn( &mut self, id: BatchId, @@ -221,7 +221,9 @@ impl WorkerPool { let inputs = store .get_batch_inputs( - transactions.iter().flat_map(|tx| tx.unauthenticated_notes()), + transactions + .iter() + .flat_map(AuthenticatedTransaction::unauthenticated_notes), ) .await .map_err(|err| (id, BuildBatchError::FetchBatchInputsFailed(err)))?; diff --git a/crates/block-producer/src/block_builder/mod.rs b/crates/block-producer/src/block_builder/mod.rs index ff79dd603..110902379 100644 --- a/crates/block-producer/src/block_builder/mod.rs +++ b/crates/block-producer/src/block_builder/mod.rs @@ -5,7 +5,7 @@ use miden_objects::{ accounts::AccountId, block::Block, notes::{NoteHeader, Nullifier}, - transaction::InputNoteCommitment, + transaction::{InputNoteCommitment, OutputNote}, }; use rand::Rng; use tokio::time::Duration; @@ -48,7 +48,7 @@ impl BlockBuilder { store, } } - /// Starts the [BlockBuilder], infinitely producing blocks at the configured interval. + /// Starts the [`BlockBuilder`], infinitely producing blocks at the configured interval. /// /// Block production is sequential and consists of /// @@ -98,7 +98,7 @@ impl BlockBuilder { info!( target: COMPONENT, num_batches = batches.len(), - batches = %format_array(batches.iter().map(|batch| batch.id())), + batches = %format_array(batches.iter().map(TransactionBatch::id)), ); let updated_account_set: BTreeSet = batches @@ -114,10 +114,8 @@ impl BlockBuilder { batches.iter().flat_map(TransactionBatch::produced_nullifiers).collect(); // Populate set of output notes from all batches - let output_notes_set: BTreeSet<_> = output_notes - .iter() - .flat_map(|batch| batch.iter().map(|note| note.id())) - .collect(); + let output_notes_set: BTreeSet<_> = + output_notes.iter().flat_map(|batch| batch.iter().map(OutputNote::id)).collect(); // Build a set of unauthenticated input notes for this block which do not have a matching // output note produced in this block diff --git a/crates/block-producer/src/block_builder/prover/block_witness.rs b/crates/block-producer/src/block_builder/prover/block_witness.rs index 57205244c..b4ff493b0 100644 --- a/crates/block-producer/src/block_builder/prover/block_witness.rs +++ b/crates/block-producer/src/block_builder/prover/block_witness.rs @@ -11,7 +11,7 @@ use miden_objects::{ }; use crate::{ - batch_builder::{batch::AccountUpdate, TransactionBatch}, + batch_builder::batch::{AccountUpdate, TransactionBatch}, block::BlockInputs, errors::{BlockProverError, BuildBlockError}, }; @@ -23,7 +23,7 @@ use crate::{ #[derive(Debug, PartialEq)] pub struct BlockWitness { pub(super) updated_accounts: Vec<(AccountId, AccountUpdateWitness)>, - /// (batch_index, created_notes_root) for batches that contain notes + /// (`batch_index`, `created_notes_root`) for batches that contain notes pub(super) batch_created_notes_roots: BTreeMap, pub(super) produced_nullifiers: BTreeMap, pub(super) chain_peaks: MmrPeaks, @@ -162,7 +162,7 @@ impl BlockWitness { block_inputs.nullifiers.keys().copied().collect(); let produced_nullifiers_from_batches: BTreeSet = - batches.iter().flat_map(|batch| batch.produced_nullifiers()).collect(); + batches.iter().flat_map(TransactionBatch::produced_nullifiers).collect(); if produced_nullifiers_from_store == produced_nullifiers_from_batches { Ok(()) @@ -218,8 +218,7 @@ impl BlockWitness { let empty_root = EmptySubtreeRoots::entry(BLOCK_NOTE_TREE_DEPTH, 0); advice_stack.extend(*empty_root); - for (batch_index, batch_created_notes_root) in self.batch_created_notes_roots.iter() - { + for (batch_index, batch_created_notes_root) in &self.batch_created_notes_roots { advice_stack.extend(batch_created_notes_root.iter()); let batch_index = Felt::try_from(*batch_index as u64) diff --git a/crates/block-producer/src/block_builder/prover/tests.rs b/crates/block-producer/src/block_builder/prover/tests.rs index 927493975..d14ec2658 100644 --- a/crates/block-producer/src/block_builder/prover/tests.rs +++ b/crates/block-producer/src/block_builder/prover/tests.rs @@ -1,6 +1,7 @@ use std::{collections::BTreeMap, iter}; use assert_matches::assert_matches; +use miden_node_proto::domain::notes::NoteAuthenticationInfo; use miden_objects::{ accounts::{ delta::AccountUpdateDetails, AccountId, AccountIdVersion, AccountStorageMode, AccountType, @@ -20,7 +21,7 @@ use miden_objects::{ use self::block_witness::AccountUpdateWitness; use super::*; use crate::{ - batch_builder::TransactionBatch, + batch_builder::batch::TransactionBatch, block::{AccountWitness, BlockInputs}, test_utils::{ block::{build_actual_block_header, build_expected_block_header, MockBlockBuilder}, @@ -57,7 +58,7 @@ fn block_witness_validation_inconsistent_account_ids() { ); let block_inputs_from_store: BlockInputs = { - let block_header = BlockHeader::mock(0, None, None, &[], Default::default()); + let block_header = BlockHeader::mock(0, None, None, &[], Digest::default()); let chain_peaks = MmrPeaks::new(0, Vec::new()).unwrap(); let accounts = BTreeMap::from_iter(vec![ @@ -69,8 +70,8 @@ fn block_witness_validation_inconsistent_account_ids() { block_header, chain_peaks, accounts, - nullifiers: Default::default(), - found_unauthenticated_notes: Default::default(), + nullifiers: BTreeMap::default(), + found_unauthenticated_notes: NoteAuthenticationInfo::default(), } }; @@ -83,7 +84,7 @@ fn block_witness_validation_inconsistent_account_ids() { ) .build(); - TransactionBatch::new([&tx], Default::default()).unwrap() + TransactionBatch::new([&tx], NoteAuthenticationInfo::default()).unwrap() }; let batch_2 = { @@ -94,7 +95,7 @@ fn block_witness_validation_inconsistent_account_ids() { ) .build(); - TransactionBatch::new([&tx], Default::default()).unwrap() + TransactionBatch::new([&tx], NoteAuthenticationInfo::default()).unwrap() }; vec![batch_1, batch_2] @@ -121,7 +122,7 @@ fn block_witness_validation_inconsistent_account_hashes() { Digest::new([Felt::new(4u64), Felt::new(3u64), Felt::new(2u64), Felt::new(1u64)]); let block_inputs_from_store: BlockInputs = { - let block_header = BlockHeader::mock(0, None, None, &[], Default::default()); + let block_header = BlockHeader::mock(0, None, None, &[], Digest::default()); let chain_peaks = MmrPeaks::new(0, Vec::new()).unwrap(); let accounts = BTreeMap::from_iter(vec![ @@ -129,18 +130,18 @@ fn block_witness_validation_inconsistent_account_hashes() { account_id_1, AccountWitness { hash: account_1_hash_store, - proof: Default::default(), + proof: MerklePath::default(), }, ), - (account_id_2, Default::default()), + (account_id_2, AccountWitness::default()), ]); BlockInputs { block_header, chain_peaks, accounts, - nullifiers: Default::default(), - found_unauthenticated_notes: Default::default(), + nullifiers: BTreeMap::default(), + found_unauthenticated_notes: NoteAuthenticationInfo::default(), } }; @@ -152,7 +153,7 @@ fn block_witness_validation_inconsistent_account_hashes() { Digest::default(), ) .build()], - Default::default(), + NoteAuthenticationInfo::default(), ) .unwrap(); let batch_2 = TransactionBatch::new( @@ -162,7 +163,7 @@ fn block_witness_validation_inconsistent_account_hashes() { Digest::default(), ) .build()], - Default::default(), + NoteAuthenticationInfo::default(), ) .unwrap(); @@ -224,7 +225,7 @@ fn block_witness_multiple_batches_per_account() { )]); let block_inputs_from_store: BlockInputs = { - let block_header = BlockHeader::mock(0, None, None, &[], Default::default()); + let block_header = BlockHeader::mock(0, None, None, &[], Digest::default()); let chain_peaks = MmrPeaks::new(0, Vec::new()).unwrap(); let x_witness = AccountWitness { @@ -241,14 +242,18 @@ fn block_witness_multiple_batches_per_account() { block_header, chain_peaks, accounts, - nullifiers: Default::default(), - found_unauthenticated_notes: Default::default(), + nullifiers: BTreeMap::default(), + found_unauthenticated_notes: NoteAuthenticationInfo::default(), } }; let batches = { - let batch_1 = TransactionBatch::new([&x_txs[0], &y_txs[1]], Default::default()).unwrap(); - let batch_2 = TransactionBatch::new([&y_txs[0], &x_txs[1]], Default::default()).unwrap(); + let batch_1 = + TransactionBatch::new([&x_txs[0], &y_txs[1]], NoteAuthenticationInfo::default()) + .unwrap(); + let batch_2 = + TransactionBatch::new([&y_txs[0], &x_txs[1]], NoteAuthenticationInfo::default()) + .unwrap(); vec![batch_1, batch_2] }; @@ -369,8 +374,8 @@ async fn compute_account_root_success() { }) .collect(); - let batch_1 = TransactionBatch::new(&txs[..2], Default::default()).unwrap(); - let batch_2 = TransactionBatch::new(&txs[2..], Default::default()).unwrap(); + let batch_1 = TransactionBatch::new(&txs[..2], NoteAuthenticationInfo::default()).unwrap(); + let batch_2 = TransactionBatch::new(&txs[2..], NoteAuthenticationInfo::default()).unwrap(); vec![batch_1, batch_2] }; @@ -538,7 +543,7 @@ async fn compute_note_root_empty_notes_success() { .unwrap(); let batches: Vec = { - let batch = TransactionBatch::new(vec![], Default::default()).unwrap(); + let batch = TransactionBatch::new(vec![], NoteAuthenticationInfo::default()).unwrap(); vec![batch] }; @@ -627,8 +632,8 @@ async fn compute_note_root_success() { }) .collect(); - let batch_1 = TransactionBatch::new(&txs[..2], Default::default()).unwrap(); - let batch_2 = TransactionBatch::new(&txs[2..], Default::default()).unwrap(); + let batch_1 = TransactionBatch::new(&txs[..2], NoteAuthenticationInfo::default()).unwrap(); + let batch_2 = TransactionBatch::new(&txs[2..], NoteAuthenticationInfo::default()).unwrap(); vec![batch_1, batch_2] }; @@ -644,6 +649,7 @@ async fn compute_note_root_success() { // The current logic is hardcoded to a depth of 6 // Specifically, we assume the block has up to 2^6 batches, and each batch up to 2^10 created // notes, where each note is stored at depth 10 in the batch tree. + #[allow(clippy::items_after_statements, reason = "assert belongs to this section")] const _: () = assert!(BLOCK_NOTE_TREE_DEPTH - BATCH_NOTE_TREE_DEPTH == 6); // The first 2 txs were put in the first batch; the 3rd was put in the second @@ -684,13 +690,13 @@ fn block_witness_validation_inconsistent_nullifiers() { let batch_1 = { let tx = MockProvenTxBuilder::with_account_index(0).nullifiers_range(0..1).build(); - TransactionBatch::new([&tx], Default::default()).unwrap() + TransactionBatch::new([&tx], NoteAuthenticationInfo::default()).unwrap() }; let batch_2 = { let tx = MockProvenTxBuilder::with_account_index(1).nullifiers_range(1..2).build(); - TransactionBatch::new([&tx], Default::default()).unwrap() + TransactionBatch::new([&tx], NoteAuthenticationInfo::default()).unwrap() }; vec![batch_1, batch_2] @@ -702,12 +708,12 @@ fn block_witness_validation_inconsistent_nullifiers() { Nullifier::from([101_u32.into(), 102_u32.into(), 103_u32.into(), 104_u32.into()]); let block_inputs_from_store: BlockInputs = { - let block_header = BlockHeader::mock(0, None, None, &[], Default::default()); + let block_header = BlockHeader::mock(0, None, None, &[], Digest::default()); let chain_peaks = MmrPeaks::new(0, Vec::new()).unwrap(); let accounts = batches .iter() - .flat_map(|batch| batch.account_initial_states()) + .flat_map(TransactionBatch::account_initial_states) .map(|(account_id, hash)| { (account_id, AccountWitness { hash, proof: MerklePath::default() }) }) @@ -741,7 +747,7 @@ fn block_witness_validation_inconsistent_nullifiers() { chain_peaks, accounts, nullifiers, - found_unauthenticated_notes: Default::default(), + found_unauthenticated_notes: NoteAuthenticationInfo::default(), } }; @@ -763,13 +769,13 @@ async fn compute_nullifier_root_empty_success() { let batch_1 = { let tx = MockProvenTxBuilder::with_account_index(0).build(); - TransactionBatch::new([&tx], Default::default()).unwrap() + TransactionBatch::new([&tx], NoteAuthenticationInfo::default()).unwrap() }; let batch_2 = { let tx = MockProvenTxBuilder::with_account_index(1).build(); - TransactionBatch::new([&tx], Default::default()).unwrap() + TransactionBatch::new([&tx], NoteAuthenticationInfo::default()).unwrap() }; vec![batch_1, batch_2] @@ -777,7 +783,7 @@ async fn compute_nullifier_root_empty_success() { let account_ids: Vec = batches .iter() - .flat_map(|batch| batch.account_initial_states()) + .flat_map(TransactionBatch::account_initial_states) .map(|(account_id, _)| account_id) .collect(); @@ -817,13 +823,13 @@ async fn compute_nullifier_root_success() { let batch_1 = { let tx = MockProvenTxBuilder::with_account_index(0).nullifiers_range(0..1).build(); - TransactionBatch::new([&tx], Default::default()).unwrap() + TransactionBatch::new([&tx], NoteAuthenticationInfo::default()).unwrap() }; let batch_2 = { let tx = MockProvenTxBuilder::with_account_index(1).nullifiers_range(1..2).build(); - TransactionBatch::new([&tx], Default::default()).unwrap() + TransactionBatch::new([&tx], NoteAuthenticationInfo::default()).unwrap() }; vec![batch_1, batch_2] @@ -831,7 +837,7 @@ async fn compute_nullifier_root_success() { let account_ids: Vec = batches .iter() - .flat_map(|batch| batch.account_initial_states()) + .flat_map(TransactionBatch::account_initial_states) .map(|(account_id, _)| account_id) .collect(); diff --git a/crates/block-producer/src/domain/transaction.rs b/crates/block-producer/src/domain/transaction.rs index c36302c5f..a8567f7fc 100644 --- a/crates/block-producer/src/domain/transaction.rs +++ b/crates/block-producer/src/domain/transaction.rs @@ -50,7 +50,7 @@ impl AuthenticatedTransaction { ) -> Result { let nullifiers_already_spent = tx .get_nullifiers() - .filter(|nullifier| inputs.nullifiers.get(nullifier).cloned().flatten().is_some()) + .filter(|nullifier| inputs.nullifiers.get(nullifier).copied().flatten().is_some()) .collect::>(); if !nullifiers_already_spent.is_empty() { return Err(VerifyTxError::InputNotesAlreadyConsumed(nullifiers_already_spent)); @@ -89,7 +89,7 @@ impl AuthenticatedTransaction { } pub fn output_notes(&self) -> impl Iterator + '_ { - self.inner.output_notes().iter().map(|note| note.id()) + self.inner.output_notes().iter().map(miden_objects::transaction::OutputNote::id) } pub fn output_note_count(&self) -> usize { @@ -105,7 +105,7 @@ impl AuthenticatedTransaction { pub fn unauthenticated_notes(&self) -> impl Iterator + '_ { self.inner .get_unauthenticated_notes() - .cloned() + .copied() .map(|header| header.id()) .filter(|note_id| !self.notes_authenticated_by_store.contains(note_id)) } @@ -134,8 +134,8 @@ impl AuthenticatedTransaction { account_id: inner.account_id(), account_hash: store_account_state, nullifiers: inner.get_nullifiers().map(|nullifier| (nullifier, None)).collect(), - found_unauthenticated_notes: Default::default(), - current_block_height: Default::default(), + found_unauthenticated_notes: BTreeSet::default(), + current_block_height: 0, }; // SAFETY: nullifiers were set to None aka are definitely unspent. Self::new(inner, inputs).unwrap() diff --git a/crates/block-producer/src/errors.rs b/crates/block-producer/src/errors.rs index 2dd8a4028..9e30d2e9c 100644 --- a/crates/block-producer/src/errors.rs +++ b/crates/block-producer/src/errors.rs @@ -102,20 +102,22 @@ pub enum AddTransactionError { impl From for tonic::Status { fn from(value: AddTransactionError) -> Self { - use AddTransactionError::*; match value { - VerificationFailed(VerifyTxError::InputNotesAlreadyConsumed(_)) - | VerificationFailed(VerifyTxError::UnauthenticatedNotesNotFound(_)) - | VerificationFailed(VerifyTxError::OutputNotesAlreadyExist(_)) - | VerificationFailed(VerifyTxError::IncorrectAccountInitialHash { .. }) - | VerificationFailed(VerifyTxError::InvalidTransactionProof(_)) - | Expired { .. } - | TransactionDeserializationFailed(_) => Self::invalid_argument(value.to_string()), + AddTransactionError::VerificationFailed( + VerifyTxError::InputNotesAlreadyConsumed(_) + | VerifyTxError::UnauthenticatedNotesNotFound(_) + | VerifyTxError::OutputNotesAlreadyExist(_) + | VerifyTxError::IncorrectAccountInitialHash { .. } + | VerifyTxError::InvalidTransactionProof(_), + ) + | AddTransactionError::Expired { .. } + | AddTransactionError::TransactionDeserializationFailed(_) => { + Self::invalid_argument(value.to_string()) + }, // Internal errors which should not be communicated to the user. - VerificationFailed(VerifyTxError::StoreConnectionFailed(_)) | StaleInputs { .. } => { - Self::internal("Internal error") - }, + AddTransactionError::VerificationFailed(VerifyTxError::StoreConnectionFailed(_)) + | AddTransactionError::StaleInputs { .. } => Self::internal("Internal error"), } } } @@ -207,7 +209,7 @@ pub enum BuildBlockError { // Store errors // ================================================================================================= -/// Errors returned by the [StoreClient](crate::store::StoreClient). +/// Errors returned by the [`StoreClient`](crate::store::StoreClient). #[derive(Debug, Error)] pub enum StoreError { #[error("gRPC client error")] diff --git a/crates/block-producer/src/mempool/batch_graph.rs b/crates/block-producer/src/mempool/batch_graph.rs index ef7822150..73948f46e 100644 --- a/crates/block-producer/src/mempool/batch_graph.rs +++ b/crates/block-producer/src/mempool/batch_graph.rs @@ -111,9 +111,9 @@ impl BatchGraph { // Reverse lookup parent batch IDs. Take care to allow for parent transactions within this // batch i.e. internal dependencies. - transactions.iter().for_each(|tx| { + for tx in &transactions { parents.remove(tx); - }); + } let parent_batches = parents .into_iter() .map(|tx| { @@ -175,7 +175,7 @@ impl BatchGraph { /// /// Returns all removed batches and their transactions. /// - /// Unlike [remove_batches](Self::remove_batches), this has no error condition as batches are + /// Unlike [`remove_batches`](Self::remove_batches), this has no error condition as batches are /// derived internally. pub fn remove_batches_with_transactions<'a>( &mut self, @@ -207,14 +207,14 @@ impl BatchGraph { /// The last point implies that batches should be removed in block order. pub fn prune_committed( &mut self, - batch_ids: BTreeSet, + batch_ids: &BTreeSet, ) -> Result, GraphError> { // This clone could be elided by moving this call to the end. This would lose the atomic // property of this method though its unclear what value (if any) that has. self.inner.prune_processed(batch_ids.clone())?; let mut transactions = Vec::new(); - for batch_id in &batch_ids { + for batch_id in batch_ids { transactions.extend(self.batches.remove(batch_id).into_iter().flatten()); } @@ -294,8 +294,8 @@ mod tests { let mut uut = BatchGraph::default(); - uut.insert(vec![tx_dup], Default::default()).unwrap(); - let err = uut.insert(vec![tx_dup, tx_non_dup], Default::default()).unwrap_err(); + uut.insert(vec![tx_dup], BTreeSet::default()).unwrap(); + let err = uut.insert(vec![tx_dup, tx_non_dup], BTreeSet::default()).unwrap_err(); let expected = BatchInsertError::DuplicateTransactions([tx_dup].into()); assert_eq!(err, expected); @@ -339,10 +339,10 @@ mod tests { let disjoint_batch_txs = (0..5).map(|_| rng.draw_tx_id()).collect(); let mut uut = BatchGraph::default(); - let parent_batch_id = uut.insert(parent_batch_txs.clone(), Default::default()).unwrap(); + let parent_batch_id = uut.insert(parent_batch_txs.clone(), BTreeSet::default()).unwrap(); let child_batch_id = uut.insert(child_batch_txs.clone(), [parent_batch_txs[0]].into()).unwrap(); - uut.insert(disjoint_batch_txs, Default::default()).unwrap(); + uut.insert(disjoint_batch_txs, BTreeSet::default()).unwrap(); let result = uut.remove_batches([parent_batch_id].into()).unwrap(); let expected = diff --git a/crates/block-producer/src/mempool/graph/mod.rs b/crates/block-producer/src/mempool/graph/mod.rs index 5b0921d49..55442deed 100644 --- a/crates/block-producer/src/mempool/graph/mod.rs +++ b/crates/block-producer/src/mempool/graph/mod.rs @@ -108,12 +108,12 @@ pub enum GraphError { impl Default for DependencyGraph { fn default() -> Self { Self { - vertices: Default::default(), - pending: Default::default(), - parents: Default::default(), - children: Default::default(), - roots: Default::default(), - processed: Default::default(), + vertices: BTreeMap::default(), + pending: BTreeSet::default(), + parents: BTreeMap::default(), + children: BTreeMap::default(), + roots: BTreeSet::default(), + processed: BTreeSet::default(), } } } @@ -146,7 +146,7 @@ impl DependencyGraph { } self.pending.insert(key); self.parents.insert(key, parents); - self.children.insert(key, Default::default()); + self.children.insert(key, BTreeSet::default()); Ok(()) } @@ -260,7 +260,7 @@ impl DependencyGraph { // No parent may be left dangling i.e. all parents must be part of this prune set. let dangling = keys .iter() - .flat_map(|key| self.parents.get(key)) + .filter_map(|key| self.parents.get(key)) .flatten() .filter(|parent| !keys.contains(parent)) .copied() diff --git a/crates/block-producer/src/mempool/graph/tests.rs b/crates/block-producer/src/mempool/graph/tests.rs index acee736f2..af8acdc18 100644 --- a/crates/block-producer/src/mempool/graph/tests.rs +++ b/crates/block-producer/src/mempool/graph/tests.rs @@ -10,7 +10,7 @@ type TestGraph = DependencyGraph; impl TestGraph { /// Alias for inserting a node with no parents. fn insert_with_no_parents(&mut self, node: u32) -> Result<(), GraphError> { - self.insert_with_parents(node, Default::default()) + self.insert_with_parents(node, BTreeSet::default()) } /// Alias for inserting a node with a single parent. @@ -41,7 +41,7 @@ impl TestGraph { self.promote(self.pending.clone()).unwrap(); } - /// Calls process_root until all nodes have been processed. + /// Calls `process_root` until all nodes have been processed. fn process_all(&mut self) { while let Some(root) = self.roots().first().copied() { // SAFETY: this is definitely a root since we just took it from there :) diff --git a/crates/block-producer/src/mempool/inflight_state/account_state.rs b/crates/block-producer/src/mempool/inflight_state/account_state.rs index dc79a339a..560241185 100644 --- a/crates/block-producer/src/mempool/inflight_state/account_state.rs +++ b/crates/block-producer/src/mempool/inflight_state/account_state.rs @@ -99,7 +99,7 @@ impl InflightAccountState { self.emptiness() } - /// This is essentially `is_empty` with the additional benefit that [AccountStatus] is marked + /// This is essentially `is_empty` with the additional benefit that [`AccountStatus`] is marked /// as `#[must_use]`, forcing callers to handle empty accounts (which should be pruned). fn emptiness(&self) -> AccountStatus { if self.states.is_empty() { @@ -115,21 +115,21 @@ impl InflightAccountState { } } -/// Describes the emptiness of an [InflightAccountState]. +/// Describes the emptiness of an [`InflightAccountState`]. /// /// Is marked as `#[must_use]` so that callers handle prune empty accounts. #[must_use] #[derive(Clone, Copy, PartialEq, Eq)] pub enum AccountStatus { - /// [InflightAccountState] contains no state and should be pruned. + /// [`InflightAccountState`] contains no state and should be pruned. Empty, - /// [InflightAccountState] contains state and should be kept. + /// [`InflightAccountState`] contains state and should be kept. NonEmpty, } impl AccountStatus { - pub fn is_empty(&self) -> bool { - *self == Self::Empty + pub fn is_empty(self) -> bool { + self == Self::Empty } } @@ -221,9 +221,9 @@ mod tests { #[test] fn reverted_txs_are_nonextant() { - let mut rng = Random::with_random_seed(); const N: usize = 5; const REVERT: usize = 2; + let mut rng = Random::with_random_seed(); let states = (0..N).map(|_| (rng.draw_digest(), rng.draw_tx_id())).collect::>(); @@ -243,9 +243,9 @@ mod tests { #[test] fn pruned_txs_are_nonextant() { - let mut rng = Random::with_random_seed(); const N: usize = 5; const PRUNE: usize = 2; + let mut rng = Random::with_random_seed(); let states = (0..N).map(|_| (rng.draw_digest(), rng.draw_tx_id())).collect::>(); @@ -266,8 +266,8 @@ mod tests { #[test] fn is_empty_after_full_commit_and_prune() { - let mut rng = Random::with_random_seed(); const N: usize = 5; + let mut rng = Random::with_random_seed(); let mut uut = InflightAccountState::default(); for _ in 0..N { uut.insert(rng.draw_digest(), rng.draw_tx_id()); @@ -276,7 +276,7 @@ mod tests { uut.commit(N); let _ = uut.prune_committed(N); - assert_eq!(uut, Default::default()); + assert_eq!(uut, InflightAccountState::default()); } #[test] @@ -290,14 +290,14 @@ mod tests { let _ = uut.revert(N); - assert_eq!(uut, Default::default()); + assert_eq!(uut, InflightAccountState::default()); } #[test] #[should_panic] fn revert_panics_on_out_of_bounds() { - let mut rng = Random::with_random_seed(); const N: usize = 5; + let mut rng = Random::with_random_seed(); let mut uut = InflightAccountState::default(); for _ in 0..N { uut.insert(rng.draw_digest(), rng.draw_tx_id()); @@ -310,8 +310,8 @@ mod tests { #[test] #[should_panic] fn commit_panics_on_out_of_bounds() { - let mut rng = Random::with_random_seed(); const N: usize = 5; + let mut rng = Random::with_random_seed(); let mut uut = InflightAccountState::default(); for _ in 0..N { uut.insert(rng.draw_digest(), rng.draw_tx_id()); diff --git a/crates/block-producer/src/mempool/inflight_state/mod.rs b/crates/block-producer/src/mempool/inflight_state/mod.rs index a68d4283b..865bac2cb 100644 --- a/crates/block-producer/src/mempool/inflight_state/mod.rs +++ b/crates/block-producer/src/mempool/inflight_state/mod.rs @@ -87,7 +87,7 @@ impl Delta { } impl InflightState { - /// Creates an [InflightState] which will retain committed state for the given + /// Creates an [`InflightState`] which will retain committed state for the given /// amount of blocks before pruning them. pub fn new( chain_tip: BlockNumber, @@ -98,11 +98,11 @@ impl InflightState { num_retained_blocks, chain_tip, expiration_slack, - accounts: Default::default(), - nullifiers: Default::default(), - output_notes: Default::default(), - transaction_deltas: Default::default(), - committed_blocks: Default::default(), + accounts: BTreeMap::default(), + nullifiers: BTreeSet::default(), + output_notes: BTreeMap::default(), + transaction_deltas: BTreeMap::default(), + committed_blocks: VecDeque::default(), } } @@ -283,7 +283,7 @@ impl InflightState { /// Panics if any transactions is not part of the uncommitted state. pub fn commit_block(&mut self, txs: impl IntoIterator) { let mut block_deltas = BTreeMap::new(); - for tx in txs.into_iter() { + for tx in txs { let delta = self.transaction_deltas.remove(&tx).expect("Transaction delta must exist"); // SAFETY: Since the delta exists, so must the account. diff --git a/crates/block-producer/src/mempool/mod.rs b/crates/block-producer/src/mempool/mod.rs index b16efcd31..22873c3fc 100644 --- a/crates/block-producer/src/mempool/mod.rs +++ b/crates/block-producer/src/mempool/mod.rs @@ -50,14 +50,13 @@ impl BlockNumber { Self(x) } - pub fn next(&self) -> Self { - let mut ret = *self; - ret.increment(); + pub fn next(mut self) -> Self { + self.increment(); - ret + self } - pub fn prev(&self) -> Option { + pub fn prev(self) -> Option { self.checked_sub(Self(1)) } @@ -65,7 +64,7 @@ impl BlockNumber { self.0 += 1; } - pub fn checked_sub(&self, rhs: Self) -> Option { + pub fn checked_sub(self, rhs: Self) -> Option { self.0.checked_sub(rhs.0).map(Self) } @@ -125,14 +124,14 @@ impl Default for BlockBudget { impl BatchBudget { /// Attempts to consume the transaction's resources from the budget. /// - /// Returns [BudgetStatus::Exceeded] if the transaction would exceed the remaining budget, - /// otherwise returns [BudgetStatus::Ok] and subtracts the resources from the budger. + /// Returns [`BudgetStatus::Exceeded`] if the transaction would exceed the remaining budget, + /// otherwise returns [`BudgetStatus::Ok`] and subtracts the resources from the budger. #[must_use] fn check_then_subtract(&mut self, tx: &AuthenticatedTransaction) -> BudgetStatus { // This type assertion reminds us to update the account check if we ever support multiple // account updates per tx. - let _: miden_objects::accounts::AccountId = tx.account_update().account_id(); const ACCOUNT_UPDATES_PER_TX: usize = 1; + let _: miden_objects::accounts::AccountId = tx.account_update().account_id(); let output_notes = tx.output_note_count(); let input_notes = tx.input_note_count(); @@ -157,8 +156,8 @@ impl BatchBudget { impl BlockBudget { /// Attempts to consume the batch's resources from the budget. /// - /// Returns [BudgetStatus::Exceeded] if the batch would exceed the remaining budget, - /// otherwise returns [BudgetStatus::Ok]. + /// Returns [`BudgetStatus::Exceeded`] if the batch would exceed the remaining budget, + /// otherwise returns [`BudgetStatus::Ok`]. #[must_use] fn check_then_subtract(&mut self, _batch: &TransactionBatch) -> BudgetStatus { if self.batches == 0 { @@ -206,7 +205,7 @@ pub struct Mempool { } impl Mempool { - /// Creates a new [SharedMempool] with the provided configuration. + /// Creates a new [`SharedMempool`] with the provided configuration. pub fn shared( chain_tip: BlockNumber, batch_budget: BatchBudget, @@ -235,10 +234,10 @@ impl Mempool { batch_budget, block_budget, state: InflightState::new(chain_tip, state_retention, expiration_slack), - block_in_progress: Default::default(), - transactions: Default::default(), - batches: Default::default(), - expirations: Default::default(), + block_in_progress: None, + transactions: TransactionGraph::default(), + batches: BatchGraph::default(), + expirations: TransactionExpirations::default(), } } @@ -353,7 +352,8 @@ impl Mempool { // Remove committed batches and transactions from graphs. let batches = self.block_in_progress.take().expect("No block in progress to commit"); - let transactions = self.batches.prune_committed(batches).expect("Batches failed to commit"); + let transactions = + self.batches.prune_committed(&batches).expect("Batches failed to commit"); self.transactions .commit_transactions(&transactions) .expect("Transaction graph malformed"); diff --git a/crates/block-producer/src/mempool/tests.rs b/crates/block-producer/src/mempool/tests.rs index 2646706a3..6a4ad8b91 100644 --- a/crates/block-producer/src/mempool/tests.rs +++ b/crates/block-producer/src/mempool/tests.rs @@ -1,3 +1,4 @@ +use miden_node_proto::domain::notes::NoteAuthenticationInfo; use pretty_assertions::assert_eq; use super::*; @@ -7,10 +8,10 @@ impl Mempool { fn for_tests() -> Self { Self::new( BlockNumber::new(0), - Default::default(), - Default::default(), + BatchBudget::default(), + BlockBudget::default(), 5, - Default::default(), + BlockNumber::default(), ) } } @@ -47,7 +48,8 @@ fn children_of_failed_batches_are_ignored() { assert_eq!(uut, reference); let proof = - TransactionBatch::new([txs[2].raw_proven_transaction()], Default::default()).unwrap(); + TransactionBatch::new([txs[2].raw_proven_transaction()], NoteAuthenticationInfo::default()) + .unwrap(); uut.batch_proved(proof); assert_eq!(uut, reference); } @@ -93,7 +95,11 @@ fn block_commit_reverts_expired_txns() { uut.add_transaction(tx_to_commit.clone()).unwrap(); uut.select_batch().unwrap(); uut.batch_proved( - TransactionBatch::new([tx_to_commit.raw_proven_transaction()], Default::default()).unwrap(), + TransactionBatch::new( + [tx_to_commit.raw_proven_transaction()], + NoteAuthenticationInfo::default(), + ) + .unwrap(), ); let (block, _) = uut.select_block(); // A reverted transaction behaves as if it never existed, the current state is the expected @@ -163,8 +169,11 @@ fn block_failure_reverts_its_transactions() { uut.add_transaction(reverted_txs[0].clone()).unwrap(); uut.select_batch().unwrap(); uut.batch_proved( - TransactionBatch::new([reverted_txs[0].raw_proven_transaction()], Default::default()) - .unwrap(), + TransactionBatch::new( + [reverted_txs[0].raw_proven_transaction()], + NoteAuthenticationInfo::default(), + ) + .unwrap(), ); // Block 1 will contain just the first batch. diff --git a/crates/block-producer/src/mempool/transaction_expiration.rs b/crates/block-producer/src/mempool/transaction_expiration.rs index 103e665f5..79dc450aa 100644 --- a/crates/block-producer/src/mempool/transaction_expiration.rs +++ b/crates/block-producer/src/mempool/transaction_expiration.rs @@ -65,7 +65,7 @@ mod tests { uut.insert(tx, block); uut.remove(std::iter::once(&tx)); - assert_eq!(uut, Default::default()); + assert_eq!(uut, TransactionExpirations::default()); } #[test] diff --git a/crates/block-producer/src/mempool/transaction_graph.rs b/crates/block-producer/src/mempool/transaction_graph.rs index 5709bfdeb..ab1eb43bf 100644 --- a/crates/block-producer/src/mempool/transaction_graph.rs +++ b/crates/block-producer/src/mempool/transaction_graph.rs @@ -53,7 +53,7 @@ impl TransactionGraph { /// /// # Errors /// - /// Follows the error conditions of [DependencyGraph::insert_pending]. + /// Follows the error conditions of [`DependencyGraph::insert_pending`]. pub fn insert( &mut self, transaction: AuthenticatedTransaction, @@ -73,8 +73,8 @@ impl TransactionGraph { /// Note: this may emit empty batches. /// /// See also: - /// - [Self::requeue_transactions] - /// - [Self::commit_transactions] + /// - [`Self::requeue_transactions`] + /// - [`Self::commit_transactions`] pub fn select_batch( &mut self, mut budget: BatchBudget, @@ -84,7 +84,7 @@ impl TransactionGraph { let mut batch = Vec::with_capacity(budget.transactions); let mut parents = BTreeSet::new(); - while let Some(root) = self.inner.roots().first().cloned() { + while let Some(root) = self.inner.roots().first().copied() { // SAFETY: Since it was a root batch, it must definitely have a processed batch // associated with it. let tx = self.inner.get(&root).unwrap().clone(); @@ -109,7 +109,7 @@ impl TransactionGraph { /// /// # Errors /// - /// Follows the error conditions of [DependencyGraph::revert_subgraphs]. + /// Follows the error conditions of [`DependencyGraph::revert_subgraphs`]. pub fn requeue_transactions( &mut self, transactions: BTreeSet, @@ -121,13 +121,13 @@ impl TransactionGraph { /// /// # Errors /// - /// Follows the error conditions of [DependencyGraph::prune_processed]. + /// Follows the error conditions of [`DependencyGraph::prune_processed`]. pub fn commit_transactions( &mut self, tx_ids: &[TransactionId], ) -> Result<(), GraphError> { // TODO: revisit this api. - let tx_ids = tx_ids.iter().cloned().collect(); + let tx_ids = tx_ids.iter().copied().collect(); self.inner.prune_processed(tx_ids)?; Ok(()) } @@ -138,7 +138,7 @@ impl TransactionGraph { /// /// # Errors /// - /// Follows the error conditions of [DependencyGraph::purge_subgraphs]. + /// Follows the error conditions of [`DependencyGraph::purge_subgraphs`]. pub fn remove_transactions( &mut self, transactions: Vec, diff --git a/crates/block-producer/src/server.rs b/crates/block-producer/src/server.rs index afd82c9e4..b0b445f2c 100644 --- a/crates/block-producer/src/server.rs +++ b/crates/block-producer/src/server.rs @@ -74,10 +74,10 @@ impl BlockProducer { info!(target: COMPONENT, "Server initialized"); Ok(Self { - batch_builder: Default::default(), + batch_builder: BatchBuilder::default(), block_builder: BlockBuilder::new(store.clone()), - batch_budget: Default::default(), - block_budget: Default::default(), + batch_budget: BatchBudget::default(), + block_budget: BlockBudget::default(), state_retention: SERVER_MEMPOOL_STATE_RETENTION, expiration_slack: SERVER_MEMPOOL_EXPIRATION_SLACK, store, diff --git a/crates/block-producer/src/store/mod.rs b/crates/block-producer/src/store/mod.rs index 5d362a0f9..d49652f7f 100644 --- a/crates/block-producer/src/store/mod.rs +++ b/crates/block-producer/src/store/mod.rs @@ -11,8 +11,8 @@ use miden_node_proto::{ generated::{ digest, requests::{ - ApplyBlockRequest, GetBlockInputsRequest, GetNoteAuthenticationInfoRequest, - GetTransactionInputsRequest, + ApplyBlockRequest, GetBlockHeaderByNumberRequest, GetBlockInputsRequest, + GetNoteAuthenticationInfoRequest, GetTransactionInputsRequest, }, responses::{GetTransactionInputsResponse, NullifierTransactionInputRecord}, store::api_client as store_client, @@ -46,7 +46,7 @@ pub struct TransactionInputs { pub account_hash: Option, /// Maps each consumed notes' nullifier to block number, where the note is consumed. /// - /// We use NonZeroU32 as the wire format uses 0 to encode none. + /// We use `NonZeroU32` as the wire format uses 0 to encode none. pub nullifiers: BTreeMap>, /// Unauthenticated notes which are present in the store. /// @@ -67,7 +67,7 @@ impl Display for TransactionInputs { let nullifiers = if nullifiers.is_empty() { "None".to_owned() } else { - format!("{{ {} }}", nullifiers) + format!("{{ {nullifiers} }}") }; f.write_fmt(format_args!( @@ -112,8 +112,8 @@ impl TryFrom for TransactionInputs { account_id, account_hash, nullifiers, - current_block_height, found_unauthenticated_notes, + current_block_height, }) } } @@ -141,7 +141,9 @@ impl StoreClient { let response = self .inner .clone() - .get_block_header_by_number(tonic::Request::new(Default::default())) + .get_block_header_by_number(tonic::Request::new( + GetBlockHeaderByNumberRequest::default(), + )) .await? .into_inner() .block_header diff --git a/crates/block-producer/src/test_utils/account.rs b/crates/block-producer/src/test_utils/account.rs index 935ba11ba..98d1b9f12 100644 --- a/crates/block-producer/src/test_utils/account.rs +++ b/crates/block-producer/src/test_utils/account.rs @@ -84,5 +84,5 @@ impl From for MockPrivateAccount { } pub fn mock_account_id(num: u8) -> AccountId { - MockPrivateAccount::<3>::from(num as u32).id + MockPrivateAccount::<3>::from(u32::from(num)).id } diff --git a/crates/block-producer/src/test_utils/batch.rs b/crates/block-producer/src/test_utils/batch.rs index 53a572b9d..d27eb9e0b 100644 --- a/crates/block-producer/src/test_utils/batch.rs +++ b/crates/block-producer/src/test_utils/batch.rs @@ -1,3 +1,5 @@ +use miden_node_proto::domain::notes::NoteAuthenticationInfo; + use crate::{batch_builder::TransactionBatch, test_utils::MockProvenTxBuilder}; pub trait TransactionBatchConstructor { @@ -15,7 +17,7 @@ impl TransactionBatchConstructor for TransactionBatch { .iter() .enumerate() .map(|(index, &num_notes)| { - let starting_note_index = starting_account_index as u64 + index as u64; + let starting_note_index = u64::from(starting_account_index) + index as u64; MockProvenTxBuilder::with_account_index(starting_account_index + index as u32) .private_notes_created_range( starting_note_index..(starting_note_index + num_notes), @@ -24,7 +26,7 @@ impl TransactionBatchConstructor for TransactionBatch { }) .collect(); - Self::new(&txs, Default::default()).unwrap() + Self::new(&txs, NoteAuthenticationInfo::default()).unwrap() } fn from_txs(starting_account_index: u32, num_txs_in_batch: u64) -> Self { @@ -36,6 +38,6 @@ impl TransactionBatchConstructor for TransactionBatch { }) .collect(); - Self::new(&txs, Default::default()).unwrap() + Self::new(&txs, NoteAuthenticationInfo::default()).unwrap() } } diff --git a/crates/block-producer/src/test_utils/block.rs b/crates/block-producer/src/test_utils/block.rs index d81376caf..54fa67631 100644 --- a/crates/block-producer/src/test_utils/block.rs +++ b/crates/block-producer/src/test_utils/block.rs @@ -126,6 +126,7 @@ impl MockBlockBuilder { } } + #[must_use] pub fn account_updates(mut self, updated_accounts: Vec) -> Self { for update in &updated_accounts { self.store_accounts @@ -137,12 +138,14 @@ impl MockBlockBuilder { self } + #[must_use] pub fn created_notes(mut self, created_notes: Vec) -> Self { self.created_notes = Some(created_notes); self } + #[must_use] pub fn produced_nullifiers(mut self, produced_nullifiers: Vec) -> Self { self.produced_nullifiers = Some(produced_nullifiers); diff --git a/crates/block-producer/src/test_utils/proven_tx.rs b/crates/block-producer/src/test_utils/proven_tx.rs index 271466341..396e0c319 100644 --- a/crates/block-producer/src/test_utils/proven_tx.rs +++ b/crates/block-producer/src/test_utils/proven_tx.rs @@ -67,30 +67,35 @@ impl MockProvenTxBuilder { } } + #[must_use] pub fn unauthenticated_notes(mut self, notes: Vec) -> Self { self.input_notes = Some(notes.into_iter().map(InputNote::unauthenticated).collect()); self } + #[must_use] pub fn nullifiers(mut self, nullifiers: Vec) -> Self { self.nullifiers = Some(nullifiers); self } + #[must_use] pub fn expiration_block_num(mut self, expiration_block_num: u32) -> Self { self.expiration_block_num = expiration_block_num; self } + #[must_use] pub fn output_notes(mut self, notes: Vec) -> Self { self.output_notes = Some(notes); self } + #[must_use] pub fn nullifiers_range(self, range: Range) -> Self { let nullifiers = range .map(|index| { @@ -103,6 +108,7 @@ impl MockProvenTxBuilder { self.nullifiers(nullifiers) } + #[must_use] pub fn private_notes_created_range(self, range: Range) -> Self { let notes = range .map(|note_index| { diff --git a/crates/block-producer/src/test_utils/store.rs b/crates/block-producer/src/test_utils/store.rs index 2314ad16b..e4eccfb18 100644 --- a/crates/block-producer/src/test_utils/store.rs +++ b/crates/block-producer/src/test_utils/store.rs @@ -72,24 +72,28 @@ impl MockStoreSuccessBuilder { } } + #[must_use] pub fn initial_notes<'a>(mut self, notes: impl Iterator + Clone) -> Self { self.notes = Some(notes.cloned().collect()); self } + #[must_use] pub fn initial_nullifiers(mut self, nullifiers: BTreeSet) -> Self { self.produced_nullifiers = Some(nullifiers); self } + #[must_use] pub fn initial_chain_mmr(mut self, chain_mmr: Mmr) -> Self { self.chain_mmr = Some(chain_mmr); self } + #[must_use] pub fn initial_block_num(mut self, block_num: u32) -> Self { self.block_num = Some(block_num); @@ -151,7 +155,7 @@ impl MockStoreSuccessBuilder { initial_block_header.block_num(), initial_block_header, )]))), - num_apply_block_called: Default::default(), + num_apply_block_called: Arc::default(), notes: Arc::new(RwLock::new(notes)), } } diff --git a/crates/proto/Cargo.toml b/crates/proto/Cargo.toml index de58f3512..445a93a78 100644 --- a/crates/proto/Cargo.toml +++ b/crates/proto/Cargo.toml @@ -11,6 +11,9 @@ authors.workspace = true homepage.workspace = true repository.workspace = true +[lints] +workspace = true + [dependencies] hex = { version = "0.4" } miden-node-utils = { workspace = true } diff --git a/crates/proto/build.rs b/crates/proto/build.rs index b1382f2ca..3ea956b4b 100644 --- a/crates/proto/build.rs +++ b/crates/proto/build.rs @@ -8,8 +8,8 @@ use protox::prost::Message; /// Generates Rust protobuf bindings from .proto files in the root directory. /// -/// This is done only if BUILD_PROTO environment variable is set to `1` to avoid running the script -/// on crates.io where repo-level .proto files are not available. +/// This is done only if `BUILD_PROTO` environment variable is set to `1` to avoid running the +/// script on crates.io where repo-level .proto files are not available. fn main() -> anyhow::Result<()> { println!("cargo::rerun-if-changed=../../proto"); println!("cargo::rerun-if-env-changed=BUILD_PROTO"); @@ -91,9 +91,11 @@ fn generate_mod_rs(directory: impl AsRef) -> std::io::Result<()> { submodules.sort(); let contents = submodules.iter().map(|f| format!("pub mod {f};\n")); - let contents = std::iter::once("// Generated by build.rs\n\n".to_owned()) - .chain(contents) - .collect::(); + let contents = std::iter::once( + "#![allow(clippy::pedantic, reason = \"generated by build.rs and tonic\")]\n\n".to_string(), + ) + .chain(contents) + .collect::(); fs::write(mod_filepath, contents) } diff --git a/crates/proto/src/domain/accounts.rs b/crates/proto/src/domain/accounts.rs index cf358ab04..1283e193a 100644 --- a/crates/proto/src/domain/accounts.rs +++ b/crates/proto/src/domain/accounts.rs @@ -21,7 +21,7 @@ impl Display for proto::account::AccountId { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { write!(f, "0x")?; for byte in &self.id { - write!(f, "{:02x}", byte)?; + write!(f, "{byte:02x}")?; } Ok(()) } @@ -89,7 +89,7 @@ impl From<&AccountInfo> for proto::account::AccountInfo { fn from(AccountInfo { summary, details }: &AccountInfo) -> Self { Self { summary: Some(summary.into()), - details: details.as_ref().map(|account| account.to_bytes()), + details: details.as_ref().map(miden_objects::utils::Serializable::to_bytes), } } } diff --git a/crates/proto/src/domain/digest.rs b/crates/proto/src/domain/digest.rs index 811ee872d..b0bf19fac 100644 --- a/crates/proto/src/domain/digest.rs +++ b/crates/proto/src/domain/digest.rs @@ -159,19 +159,19 @@ impl TryFrom for [Felt; 4] { type Error = ConversionError; fn try_from(value: proto::Digest) -> Result { - if ![value.d0, value.d1, value.d2, value.d3] + if [value.d0, value.d1, value.d2, value.d3] .iter() - .all(|v| *v < ::MODULUS) + .any(|v| *v >= ::MODULUS) { - Err(ConversionError::NotAValidFelt) - } else { - Ok([ - Felt::new(value.d0), - Felt::new(value.d1), - Felt::new(value.d2), - Felt::new(value.d3), - ]) + return Err(ConversionError::NotAValidFelt); } + + Ok([ + Felt::new(value.d0), + Felt::new(value.d1), + Felt::new(value.d2), + Felt::new(value.d3), + ]) } } @@ -212,10 +212,10 @@ mod test { #[test] fn hex_digest() { let digest = proto::Digest { - d0: 3488802789098113751, - d1: 5271242459988994564, - d2: 17816570245237064784, - d3: 10910963388447438895, + d0: 0x306A_B7A6_F795_CAD7, + d1: 0x4927_3716_D099_AA04, + d2: 0xF741_2C3D_E726_4450, + d3: 0x976B_8764_9DB3_B82F, }; let encoded: String = ToHex::encode_hex(&digest); let round_trip: Result = FromHex::from_hex::<&[u8]>(encoded.as_ref()); diff --git a/crates/proto/src/domain/mod.rs b/crates/proto/src/domain/mod.rs index 43da3d479..b3e682664 100644 --- a/crates/proto/src/domain/mod.rs +++ b/crates/proto/src/domain/mod.rs @@ -15,7 +15,7 @@ where From: Into, R: FromIterator, { - from.into_iter().map(|e| e.into()).collect() + from.into_iter().map(Into::into).collect() } pub fn try_convert(from: T) -> Result @@ -24,5 +24,5 @@ where From: TryInto, R: FromIterator, { - from.into_iter().map(|e| e.try_into()).collect() + from.into_iter().map(TryInto::try_into).collect() } diff --git a/crates/proto/src/domain/notes.rs b/crates/proto/src/domain/notes.rs index 4dafb028f..4ea9e8652 100644 --- a/crates/proto/src/domain/notes.rs +++ b/crates/proto/src/domain/notes.rs @@ -21,7 +21,7 @@ impl TryFrom for NoteMetadata { .sender .ok_or_else(|| proto::NoteMetadata::missing_field(stringify!(sender)))? .try_into()?; - let note_type = NoteType::try_from(value.note_type as u64)?; + let note_type = NoteType::try_from(u64::from(value.note_type))?; let tag = NoteTag::from(value.tag); let execution_hint = NoteExecutionHint::try_from(value.execution_hint)?; diff --git a/crates/proto/src/generated/mod.rs b/crates/proto/src/generated/mod.rs index 13d511099..dc7675d92 100644 --- a/crates/proto/src/generated/mod.rs +++ b/crates/proto/src/generated/mod.rs @@ -1,4 +1,4 @@ -// Generated by build.rs +#![allow(clippy::pedantic, reason = "generated by build.rs and tonic")] pub mod account; pub mod block; diff --git a/crates/rpc-proto/Cargo.toml b/crates/rpc-proto/Cargo.toml index 39da27233..a9015ed67 100644 --- a/crates/rpc-proto/Cargo.toml +++ b/crates/rpc-proto/Cargo.toml @@ -12,6 +12,9 @@ authors.workspace = true homepage.workspace = true repository.workspace = true +[lints] +workspace = true + [features] default = ["std"] std = [] diff --git a/crates/rpc-proto/build.rs b/crates/rpc-proto/build.rs index 4f6d911cc..48a3d6df2 100644 --- a/crates/rpc-proto/build.rs +++ b/crates/rpc-proto/build.rs @@ -15,10 +15,10 @@ const DOC_COMMENT: &str = // BUILD SCRIPT // ================================================================================================ -/// Copies .proto files to the local directory and re-builds src/proto_files.rs file. +/// Copies .proto files to the local directory and re-builds `src/proto_files.rs` file. /// -/// This is done only if BUILD_PROTO environment variable is set to `1` to avoid running the script -/// on crates.io where repo-level .proto files are not available. +/// This is done only if `BUILD_PROTO` environment variable is set to `1` to avoid running the +/// script on crates.io where repo-level .proto files are not available. fn main() -> io::Result<()> { println!("cargo::rerun-if-changed=../../proto"); println!("cargo::rerun-if-env-changed=BUILD_PROTO"); diff --git a/crates/rpc-proto/src/lib.rs b/crates/rpc-proto/src/lib.rs index 17711f2bc..d48a1afa9 100644 --- a/crates/rpc-proto/src/lib.rs +++ b/crates/rpc-proto/src/lib.rs @@ -17,8 +17,7 @@ pub fn write_proto(target_dir: &std::path::Path) -> Result<(), std::string::Stri }; if !target_dir.exists() { - fs::create_dir_all(target_dir) - .map_err(|err| format!("Error creating directory: {}", err))?; + fs::create_dir_all(target_dir).map_err(|err| format!("Error creating directory: {err}"))?; } else if !target_dir.is_dir() { return Err("The target path exists but is not a directory".to_string()); } @@ -26,10 +25,10 @@ pub fn write_proto(target_dir: &std::path::Path) -> Result<(), std::string::Stri for (file_name, file_content) in PROTO_FILES { let mut file_path = target_dir.to_path_buf(); file_path.push(file_name); - let mut file = File::create(&file_path) - .map_err(|err| format!("Error creating {}: {}", file_name, err))?; + let mut file = + File::create(&file_path).map_err(|err| format!("Error creating {file_name}: {err}"))?; file.write_all(file_content.as_bytes()) - .map_err(|err| format!("Error writing {}: {}", file_name, err))?; + .map_err(|err| format!("Error writing {file_name}: {err}"))?; } Ok(()) diff --git a/crates/rpc/Cargo.toml b/crates/rpc/Cargo.toml index c8e8ab167..3dfba5246 100644 --- a/crates/rpc/Cargo.toml +++ b/crates/rpc/Cargo.toml @@ -11,6 +11,9 @@ authors.workspace = true homepage.workspace = true repository.workspace = true +[lints] +workspace = true + [dependencies] miden-node-proto = { workspace = true } miden-node-utils = { workspace = true } diff --git a/crates/rpc/src/server/api.rs b/crates/rpc/src/server/api.rs index 96bca8096..6afad672f 100644 --- a/crates/rpc/src/server/api.rs +++ b/crates/rpc/src/server/api.rs @@ -72,7 +72,7 @@ impl api_server::Api for RpcApi { debug!(target: COMPONENT, request = ?request.get_ref()); // validate all the nullifiers from the user request - for nullifier in request.get_ref().nullifiers.iter() { + for nullifier in &request.get_ref().nullifiers { let _: Digest = nullifier .try_into() .or(Err(Status::invalid_argument("Digest field is not in the modulus range")))?; @@ -162,7 +162,7 @@ impl api_server::Api for RpcApi { let note_ids = request.get_ref().note_ids.clone(); let _: Vec = try_convert(note_ids) - .map_err(|err| Status::invalid_argument(format!("Invalid NoteId: {}", err)))?; + .map_err(|err| Status::invalid_argument(format!("Invalid NoteId: {err}")))?; self.store.clone().get_notes_by_id(request).await } diff --git a/crates/store/Cargo.toml b/crates/store/Cargo.toml index 6ca51bb4b..8eec38f27 100644 --- a/crates/store/Cargo.toml +++ b/crates/store/Cargo.toml @@ -11,6 +11,9 @@ authors.workspace = true homepage.workspace = true repository.workspace = true +[lints] +workspace = true + [dependencies] deadpool-sqlite = { version = "0.9.0", features = ["rt_tokio_1"] } hex = { version = "0.4" } diff --git a/crates/store/src/config.rs b/crates/store/src/config.rs index 447b5721d..3cac20a77 100644 --- a/crates/store/src/config.rs +++ b/crates/store/src/config.rs @@ -14,7 +14,7 @@ use serde::{Deserialize, Serialize}; pub struct StoreConfig { /// Defines the listening socket. pub endpoint: Endpoint, - /// SQLite database file + /// `SQLite` database file pub database_filepath: PathBuf, /// Genesis file pub genesis_filepath: PathBuf, diff --git a/crates/store/src/db/mod.rs b/crates/store/src/db/mod.rs index 64e0ebdb0..fc466d502 100644 --- a/crates/store/src/db/mod.rs +++ b/crates/store/src/db/mod.rs @@ -256,7 +256,7 @@ impl Db { self.pool .get() .await? - .interact(move |conn| sql::select_block_headers(conn, blocks)) + .interact(move |conn| sql::select_block_headers(conn, &blocks)) .await .map_err(|err| { DatabaseError::InteractError(format!( diff --git a/crates/store/src/db/sql/mod.rs b/crates/store/src/db/sql/mod.rs index fa93a26ec..a6539b7c8 100644 --- a/crates/store/src/db/sql/mod.rs +++ b/crates/store/src/db/sql/mod.rs @@ -66,7 +66,7 @@ pub fn select_all_accounts(conn: &mut Connection) -> Result> { let mut accounts = vec![]; while let Some(row) = rows.next()? { - accounts.push(account_info_from_row(row)?) + accounts.push(account_info_from_row(row)?); } Ok(accounts) } @@ -93,12 +93,12 @@ pub fn select_all_account_hashes(conn: &mut Connection) -> Result (None, None), @@ -571,9 +572,12 @@ pub fn insert_nullifiers_for_block( )?; let mut count = 0; - for nullifier in nullifiers.iter() { - count += - stmt.execute(params![nullifier.to_bytes(), get_nullifier_prefix(nullifier), block_num])? + for nullifier in nullifiers { + count += stmt.execute(params![ + nullifier.to_bytes(), + get_nullifier_prefix(nullifier), + block_num + ])?; } Ok(count) } @@ -606,7 +610,7 @@ pub fn select_all_nullifiers(conn: &mut Connection) -> Result Result> { metadata, details, merkle_path, - }) + }); } Ok(notes) } @@ -784,8 +788,8 @@ pub fn insert_notes(transaction: &Transaction, notes: &[NoteRecord]) -> Result Result< note_id: note_id.into(), metadata, merkle_path, - }) + }); } Ok(notes) } -/// Select note inclusion proofs matching the NoteId, using the given [Connection]. +/// Select note inclusion proofs matching the `NoteId`, using the given [Connection]. /// /// # Returns /// @@ -985,7 +989,7 @@ pub fn select_note_inclusion_proofs( // BLOCK CHAIN QUERIES // ================================================================================================ -/// Insert a [BlockHeader] to the DB using the given [Transaction]. +/// Insert a [`BlockHeader`] to the DB using the given [Transaction]. /// /// # Returns /// @@ -1001,7 +1005,7 @@ pub fn insert_block_header(transaction: &Transaction, block_header: &BlockHeader Ok(stmt.execute(params![block_header.block_num(), block_header.to_bytes()])?) } -/// Select a [BlockHeader] from the DB by its `block_num` using the given [Connection]. +/// Select a [`BlockHeader`] from the DB by its `block_num` using the given [Connection]. /// /// # Returns /// @@ -1012,18 +1016,15 @@ pub fn select_block_header_by_block_num( block_number: Option, ) -> Result> { let mut stmt; - let mut rows = match block_number { - Some(block_number) => { - stmt = - conn.prepare_cached("SELECT block_header FROM block_headers WHERE block_num = ?1")?; - stmt.query([block_number])? - }, - None => { - stmt = conn.prepare_cached( - "SELECT block_header FROM block_headers ORDER BY block_num DESC LIMIT 1", - )?; - stmt.query([])? - }, + let mut rows = if let Some(block_number) = block_number { + stmt = + conn.prepare_cached("SELECT block_header FROM block_headers WHERE block_num = ?1")?; + stmt.query([block_number])? + } else { + stmt = conn.prepare_cached( + "SELECT block_header FROM block_headers ORDER BY block_num DESC LIMIT 1", + )?; + stmt.query([])? }; match rows.next()? { @@ -1043,10 +1044,10 @@ pub fn select_block_header_by_block_num( /// /// # Returns /// -/// A vector of [BlockHeader] or an error. +/// A vector of [`BlockHeader`] or an error. pub fn select_block_headers( conn: &mut Connection, - blocks: Vec, + blocks: &[BlockNumber], ) -> Result> { let mut headers = Vec::with_capacity(blocks.len()); @@ -1068,7 +1069,7 @@ pub fn select_block_headers( /// /// # Returns /// -/// A vector of [BlockHeader] or an error. +/// A vector of [`BlockHeader`] or an error. pub fn select_all_block_headers(conn: &mut Connection) -> Result> { let mut stmt = conn.prepare_cached("SELECT block_header FROM block_headers ORDER BY block_num ASC;")?; @@ -1109,7 +1110,7 @@ pub fn insert_transactions( let account_id = update.account_id(); for transaction_id in update.transactions() { count += - stmt.execute(params![transaction_id.to_bytes(), account_id.to_bytes(), block_num])? + stmt.execute(params![transaction_id.to_bytes(), account_id.to_bytes(), block_num])?; } } Ok(count) @@ -1120,7 +1121,7 @@ pub fn insert_transactions( /// /// # Returns /// -/// The vector of [RpoDigest] with the transaction IDs. +/// The vector of [`RpoDigest`] with the transaction IDs. pub fn select_transactions_by_accounts_and_block_range( conn: &mut Connection, block_start: BlockNumber, diff --git a/crates/store/src/db/sql/utils.rs b/crates/store/src/db/sql/utils.rs index 03faf052c..0787acd28 100644 --- a/crates/store/src/db/sql/utils.rs +++ b/crates/store/src/db/sql/utils.rs @@ -50,7 +50,7 @@ macro_rules! subst { /// insert_sql!(users { id, first_name, last_name, age }); /// ``` /// which generates: -/// "INSERT INTO users (id, first_name, last_name, age) VALUES (?, ?, ?, ?)" +/// "INSERT INTO users (id, `first_name`, `last_name`, age) VALUES (?, ?, ?, ?)" macro_rules! insert_sql { ($table:ident { $first_field:ident $(, $($field:ident),+)? $(,)? }) => { concat!( @@ -71,6 +71,10 @@ macro_rules! insert_sql { /// Sqlite uses `i64` as its internal representation format. Note that the `as` operator performs a /// lossless conversion from `u64` to `i64`. pub fn u64_to_value(v: u64) -> Value { + #[allow( + clippy::cast_possible_wrap, + reason = "We store u64 as i64 as sqlite only allows the latter." + )] Value::Integer(v as i64) } @@ -83,6 +87,10 @@ pub fn column_value_as_u64( index: I, ) -> rusqlite::Result { let value: i64 = row.get(index)?; + #[allow( + clippy::cast_sign_loss, + reason = "We store u64 as i64 as sqlite only allows the latter." + )] Ok(value as u64) } diff --git a/crates/store/src/db/tests.rs b/crates/store/src/db/tests.rs index b931e850e..0ae53e9eb 100644 --- a/crates/store/src/db/tests.rs +++ b/crates/store/src/db/tests.rs @@ -1,3 +1,6 @@ +#![allow(clippy::similar_names, reason = "naming dummy test values is hard")] +#![allow(clippy::too_many_lines, reason = "test code can be long")] + use miden_lib::transaction::TransactionKernel; use miden_node_proto::domain::accounts::AccountSummary; use miden_objects::{ @@ -173,13 +176,13 @@ fn sql_select_notes() { let note = NoteRecord { block_num, note_index: BlockNoteIndex::new(0, i as usize).unwrap(), - note_id: num_to_rpo_digest(i as u64), + note_id: num_to_rpo_digest(u64::from(i)), metadata: NoteMetadata::new( ACCOUNT_ID_OFF_CHAIN_SENDER.try_into().unwrap(), NoteType::Public, i.into(), NoteExecutionHint::none(), - Default::default(), + Felt::default(), ) .unwrap(), details: Some(vec![1, 2, 3]), @@ -219,7 +222,7 @@ fn sql_select_notes_different_execution_hints() { NoteType::Public, 0.into(), NoteExecutionHint::none(), - Default::default(), + Felt::default(), ) .unwrap(), details: Some(vec![1, 2, 3]), @@ -243,7 +246,7 @@ fn sql_select_notes_different_execution_hints() { NoteType::Public, 1.into(), NoteExecutionHint::always(), - Default::default(), + Felt::default(), ) .unwrap(), details: Some(vec![1, 2, 3]), @@ -267,7 +270,7 @@ fn sql_select_notes_different_execution_hints() { NoteType::Public, 2.into(), NoteExecutionHint::after_block(12).unwrap(), - Default::default(), + Felt::default(), ) .unwrap(), details: Some(vec![1, 2, 3]), @@ -302,7 +305,7 @@ fn sql_select_accounts() { AccountType::RegularAccountImmutableCode, miden_objects::accounts::AccountStorageMode::Private, ); - let account_hash = num_to_rpo_digest(i as u64); + let account_hash = num_to_rpo_digest(u64::from(i)); state.push(AccountInfo { summary: AccountSummary { account_id, account_hash, block_num }, details: None, @@ -606,8 +609,8 @@ fn sql_select_nullifiers_by_block_range() { #[test] fn select_nullifiers_by_prefix() { - let mut conn = create_db(); const PREFIX_LEN: u32 = 16; + let mut conn = create_db(); // test empty table let nullifiers = sql::select_nullifiers_by_prefix(&mut conn, PREFIX_LEN, &[]).unwrap(); assert!(nullifiers.is_empty()); @@ -869,7 +872,7 @@ fn notes() { .unwrap(); let values = [(note_index, note_id.into(), note_metadata)]; - let notes_db = BlockNoteTree::with_entries(values.iter().cloned()).unwrap(); + let notes_db = BlockNoteTree::with_entries(values.iter().copied()).unwrap(); let details = Some(vec![1, 2, 3]); let merkle_path = notes_db.get_note_path(note_index); @@ -882,7 +885,7 @@ fn notes() { NoteType::Public, tag.into(), NoteExecutionHint::none(), - Default::default(), + Felt::default(), ) .unwrap(), details, @@ -948,7 +951,7 @@ fn notes() { let note_0 = res[0].clone(); let note_1 = res[1].clone(); assert_eq!(note_0.details, Some(vec![1, 2, 3])); - assert_eq!(note_1.details, None) + assert_eq!(note_1.details, None); } // UTILITIES diff --git a/crates/store/src/genesis.rs b/crates/store/src/genesis.rs index 98bf5379f..ada75543e 100644 --- a/crates/store/src/genesis.rs +++ b/crates/store/src/genesis.rs @@ -74,7 +74,7 @@ impl GenesisState { impl Serializable for GenesisState { fn write_into(&self, target: &mut W) { - assert!(self.accounts.len() <= u64::MAX as usize, "too many accounts in GenesisState"); + assert!(u64::try_from(self.accounts.len()).is_ok(), "too many accounts in GenesisState"); target.write_usize(self.accounts.len()); target.write_many(&self.accounts); diff --git a/crates/store/src/nullifier_tree.rs b/crates/store/src/nullifier_tree.rs index 282b27f44..81c197330 100644 --- a/crates/store/src/nullifier_tree.rs +++ b/crates/store/src/nullifier_tree.rs @@ -94,7 +94,7 @@ mod tests { let block_num = 123; let nullifier_value = NullifierTree::block_num_to_leaf_value(block_num); - assert_eq!(nullifier_value, [Felt::from(block_num), ZERO, ZERO, ZERO]) + assert_eq!(nullifier_value, [Felt::from(block_num), ZERO, ZERO, ZERO]); } #[test] diff --git a/crates/store/src/server/api.rs b/crates/store/src/server/api.rs index 006561592..efdd68438 100644 --- a/crates/store/src/server/api.rs +++ b/crates/store/src/server/api.rs @@ -259,7 +259,7 @@ impl api_server::Api for StoreApi { let note_ids = request.into_inner().note_ids; let note_ids: Vec = try_convert(note_ids) - .map_err(|err| Status::invalid_argument(format!("Invalid NoteId: {}", err)))?; + .map_err(|err| Status::invalid_argument(format!("Invalid NoteId: {err}")))?; let note_ids: Vec = note_ids.into_iter().map(From::from).collect(); @@ -291,7 +291,7 @@ impl api_server::Api for StoreApi { let note_ids = request.into_inner().note_ids; let note_ids: Vec = try_convert(note_ids) - .map_err(|err| Status::invalid_argument(format!("Invalid NoteId: {}", err)))?; + .map_err(|err| Status::invalid_argument(format!("Invalid NoteId: {err}")))?; let note_ids = note_ids.into_iter().map(From::from).collect(); @@ -485,11 +485,11 @@ impl api_server::Api for StoreApi { let include_headers = include_headers.unwrap_or_default(); let request_code_commitments: BTreeSet = try_convert(code_commitments) - .map_err(|err| Status::invalid_argument(format!("Invalid code commitment: {}", err)))?; + .map_err(|err| Status::invalid_argument(format!("Invalid code commitment: {err}")))?; let account_requests: Vec = try_convert(account_requests).map_err(|err| { - Status::invalid_argument(format!("Invalid account proofs request: {}", err)) + Status::invalid_argument(format!("Invalid account proofs request: {err}")) })?; let (block_num, infos) = self @@ -545,7 +545,7 @@ fn invalid_argument(err: E) -> Status { fn read_account_id(id: Option) -> Result { id.ok_or(invalid_argument("missing account ID"))? .try_into() - .map_err(|err| invalid_argument(format!("invalid account ID: {}", err))) + .map_err(|err| invalid_argument(format!("invalid account ID: {err}"))) } #[instrument(target = COMPONENT, skip_all, err)] @@ -564,7 +564,7 @@ fn read_account_ids( fn validate_nullifiers(nullifiers: &[generated::digest::Digest]) -> Result, Status> { nullifiers .iter() - .cloned() + .copied() .map(TryInto::try_into) .collect::>() .map_err(|_| invalid_argument("Digest field is not in the modulus range")) diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index ab1e0045b..176d6cffd 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -233,7 +233,7 @@ impl State { .nullifiers() .iter() .filter(|&n| inner.nullifier_tree.get_block_num(n).is_some()) - .cloned() + .copied() .collect(); if !duplicate_nullifiers.is_empty() { return Err(InvalidBlockError::DuplicatedNullifiers(duplicate_nullifiers).into()); @@ -290,10 +290,10 @@ impl State { let details = match note { OutputNote::Full(note) => Some(note.to_bytes()), OutputNote::Header(_) => None, - note => { + note @ OutputNote::Partial(_) => { return Err(InvalidBlockError::InvalidOutputNoteType(Box::new( note.clone(), - ))) + ))); }, }; @@ -422,10 +422,10 @@ impl State { nullifiers.iter().map(|n| inner.nullifier_tree.open(n)).collect() } - /// Queries a list of [NoteRecord] from the database. + /// Queries a list of [`NoteRecord`] from the database. /// - /// If the provided list of [NoteId] given is empty or no [NoteRecord] matches the provided - /// [NoteId] an empty list is returned. + /// If the provided list of [`NoteId`] given is empty or no [`NoteRecord`] matches the provided + /// [`NoteId`] an empty list is returned. pub async fn get_notes_by_id( &self, note_ids: Vec, @@ -607,7 +607,7 @@ impl State { })?; let account_states = account_ids .iter() - .cloned() + .copied() .map(|account_id| { let ValuePath { value: account_hash, path: proof } = inner.account_tree.open(&LeafIndex::new_max_depth(account_id.prefix().into())); @@ -691,7 +691,7 @@ impl State { let account_ids: Vec = account_requests.iter().map(|req| req.account_id).collect(); - let state_headers = if !include_headers { + let state_headers = if include_headers.not() { BTreeMap::::default() } else { let infos = self.db.select_accounts_by_ids(account_ids.clone()).await?; @@ -724,7 +724,7 @@ impl State { let proof = storage_map.open(map_key); let slot_map_key = StorageSlotMapProof { - storage_slot: *storage_index as u32, + storage_slot: u32::from(*storage_index), smt_proof: proof.to_bytes(), }; storage_slot_map_keys.push(slot_map_key); diff --git a/crates/test-macro/Cargo.toml b/crates/test-macro/Cargo.toml index fdd7b06b0..4f1b3b15b 100644 --- a/crates/test-macro/Cargo.toml +++ b/crates/test-macro/Cargo.toml @@ -11,6 +11,9 @@ authors.workspace = true homepage.workspace = true repository.workspace = true +[lints] +workspace = true + [dependencies] quote = { version = "1.0" } syn = { version = "2.0" , features = ["full", "extra-traits"]} diff --git a/crates/utils/Cargo.toml b/crates/utils/Cargo.toml index 75f19675b..001f0d463 100644 --- a/crates/utils/Cargo.toml +++ b/crates/utils/Cargo.toml @@ -11,6 +11,9 @@ authors.workspace = true homepage.workspace = true repository.workspace = true +[lints] +workspace = true + [features] # Enables depedencies intended for build script generation of version metadata. vergen = ["dep:vergen", "dep:vergen-gitcl"] diff --git a/crates/utils/src/formatting.rs b/crates/utils/src/formatting.rs index 642eaf2ab..38e980b14 100644 --- a/crates/utils/src/formatting.rs +++ b/crates/utils/src/formatting.rs @@ -14,7 +14,7 @@ pub fn format_account_id(id: u64) -> String { } pub fn format_opt(opt: Option<&T>) -> String { - opt.map(ToString::to_string).unwrap_or("None".to_owned()) + opt.map_or("None".to_owned(), ToString::to_string) } pub fn format_input_notes(notes: &InputNotes) -> String { @@ -43,7 +43,7 @@ pub fn format_map<'a, K: Display + 'a, V: Display + 'a>( if map_str.is_empty() { "None".to_owned() } else { - format!("{{ {} }}", map_str) + format!("{{ {map_str} }}") } } @@ -52,7 +52,7 @@ pub fn format_array(list: impl IntoIterator) -> String { if comma_separated.is_empty() { "None".to_owned() } else { - format!("[{}]", comma_separated) + format!("[{comma_separated}]") } } diff --git a/crates/utils/src/version/mod.rs b/crates/utils/src/version/mod.rs index 18f2c5036..98f3c096c 100644 --- a/crates/utils/src/version/mod.rs +++ b/crates/utils/src/version/mod.rs @@ -6,7 +6,7 @@ pub use vergen::vergen; /// /// The build metadata can be embedded at compile time using the `vergen` function /// available from the `vergen` feature. See that functions description for a list -/// of the environment variables emitted which map nicely to [LongVersion]. +/// of the environment variables emitted which map nicely to [`LongVersion`]. /// /// Unfortunately these values must be transferred manually by the end user since the /// env variables are only available once the caller's build script has run - which is From e8f83147ba0277a089bf07eb8443f6bd4526727b Mon Sep 17 00:00:00 2001 From: igamigo Date: Tue, 21 Jan 2025 11:13:36 -0300 Subject: [PATCH 46/50] refactor: Use miden-objects's `BlockNumber` (#632) * refactor: Use miden-base's BlockNumber * refactor: Use native methods * feat: Use BlockNumber::GENESIS, update imports --- Cargo.lock | 50 +++--- bin/faucet/src/client.rs | 9 +- bin/faucet/src/store.rs | 5 +- .../block-producer/src/batch_builder/batch.rs | 2 +- crates/block-producer/src/block.rs | 5 +- .../block-producer/src/block_builder/mod.rs | 6 +- .../src/block_builder/prover/block_witness.rs | 4 +- .../src/block_builder/prover/mod.rs | 6 +- .../src/block_builder/prover/tests.rs | 4 +- .../block-producer/src/domain/transaction.rs | 13 +- crates/block-producer/src/errors.rs | 3 +- crates/block-producer/src/lib.rs | 4 +- .../src/mempool/inflight_state/mod.rs | 69 ++++---- crates/block-producer/src/mempool/mod.rs | 67 ++------ crates/block-producer/src/mempool/tests.rs | 14 +- .../src/mempool/transaction_expiration.rs | 8 +- crates/block-producer/src/server.rs | 20 ++- crates/block-producer/src/store/mod.rs | 8 +- crates/block-producer/src/test_utils/block.rs | 4 +- .../src/test_utils/proven_tx.rs | 7 +- crates/block-producer/src/test_utils/store.rs | 16 +- crates/proto/src/domain/accounts.rs | 5 +- crates/proto/src/domain/blocks.rs | 15 +- crates/proto/src/domain/notes.rs | 4 +- crates/store/src/blocks.rs | 27 +++- crates/store/src/db/mod.rs | 10 +- crates/store/src/db/sql/mod.rs | 96 +++++++----- crates/store/src/db/sql/utils.rs | 12 +- crates/store/src/db/tests.rs | 148 ++++++++++-------- crates/store/src/errors.rs | 7 +- crates/store/src/genesis.rs | 6 +- crates/store/src/lib.rs | 1 - crates/store/src/nullifier_tree.rs | 12 +- crates/store/src/server/api.rs | 41 +++-- crates/store/src/state.rs | 40 ++--- crates/store/src/types.rs | 1 - 36 files changed, 393 insertions(+), 356 deletions(-) delete mode 100644 crates/store/src/types.rs diff --git a/Cargo.lock b/Cargo.lock index ebdb1852a..2f78ddeb6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -409,7 +409,7 @@ checksum = "8769706aad5d996120af43197bf46ef6ad0fda35216b4505f926a365a232d924" dependencies = [ "camino", "cargo-platform", - "semver 1.0.24", + "semver 1.0.25", "serde", "serde_json", "thiserror 2.0.11", @@ -417,9 +417,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.9" +version = "1.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8293772165d9345bdaaa39b45b2109591e63fe5e6fbc23c6ff930a048aa310b" +checksum = "13208fcbb66eaeffe09b99fffbe1af420f00a7b35aa99ad683dfc1aa76145229" dependencies = [ "jobserver", "libc", @@ -478,9 +478,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.26" +version = "4.5.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8eb5e908ef3a6efbe1ed62520fb7287959888c88485abe072543190ecc66783" +checksum = "769b0145982b4b48713e01ec42d61614425f27b7058bda7180a3a41f30104796" dependencies = [ "clap_builder", "clap_derive", @@ -488,9 +488,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.26" +version = "4.5.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96b01801b5fc6a0a232407abc821660c9c6d25a1cafc0d4f85f29fb8d9afc121" +checksum = "1b26884eb4b57140e4d2d93652abfa49498b938b3c9179f9fc487b0acc3edad7" dependencies = [ "anstream", "anstyle", @@ -949,7 +949,7 @@ dependencies = [ "futures-core", "futures-sink", "http", - "indexmap 2.7.0", + "indexmap 2.7.1", "slab", "tokio", "tokio-util", @@ -1150,9 +1150,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.7.0" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62f822373a4fe84d4bb149bf54e584a7f4abec90e072ed49cda0edea5b95471f" +checksum = "8c9c992b02b5b4c94ea26e32fe5bccb7aa7d9f390ab5c1221ff895bc7ea8b652" dependencies = [ "equivalent", "hashbrown 0.15.2", @@ -1518,7 +1518,7 @@ dependencies = [ [[package]] name = "miden-lib" version = "0.7.0" -source = "git+https://github.com/0xPolygonMiden/miden-base.git?branch=next#d09a78e8674f983f30e38b5e4a2d21bed592bc95" +source = "git+https://github.com/0xPolygonMiden/miden-base.git?branch=next#a0521f17421c2db2968be404b20280457b79d195" dependencies = [ "miden-assembly", "miden-objects", @@ -1703,7 +1703,7 @@ dependencies = [ [[package]] name = "miden-objects" version = "0.7.0" -source = "git+https://github.com/0xPolygonMiden/miden-base.git?branch=next#d09a78e8674f983f30e38b5e4a2d21bed592bc95" +source = "git+https://github.com/0xPolygonMiden/miden-base.git?branch=next#a0521f17421c2db2968be404b20280457b79d195" dependencies = [ "getrandom", "miden-assembly", @@ -1713,7 +1713,7 @@ dependencies = [ "miden-verifier", "rand", "rand_xoshiro", - "semver 1.0.24", + "semver 1.0.25", "serde", "thiserror 2.0.11", "toml", @@ -1781,7 +1781,7 @@ dependencies = [ [[package]] name = "miden-tx" version = "0.7.0" -source = "git+https://github.com/0xPolygonMiden/miden-base.git?branch=next#d09a78e8674f983f30e38b5e4a2d21bed592bc95" +source = "git+https://github.com/0xPolygonMiden/miden-base.git?branch=next#a0521f17421c2db2968be404b20280457b79d195" dependencies = [ "async-trait", "miden-lib", @@ -2115,7 +2115,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" dependencies = [ "fixedbitset", - "indexmap 2.7.0", + "indexmap 2.7.1", ] [[package]] @@ -2538,7 +2538,7 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" dependencies = [ - "semver 1.0.24", + "semver 1.0.25", ] [[package]] @@ -2610,9 +2610,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.24" +version = "1.0.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3cb6eb87a131f756572d7fb904f6e7b68633f09cca868c5df1c4b8d1a694bbba" +checksum = "f79dfe2d285b0488816f30e700a7438c5a73d816b5b7d3ac72fbc48b0d185e03" dependencies = [ "serde", ] @@ -2645,9 +2645,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.135" +version = "1.0.137" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b0d7ba2887406110130a978386c4e1befb98c674b4fba677954e4db976630d9" +checksum = "930cfb6e6abf99298aaad7d29abbef7a9999a9a8806a40088f55f0dcec03146b" dependencies = [ "itoa", "memchr", @@ -3062,7 +3062,7 @@ version = "0.22.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" dependencies = [ - "indexmap 2.7.0", + "indexmap 2.7.1", "serde", "serde_spanned", "toml_datetime", @@ -3309,9 +3309,9 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "trybuild" -version = "1.0.101" +version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8dcd332a5496c026f1e14b7f3d2b7bd98e509660c04239c58b0ba38a12daded4" +checksum = "9f14b5c02a137632f68194ec657ecb92304138948e8957c932127eb1b58c23be" dependencies = [ "dissimilar", "glob", @@ -3388,9 +3388,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "valuable" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" [[package]] name = "vcpkg" diff --git a/bin/faucet/src/client.rs b/bin/faucet/src/client.rs index 562ff483d..10b911e52 100644 --- a/bin/faucet/src/client.rs +++ b/bin/faucet/src/client.rs @@ -11,6 +11,7 @@ use miden_node_proto::generated::{ use miden_objects::{ accounts::{Account, AccountData, AccountId, AuthSecretKey}, assets::FungibleAsset, + block::{BlockHeader, BlockNumber}, crypto::{ merkle::{MmrPeaks, PartialMmr}, rand::RpoRandomCoin, @@ -19,7 +20,7 @@ use miden_objects::{ transaction::{ChainMmr, ExecutedTransaction, TransactionArgs, TransactionScript}, utils::Deserializable, vm::AdviceMap, - BlockHeader, Felt, + Felt, }; use miden_tx::{ auth::BasicAuthenticator, utils::Serializable, LocalTransactionProver, ProvingOptions, @@ -146,7 +147,7 @@ impl FaucetClient { let executed_tx = self .executor - .execute_transaction(self.id, 0, &[], transaction_args) + .execute_transaction(self.id, 0.into(), &[], transaction_args) .context("Failed to execute transaction")?; Ok((executed_tx, output_note)) @@ -156,7 +157,7 @@ impl FaucetClient { pub async fn prove_and_submit_transaction( &mut self, executed_tx: ExecutedTransaction, - ) -> Result { + ) -> Result { // Prepare request with proven transaction. // This is needed to be in a separated code block in order to release reference to avoid // borrow checker error. @@ -178,7 +179,7 @@ impl FaucetClient { .await .context("Failed to submit proven transaction")?; - Ok(response.into_inner().block_height) + Ok(response.into_inner().block_height.into()) } /// Returns a reference to the data store. diff --git a/bin/faucet/src/store.rs b/bin/faucet/src/store.rs index a4a3c831e..6e36eabfa 100644 --- a/bin/faucet/src/store.rs +++ b/bin/faucet/src/store.rs @@ -2,9 +2,10 @@ use std::sync::Mutex; use miden_objects::{ accounts::{Account, AccountId}, + block::{BlockHeader, BlockNumber}, notes::NoteId, transaction::{ChainMmr, InputNotes, TransactionInputs}, - BlockHeader, Word, + Word, }; use miden_tx::{DataStore, DataStoreError}; @@ -49,7 +50,7 @@ impl DataStore for FaucetDataStore { fn get_transaction_inputs( &self, account_id: AccountId, - _block_ref: u32, + _block_ref: BlockNumber, _notes: &[NoteId], ) -> Result { let account = self.faucet_account.lock().expect("Poisoned lock"); diff --git a/crates/block-producer/src/batch_builder/batch.rs b/crates/block-producer/src/batch_builder/batch.rs index 28cc3fa26..7806e3557 100644 --- a/crates/block-producer/src/batch_builder/batch.rs +++ b/crates/block-producer/src/batch_builder/batch.rs @@ -420,7 +420,7 @@ mod tests { let txs = mock_proven_txs(); let found_unauthenticated_notes = BTreeMap::from_iter([( mock_note(5).id(), - NoteInclusionProof::new(0, 0, MerklePath::default()).unwrap(), + NoteInclusionProof::new(0.into(), 0, MerklePath::default()).unwrap(), )]); let found_unauthenticated_notes = NoteAuthenticationInfo { note_proofs: found_unauthenticated_notes, diff --git a/crates/block-producer/src/block.rs b/crates/block-producer/src/block.rs index 5d47d3efd..dd1460664 100644 --- a/crates/block-producer/src/block.rs +++ b/crates/block-producer/src/block.rs @@ -8,9 +8,10 @@ use miden_node_proto::{ }; use miden_objects::{ accounts::AccountId, + block::BlockHeader, crypto::merkle::{MerklePath, MmrPeaks, SmtProof}, notes::Nullifier, - BlockHeader, Digest, + Digest, }; // BLOCK INPUTS @@ -55,7 +56,7 @@ impl TryFrom for BlockInputs { // what is currently in the chain MMR (i.e., chain MMR with block_num = 1 has 2 leave); // this is because GetBlockInputs returns the state of the chain MMR as of one block // ago so that block_header.chain_root matches the hash of MMR peaks. - let num_leaves = block_header.block_num() as usize; + let num_leaves = block_header.block_num().as_usize(); MmrPeaks::new( num_leaves, diff --git a/crates/block-producer/src/block_builder/mod.rs b/crates/block-producer/src/block_builder/mod.rs index 110902379..5066d267a 100644 --- a/crates/block-producer/src/block_builder/mod.rs +++ b/crates/block-producer/src/block_builder/mod.rs @@ -149,7 +149,6 @@ impl BlockBuilder { let (block_header_witness, updated_accounts) = BlockWitness::new(block_inputs, batches)?; let new_block_header = self.block_kernel.prove(block_header_witness)?; - let block_num = new_block_header.block_num(); // TODO: return an error? let block = @@ -157,8 +156,9 @@ impl BlockBuilder { .expect("invalid block components"); let block_hash = block.hash(); + let block_num = new_block_header.block_num(); - info!(target: COMPONENT, block_num, %block_hash, "block built"); + info!(target: COMPONENT, %block_num, %block_hash, "block built"); debug!(target: COMPONENT, ?block); self.store @@ -166,7 +166,7 @@ impl BlockBuilder { .await .map_err(BuildBlockError::StoreApplyBlockFailed)?; - info!(target: COMPONENT, block_num, %block_hash, "block committed"); + info!(target: COMPONENT, %block_num, %block_hash, "block committed"); Ok(()) } diff --git a/crates/block-producer/src/block_builder/prover/block_witness.rs b/crates/block-producer/src/block_builder/prover/block_witness.rs index b4ff493b0..5101a421d 100644 --- a/crates/block-producer/src/block_builder/prover/block_witness.rs +++ b/crates/block-producer/src/block_builder/prover/block_witness.rs @@ -2,12 +2,12 @@ use std::collections::{BTreeMap, BTreeSet}; use miden_objects::{ accounts::{delta::AccountUpdateDetails, AccountId}, - block::BlockAccountUpdate, + block::{BlockAccountUpdate, BlockHeader}, crypto::merkle::{EmptySubtreeRoots, MerklePath, MerkleStore, MmrPeaks, SmtProof}, notes::Nullifier, transaction::TransactionId, vm::{AdviceInputs, StackInputs}, - BlockHeader, Digest, Felt, BLOCK_NOTE_TREE_DEPTH, MAX_BATCHES_PER_BLOCK, ZERO, + Digest, Felt, BLOCK_NOTE_TREE_DEPTH, MAX_BATCHES_PER_BLOCK, ZERO, }; use crate::{ diff --git a/crates/block-producer/src/block_builder/prover/mod.rs b/crates/block-producer/src/block_builder/prover/mod.rs index cede229eb..0ef259f7c 100644 --- a/crates/block-producer/src/block_builder/prover/mod.rs +++ b/crates/block-producer/src/block_builder/prover/mod.rs @@ -1,7 +1,11 @@ use std::time::{SystemTime, UNIX_EPOCH}; use miden_lib::transaction::TransactionKernel; -use miden_objects::{assembly::Assembler, block::compute_tx_hash, BlockHeader, Digest}; +use miden_objects::{ + assembly::Assembler, + block::{compute_tx_hash, BlockHeader}, + Digest, +}; use miden_processor::{execute, DefaultHost, ExecutionOptions, MemAdviceProvider, Program}; use miden_stdlib::StdLibrary; diff --git a/crates/block-producer/src/block_builder/prover/tests.rs b/crates/block-producer/src/block_builder/prover/tests.rs index d14ec2658..b739f1132 100644 --- a/crates/block-producer/src/block_builder/prover/tests.rs +++ b/crates/block-producer/src/block_builder/prover/tests.rs @@ -6,7 +6,7 @@ use miden_objects::{ accounts::{ delta::AccountUpdateDetails, AccountId, AccountIdVersion, AccountStorageMode, AccountType, }, - block::{BlockAccountUpdate, BlockNoteIndex, BlockNoteTree}, + block::{BlockAccountUpdate, BlockNoteIndex, BlockNoteTree, BlockNumber}, crypto::merkle::{ EmptySubtreeRoots, LeafIndex, MerklePath, Mmr, MmrPeaks, Smt, SmtLeaf, SmtProof, SMT_DEPTH, }, @@ -848,7 +848,7 @@ async fn compute_nullifier_root_success() { // Set up store // --------------------------------------------------------------------------------------------- - let initial_block_num = 42; + let initial_block_num = BlockNumber::from(42); let store = MockStoreSuccessBuilder::from_batches(batches.iter()) .initial_block_num(initial_block_num) diff --git a/crates/block-producer/src/domain/transaction.rs b/crates/block-producer/src/domain/transaction.rs index a8567f7fc..c23644d54 100644 --- a/crates/block-producer/src/domain/transaction.rs +++ b/crates/block-producer/src/domain/transaction.rs @@ -2,12 +2,13 @@ use std::{collections::BTreeSet, sync::Arc}; use miden_objects::{ accounts::AccountId, + block::BlockNumber, notes::{NoteId, Nullifier}, transaction::{ProvenTransaction, TransactionId, TxAccountUpdate}, Digest, }; -use crate::{errors::VerifyTxError, mempool::BlockNumber, store::TransactionInputs}; +use crate::{errors::VerifyTxError, store::TransactionInputs}; /// A transaction who's proof has been verified, and which has been authenticated against the store. /// @@ -59,7 +60,7 @@ impl AuthenticatedTransaction { Ok(AuthenticatedTransaction { inner: Arc::new(tx), notes_authenticated_by_store: inputs.found_unauthenticated_notes, - authentication_height: BlockNumber::new(inputs.current_block_height), + authentication_height: inputs.current_block_height, store_account_state: inputs.account_hash, }) } @@ -115,7 +116,7 @@ impl AuthenticatedTransaction { } pub fn expires_at(&self) -> BlockNumber { - BlockNumber::new(self.inner.expiration_block_num()) + self.inner.expiration_block_num() } } @@ -135,15 +136,15 @@ impl AuthenticatedTransaction { account_hash: store_account_state, nullifiers: inner.get_nullifiers().map(|nullifier| (nullifier, None)).collect(), found_unauthenticated_notes: BTreeSet::default(), - current_block_height: 0, + current_block_height: 0.into(), }; // SAFETY: nullifiers were set to None aka are definitely unspent. Self::new(inner, inputs).unwrap() } /// Overrides the authentication height with the given value. - pub fn with_authentication_height(mut self, height: u32) -> Self { - self.authentication_height = BlockNumber::new(height); + pub fn with_authentication_height(mut self, height: BlockNumber) -> Self { + self.authentication_height = height; self } diff --git a/crates/block-producer/src/errors.rs b/crates/block-producer/src/errors.rs index 9e30d2e9c..e3a6a670c 100644 --- a/crates/block-producer/src/errors.rs +++ b/crates/block-producer/src/errors.rs @@ -2,6 +2,7 @@ use miden_node_proto::errors::ConversionError; use miden_node_utils::formatting::format_opt; use miden_objects::{ accounts::AccountId, + block::BlockNumber, crypto::merkle::MerkleError, notes::{NoteId, Nullifier}, transaction::TransactionId, @@ -11,8 +12,6 @@ use miden_processor::ExecutionError; use thiserror::Error; use tokio::task::JoinError; -use crate::mempool::BlockNumber; - // Block-producer errors // ================================================================================================= diff --git a/crates/block-producer/src/lib.rs b/crates/block-producer/src/lib.rs index 809d8c0ca..1cb3b62c9 100644 --- a/crates/block-producer/src/lib.rs +++ b/crates/block-producer/src/lib.rs @@ -1,7 +1,5 @@ use std::time::Duration; -use mempool::BlockNumber; - #[cfg(test)] pub mod test_utils; @@ -44,7 +42,7 @@ const SERVER_MEMPOOL_STATE_RETENTION: usize = 5; /// chain tip and the transaction's expiration block. /// /// This rejects transactions which would likely expire before making it into a block. -const SERVER_MEMPOOL_EXPIRATION_SLACK: BlockNumber = BlockNumber::new(2); +const SERVER_MEMPOOL_EXPIRATION_SLACK: u32 = 2; const _: () = assert!( SERVER_MAX_BATCHES_PER_BLOCK <= miden_objects::MAX_BATCHES_PER_BLOCK, diff --git a/crates/block-producer/src/mempool/inflight_state/mod.rs b/crates/block-producer/src/mempool/inflight_state/mod.rs index 865bac2cb..5734eeeb7 100644 --- a/crates/block-producer/src/mempool/inflight_state/mod.rs +++ b/crates/block-producer/src/mempool/inflight_state/mod.rs @@ -2,6 +2,7 @@ use std::collections::{BTreeMap, BTreeSet, VecDeque}; use miden_objects::{ accounts::AccountId, + block::BlockNumber, notes::{NoteId, Nullifier}, transaction::TransactionId, }; @@ -15,8 +16,6 @@ mod account_state; use account_state::InflightAccountState; -use super::BlockNumber; - // IN-FLIGHT STATE // ================================================================================================ @@ -62,7 +61,7 @@ pub struct InflightState { /// before rejecting it. /// /// A new transaction is rejected if its expiration block is this close to the chain tip. - expiration_slack: BlockNumber, + expiration_slack: u32, } /// A summary of a transaction's impact on the state. @@ -89,11 +88,7 @@ impl Delta { impl InflightState { /// Creates an [`InflightState`] which will retain committed state for the given /// amount of blocks before pruning them. - pub fn new( - chain_tip: BlockNumber, - num_retained_blocks: usize, - expiration_slack: BlockNumber, - ) -> Self { + pub fn new(chain_tip: BlockNumber, num_retained_blocks: usize, expiration_slack: u32) -> Self { Self { num_retained_blocks, chain_tip, @@ -129,7 +124,7 @@ impl InflightState { .try_into() .expect("We should not be storing many blocks"); self.chain_tip - .checked_sub(BlockNumber::new(committed_len)) + .checked_sub(committed_len) .expect("Chain height cannot be less than number of committed blocks") } @@ -298,7 +293,7 @@ impl InflightState { self.committed_blocks.push_back(block_deltas); self.prune_block(); - self.chain_tip.increment(); + self.chain_tip = self.chain_tip.child(); } /// Prunes the state from the oldest committed block _IFF_ there are more than the number we @@ -383,14 +378,14 @@ mod tests { #[test] fn rejects_expired_transaction() { - let chain_tip = BlockNumber::new(123); - let mut uut = InflightState::new(chain_tip, 5, BlockNumber::new(0)); + let chain_tip = BlockNumber::from(123); + let mut uut = InflightState::new(chain_tip, 5, 0u32); let expired = MockProvenTxBuilder::with_account_index(0) - .expiration_block_num(chain_tip.into_inner()) + .expiration_block_num(chain_tip) .build(); - let expired = AuthenticatedTransaction::from_inner(expired) - .with_authentication_height(chain_tip.into_inner()); + let expired = + AuthenticatedTransaction::from_inner(expired).with_authentication_height(chain_tip); let err = uut.add_transaction(&expired).unwrap_err(); assert_matches!(err, AddTransactionError::Expired { .. }); @@ -399,24 +394,24 @@ mod tests { /// Ensures that the specified expiration slack is adhered to. #[test] fn expiration_slack_is_respected() { - let slack = BlockNumber::new(3); - let chain_tip = BlockNumber::new(123); - let expiration_limit = BlockNumber::new(chain_tip.into_inner() + slack.into_inner()); + let slack = 3; + let chain_tip = BlockNumber::from(123); + let expiration_limit = chain_tip + slack; let mut uut = InflightState::new(chain_tip, 5, slack); let unexpired = MockProvenTxBuilder::with_account_index(0) - .expiration_block_num(expiration_limit.next().into_inner()) + .expiration_block_num(expiration_limit + 1) .build(); - let unexpired = AuthenticatedTransaction::from_inner(unexpired) - .with_authentication_height(chain_tip.into_inner()); + let unexpired = + AuthenticatedTransaction::from_inner(unexpired).with_authentication_height(chain_tip); uut.add_transaction(&unexpired).unwrap(); let expired = MockProvenTxBuilder::with_account_index(1) - .expiration_block_num(expiration_limit.into_inner()) + .expiration_block_num(expiration_limit) .build(); - let expired = AuthenticatedTransaction::from_inner(expired) - .with_authentication_height(chain_tip.into_inner()); + let expired = + AuthenticatedTransaction::from_inner(expired).with_authentication_height(chain_tip); let err = uut.add_transaction(&expired).unwrap_err(); assert_matches!(err, AddTransactionError::Expired { .. }); @@ -439,7 +434,7 @@ mod tests { .unauthenticated_notes(vec![mock_note(note_seed)]) .build(); - let mut uut = InflightState::new(BlockNumber::default(), 1, BlockNumber::new(0)); + let mut uut = InflightState::new(BlockNumber::default(), 1, 0u32); uut.add_transaction(&AuthenticatedTransaction::from_inner(tx0)).unwrap(); uut.add_transaction(&AuthenticatedTransaction::from_inner(tx1)).unwrap(); @@ -466,7 +461,7 @@ mod tests { .output_notes(vec![note.clone()]) .build(); - let mut uut = InflightState::new(BlockNumber::default(), 1, BlockNumber::new(0)); + let mut uut = InflightState::new(BlockNumber::default(), 1, 0u32); uut.add_transaction(&AuthenticatedTransaction::from_inner(tx0)).unwrap(); let err = uut.add_transaction(&AuthenticatedTransaction::from_inner(tx1)).unwrap_err(); @@ -486,7 +481,7 @@ mod tests { let tx = MockProvenTxBuilder::with_account(account, states[0], states[1]).build(); - let mut uut = InflightState::new(BlockNumber::default(), 1, BlockNumber::new(0)); + let mut uut = InflightState::new(BlockNumber::default(), 1, 0u32); let err = uut .add_transaction(&AuthenticatedTransaction::from_inner(tx).with_store_state(states[2])) .unwrap_err(); @@ -508,7 +503,7 @@ mod tests { let tx0 = MockProvenTxBuilder::with_account(account, states[0], states[1]).build(); let tx1 = MockProvenTxBuilder::with_account(account, states[1], states[2]).build(); - let mut uut = InflightState::new(BlockNumber::default(), 1, BlockNumber::new(0)); + let mut uut = InflightState::new(BlockNumber::default(), 1, 0u32); uut.add_transaction(&AuthenticatedTransaction::from_inner(tx0)).unwrap(); uut.add_transaction(&AuthenticatedTransaction::from_inner(tx1).with_empty_store_state()) .unwrap(); @@ -525,7 +520,7 @@ mod tests { ) .build(); - let mut uut = InflightState::new(BlockNumber::default(), 1, BlockNumber::new(0)); + let mut uut = InflightState::new(BlockNumber::default(), 1, 0u32); uut.add_transaction(&AuthenticatedTransaction::from_inner(tx).with_empty_store_state()) .unwrap(); } @@ -538,7 +533,7 @@ mod tests { let tx0 = MockProvenTxBuilder::with_account(account, states[0], states[1]).build(); let tx1 = MockProvenTxBuilder::with_account(account, states[1], states[2]).build(); - let mut uut = InflightState::new(BlockNumber::default(), 1, BlockNumber::new(0)); + let mut uut = InflightState::new(BlockNumber::default(), 1, 0u32); uut.add_transaction(&AuthenticatedTransaction::from_inner(tx0)).unwrap(); // Feed in an old state via input. This should be ignored, and the previous tx's final @@ -564,7 +559,7 @@ mod tests { .unauthenticated_notes(vec![mock_note(note_seed)]) .build(); - let mut uut = InflightState::new(BlockNumber::default(), 1, BlockNumber::new(0)); + let mut uut = InflightState::new(BlockNumber::default(), 1, 0u32); uut.add_transaction(&AuthenticatedTransaction::from_inner(tx0.clone())).unwrap(); uut.add_transaction(&AuthenticatedTransaction::from_inner(tx1.clone())).unwrap(); @@ -595,7 +590,7 @@ mod tests { .unauthenticated_notes(vec![mock_note(note_seed)]) .build(); - let mut uut = InflightState::new(BlockNumber::default(), 1, BlockNumber::new(0)); + let mut uut = InflightState::new(BlockNumber::default(), 1, 0u32); uut.add_transaction(&tx0.clone()).unwrap(); uut.add_transaction(&tx1.clone()).unwrap(); @@ -637,7 +632,7 @@ mod tests { for i in 0..states.len() { // Insert all txs and then revert the last `i` of them. // This should match only inserting the first `N-i` of them. - let mut reverted = InflightState::new(BlockNumber::default(), 1, BlockNumber::new(0)); + let mut reverted = InflightState::new(BlockNumber::default(), 1, 0u32); for (idx, tx) in txs.iter().enumerate() { reverted.add_transaction(tx).unwrap_or_else(|err| { panic!("Inserting tx #{idx} in iteration {i} should succeed: {err}") @@ -647,7 +642,7 @@ mod tests { txs.iter().rev().take(i).rev().map(AuthenticatedTransaction::id).collect(), ); - let mut inserted = InflightState::new(BlockNumber::default(), 1, BlockNumber::new(0)); + let mut inserted = InflightState::new(BlockNumber::default(), 1, 0u32); for (idx, tx) in txs.iter().rev().skip(i).rev().enumerate() { inserted.add_transaction(tx).unwrap_or_else(|err| { panic!("Inserting tx #{idx} in iteration {i} should succeed: {err}") @@ -691,7 +686,7 @@ mod tests { // This should match only inserting the final `N-i` transactions. // Note: we force all committed state to immedietely be pruned by setting // it to zero. - let mut committed = InflightState::new(BlockNumber::default(), 0, BlockNumber::new(0)); + let mut committed = InflightState::new(BlockNumber::default(), 0, 0u32); for (idx, tx) in txs.iter().enumerate() { committed.add_transaction(tx).unwrap_or_else(|err| { panic!("Inserting tx #{idx} in iteration {i} should succeed: {err}") @@ -699,10 +694,10 @@ mod tests { } committed.commit_block(txs.iter().take(i).map(AuthenticatedTransaction::id)); - let mut inserted = InflightState::new(BlockNumber::new(1), 0, BlockNumber::new(0)); + let mut inserted = InflightState::new(BlockNumber::from(1), 0, 0u32); for (idx, tx) in txs.iter().skip(i).enumerate() { // We need to adjust the height since we are effectively at block "1" now. - let tx = tx.clone().with_authentication_height(1); + let tx = tx.clone().with_authentication_height(1.into()); inserted.add_transaction(&tx).unwrap_or_else(|err| { panic!("Inserting tx #{idx} in iteration {i} should succeed: {err}") }); diff --git a/crates/block-producer/src/mempool/mod.rs b/crates/block-producer/src/mempool/mod.rs index 22873c3fc..a772dfb50 100644 --- a/crates/block-producer/src/mempool/mod.rs +++ b/crates/block-producer/src/mempool/mod.rs @@ -1,11 +1,11 @@ -use std::{collections::BTreeSet, fmt::Display, sync::Arc}; +use std::{collections::BTreeSet, sync::Arc}; use batch_graph::BatchGraph; use graph::GraphError; use inflight_state::InflightState; use miden_objects::{ - transaction::TransactionId, MAX_ACCOUNTS_PER_BATCH, MAX_INPUT_NOTES_PER_BATCH, - MAX_OUTPUT_NOTES_PER_BATCH, + block::BlockNumber, transaction::TransactionId, MAX_ACCOUNTS_PER_BATCH, + MAX_INPUT_NOTES_PER_BATCH, MAX_OUTPUT_NOTES_PER_BATCH, }; use tokio::sync::Mutex; use tracing::instrument; @@ -28,51 +28,6 @@ mod transaction_graph; #[cfg(test)] mod tests; -#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] -pub struct BlockNumber(u32); - -impl Display for BlockNumber { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - self.0.fmt(f) - } -} - -impl std::ops::Add for BlockNumber { - type Output = Self; - - fn add(self, rhs: Self) -> Self::Output { - BlockNumber::new(self.0 + rhs.0) - } -} - -impl BlockNumber { - pub const fn new(x: u32) -> Self { - Self(x) - } - - pub fn next(mut self) -> Self { - self.increment(); - - self - } - - pub fn prev(self) -> Option { - self.checked_sub(Self(1)) - } - - pub fn increment(&mut self) { - self.0 += 1; - } - - pub fn checked_sub(self, rhs: Self) -> Option { - self.0.checked_sub(rhs.0).map(Self) - } - - pub fn into_inner(self) -> u32 { - self.0 - } -} - // MEMPOOL BUDGET // ================================================================================================ @@ -211,7 +166,7 @@ impl Mempool { batch_budget: BatchBudget, block_budget: BlockBudget, state_retention: usize, - expiration_slack: BlockNumber, + expiration_slack: u32, ) -> SharedMempool { Arc::new(Mutex::new(Self::new( chain_tip, @@ -227,7 +182,7 @@ impl Mempool { batch_budget: BatchBudget, block_budget: BlockBudget, state_retention: usize, - expiration_slack: BlockNumber, + expiration_slack: u32, ) -> Mempool { Self { chain_tip, @@ -254,7 +209,7 @@ impl Mempool { pub fn add_transaction( &mut self, transaction: AuthenticatedTransaction, - ) -> Result { + ) -> Result { // Add transaction to inflight state. let parents = self.state.add_transaction(&transaction)?; @@ -264,7 +219,7 @@ impl Mempool { .insert(transaction, parents) .expect("Transaction should insert after passing inflight state"); - Ok(self.chain_tip.0) + Ok(self.chain_tip) } /// Returns a set of transactions for the next batch. @@ -338,7 +293,7 @@ impl Mempool { let batches = self.batches.select_block(self.block_budget); self.block_in_progress = Some(batches.iter().map(TransactionBatch::id).collect()); - (self.chain_tip.next(), batches) + (self.chain_tip.child(), batches) } /// Notify the pool that the block was successfully completed. @@ -348,7 +303,7 @@ impl Mempool { /// Panics if blocks are completed out-of-order or if there is no block in flight. #[instrument(target = COMPONENT, skip_all, fields(block_number))] pub fn block_committed(&mut self, block_number: BlockNumber) { - assert_eq!(block_number, self.chain_tip.next(), "Blocks must be submitted sequentially"); + assert_eq!(block_number, self.chain_tip.child(), "Blocks must be submitted sequentially"); // Remove committed batches and transactions from graphs. let batches = self.block_in_progress.take().expect("No block in progress to commit"); @@ -363,7 +318,7 @@ impl Mempool { // Inform inflight state about committed data. self.state.commit_block(transactions); - self.chain_tip.increment(); + self.chain_tip = self.chain_tip.child(); // Revert expired transactions and their descendents. let expired = self.expirations.get(block_number); @@ -379,7 +334,7 @@ impl Mempool { /// inflight block. #[instrument(target = COMPONENT, skip_all, fields(block_number))] pub fn block_failed(&mut self, block_number: BlockNumber) { - assert_eq!(block_number, self.chain_tip.next(), "Blocks must be submitted sequentially"); + assert_eq!(block_number, self.chain_tip.child(), "Blocks must be submitted sequentially"); let batches = self.block_in_progress.take().expect("No block in progress to be failed"); diff --git a/crates/block-producer/src/mempool/tests.rs b/crates/block-producer/src/mempool/tests.rs index 6a4ad8b91..1e68933d1 100644 --- a/crates/block-producer/src/mempool/tests.rs +++ b/crates/block-producer/src/mempool/tests.rs @@ -1,4 +1,5 @@ use miden_node_proto::domain::notes::NoteAuthenticationInfo; +use miden_objects::block::BlockNumber; use pretty_assertions::assert_eq; use super::*; @@ -7,11 +8,11 @@ use crate::test_utils::MockProvenTxBuilder; impl Mempool { fn for_tests() -> Self { Self::new( - BlockNumber::new(0), + BlockNumber::GENESIS, BatchBudget::default(), BlockBudget::default(), 5, - BlockNumber::default(), + u32::default(), ) } } @@ -107,9 +108,8 @@ fn block_commit_reverts_expired_txns() { let mut reference = uut.clone(); // Add a new transaction which will expire when the pending block is committed. - let tx_to_revert = MockProvenTxBuilder::with_account_index(1) - .expiration_block_num(block.into_inner()) - .build(); + let tx_to_revert = + MockProvenTxBuilder::with_account_index(1).expiration_block_num(block).build(); let tx_to_revert = AuthenticatedTransaction::from_inner(tx_to_revert); uut.add_transaction(tx_to_revert).unwrap(); @@ -136,13 +136,13 @@ fn blocks_must_be_committed_sequentially() { let mut uut = Mempool::for_tests(); let (block, _) = uut.select_block(); - uut.block_committed(block.next()); + uut.block_committed(block + 1); } #[test] #[should_panic] fn block_commitment_is_rejected_if_no_block_is_in_flight() { - Mempool::for_tests().block_committed(BlockNumber::new(1)); + Mempool::for_tests().block_committed(BlockNumber::from(1)); } #[test] diff --git a/crates/block-producer/src/mempool/transaction_expiration.rs b/crates/block-producer/src/mempool/transaction_expiration.rs index 79dc450aa..e806a415c 100644 --- a/crates/block-producer/src/mempool/transaction_expiration.rs +++ b/crates/block-producer/src/mempool/transaction_expiration.rs @@ -1,8 +1,6 @@ use std::collections::{btree_map::Entry, BTreeMap, BTreeSet}; -use miden_objects::transaction::TransactionId; - -use super::BlockNumber; +use miden_objects::{block::BlockNumber, transaction::TransactionId}; /// Tracks transactions and their expiration block heights. /// @@ -59,7 +57,7 @@ mod tests { #[test] fn remove_prunes_empty_block_maps() { let tx = Random::with_random_seed().draw_tx_id(); - let block = BlockNumber::new(123); + let block = BlockNumber::from(123); let mut uut = TransactionExpirations::default(); uut.insert(tx, block); @@ -70,6 +68,6 @@ mod tests { #[test] fn get_empty() { - assert!(TransactionExpirations::default().get(BlockNumber(123)).is_empty()); + assert!(TransactionExpirations::default().get(BlockNumber::from(123)).is_empty()); } } diff --git a/crates/block-producer/src/server.rs b/crates/block-producer/src/server.rs index b0b445f2c..5c7a42ec5 100644 --- a/crates/block-producer/src/server.rs +++ b/crates/block-producer/src/server.rs @@ -8,7 +8,9 @@ use miden_node_utils::{ errors::ApiError, formatting::{format_input_notes, format_output_notes}, }; -use miden_objects::{transaction::ProvenTransaction, utils::serde::Deserializable}; +use miden_objects::{ + block::BlockNumber, transaction::ProvenTransaction, utils::serde::Deserializable, +}; use tokio::{net::TcpListener, sync::Mutex}; use tokio_stream::wrappers::TcpListenerStream; use tonic::Status; @@ -20,7 +22,7 @@ use crate::{ config::BlockProducerConfig, domain::transaction::AuthenticatedTransaction, errors::{AddTransactionError, BlockProducerError, VerifyTxError}, - mempool::{BatchBudget, BlockBudget, BlockNumber, Mempool, SharedMempool}, + mempool::{BatchBudget, BlockBudget, Mempool, SharedMempool}, store::StoreClient, COMPONENT, SERVER_MEMPOOL_EXPIRATION_SLACK, SERVER_MEMPOOL_STATE_RETENTION, }; @@ -37,7 +39,7 @@ pub struct BlockProducer { batch_budget: BatchBudget, block_budget: BlockBudget, state_retention: usize, - expiration_slack: BlockNumber, + expiration_slack: u32, rpc_listener: TcpListener, store: StoreClient, chain_tip: BlockNumber, @@ -60,7 +62,7 @@ impl BlockProducer { .latest_header() .await .map_err(|err| ApiError::DatabaseConnectionFailed(err.to_string()))?; - let chain_tip = BlockNumber::new(latest_header.block_num()); + let chain_tip = latest_header.block_num(); let rpc_listener = config .endpoint @@ -245,12 +247,8 @@ impl BlockProducerRpcServer { // SAFETY: we assume that the rpc component has verified the transaction proof already. let tx = AuthenticatedTransaction::new(tx, inputs)?; - self.mempool - .lock() - .await - .lock() - .await - .add_transaction(tx) - .map(|block_height| SubmitProvenTransactionResponse { block_height }) + self.mempool.lock().await.lock().await.add_transaction(tx).map(|block_height| { + SubmitProvenTransactionResponse { block_height: block_height.as_u32() } + }) } } diff --git a/crates/block-producer/src/store/mod.rs b/crates/block-producer/src/store/mod.rs index d49652f7f..9d2f76bd3 100644 --- a/crates/block-producer/src/store/mod.rs +++ b/crates/block-producer/src/store/mod.rs @@ -22,11 +22,11 @@ use miden_node_proto::{ use miden_node_utils::formatting::format_opt; use miden_objects::{ accounts::AccountId, - block::Block, + block::{Block, BlockHeader, BlockNumber}, notes::{NoteId, Nullifier}, transaction::ProvenTransaction, utils::Serializable, - BlockHeader, Digest, + Digest, }; use miden_processor::crypto::RpoDigest; use tonic::transport::Channel; @@ -53,7 +53,7 @@ pub struct TransactionInputs { /// These are notes which were committed _after_ the transaction was created. pub found_unauthenticated_notes: BTreeSet, /// The current block height. - pub current_block_height: u32, + pub current_block_height: BlockNumber, } impl Display for TransactionInputs { @@ -106,7 +106,7 @@ impl TryFrom for TransactionInputs { .map(|digest| Ok(RpoDigest::try_from(digest)?.into())) .collect::>()?; - let current_block_height = response.block_height; + let current_block_height = response.block_height.into(); Ok(Self { account_id, diff --git a/crates/block-producer/src/test_utils/block.rs b/crates/block-producer/src/test_utils/block.rs index 54fa67631..27276a6ee 100644 --- a/crates/block-producer/src/test_utils/block.rs +++ b/crates/block-producer/src/test_utils/block.rs @@ -1,11 +1,11 @@ use std::iter; use miden_objects::{ - block::{Block, BlockAccountUpdate, BlockNoteIndex, BlockNoteTree, NoteBatch}, + block::{Block, BlockAccountUpdate, BlockHeader, BlockNoteIndex, BlockNoteTree, NoteBatch}, crypto::merkle::{Mmr, SimpleSmt}, notes::Nullifier, transaction::OutputNote, - BlockHeader, Digest, ACCOUNT_TREE_DEPTH, + Digest, ACCOUNT_TREE_DEPTH, }; use super::MockStoreSuccess; diff --git a/crates/block-producer/src/test_utils/proven_tx.rs b/crates/block-producer/src/test_utils/proven_tx.rs index 396e0c319..55a69dba7 100644 --- a/crates/block-producer/src/test_utils/proven_tx.rs +++ b/crates/block-producer/src/test_utils/proven_tx.rs @@ -4,6 +4,7 @@ use itertools::Itertools; use miden_air::HashFunction; use miden_objects::{ accounts::AccountId, + block::BlockNumber, notes::{Note, NoteExecutionHint, NoteHeader, NoteMetadata, NoteType, Nullifier}, transaction::{InputNote, OutputNote, ProvenTransaction, ProvenTransactionBuilder}, vm::ExecutionProof, @@ -19,7 +20,7 @@ pub struct MockProvenTxBuilder { account_id: AccountId, initial_account_hash: Digest, final_account_hash: Digest, - expiration_block_num: u32, + expiration_block_num: BlockNumber, output_notes: Option>, input_notes: Option>, nullifiers: Option>, @@ -60,7 +61,7 @@ impl MockProvenTxBuilder { account_id, initial_account_hash, final_account_hash, - expiration_block_num: u32::MAX, + expiration_block_num: u32::MAX.into(), output_notes: None, input_notes: None, nullifiers: None, @@ -82,7 +83,7 @@ impl MockProvenTxBuilder { } #[must_use] - pub fn expiration_block_num(mut self, expiration_block_num: u32) -> Self { + pub fn expiration_block_num(mut self, expiration_block_num: BlockNumber) -> Self { self.expiration_block_num = expiration_block_num; self diff --git a/crates/block-producer/src/test_utils/store.rs b/crates/block-producer/src/test_utils/store.rs index e4eccfb18..63dd1f31f 100644 --- a/crates/block-producer/src/test_utils/store.rs +++ b/crates/block-producer/src/test_utils/store.rs @@ -5,11 +5,11 @@ use std::{ use miden_node_proto::domain::{blocks::BlockInclusionProof, notes::NoteAuthenticationInfo}; use miden_objects::{ - block::{Block, NoteBatch}, + block::{Block, BlockHeader, BlockNumber, NoteBatch}, crypto::merkle::{Mmr, SimpleSmt, Smt, ValuePath}, notes::{NoteId, NoteInclusionProof, Nullifier}, transaction::ProvenTransaction, - BlockHeader, ACCOUNT_TREE_DEPTH, EMPTY_WORD, ZERO, + ACCOUNT_TREE_DEPTH, EMPTY_WORD, ZERO, }; use tokio::sync::RwLock; @@ -31,7 +31,7 @@ pub struct MockStoreSuccessBuilder { notes: Option>, produced_nullifiers: Option>, chain_mmr: Option, - block_num: Option, + block_num: Option, } impl MockStoreSuccessBuilder { @@ -94,14 +94,14 @@ impl MockStoreSuccessBuilder { } #[must_use] - pub fn initial_block_num(mut self, block_num: u32) -> Self { + pub fn initial_block_num(mut self, block_num: BlockNumber) -> Self { self.block_num = Some(block_num); self } pub fn build(self) -> MockStoreSuccess { - let block_num = self.block_num.unwrap_or(1); + let block_num = self.block_num.unwrap_or(1.into()); let accounts_smt = self.accounts.unwrap_or(SimpleSmt::new().unwrap()); let notes = self.notes.unwrap_or_default(); let block_note_tree = note_created_smt_from_note_batches(notes.iter()); @@ -172,7 +172,7 @@ pub struct MockStoreSuccess { pub chain_mmr: Arc>, /// The chains block headers. - pub block_headers: Arc>>, + pub block_headers: Arc>>, /// The number of times `apply_block()` was called pub num_apply_block_called: Arc>, @@ -282,7 +282,7 @@ impl MockStoreSuccess { account_hash, nullifiers, found_unauthenticated_notes, - current_block_height: 0, + current_block_height: 0.into(), }) } @@ -331,7 +331,7 @@ impl MockStoreSuccess { .map(|note_proof| { let block_num = note_proof.location().block_num(); let block_header = *locked_headers.get(&block_num).unwrap(); - let mmr_path = locked_chain_mmr.open(block_num as usize).unwrap().merkle_path; + let mmr_path = locked_chain_mmr.open(block_num.as_usize()).unwrap().merkle_path; BlockInclusionProof { block_header, mmr_path, chain_length } }) diff --git a/crates/proto/src/domain/accounts.rs b/crates/proto/src/domain/accounts.rs index 1283e193a..a5e387a3f 100644 --- a/crates/proto/src/domain/accounts.rs +++ b/crates/proto/src/domain/accounts.rs @@ -3,6 +3,7 @@ use std::fmt::{Debug, Display, Formatter}; use miden_node_utils::formatting::format_opt; use miden_objects::{ accounts::{Account, AccountHeader, AccountId}, + block::BlockNumber, crypto::{hash::rpo::RpoDigest, merkle::MerklePath}, utils::{Deserializable, Serializable}, Digest, @@ -66,7 +67,7 @@ impl TryFrom for AccountId { pub struct AccountSummary { pub account_id: AccountId, pub account_hash: RpoDigest, - pub block_num: u32, + pub block_num: BlockNumber, } impl From<&AccountSummary> for proto::account::AccountSummary { @@ -74,7 +75,7 @@ impl From<&AccountSummary> for proto::account::AccountSummary { Self { account_id: Some(update.account_id.into()), account_hash: Some(update.account_hash.into()), - block_num: update.block_num, + block_num: update.block_num.as_u32(), } } } diff --git a/crates/proto/src/domain/blocks.rs b/crates/proto/src/domain/blocks.rs index 40f8b1032..fa7e4bcfb 100644 --- a/crates/proto/src/domain/blocks.rs +++ b/crates/proto/src/domain/blocks.rs @@ -1,4 +1,7 @@ -use miden_objects::{crypto::merkle::MerklePath, BlockHeader}; +use miden_objects::{ + block::{BlockHeader, BlockNumber}, + crypto::merkle::MerklePath, +}; use crate::{ errors::{ConversionError, MissingFieldHelper}, @@ -13,7 +16,7 @@ impl From<&BlockHeader> for proto::BlockHeader { Self { version: header.version(), prev_hash: Some(header.prev_hash().into()), - block_num: header.block_num(), + block_num: header.block_num().as_u32(), chain_root: Some(header.chain_root().into()), account_root: Some(header.account_root().into()), nullifier_root: Some(header.nullifier_root().into()), @@ -50,7 +53,7 @@ impl TryFrom for BlockHeader { .prev_hash .ok_or(proto::BlockHeader::missing_field(stringify!(prev_hash)))? .try_into()?, - value.block_num, + value.block_num.into(), value .chain_root .ok_or(proto::BlockHeader::missing_field(stringify!(chain_root)))? @@ -89,7 +92,7 @@ impl TryFrom for BlockHeader { pub struct BlockInclusionProof { pub block_header: BlockHeader, pub mmr_path: MerklePath, - pub chain_length: u32, + pub chain_length: BlockNumber, } impl From for proto::BlockInclusionProof { @@ -97,7 +100,7 @@ impl From for proto::BlockInclusionProof { Self { block_header: Some(value.block_header.into()), mmr_path: Some((&value.mmr_path).into()), - chain_length: value.chain_length, + chain_length: value.chain_length.as_u32(), } } } @@ -115,7 +118,7 @@ impl TryFrom for BlockInclusionProof { .mmr_path .ok_or(proto::BlockInclusionProof::missing_field("mmr_path"))?) .try_into()?, - chain_length: value.chain_length, + chain_length: value.chain_length.into(), }; Ok(result) diff --git a/crates/proto/src/domain/notes.rs b/crates/proto/src/domain/notes.rs index 4ea9e8652..6535c3a47 100644 --- a/crates/proto/src/domain/notes.rs +++ b/crates/proto/src/domain/notes.rs @@ -54,7 +54,7 @@ impl From<(&NoteId, &NoteInclusionProof)> for proto::NoteInclusionInBlockProof { fn from((note_id, proof): (&NoteId, &NoteInclusionProof)) -> Self { Self { note_id: Some(note_id.into()), - block_num: proof.location().block_num(), + block_num: proof.location().block_num().as_u32(), note_index_in_block: proof.location().node_index_in_block().into(), merkle_path: Some(Into::into(proof.note_path())), } @@ -76,7 +76,7 @@ impl TryFrom<&proto::NoteInclusionInBlockProof> for (NoteId, NoteInclusionProof) )? .into(), NoteInclusionProof::new( - proof.block_num, + proof.block_num.into(), proof.note_index_in_block.try_into()?, proof .merkle_path diff --git a/crates/store/src/blocks.rs b/crates/store/src/blocks.rs index 41ffc3af0..df2147abf 100644 --- a/crates/store/src/blocks.rs +++ b/crates/store/src/blocks.rs @@ -1,5 +1,7 @@ use std::{io::ErrorKind, path::PathBuf}; +use miden_objects::block::BlockNumber; + #[derive(Debug)] pub struct BlockStore { store_dir: PathBuf, @@ -12,7 +14,10 @@ impl BlockStore { Ok(Self { store_dir }) } - pub async fn load_block(&self, block_num: u32) -> Result>, std::io::Error> { + pub async fn load_block( + &self, + block_num: BlockNumber, + ) -> Result>, std::io::Error> { match tokio::fs::read(self.block_path(block_num)).await { Ok(data) => Ok(Some(data)), Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(None), @@ -20,7 +25,11 @@ impl BlockStore { } } - pub async fn save_block(&self, block_num: u32, data: &[u8]) -> Result<(), std::io::Error> { + pub async fn save_block( + &self, + block_num: BlockNumber, + data: &[u8], + ) -> Result<(), std::io::Error> { let (epoch_path, block_path) = self.epoch_block_path(block_num)?; if !epoch_path.exists() { tokio::fs::create_dir_all(epoch_path).await?; @@ -29,7 +38,11 @@ impl BlockStore { tokio::fs::write(block_path, data).await } - pub fn save_block_blocking(&self, block_num: u32, data: &[u8]) -> Result<(), std::io::Error> { + pub fn save_block_blocking( + &self, + block_num: BlockNumber, + data: &[u8], + ) -> Result<(), std::io::Error> { let (epoch_path, block_path) = self.epoch_block_path(block_num)?; if !epoch_path.exists() { std::fs::create_dir_all(epoch_path)?; @@ -41,13 +54,17 @@ impl BlockStore { // HELPER FUNCTIONS // -------------------------------------------------------------------------------------------- - fn block_path(&self, block_num: u32) -> PathBuf { + fn block_path(&self, block_num: BlockNumber) -> PathBuf { + let block_num = block_num.as_u32(); let epoch = block_num >> 16; let epoch_dir = self.store_dir.join(format!("{epoch:04x}")); epoch_dir.join(format!("block_{block_num:08x}.dat")) } - fn epoch_block_path(&self, block_num: u32) -> Result<(PathBuf, PathBuf), std::io::Error> { + fn epoch_block_path( + &self, + block_num: BlockNumber, + ) -> Result<(PathBuf, PathBuf), std::io::Error> { let block_path = self.block_path(block_num); let epoch_path = block_path.parent().ok_or(std::io::Error::from(ErrorKind::NotFound))?; diff --git a/crates/store/src/db/mod.rs b/crates/store/src/db/mod.rs index fc466d502..ba589a1e9 100644 --- a/crates/store/src/db/mod.rs +++ b/crates/store/src/db/mod.rs @@ -11,12 +11,11 @@ use miden_node_proto::{ }; use miden_objects::{ accounts::{AccountDelta, AccountId}, - block::{Block, BlockNoteIndex}, + block::{Block, BlockHeader, BlockNoteIndex, BlockNumber}, crypto::{hash::rpo::RpoDigest, merkle::MerklePath, utils::Deserializable}, notes::{NoteId, NoteInclusionProof, NoteMetadata, Nullifier}, transaction::TransactionId, utils::Serializable, - BlockHeader, GENESIS_BLOCK, }; use rusqlite::vtab::array; use tokio::sync::oneshot; @@ -28,7 +27,6 @@ use crate::{ db::migrations::apply_migrations, errors::{DatabaseError, DatabaseSetupError, GenesisError, NoteSyncError, StateSyncError}, genesis::GenesisState, - types::BlockNumber, COMPONENT, SQL_STATEMENT_CACHE_CAPACITY, }; @@ -71,7 +69,7 @@ pub struct NoteRecord { impl From for proto::Note { fn from(note: NoteRecord) -> Self { Self { - block_num: note.block_num, + block_num: note.block_num.as_u32(), note_index: note.note_index.leaf_index_value().into(), note_id: Some(note.note_id.into()), metadata: Some(note.metadata.into()), @@ -490,7 +488,7 @@ impl Db { }; let maybe_block_header_in_store = self - .select_block_header_by_block_num(Some(GENESIS_BLOCK)) + .select_block_header_by_block_num(Some(BlockNumber::GENESIS)) .await .map_err(|err| GenesisError::SelectBlockHeaderByBlockNumError(err.into()))?; @@ -527,7 +525,7 @@ impl Db { genesis_block.updated_accounts(), )?; - block_store.save_block_blocking(0, &genesis_block.to_bytes())?; + block_store.save_block_blocking(0.into(), &genesis_block.to_bytes())?; transaction.commit()?; diff --git a/crates/store/src/db/sql/mod.rs b/crates/store/src/db/sql/mod.rs index a6539b7c8..ca1592429 100644 --- a/crates/store/src/db/sql/mod.rs +++ b/crates/store/src/db/sql/mod.rs @@ -17,15 +17,15 @@ use miden_objects::{ StorageMapDelta, }, assets::NonFungibleAsset, - block::{BlockAccountUpdate, BlockNoteIndex}, + block::{BlockAccountUpdate, BlockHeader, BlockNoteIndex, BlockNumber}, crypto::{hash::rpo::RpoDigest, merkle::MerklePath}, notes::{NoteId, NoteInclusionProof, NoteMetadata, NoteType, Nullifier}, transaction::TransactionId, utils::serde::{Deserializable, Serializable}, - BlockHeader, Digest, Word, + Digest, Word, }; use rusqlite::{params, types::Value, Connection, Transaction}; -use utils::read_from_blob_column; +use utils::{read_block_number, read_from_blob_column}; use super::{ NoteRecord, NoteSyncRecord, NoteSyncUpdate, NullifierInfo, Result, StateSyncUpdate, @@ -37,7 +37,6 @@ use crate::{ get_nullifier_prefix, u64_to_value, }, errors::{DatabaseError, NoteSyncError, StateSyncError}, - types::BlockNumber, }; // ACCOUNT QUERIES // ================================================================================================ @@ -127,7 +126,8 @@ pub fn select_accounts_by_block_range( .copied() .map(|account_id| account_id.to_bytes().into()) .collect(); - let mut rows = stmt.query(params![block_start, block_end, Rc::new(account_ids)])?; + let mut rows = + stmt.query(params![block_start.as_u32(), block_end.as_u32(), Rc::new(account_ids)])?; let mut result = Vec::new(); while let Some(row) = rows.next()? { @@ -309,15 +309,20 @@ pub fn select_account_delta( let account_id = account_id.to_bytes(); let nonce = match select_nonce_stmt - .query_row(params![account_id, block_start, block_end], |row| row.get::<_, u64>(0)) - { + .query_row(params![account_id, block_start.as_u32(), block_end.as_u32()], |row| { + row.get::<_, u64>(0) + }) { Ok(nonce) => nonce.try_into().map_err(DatabaseError::InvalidFelt)?, Err(rusqlite::Error::QueryReturnedNoRows) => return Ok(None), Err(e) => return Err(e.into()), }; let mut storage_scalars = BTreeMap::new(); - let mut rows = select_slot_updates_stmt.query(params![account_id, block_start, block_end])?; + let mut rows = select_slot_updates_stmt.query(params![ + account_id, + block_start.as_u32(), + block_end.as_u32() + ])?; while let Some(row) = rows.next()? { let slot = row.get(0)?; let value_data = row.get_ref(1)?.as_blob()?; @@ -326,8 +331,11 @@ pub fn select_account_delta( } let mut storage_maps = BTreeMap::new(); - let mut rows = - select_storage_map_updates_stmt.query(params![account_id, block_start, block_end])?; + let mut rows = select_storage_map_updates_stmt.query(params![ + account_id, + block_start.as_u32(), + block_end.as_u32() + ])?; while let Some(row) = rows.next()? { let slot = row.get(0)?; let key_data = row.get_ref(1)?.as_blob()?; @@ -346,8 +354,11 @@ pub fn select_account_delta( } let mut fungible = BTreeMap::new(); - let mut rows = - select_fungible_asset_deltas_stmt.query(params![account_id, block_start, block_end])?; + let mut rows = select_fungible_asset_deltas_stmt.query(params![ + account_id, + block_start.as_u32(), + block_end.as_u32() + ])?; while let Some(row) = rows.next()? { let faucet_id: AccountId = read_from_blob_column(row, 0)?; let value = row.get(1)?; @@ -357,8 +368,8 @@ pub fn select_account_delta( let mut non_fungible_delta = NonFungibleAssetDelta::default(); let mut rows = select_non_fungible_asset_updates_stmt.query(params![ account_id, - block_start, - block_end + block_start.as_u32(), + block_end.as_u32() ])?; while let Some(row) = rows.next()? { let vault_key_data = row.get_ref(1)?.as_blob()?; @@ -438,7 +449,7 @@ pub fn upsert_accounts( let inserted = upsert_stmt.execute(params![ account_id.to_bytes(), update.new_state_hash().to_bytes(), - block_num, + block_num.as_u32(), full_account.as_ref().map(|account| account.to_bytes()), ])?; @@ -499,14 +510,14 @@ fn insert_account_delta( insert_acc_delta_stmt.execute(params![ account_id.to_bytes(), - block_number, + block_number.as_u32(), delta.nonce().map(Into::::into).unwrap_or_default() ])?; for (&slot, value) in delta.storage().values() { insert_slot_update_stmt.execute(params![ account_id.to_bytes(), - block_number, + block_number.as_u32(), slot, value.to_bytes() ])?; @@ -516,7 +527,7 @@ fn insert_account_delta( for (key, value) in map_delta.leaves() { insert_storage_map_update_stmt.execute(params![ account_id.to_bytes(), - block_number, + block_number.as_u32(), slot, key.to_bytes(), value.to_bytes(), @@ -527,7 +538,7 @@ fn insert_account_delta( for (&faucet_id, &delta) in delta.vault().fungible().iter() { insert_fungible_asset_delta_stmt.execute(params![ account_id.to_bytes(), - block_number, + block_number.as_u32(), faucet_id.to_bytes(), delta, ])?; @@ -540,7 +551,7 @@ fn insert_account_delta( }; insert_non_fungible_asset_update_stmt.execute(params![ account_id.to_bytes(), - block_number, + block_number.as_u32(), asset.to_bytes(), is_remove, ])?; @@ -576,7 +587,7 @@ pub fn insert_nullifiers_for_block( count += stmt.execute(params![ nullifier.to_bytes(), get_nullifier_prefix(nullifier), - block_num + block_num.as_u32() ])?; } Ok(count) @@ -596,7 +607,7 @@ pub fn select_all_nullifiers(conn: &mut Connection) -> Result Result> { NoteMetadata::new(sender, note_type, tag.into(), execution_hint.try_into()?, aux)?; notes.push(NoteRecord { - block_num: row.get(0)?, + block_num: read_block_number(row, 0)?, note_index: BlockNoteIndex::new(row.get(1)?, row.get(2)?)?, note_id, metadata, @@ -791,7 +803,7 @@ pub fn insert_notes(transaction: &Transaction, notes: &[NoteRecord]) -> Result(4)?; @@ -922,7 +934,7 @@ pub fn select_notes_by_id(conn: &mut Connection, note_ids: &[NoteId]) -> Result< NoteMetadata::new(sender, note_type, tag.into(), execution_hint.try_into()?, aux)?; notes.push(NoteRecord { - block_num: row.get(0)?, + block_num: read_block_number(row, 0)?, note_index: BlockNoteIndex::new(row.get(1)?, row.get(2)?)?, details, note_id: note_id.into(), @@ -966,7 +978,7 @@ pub fn select_note_inclusion_proofs( let mut result = BTreeMap::new(); let mut rows = select_notes_stmt.query(params![Rc::new(note_ids)])?; while let Some(row) = rows.next()? { - let block_num = row.get(0)?; + let block_num: u32 = row.get(0)?; let note_id_data = row.get_ref(1)?.as_blob()?; let note_id = NoteId::read_from_bytes(note_id_data)?; @@ -978,7 +990,7 @@ pub fn select_note_inclusion_proofs( let merkle_path_data = row.get_ref(4)?.as_blob()?; let merkle_path = MerklePath::read_from_bytes(merkle_path_data)?; - let proof = NoteInclusionProof::new(block_num, node_index_in_block, merkle_path)?; + let proof = NoteInclusionProof::new(block_num.into(), node_index_in_block, merkle_path)?; result.insert(note_id, proof); } @@ -1002,7 +1014,7 @@ pub fn select_note_inclusion_proofs( pub fn insert_block_header(transaction: &Transaction, block_header: &BlockHeader) -> Result { let mut stmt = transaction .prepare_cached("INSERT INTO block_headers (block_num, block_header) VALUES (?1, ?2);")?; - Ok(stmt.execute(params![block_header.block_num(), block_header.to_bytes()])?) + Ok(stmt.execute(params![block_header.block_num().as_u32(), block_header.to_bytes()])?) } /// Select a [`BlockHeader`] from the DB by its `block_num` using the given [Connection]. @@ -1019,7 +1031,7 @@ pub fn select_block_header_by_block_num( let mut rows = if let Some(block_number) = block_number { stmt = conn.prepare_cached("SELECT block_header FROM block_headers WHERE block_num = ?1")?; - stmt.query([block_number])? + stmt.query([block_number.as_u32()])? } else { stmt = conn.prepare_cached( "SELECT block_header FROM block_headers ORDER BY block_num DESC LIMIT 1", @@ -1051,7 +1063,7 @@ pub fn select_block_headers( ) -> Result> { let mut headers = Vec::with_capacity(blocks.len()); - let blocks: Vec = blocks.iter().copied().map(Into::into).collect(); + let blocks: Vec = blocks.iter().copied().map(|b| b.as_u32().into()).collect(); let mut stmt = conn .prepare_cached("SELECT block_header FROM block_headers WHERE block_num IN rarray(?1);")?; let mut rows = stmt.query(params![Rc::new(blocks)])?; @@ -1109,8 +1121,11 @@ pub fn insert_transactions( for update in accounts { let account_id = update.account_id(); for transaction_id in update.transactions() { - count += - stmt.execute(params![transaction_id.to_bytes(), account_id.to_bytes(), block_num])?; + count += stmt.execute(params![ + transaction_id.to_bytes(), + account_id.to_bytes(), + block_num.as_u32() + ])?; } } Ok(count) @@ -1151,12 +1166,13 @@ pub fn select_transactions_by_accounts_and_block_range( ", )?; - let mut rows = stmt.query(params![block_start, block_end, Rc::new(account_ids)])?; + let mut rows = + stmt.query(params![block_start.as_u32(), block_end.as_u32(), Rc::new(account_ids)])?; let mut result = vec![]; while let Some(row) = rows.next()? { let account_id = read_from_blob_column(row, 0)?; - let block_num = row.get(1)?; + let block_num = read_block_number(row, 1)?; let transaction_id_data = row.get_ref(2)?.as_blob()?; let transaction_id = TransactionId::read_from_bytes(transaction_id_data)?; diff --git a/crates/store/src/db/sql/utils.rs b/crates/store/src/db/sql/utils.rs index 0787acd28..58dd72afa 100644 --- a/crates/store/src/db/sql/utils.rs +++ b/crates/store/src/db/sql/utils.rs @@ -1,6 +1,7 @@ use miden_node_proto::domain::accounts::{AccountInfo, AccountSummary}; use miden_objects::{ accounts::{Account, AccountDelta, AccountId}, + block::BlockNumber, crypto::hash::rpo::RpoDigest, notes::Nullifier, utils::Deserializable, @@ -94,6 +95,15 @@ pub fn column_value_as_u64( Ok(value as u64) } +/// Gets a `BlockNum` value from the database. +pub fn read_block_number( + row: &rusqlite::Row<'_>, + index: I, +) -> rusqlite::Result { + let value: u32 = row.get(index)?; + Ok(value.into()) +} + /// Gets a blob value from the database and tries to deserialize it into the necessary type. pub fn read_from_blob_column(row: &rusqlite::Row<'_>, index: I) -> rusqlite::Result where @@ -118,7 +128,7 @@ pub fn account_summary_from_row(row: &rusqlite::Row<'_>) -> crate::db::Result Connection { conn } -fn create_block(conn: &mut Connection, block_num: u32) { +fn create_block(conn: &mut Connection, block_num: BlockNumber) { let block_header = BlockHeader::new( 1_u8.into(), num_to_rpo_digest(2), @@ -57,7 +57,7 @@ fn sql_insert_nullifiers_for_block() { let nullifiers = [num_to_nullifier(1 << 48)]; - let block_num = 1; + let block_num = 1.into(); create_block(&mut conn, block_num); // Insert a new nullifier succeeds @@ -89,7 +89,7 @@ fn sql_insert_nullifiers_for_block() { // test inserting multiple nullifiers { let nullifiers: Vec<_> = (0..10).map(num_to_nullifier).collect(); - let block_num = 1; + let block_num = 1.into(); let transaction = conn.transaction().unwrap(); let res = sql::insert_nullifiers_for_block(&transaction, &nullifiers, block_num); transaction.commit().unwrap(); @@ -111,8 +111,8 @@ fn sql_select_transactions() { fn query_transactions(conn: &mut Connection) -> Vec { sql::select_transactions_by_accounts_and_block_range( conn, - 0, - 2, + 0.into(), + 2.into(), &[AccountId::try_from(ACCOUNT_ID_OFF_CHAIN_SENDER).unwrap()], ) .unwrap() @@ -137,7 +137,7 @@ fn sql_select_transactions() { fn sql_select_nullifiers() { let mut conn = create_db(); - let block_num = 1; + let block_num = 1.into(); create_block(&mut conn, block_num); // test querying empty table @@ -163,7 +163,7 @@ fn sql_select_nullifiers() { fn sql_select_notes() { let mut conn = create_db(); - let block_num = 1; + let block_num = BlockNumber::from(1); create_block(&mut conn, block_num); // test querying empty table @@ -203,7 +203,7 @@ fn sql_select_notes() { fn sql_select_notes_different_execution_hints() { let mut conn = create_db(); - let block_num = 1; + let block_num = 1.into(); create_block(&mut conn, block_num); // test querying empty table @@ -269,7 +269,7 @@ fn sql_select_notes_different_execution_hints() { ACCOUNT_ID_OFF_CHAIN_SENDER.try_into().unwrap(), NoteType::Public, 2.into(), - NoteExecutionHint::after_block(12).unwrap(), + NoteExecutionHint::after_block(12.into()).unwrap(), Felt::default(), ) .unwrap(), @@ -283,14 +283,17 @@ fn sql_select_notes_different_execution_hints() { assert_eq!(res.unwrap(), 1, "One element must have been inserted"); transaction.commit().unwrap(); let note = &sql::select_notes_by_id(&mut conn, &[num_to_rpo_digest(2).into()]).unwrap()[0]; - assert_eq!(note.metadata.execution_hint(), NoteExecutionHint::after_block(12).unwrap()); + assert_eq!( + note.metadata.execution_hint(), + NoteExecutionHint::after_block(12.into()).unwrap() + ); } #[test] fn sql_select_accounts() { let mut conn = create_db(); - let block_num = 1; + let block_num = 1.into(); create_block(&mut conn, block_num); // test querying empty table @@ -333,7 +336,7 @@ fn sql_select_accounts() { fn sql_public_account_details() { let mut conn = create_db(); - create_block(&mut conn, 1); + create_block(&mut conn, 1.into()); let fungible_faucet_id = AccountId::try_from(ACCOUNT_ID_FUNGIBLE_FAUCET_ON_CHAIN).unwrap(); let non_fungible_faucet_id = @@ -365,7 +368,7 @@ fn sql_public_account_details() { AccountUpdateDetails::New(account.clone()), vec![], )], - 1, + 1.into(), ) .unwrap(); @@ -380,9 +383,10 @@ fn sql_public_account_details() { let account_read = accounts_in_db.pop().unwrap().details.unwrap(); assert_eq!(account_read, account); - create_block(&mut conn, 2); + create_block(&mut conn, 2.into()); - let read_delta = sql::select_account_delta(&mut conn, account.id(), 1, 2).unwrap(); + let read_delta = + sql::select_account_delta(&mut conn, account.id(), 1.into(), 2.into()).unwrap(); assert_eq!(read_delta, None); @@ -411,7 +415,7 @@ fn sql_public_account_details() { AccountUpdateDetails::Delta(delta2.clone()), vec![], )], - 2, + 2.into(), ) .unwrap(); @@ -430,10 +434,11 @@ fn sql_public_account_details() { assert_eq!(account_read.nonce(), account.nonce()); assert_eq!(account_read.storage(), account.storage()); - let read_delta = sql::select_account_delta(&mut conn, account.id(), 1, 2).unwrap(); + let read_delta = + sql::select_account_delta(&mut conn, account.id(), 1.into(), 2.into()).unwrap(); assert_eq!(read_delta.as_ref(), Some(&delta2)); - create_block(&mut conn, 3); + create_block(&mut conn, 3.into()); let storage_delta3 = AccountStorageDelta::from_iters([5], [], []); @@ -455,7 +460,7 @@ fn sql_public_account_details() { AccountUpdateDetails::Delta(delta3.clone()), vec![], )], - 3, + 3.into(), ) .unwrap(); @@ -473,7 +478,8 @@ fn sql_public_account_details() { assert_eq!(account_read.vault(), account.vault()); assert_eq!(account_read.nonce(), account.nonce()); - let read_delta = sql::select_account_delta(&mut conn, account.id(), 1, 3).unwrap(); + let read_delta = + sql::select_account_delta(&mut conn, account.id(), 1.into(), 3.into()).unwrap(); delta2.merge(delta3).unwrap(); @@ -485,12 +491,13 @@ fn sql_select_nullifiers_by_block_range() { let mut conn = create_db(); // test empty table - let nullifiers = sql::select_nullifiers_by_block_range(&mut conn, 0, u32::MAX, &[]).unwrap(); + let nullifiers = + sql::select_nullifiers_by_block_range(&mut conn, 0.into(), u32::MAX.into(), &[]).unwrap(); assert!(nullifiers.is_empty()); // test single item let nullifier1 = num_to_nullifier(1 << 48); - let block_number1 = 1; + let block_number1 = 1.into(); create_block(&mut conn, block_number1); let transaction = conn.transaction().unwrap(); @@ -499,8 +506,8 @@ fn sql_select_nullifiers_by_block_range() { let nullifiers = sql::select_nullifiers_by_block_range( &mut conn, - 0, - u32::MAX, + 0.into(), + u32::MAX.into(), &[sql::utils::get_nullifier_prefix(&nullifier1)], ) .unwrap(); @@ -514,7 +521,7 @@ fn sql_select_nullifiers_by_block_range() { // test two elements let nullifier2 = num_to_nullifier(2 << 48); - let block_number2 = 2; + let block_number2 = 2.into(); create_block(&mut conn, block_number2); let transaction = conn.transaction().unwrap(); @@ -527,8 +534,8 @@ fn sql_select_nullifiers_by_block_range() { // only the nullifiers matching the prefix are included let nullifiers = sql::select_nullifiers_by_block_range( &mut conn, - 0, - u32::MAX, + 0.into(), + u32::MAX.into(), &[sql::utils::get_nullifier_prefix(&nullifier1)], ) .unwrap(); @@ -541,8 +548,8 @@ fn sql_select_nullifiers_by_block_range() { ); let nullifiers = sql::select_nullifiers_by_block_range( &mut conn, - 0, - u32::MAX, + 0.into(), + u32::MAX.into(), &[sql::utils::get_nullifier_prefix(&nullifier2)], ) .unwrap(); @@ -557,8 +564,8 @@ fn sql_select_nullifiers_by_block_range() { // Nullifiers created at block_end are included let nullifiers = sql::select_nullifiers_by_block_range( &mut conn, - 0, - 1, + 0.into(), + 1.into(), &[ sql::utils::get_nullifier_prefix(&nullifier1), sql::utils::get_nullifier_prefix(&nullifier2), @@ -576,8 +583,8 @@ fn sql_select_nullifiers_by_block_range() { // Nullifiers created at block_start are not included let nullifiers = sql::select_nullifiers_by_block_range( &mut conn, - 1, - u32::MAX, + 1.into(), + u32::MAX.into(), &[ sql::utils::get_nullifier_prefix(&nullifier1), sql::utils::get_nullifier_prefix(&nullifier2), @@ -596,8 +603,8 @@ fn sql_select_nullifiers_by_block_range() { // when the client requests a sync update, and it is already tracking the chain tip. let nullifiers = sql::select_nullifiers_by_block_range( &mut conn, - 2, - 2, + 2.into(), + 2.into(), &[ sql::utils::get_nullifier_prefix(&nullifier1), sql::utils::get_nullifier_prefix(&nullifier2), @@ -617,7 +624,7 @@ fn select_nullifiers_by_prefix() { // test single item let nullifier1 = num_to_nullifier(1 << 48); - let block_number1 = 1; + let block_number1 = 1.into(); create_block(&mut conn, block_number1); let transaction = conn.transaction().unwrap(); @@ -640,7 +647,7 @@ fn select_nullifiers_by_prefix() { // test two elements let nullifier2 = num_to_nullifier(2 << 48); - let block_number2 = 2; + let block_number2 = 2.into(); create_block(&mut conn, block_number2); let transaction = conn.transaction().unwrap(); @@ -718,7 +725,7 @@ fn db_block_header() { // test querying empty table let block_number = 1; - let res = sql::select_block_header_by_block_num(&mut conn, Some(block_number)).unwrap(); + let res = sql::select_block_header_by_block_num(&mut conn, Some(block_number.into())).unwrap(); assert!(res.is_none()); let res = sql::select_block_header_by_block_num(&mut conn, None).unwrap(); @@ -730,7 +737,7 @@ fn db_block_header() { let block_header = BlockHeader::new( 1_u8.into(), num_to_rpo_digest(2), - 3, + 3.into(), num_to_rpo_digest(4), num_to_rpo_digest(5), num_to_rpo_digest(6), @@ -747,7 +754,7 @@ fn db_block_header() { // test fetch unknown block header let block_number = 1; - let res = sql::select_block_header_by_block_num(&mut conn, Some(block_number)).unwrap(); + let res = sql::select_block_header_by_block_num(&mut conn, Some(block_number.into())).unwrap(); assert!(res.is_none()); // test fetch block header by block number @@ -762,7 +769,7 @@ fn db_block_header() { let block_header2 = BlockHeader::new( 11_u8.into(), num_to_rpo_digest(12), - 13, + 13.into(), num_to_rpo_digest(14), num_to_rpo_digest(15), num_to_rpo_digest(16), @@ -788,7 +795,7 @@ fn db_block_header() { fn db_account() { let mut conn = create_db(); - let block_num = 1; + let block_num = 1.into(); create_block(&mut conn, block_num); // test empty table @@ -797,7 +804,9 @@ fn db_account() { .iter() .map(|acc_id| (*acc_id).try_into().unwrap()) .collect(); - let res = sql::select_accounts_by_block_range(&mut conn, 0, u32::MAX, &account_ids).unwrap(); + let res = + sql::select_accounts_by_block_range(&mut conn, 0.into(), u32::MAX.into(), &account_ids) + .unwrap(); assert!(res.is_empty()); // test insertion @@ -821,7 +830,9 @@ fn db_account() { assert_eq!(row_count, 1); // test successful query - let res = sql::select_accounts_by_block_range(&mut conn, 0, u32::MAX, &account_ids).unwrap(); + let res = + sql::select_accounts_by_block_range(&mut conn, 0.into(), u32::MAX.into(), &account_ids) + .unwrap(); assert_eq!( res, vec![AccountSummary { @@ -832,15 +843,20 @@ fn db_account() { ); // test query for update outside the block range - let res = sql::select_accounts_by_block_range(&mut conn, block_num + 1, u32::MAX, &account_ids) - .unwrap(); + let res = sql::select_accounts_by_block_range( + &mut conn, + block_num + 1, + u32::MAX.into(), + &account_ids, + ) + .unwrap(); assert!(res.is_empty()); // test query with unknown accounts let res = sql::select_accounts_by_block_range( &mut conn, block_num + 1, - u32::MAX, + u32::MAX.into(), &[6.try_into().unwrap(), 7.try_into().unwrap(), 8.try_into().unwrap()], ) .unwrap(); @@ -851,15 +867,16 @@ fn db_account() { fn notes() { let mut conn = create_db(); - let block_num_1 = 1; + let block_num_1 = 1.into(); create_block(&mut conn, block_num_1); // test empty table - let res = sql::select_notes_since_block_by_tag_and_sender(&mut conn, &[], &[], 0).unwrap(); + let res = + sql::select_notes_since_block_by_tag_and_sender(&mut conn, &[], &[], 0.into()).unwrap(); assert!(res.is_empty()); - let res = - sql::select_notes_since_block_by_tag_and_sender(&mut conn, &[1, 2, 3], &[], 0).unwrap(); + let res = sql::select_notes_since_block_by_tag_and_sender(&mut conn, &[1, 2, 3], &[], 0.into()) + .unwrap(); assert!(res.is_empty()); // test insertion @@ -897,7 +914,8 @@ fn notes() { transaction.commit().unwrap(); // test empty tags - let res = sql::select_notes_since_block_by_tag_and_sender(&mut conn, &[], &[], 0).unwrap(); + let res = + sql::select_notes_since_block_by_tag_and_sender(&mut conn, &[], &[], 0.into()).unwrap(); assert!(res.is_empty()); // test no updates @@ -906,9 +924,13 @@ fn notes() { assert!(res.is_empty()); // test match - let res = - sql::select_notes_since_block_by_tag_and_sender(&mut conn, &[tag], &[], block_num_1 - 1) - .unwrap(); + let res = sql::select_notes_since_block_by_tag_and_sender( + &mut conn, + &[tag], + &[], + block_num_1.parent().unwrap(), + ) + .unwrap(); assert_eq!(res, vec![note.clone().into()]); let block_num_2 = note.block_num + 1; @@ -929,9 +951,13 @@ fn notes() { transaction.commit().unwrap(); // only first note is returned - let res = - sql::select_notes_since_block_by_tag_and_sender(&mut conn, &[tag], &[], block_num_1 - 1) - .unwrap(); + let res = sql::select_notes_since_block_by_tag_and_sender( + &mut conn, + &[tag], + &[], + block_num_1.parent().unwrap(), + ) + .unwrap(); assert_eq!(res, vec![note.clone().into()]); // only the second note is returned @@ -978,7 +1004,7 @@ fn mock_block_account_update(account_id: AccountId, num: u64) -> BlockAccountUpd } fn insert_transactions(conn: &mut Connection) -> usize { - let block_num = 1; + let block_num = 1.into(); create_block(conn, block_num); let transaction = conn.transaction().unwrap(); diff --git a/crates/store/src/errors.rs b/crates/store/src/errors.rs index 6d3e14ed9..72fa993e6 100644 --- a/crates/store/src/errors.rs +++ b/crates/store/src/errors.rs @@ -3,6 +3,7 @@ use std::io; use deadpool_sqlite::{InteractError, PoolError}; use miden_objects::{ accounts::AccountId, + block::{BlockHeader, BlockNumber}, crypto::{ hash::rpo::RpoDigest, merkle::{MerkleError, MmrError}, @@ -10,15 +11,13 @@ use miden_objects::{ }, notes::Nullifier, transaction::OutputNote, - AccountDeltaError, AccountError, BlockError, BlockHeader, NoteError, + AccountDeltaError, AccountError, BlockError, NoteError, }; use rusqlite::types::FromSqlError; use thiserror::Error; use tokio::sync::oneshot::error::RecvError; use tonic::Status; -use crate::types::BlockNumber; - // INTERNAL ERRORS // ================================================================================================= @@ -240,7 +239,7 @@ pub enum GetBlockInputsError { #[error("failed to get MMR peaks for forest ({forest}): {error}")] FailedToGetMmrPeaksForForest { forest: usize, error: MmrError }, #[error("chain MMR forest expected to be 1 less than latest header's block num. Chain MMR forest: {forest}, block num: {block_num}")] - IncorrectChainMmrForestNumber { forest: usize, block_num: u32 }, + IncorrectChainMmrForestNumber { forest: usize, block_num: BlockNumber }, #[error("note inclusion proof MMR error")] NoteInclusionMmr(#[from] MmrError), } diff --git a/crates/store/src/genesis.rs b/crates/store/src/genesis.rs index ada75543e..402d820d7 100644 --- a/crates/store/src/genesis.rs +++ b/crates/store/src/genesis.rs @@ -1,10 +1,10 @@ use miden_lib::transaction::TransactionKernel; use miden_objects::{ accounts::{delta::AccountUpdateDetails, Account}, - block::{Block, BlockAccountUpdate}, + block::{Block, BlockAccountUpdate, BlockHeader, BlockNumber}, crypto::merkle::{EmptySubtreeRoots, MmrPeaks, SimpleSmt, Smt}, utils::serde::{ByteReader, ByteWriter, Deserializable, DeserializationError, Serializable}, - BlockHeader, Digest, ACCOUNT_TREE_DEPTH, BLOCK_NOTE_TREE_DEPTH, GENESIS_BLOCK, + Digest, ACCOUNT_TREE_DEPTH, BLOCK_NOTE_TREE_DEPTH, }; use crate::errors::GenesisError; @@ -54,7 +54,7 @@ impl GenesisState { let header = BlockHeader::new( self.version, Digest::default(), - GENESIS_BLOCK, + BlockNumber::GENESIS, MmrPeaks::new(0, Vec::new()).unwrap().hash_peaks(), account_smt.root(), Smt::default().root(), diff --git a/crates/store/src/lib.rs b/crates/store/src/lib.rs index 95767833e..6adb4eae1 100644 --- a/crates/store/src/lib.rs +++ b/crates/store/src/lib.rs @@ -6,7 +6,6 @@ pub mod genesis; mod nullifier_tree; pub mod server; pub mod state; -pub mod types; // CONSTANTS // ================================================================================================= diff --git a/crates/store/src/nullifier_tree.rs b/crates/store/src/nullifier_tree.rs index 81c197330..8f275428f 100644 --- a/crates/store/src/nullifier_tree.rs +++ b/crates/store/src/nullifier_tree.rs @@ -1,4 +1,5 @@ use miden_objects::{ + block::BlockNumber, crypto::{ hash::rpo::RpoDigest, merkle::{MutationSet, Smt, SmtProof, SMT_DEPTH}, @@ -7,7 +8,7 @@ use miden_objects::{ Felt, FieldElement, Word, }; -use crate::{errors::NullifierTreeError, types::BlockNumber}; +use crate::errors::NullifierTreeError; /// Nullifier SMT. #[derive(Debug, Clone)] @@ -79,7 +80,10 @@ impl NullifierTree { /// There are no nullifiers in the genesis block. The value zero is instead used to signal /// absence of a value. fn leaf_value_to_block_num(value: Word) -> BlockNumber { - value[0].as_int().try_into().expect("invalid block number found in store") + let block_num: u32 = + value[0].as_int().try_into().expect("invalid block number found in store"); + + block_num.into() } } @@ -92,7 +96,7 @@ mod tests { #[test] fn leaf_value_encoding() { let block_num = 123; - let nullifier_value = NullifierTree::block_num_to_leaf_value(block_num); + let nullifier_value = NullifierTree::block_num_to_leaf_value(block_num.into()); assert_eq!(nullifier_value, [Felt::from(block_num), ZERO, ZERO, ZERO]); } @@ -103,6 +107,6 @@ mod tests { let nullifier_value = [Felt::from(block_num), ZERO, ZERO, ZERO]; let decoded_block_num = NullifierTree::leaf_value_to_block_num(nullifier_value); - assert_eq!(decoded_block_num, block_num); + assert_eq!(decoded_block_num, block_num.into()); } } diff --git a/crates/store/src/server/api.rs b/crates/store/src/server/api.rs index efdd68438..fa9978e31 100644 --- a/crates/store/src/server/api.rs +++ b/crates/store/src/server/api.rs @@ -33,7 +33,7 @@ use miden_node_proto::{ }; use miden_objects::{ accounts::AccountId, - block::Block, + block::{Block, BlockNumber}, crypto::hash::rpo::RpoDigest, notes::{NoteId, Nullifier}, utils::{Deserializable, Serializable}, @@ -72,7 +72,7 @@ impl api_server::Api for StoreApi { info!(target: COMPONENT, ?request); let request = request.into_inner(); - let block_num = request.block_num; + let block_num = request.block_num.map(BlockNumber::from); let (block_header, mmr_proof) = self .state .get_block_header(block_num, request.include_mmr_proof.unwrap_or(false)) @@ -138,7 +138,7 @@ impl api_server::Api for StoreApi { .into_iter() .map(|nullifier_info| NullifierUpdate { nullifier: Some(nullifier_info.nullifier.into()), - block_num: nullifier_info.block_num, + block_num: nullifier_info.block_num.as_u32(), }) .collect(); @@ -164,7 +164,12 @@ impl api_server::Api for StoreApi { let (state, delta) = self .state - .sync_state(request.block_num, account_ids, request.note_tags, request.nullifiers) + .sync_state( + request.block_num.into(), + account_ids, + request.note_tags, + request.nullifiers, + ) .await .map_err(internal_error)?; @@ -174,7 +179,7 @@ impl api_server::Api for StoreApi { .map(|account_info| AccountSummary { account_id: Some(account_info.account_id.into()), account_hash: Some(account_info.account_hash.into()), - block_num: account_info.block_num, + block_num: account_info.block_num.as_u32(), }) .collect(); @@ -183,7 +188,7 @@ impl api_server::Api for StoreApi { .into_iter() .map(|transaction_summary| TransactionSummary { account_id: Some(transaction_summary.account_id.into()), - block_num: transaction_summary.block_num, + block_num: transaction_summary.block_num.as_u32(), transaction_id: Some(transaction_summary.transaction_id.into()), }) .collect(); @@ -195,12 +200,12 @@ impl api_server::Api for StoreApi { .into_iter() .map(|nullifier_info| NullifierUpdate { nullifier: Some(nullifier_info.nullifier.into()), - block_num: nullifier_info.block_num, + block_num: nullifier_info.block_num.as_u32(), }) .collect(); Ok(Response::new(SyncStateResponse { - chain_tip: self.state.latest_block_num().await, + chain_tip: self.state.latest_block_num().await.as_u32(), block_header: Some(state.block_header.into()), mmr_delta: Some(delta.into()), accounts, @@ -226,14 +231,14 @@ impl api_server::Api for StoreApi { let (state, mmr_proof) = self .state - .sync_notes(request.block_num, request.note_tags) + .sync_notes(request.block_num.into(), request.note_tags) .await .map_err(internal_error)?; let notes = state.notes.into_iter().map(Into::into).collect(); Ok(Response::new(SyncNoteResponse { - chain_tip: self.state.latest_block_num().await, + chain_tip: self.state.latest_block_num().await.as_u32(), block_header: Some(state.block_header.into()), mmr_path: Some((&mmr_proof.merkle_path).into()), notes, @@ -354,7 +359,7 @@ impl api_server::Api for StoreApi { Status::invalid_argument(format!("Block deserialization error: {err}")) })?; - let block_num = block.header().block_num(); + let block_num = block.header().block_num().as_u32(); info!( target: COMPONENT, @@ -421,7 +426,7 @@ impl api_server::Api for StoreApi { .get_transaction_inputs(account_id, &nullifiers, unauthenticated_notes) .await?; - let block_height = self.state.latest_block_num().await; + let block_height = self.state.latest_block_num().await.as_u32(); Ok(Response::new(GetTransactionInputsResponse { account_state: Some(AccountTransactionInputRecord { @@ -433,7 +438,7 @@ impl api_server::Api for StoreApi { .into_iter() .map(|nullifier| NullifierTransactionInputRecord { nullifier: Some(nullifier.nullifier.into()), - block_num: nullifier.block_num, + block_num: nullifier.block_num.as_u32(), }) .collect(), found_unauthenticated_notes: tx_inputs @@ -460,7 +465,7 @@ impl api_server::Api for StoreApi { debug!(target: COMPONENT, ?request); - let block = self.state.load_block(request.block_num).await?; + let block = self.state.load_block(request.block_num.into()).await?; Ok(Response::new(GetBlockByNumberResponse { block })) } @@ -498,7 +503,7 @@ impl api_server::Api for StoreApi { .await?; Ok(Response::new(GetAccountProofsResponse { - block_num, + block_num: block_num.as_u32(), account_proofs: infos.into_iter().map(Into::into).collect(), })) } @@ -521,7 +526,11 @@ impl api_server::Api for StoreApi { let account_id = read_account_id(request.account_id)?; let delta = self .state - .get_account_state_delta(account_id, request.from_block_num, request.to_block_num) + .get_account_state_delta( + account_id, + request.from_block_num.into(), + request.to_block_num.into(), + ) .await? .map(|delta| delta.to_bytes()); diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index 176d6cffd..92da06f41 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -24,7 +24,7 @@ use miden_node_proto::{ use miden_node_utils::formatting::format_array; use miden_objects::{ accounts::{AccountDelta, AccountHeader, AccountId, StorageSlot}, - block::Block, + block::{Block, BlockHeader, BlockNumber}, crypto::{ hash::rpo::RpoDigest, merkle::{ @@ -34,7 +34,7 @@ use miden_objects::{ notes::{NoteId, Nullifier}, transaction::OutputNote, utils::Serializable, - AccountError, BlockHeader, ACCOUNT_TREE_DEPTH, + AccountError, ACCOUNT_TREE_DEPTH, }; use tokio::{ sync::{oneshot, Mutex, RwLock}, @@ -51,7 +51,6 @@ use crate::{ StateSyncError, }, nullifier_tree::NullifierTree, - types::BlockNumber, COMPONENT, }; // STRUCTURES @@ -105,9 +104,11 @@ struct InnerState { impl InnerState { /// Returns the latest block number. fn latest_block_num(&self) -> BlockNumber { - (self.chain_mmr.forest() - 1) + let block_number: u32 = (self.chain_mmr.forest() - 1) .try_into() - .expect("chain_mmr always has, at least, the genesis block") + .expect("chain_mmr always has, at least, the genesis block"); + + block_number.into() } } @@ -374,7 +375,7 @@ impl State { inner.chain_mmr.add(block_hash); } - info!(%block_hash, block_num, COMPONENT, "apply_block successful"); + info!(%block_hash, block_num = block_num.as_u32(), COMPONENT, "apply_block successful"); Ok(()) } @@ -393,7 +394,7 @@ impl State { if let Some(header) = block_header { let mmr_proof = if include_mmr_proof { let inner = self.inner.read().await; - let mmr_proof = inner.chain_mmr.open(header.block_num() as usize)?; + let mmr_proof = inner.chain_mmr.open(header.block_num().as_usize())?; Some(mmr_proof) } else { None @@ -464,16 +465,16 @@ impl State { let paths = blocks .iter() .map(|&block_num| { - let proof = state.chain_mmr.open(block_num as usize)?.merkle_path; + let proof = state.chain_mmr.open(block_num.as_usize())?.merkle_path; Ok::<_, MmrError>((block_num, proof)) }) .collect::, MmrError>>()?; - let chain_length = BlockNumber::try_from(chain_length) - .expect("Forest is a chain length so should fit into block number"); + let chain_length = u32::try_from(chain_length) + .expect("Forest is a chain length so should fit into a u32"); - (chain_length, paths) + (chain_length.into(), paths) }; let headers = self.db.select_block_headers(blocks).await?; @@ -525,7 +526,10 @@ impl State { let delta = if block_num == state_sync.block_header.block_num() { // The client is in sync with the chain tip. - MmrDelta { forest: block_num as usize, data: vec![] } + MmrDelta { + forest: block_num.as_usize(), + data: vec![], + } } else { // Important notes about the boundary conditions: // @@ -536,8 +540,8 @@ impl State { // - Mmr::get_delta is inclusive, whereas the sync_state request block_num is defined to // be // exclusive, so the from_forest has to be adjusted with a +1 - let from_forest = (block_num + 1) as usize; - let to_forest = state_sync.block_header.block_num() as usize; + let from_forest = (block_num + 1).as_usize(); + let to_forest = state_sync.block_header.block_num().as_usize(); inner .chain_mmr .get_delta(from_forest, to_forest) @@ -568,7 +572,7 @@ impl State { let note_sync = self.db.get_note_sync(block_num, note_tags).await?; - let mmr_proof = inner.chain_mmr.open(note_sync.block_header.block_num() as usize)?; + let mmr_proof = inner.chain_mmr.open(note_sync.block_header.block_num().as_usize())?; Ok((note_sync, mmr_proof)) } @@ -589,7 +593,7 @@ impl State { .ok_or(GetBlockInputsError::DbBlockHeaderEmpty)?; // sanity check - if inner.chain_mmr.forest() != latest.block_num() as usize + 1 { + if inner.chain_mmr.forest() != latest.block_num().as_usize() + 1 { return Err(GetBlockInputsError::IncorrectChainMmrForestNumber { forest: inner.chain_mmr.forest(), block_num: latest.block_num(), @@ -599,9 +603,9 @@ impl State { // using current block number gets us the peaks of the chain MMR as of one block ago; // this is done so that latest.chain_root matches the returned peaks let chain_peaks = - inner.chain_mmr.peaks_at(latest.block_num() as usize).map_err(|error| { + inner.chain_mmr.peaks_at(latest.block_num().as_usize()).map_err(|error| { GetBlockInputsError::FailedToGetMmrPeaksForForest { - forest: latest.block_num() as usize, + forest: latest.block_num().as_usize(), error, } })?; diff --git a/crates/store/src/types.rs b/crates/store/src/types.rs deleted file mode 100644 index f75d6ec60..000000000 --- a/crates/store/src/types.rs +++ /dev/null @@ -1 +0,0 @@ -pub type BlockNumber = u32; From fc35e175e7fdbbb65be1db0272284ef4ea802290 Mon Sep 17 00:00:00 2001 From: Mirko <48352201+Mirko-von-Leipzig@users.noreply.github.com> Date: Tue, 21 Jan 2025 20:32:40 +0200 Subject: [PATCH 47/50] docs: align readme rpc docs (#631) --- crates/block-producer/README.md | 53 ++++--- crates/proto/src/generated/requests.rs | 2 +- crates/proto/src/generated/rpc.rs | 34 +++-- crates/proto/src/generated/store.rs | 34 +++-- crates/rpc-proto/proto/requests.proto | 2 +- crates/rpc-proto/proto/rpc.proto | 17 ++- crates/rpc-proto/proto/store.proto | 17 ++- crates/rpc/README.md | 144 +++++++++---------- crates/store/README.md | 185 +++++++++++-------------- proto/requests.proto | 2 +- proto/rpc.proto | 17 ++- proto/store.proto | 17 ++- 12 files changed, 285 insertions(+), 239 deletions(-) diff --git a/crates/block-producer/README.md b/crates/block-producer/README.md index cedb36b96..2ec0b1366 100644 --- a/crates/block-producer/README.md +++ b/crates/block-producer/README.md @@ -1,35 +1,54 @@ # Miden block producer -The **Block producer** receives transactions from the RPC component, processes them, creates block containing those transactions before sending created blocks to the store. +Contains code definining the [Miden node's block-producer](/README.md#architecture) component. It is responsible for +ordering transactions into blocks and submitting these for inclusion in the blockchain. -**Block Producer** is one of components of the [Miden node](..). +It serves a small [rRPC](htts://grpc.io) API which the node's RPC component uses to submit new transactions. In turn, +the `block-producer` uses the store's gRPC API to submit blocks and query chain state. -## Architecture +For more information on the installation and operation of this component, please see the [node's readme](/README.md). -`TODO` +## API -## Usage +The full gRPC API can be found [here](../../proto/block_producer.proto). -### Installing the Block Producer +--- -The Block Producer can be installed and run as part of [Miden node](../README.md#installing-the-node). +### SubmitProvenTransaction -## API +Submits a proven transaction to the block-producer, returning the current chain height if successful. -The **Block Producer** serves connections using the [gRPC protocol](https://grpc.io) on a port, set in the previously mentioned configuration file. -Here is a brief description of supported methods. +The block-producer does _not_ verify the transaction's proof as it assumes the RPC component has done so. This is done +to minimize the performance impact of new transactions on the block-producer. -### SubmitProvenTransaction +Transactions are verified before being added to the block-producer's mempool. Transaction which fail verification are +rejected and an error is returned. Possible reasons for verification failure include -Submits proven transaction to the Miden network. +- current account state does not match the transaction's initial account state +- transaction attempts to consume non-existing, or already consumed notes +- transaction attempts to create a duplicate note +- invalid transaction proof (checked by the RPC component) -**Parameters** +Verified transactions are added to the mempool however they are still _not guaranteed_ to make it into a block. +Transactions may be evicted from the mempool if their preconditions no longer hold. Currently the only precondition is +transaction expiration height. Furthermore, as a defense against bugs the mempool may evict transactions it deems buggy +e.g. cause block proofs to fail due to some bug in the VM, compiler, prover etc. -* `transaction`: `bytes` - transaction encoded using [winter_utils::Serializable](https://github.com/facebook/winterfell/blob/main/utils/core/src/serde/mod.rs#L26) implementation for [miden_objects::transaction::proven_tx::ProvenTransaction](https://github.com/0xPolygonMiden/miden-base/blob/main/objects/src/transaction/proven_tx.rs#L22). +Since transactions can depend on other transactions in the mempool this means a specific transaction may be evicted if: -**Returns** +- it's own expiration height is exceeded, or +- it is deemed buggy by the mempool, or +- any ancestor transaction in the mempool is evicted -This method doesn't return any data. +This list will be extended in the future e.g. eviction due to gas price fluctuations. + +Note that since the RPC response only indicates admission into the mempool, its not directly possible to know if the +transaction was evicted. The best way to ensure this is to effectively add a timeout to the transaction by setting the +transaction's expiration height. Once the blockchain advances beyond this point without including the transaction you +can know for certain it was evicted. + +--- ## License -This project is [MIT licensed](../../LICENSE). \ No newline at end of file + +This project is [MIT licensed](../../LICENSE). diff --git a/crates/proto/src/generated/requests.rs b/crates/proto/src/generated/requests.rs index 6cdccab33..a38b5f656 100644 --- a/crates/proto/src/generated/requests.rs +++ b/crates/proto/src/generated/requests.rs @@ -18,7 +18,7 @@ pub struct CheckNullifiersByPrefixRequest { #[prost(uint32, repeated, tag = "2")] pub nullifiers: ::prost::alloc::vec::Vec, } -/// Get a list of proofs for given nullifier hashes, each proof as a sparse Merkle Tree. +/// Returns a nullifier proof for each of the requested nullifiers. #[derive(Clone, PartialEq, ::prost::Message)] pub struct CheckNullifiersRequest { /// List of nullifiers to return proofs for. diff --git a/crates/proto/src/generated/rpc.rs b/crates/proto/src/generated/rpc.rs index f9af9ad7c..cab7a3998 100644 --- a/crates/proto/src/generated/rpc.rs +++ b/crates/proto/src/generated/rpc.rs @@ -90,7 +90,7 @@ pub mod api_client { self.inner = self.inner.max_encoding_message_size(limit); self } - /// Gets a list of proofs for given nullifier hashes, each proof as a sparse Merkle Tree. + /// Returns a nullifier proof for each of the requested nullifiers. pub async fn check_nullifiers( &mut self, request: impl tonic::IntoRequest< @@ -115,6 +115,8 @@ pub mod api_client { self.inner.unary(req, path, codec).await } /// Returns a list of nullifiers that match the specified prefixes and are recorded in the node. + /// + /// Note that only 16-bit prefixes are supported at this time. pub async fn check_nullifiers_by_prefix( &mut self, request: impl tonic::IntoRequest< @@ -219,7 +221,7 @@ pub mod api_client { .insert(GrpcMethod::new("rpc.Api", "GetAccountStateDelta")); self.inner.unary(req, path, codec).await } - /// Retrieves block data by given block number. + /// Returns raw block data for the specified block number. pub async fn get_block_by_number( &mut self, request: impl tonic::IntoRequest< @@ -320,10 +322,15 @@ pub mod api_client { .insert(GrpcMethod::new("rpc.Api", "SubmitProvenTransaction")); self.inner.unary(req, path, codec).await } - /// Note synchronization request. + /// Returns info which can be used by the client to sync up to the tip of chain for the notes they are interested in. + /// + /// Client specifies the `note_tags` they are interested in, and the block height from which to search for new for + /// matching notes for. The request will then return the next block containing any note matching the provided tags. + /// + /// The response includes each note's metadata and inclusion proof. /// - /// Specifies note tags that client is interested in. The server will return the first block which - /// contains a note matching `note_tags` or the chain tip. + /// A basic note sync can be implemented by repeatedly requesting the previous response's block until reaching the + /// tip of the chain. pub async fn sync_notes( &mut self, request: impl tonic::IntoRequest, @@ -396,7 +403,7 @@ pub mod api_server { /// Generated trait containing gRPC methods that should be implemented for use with ApiServer. #[async_trait] pub trait Api: std::marker::Send + std::marker::Sync + 'static { - /// Gets a list of proofs for given nullifier hashes, each proof as a sparse Merkle Tree. + /// Returns a nullifier proof for each of the requested nullifiers. async fn check_nullifiers( &self, request: tonic::Request, @@ -405,6 +412,8 @@ pub mod api_server { tonic::Status, >; /// Returns a list of nullifiers that match the specified prefixes and are recorded in the node. + /// + /// Note that only 16-bit prefixes are supported at this time. async fn check_nullifiers_by_prefix( &self, request: tonic::Request< @@ -439,7 +448,7 @@ pub mod api_server { tonic::Response, tonic::Status, >; - /// Retrieves block data by given block number. + /// Returns raw block data for the specified block number. async fn get_block_by_number( &self, request: tonic::Request, @@ -476,10 +485,15 @@ pub mod api_server { tonic::Response, tonic::Status, >; - /// Note synchronization request. + /// Returns info which can be used by the client to sync up to the tip of chain for the notes they are interested in. + /// + /// Client specifies the `note_tags` they are interested in, and the block height from which to search for new for + /// matching notes for. The request will then return the next block containing any note matching the provided tags. + /// + /// The response includes each note's metadata and inclusion proof. /// - /// Specifies note tags that client is interested in. The server will return the first block which - /// contains a note matching `note_tags` or the chain tip. + /// A basic note sync can be implemented by repeatedly requesting the previous response's block until reaching the + /// tip of the chain. async fn sync_notes( &self, request: tonic::Request, diff --git a/crates/proto/src/generated/store.rs b/crates/proto/src/generated/store.rs index 6b1aa2cee..815a65235 100644 --- a/crates/proto/src/generated/store.rs +++ b/crates/proto/src/generated/store.rs @@ -112,7 +112,7 @@ pub mod api_client { req.extensions_mut().insert(GrpcMethod::new("store.Api", "ApplyBlock")); self.inner.unary(req, path, codec).await } - /// Gets a list of proofs for given nullifier hashes, each proof as a sparse Merkle Tree. + /// Returns a nullifier proof for each of the requested nullifiers. pub async fn check_nullifiers( &mut self, request: impl tonic::IntoRequest< @@ -139,6 +139,8 @@ pub mod api_client { self.inner.unary(req, path, codec).await } /// Returns a list of nullifiers that match the specified prefixes and are recorded in the node. + /// + /// Note that only 16-bit prefixes are supported at this time. pub async fn check_nullifiers_by_prefix( &mut self, request: impl tonic::IntoRequest< @@ -247,7 +249,7 @@ pub mod api_client { .insert(GrpcMethod::new("store.Api", "GetAccountStateDelta")); self.inner.unary(req, path, codec).await } - /// Retrieves block data by given block number. + /// Returns raw block data for the specified block number. pub async fn get_block_by_number( &mut self, request: impl tonic::IntoRequest< @@ -402,10 +404,15 @@ pub mod api_client { .insert(GrpcMethod::new("store.Api", "GetTransactionInputs")); self.inner.unary(req, path, codec).await } - /// Note synchronization request. + /// Returns info which can be used by the client to sync up to the tip of chain for the notes they are interested in. + /// + /// Client specifies the `note_tags` they are interested in, and the block height from which to search for new for + /// matching notes for. The request will then return the next block containing any note matching the provided tags. + /// + /// The response includes each note's metadata and inclusion proof. /// - /// Specifies note tags that client is interested in. The server will return the first block which - /// contains a note matching `note_tags` or the chain tip. + /// A basic note sync can be implemented by repeatedly requesting the previous response's block until reaching the + /// tip of the chain. pub async fn sync_notes( &mut self, request: impl tonic::IntoRequest, @@ -486,7 +493,7 @@ pub mod api_server { tonic::Response, tonic::Status, >; - /// Gets a list of proofs for given nullifier hashes, each proof as a sparse Merkle Tree. + /// Returns a nullifier proof for each of the requested nullifiers. async fn check_nullifiers( &self, request: tonic::Request, @@ -495,6 +502,8 @@ pub mod api_server { tonic::Status, >; /// Returns a list of nullifiers that match the specified prefixes and are recorded in the node. + /// + /// Note that only 16-bit prefixes are supported at this time. async fn check_nullifiers_by_prefix( &self, request: tonic::Request< @@ -529,7 +538,7 @@ pub mod api_server { tonic::Response, tonic::Status, >; - /// Retrieves block data by given block number. + /// Returns raw block data for the specified block number. async fn get_block_by_number( &self, request: tonic::Request, @@ -582,10 +591,15 @@ pub mod api_server { tonic::Response, tonic::Status, >; - /// Note synchronization request. + /// Returns info which can be used by the client to sync up to the tip of chain for the notes they are interested in. + /// + /// Client specifies the `note_tags` they are interested in, and the block height from which to search for new for + /// matching notes for. The request will then return the next block containing any note matching the provided tags. + /// + /// The response includes each note's metadata and inclusion proof. /// - /// Specifies note tags that client is interested in. The server will return the first block which - /// contains a note matching `note_tags` or the chain tip. + /// A basic note sync can be implemented by repeatedly requesting the previous response's block until reaching the + /// tip of the chain. async fn sync_notes( &self, request: tonic::Request, diff --git a/crates/rpc-proto/proto/requests.proto b/crates/rpc-proto/proto/requests.proto index b8e2e0817..1230eac78 100644 --- a/crates/rpc-proto/proto/requests.proto +++ b/crates/rpc-proto/proto/requests.proto @@ -20,7 +20,7 @@ message CheckNullifiersByPrefixRequest { repeated uint32 nullifiers = 2; } -// Get a list of proofs for given nullifier hashes, each proof as a sparse Merkle Tree. +// Returns a nullifier proof for each of the requested nullifiers. message CheckNullifiersRequest { // List of nullifiers to return proofs for. repeated digest.Digest nullifiers = 1; diff --git a/crates/rpc-proto/proto/rpc.proto b/crates/rpc-proto/proto/rpc.proto index bd5d359ad..82da3e20c 100644 --- a/crates/rpc-proto/proto/rpc.proto +++ b/crates/rpc-proto/proto/rpc.proto @@ -6,10 +6,12 @@ import "requests.proto"; import "responses.proto"; service Api { - // Gets a list of proofs for given nullifier hashes, each proof as a sparse Merkle Tree. + // Returns a nullifier proof for each of the requested nullifiers. rpc CheckNullifiers(requests.CheckNullifiersRequest) returns (responses.CheckNullifiersResponse) {} // Returns a list of nullifiers that match the specified prefixes and are recorded in the node. + // + // Note that only 16-bit prefixes are supported at this time. rpc CheckNullifiersByPrefix(requests.CheckNullifiersByPrefixRequest) returns (responses.CheckNullifiersByPrefixResponse) {} // Returns the latest state of an account with the specified ID. @@ -22,7 +24,7 @@ service Api { // `to_block_num` (inclusive). rpc GetAccountStateDelta(requests.GetAccountStateDeltaRequest) returns (responses.GetAccountStateDeltaResponse) {} - // Retrieves block data by given block number. + // Returns raw block data for the specified block number. rpc GetBlockByNumber(requests.GetBlockByNumberRequest) returns (responses.GetBlockByNumberResponse) {} // Retrieves block header by given block number. Optionally, it also returns the MMR path @@ -35,10 +37,15 @@ service Api { // Submits proven transaction to the Miden network. rpc SubmitProvenTransaction(requests.SubmitProvenTransactionRequest) returns (responses.SubmitProvenTransactionResponse) {} - // Note synchronization request. + // Returns info which can be used by the client to sync up to the tip of chain for the notes they are interested in. + // + // Client specifies the `note_tags` they are interested in, and the block height from which to search for new for + // matching notes for. The request will then return the next block containing any note matching the provided tags. + // + // The response includes each note's metadata and inclusion proof. // - // Specifies note tags that client is interested in. The server will return the first block which - // contains a note matching `note_tags` or the chain tip. + // A basic note sync can be implemented by repeatedly requesting the previous response's block until reaching the + // tip of the chain. rpc SyncNotes(requests.SyncNoteRequest) returns (responses.SyncNoteResponse) {} // Returns info which can be used by the client to sync up to the latest state of the chain diff --git a/crates/rpc-proto/proto/store.proto b/crates/rpc-proto/proto/store.proto index 5ed980aba..0562b8c54 100644 --- a/crates/rpc-proto/proto/store.proto +++ b/crates/rpc-proto/proto/store.proto @@ -11,10 +11,12 @@ service Api { // Applies changes of a new block to the DB and in-memory data structures. rpc ApplyBlock(requests.ApplyBlockRequest) returns (responses.ApplyBlockResponse) {} - // Gets a list of proofs for given nullifier hashes, each proof as a sparse Merkle Tree. + // Returns a nullifier proof for each of the requested nullifiers. rpc CheckNullifiers(requests.CheckNullifiersRequest) returns (responses.CheckNullifiersResponse) {} // Returns a list of nullifiers that match the specified prefixes and are recorded in the node. + // + // Note that only 16-bit prefixes are supported at this time. rpc CheckNullifiersByPrefix(requests.CheckNullifiersByPrefixRequest) returns (responses.CheckNullifiersByPrefixResponse) {} // Returns the latest state of an account with the specified ID. @@ -27,7 +29,7 @@ service Api { // `to_block_num` (inclusive). rpc GetAccountStateDelta(requests.GetAccountStateDeltaRequest) returns (responses.GetAccountStateDeltaResponse) {} - // Retrieves block data by given block number. + // Returns raw block data for the specified block number. rpc GetBlockByNumber(requests.GetBlockByNumberRequest) returns (responses.GetBlockByNumberResponse) {} // Retrieves block header by given block number. Optionally, it also returns the MMR path @@ -46,10 +48,15 @@ service Api { // Returns data required to validate a new transaction. rpc GetTransactionInputs(requests.GetTransactionInputsRequest) returns (responses.GetTransactionInputsResponse) {} - // Note synchronization request. + // Returns info which can be used by the client to sync up to the tip of chain for the notes they are interested in. + // + // Client specifies the `note_tags` they are interested in, and the block height from which to search for new for + // matching notes for. The request will then return the next block containing any note matching the provided tags. + // + // The response includes each note's metadata and inclusion proof. // - // Specifies note tags that client is interested in. The server will return the first block which - // contains a note matching `note_tags` or the chain tip. + // A basic note sync can be implemented by repeatedly requesting the previous response's block until reaching the + // tip of the chain. rpc SyncNotes(requests.SyncNoteRequest) returns (responses.SyncNoteResponse) {} // Returns info which can be used by the client to sync up to the latest state of the chain diff --git a/crates/rpc/README.md b/crates/rpc/README.md index 500e56ebd..e8e7e6492 100644 --- a/crates/rpc/README.md +++ b/crates/rpc/README.md @@ -1,129 +1,121 @@ # Miden node RPC -The **RPC** is an externally-facing component through which clients can interact with the node. It receives client requests -(e.g., to synchronize with the latest state of the chain, or to submit transactions), performs basic validation, -and forwards the requests to the appropriate components. -**RPC** is one of components of the [Miden node](..). +Contains the code defining the [Miden node's RPC component](/README.md#architecture). This component serves the +user-facing [gRPC](https://grpc.io) methods used to submit transactions and sync with the state of the network. -## Architecture +This is the **only** set of node RPC methods intended to be publicly available. -`TODO` +For more information on the installation and operation of this component, please see the [node's readme](/README.md). -## Usage +## API overview -### Installing the RPC +The full gRPC method definitions can be found in the [rpc-proto](../rpc-proto/README.md) crate. -The RPC can be installed and run as part of [Miden node](../README.md#installing-the-node). + -## API +- [CheckNullifiers](#checknullifiers) +- [CheckNullifiersByPrefix](#checknullifiersbyprefix) +- [GetAccountDetails](#getaccountdetails) +- [GetAccountProofs](#getaccountproofs) +- [GetAccountStateDelta](#getaccountstatedelta) +- [GetBlockByNumber](#getblockbynumber) +- [GetBlockHeaderByNumber](#getblockheaderbynumber) +- [GetNotesById](#getnotesbyid) +- [SubmitProvenTransaction](#submitproventransaction) +- [SyncNotes](#syncnotes) +- [SyncState](#syncstate) -The **RPC** serves connections using the [gRPC protocol](https://grpc.io) on a port, set in the previously mentioned configuration file. -Here is a brief description of supported methods. + -### CheckNullifiers - -Gets a list of proofs for given nullifier hashes, each proof as a sparse Merkle Tree. - -**Parameters:** +--- -- `nullifiers`: `[Digest]` – array of nullifier hashes. - -**Returns:** - -- `proofs`: `[NullifierProof]` – array of nullifier proofs, positions correspond to the ones in request. +### CheckNullifiers -### GetBlockHeaderByNumber +Returns a nullifier proof for each of the requested nullifiers. -Retrieves block header by given block number, optionally alongside a Merkle path and the current chain length to validate its inclusion. +--- -**Parameters** +### CheckNullifiersByPrefix -- `block_num`: `uint32` _(optional)_ – the block number of the target block. If not provided, the latest known block will be returned. +Returns a list of nullifiers that match the specified prefixes and are recorded in the node. -**Returns:** +Only 16-bit prefixes are supported at this time. -- `block_header`: `BlockHeader` – block header. +--- -### GetBlockByNumber +### GetAccountDetails -Retrieves block data by given block number. +Returns the latest state of an account with the specified ID. -**Parameters** +--- -- `block_num`: `uint32` – the block number of the target block. +### GetAccountProofs -**Returns:** +Returns the latest state proofs of the specified accounts. -- `block`: `Block` – block data encoded using [winter_utils::Serializable](https://github.com/facebook/winterfell/blob/main/utils/core/src/serde/mod.rs#L26) implementation for [miden_objects::block::Block](https://github.com/0xPolygonMiden/miden-base/blob/main/objects/src/block/mod.rs#L43). +--- -### GetNotesById +### GetAccountStateDelta -Returns a list of notes matching the provided note IDs. +Returns delta of the account states in the range from `from_block_num` (exclusive) to `to_block_num` (inclusive). -**Parameters** +--- -- `note_ids`: `[NoteId]` - list of IDs of the notes we want to query. +### GetBlockByNumber -**Returns** +Returns raw block data for the specified block number. -- `notes`: `[Note]` - List of notes matching the list of requested NoteIds. +--- -### GetAccountDetails +### GetBlockHeaderByNumber -Returns the latest state of an account with the specified ID. +Retrieves block header by given block number. Optionally, it also returns the MMR path and current chain length to +authenticate the block's inclusion. -**Parameters** +--- -- `account_id`: `AccountId` – account ID. +### GetNotesById -**Returns** +Returns a list of notes matching the provided note IDs. -- `account`: `AccountInfo` – latest state of the account. For public accounts, this will include full details describing the current account state. For private accounts, only the hash of the latest state and the time of the last update is returned. +--- -### SyncState +### SubmitProvenTransaction -Returns info which can be used by the client to sync up to the latest state of the chain -for the objects (accounts, notes, nullifiers) the client is interested in. +Submits proven transaction to the Miden network. -This request returns the next block containing requested data. It also returns `chain_tip` which is the latest block number in the chain. -Client is expected to repeat these requests in a loop until `response.block_header.block_num == response.chain_tip`, at which point the client is fully synchronized with the chain. +--- -Each request also returns info about new notes, nullifiers etc. created. It also returns Chain MMR delta that can be used to update the state of Chain MMR. -This includes both chain MMR peaks and chain MMR nodes. +### SyncNotes -For preserving some degree of privacy, note tags and nullifiers filters contain only high part of hashes. Thus, returned data -contains excessive notes and nullifiers, client can make additional filtering of that data on its side. +Returns info which can be used by the client to sync up to the tip of chain for the notes they are interested in. -**Parameters** +Client specifies the `note_tags` they are interested in, and the block height from which to search for new for matching +notes for. The request will then return the next block containing any note matching the provided tags. -- `block_num`: `uint32` – send updates to the client starting at this block. -- `account_ids`: `[AccountId]` – accounts filter. -- `note_tags`: `[uint32]` – note tags filter. Corresponds to the high 16 bits of the real values. -- `nullifiers`: `[uint32]` – nullifiers filter. Corresponds to the high 16 bits of the real values. +The response includes each note's metadata and inclusion proof. -**Returns** +A basic note sync can be implemented by repeatedly requesting the previous response's block until reaching the tip of +the chain. -- `chain_tip`: `uint32` – number of the latest block in the chain. -- `block_header`: `BlockHeader` – block header of the block with the first note matching the specified criteria. -- `mmr_delta`: `MmrDelta` – data needed to update the partial MMR from `request.block_num + 1` to `response.block_header.block_num`. -- `accounts`: `[AccountSummary]` – account summaries for accounts updated after `request.block_num + 1` but not after `response.block_header.block_num`. -- `transactions`: `[TransactionSummary]` – transaction summaries for transactions included after `request.block_num + 1` but not after `response.block_header.block_num`. - - Each `TransactionSummary` consists of the `transaction_id` the transaction identifier, `account_id` of the account that executed that transaction, `block_num` the block number in which the transaction was included. -- `notes`: `[NoteSyncRecord]` – a list of all notes together with the Merkle paths from `response.block_header.note_root`. -- `nullifiers`: `[NullifierUpdate]` – a list of nullifiers created between `request.block_num + 1` and `response.block_header.block_num`. - - Each `NullifierUpdate` consists of the `nullifier` and `block_num` the block number in which the note corresponding to that nullifier was consumed. +--- -### SubmitProvenTransaction +### SyncState -Submits proven transaction to the Miden network. +Returns info which can be used by the client to sync up to the latest state of the chain for the objects (accounts, +notes, nullifiers) the client is interested in. -**Parameters** +This request returns the next block containing requested data. It also returns `chain_tip` which is the latest block +number in the chain. Client is expected to repeat these requests in a loop until +`response.block_header.block_num == response.chain_tip`, at which point the client is fully synchronized with the chain. -- `transaction`: `bytes` - transaction encoded using [winter_utils::Serializable](https://github.com/facebook/winterfell/blob/main/utils/core/src/serde/mod.rs#L26) implementation for [miden_objects::transaction::proven_tx::ProvenTransaction](https://github.com/0xPolygonMiden/miden-base/blob/main/objects/src/transaction/proven_tx.rs#L22). +Each request also returns info about new notes, nullifiers etc. created. It also returns Chain MMR delta that can be +used to update the state of Chain MMR. This includes both chain MMR peaks and chain MMR nodes. -**Returns** +For preserving some degree of privacy, note tags and nullifiers filters contain only high part of hashes. Thus, returned +data contains excessive notes and nullifiers, client can make additional filtering of that data on its side. -This method doesn't return any data. +--- ## License diff --git a/crates/store/README.md b/crates/store/README.md index 2932726ca..ecfe709fb 100644 --- a/crates/store/README.md +++ b/crates/store/README.md @@ -1,163 +1,142 @@ # Miden node store -The **Store** maintains the state of the chain. It serves as the "source of truth" for the chain - i.e., if it is not in -the store, the node does not consider it to be part of the chain. -Incoming requests to the store are trusted because they are validated in the RPC component. - -**Store** is one of components of the [Miden node](..). - -## Architecture - -`TODO` - -## Usage - -### Installing the Store - -The Store can be installed and run as part of [Miden node](../README.md#installing-the-node). - -## API - -The **Store** serves connections using the [gRPC protocol](https://grpc.io) on a port, set in the previously mentioned configuration file. -Here is a brief description of supported methods. +Contains the code defining the [Miden node's store component](/README.md#architecture). This component stores the +network's state and acts as the networks source of truth. It serves a [gRPC](https://grpc.io) API which allow the other +node components to interact with the store. This API is **internal** only and is considered trusted i.e. the node +operator must take care that the store's API endpoint is **only** exposed to the other node components. + +For more information on the installation and operation of this component, please see the [node's readme](/README.md). + +## API overview + +The full gRPC API can be found [here](../../proto/store.proto). + + +- [ApplyBlock](#applyblock) +- [CheckNullifiers](#checknullifiers) +- [CheckNullifiersByPrefix](#checknullifiersbyprefix) +- [GetAccountDetails](#getaccountdetails) +- [GetAccountProofs](#getaccountproofs) +- [GetAccountStateDelta](#getaccountstatedelta) +- [GetBlockByNumber](#getblockbynumber) +- [GetBlockHeaderByNumber](#getblockheaderbynumber) +- [GetBlockInputs](#getblockinputs) +- [GetNoteAuthenticationInfo](#getnoteauthenticationinfo) +- [GetNotesById](#getnotesbyid) +- [GetTransactionInputs](#gettransactioninputs) +- [SyncNotes](#syncnotes) +- [SyncState](#syncstate) + + +--- ### ApplyBlock -Applies changes of a new block to the DB and in-memory data structures. - -**Parameters** - -- `block`: `BlockHeader` – block header ([src](../proto/proto/block_header.proto)). -- `accounts`: `[AccountUpdate]` – a list of account updates. -- `nullifiers`: `[Digest]` – a list of nullifier hashes. -- `notes`: `[NoteCreated]` – a list of notes created. - -**Returns** +Applies changes of a new block to the DB and in-memory data structures. Raw block data is also stored as a flat file. -This method doesn't return any data. +--- ### CheckNullifiers -Get a list of proofs for given nullifier hashes, each proof as a sparse Merkle Tree +Returns a nullifier proof for each of the requested nullifiers. -**Parameters:** +--- -- `nullifiers`: `[Digest]` – array of nullifier hashes. +### CheckNullifiersByPrefix -**Returns:** +Returns a list of nullifiers that match the specified prefixes and are recorded in the node. -- `proofs`: `[NullifierProof]` – array of nullifier proofs, positions correspond to the ones in request. +Only 16-bit prefixes are supported at this time. -### GetBlockHeaderByNumber - -Retrieves block header by given block number. Optionally, it also returns the MMR path and current chain length to authenticate the block's inclusion. +--- -**Parameters** +### GetAccountDetails -- `block_num`: `uint32` _(optional)_ – the block number of the target block. If not provided, the latest known block will be returned. +Returns the latest state of an account with the specified ID. -**Returns:** +--- -- `block_header`: `BlockHeader` – block header. +### GetAccountProofs -### GetBlockByNumber +Returns the latest state proofs of the specified accounts. -Retrieves block data by given block number. +--- -**Parameters** +### GetAccountStateDelta -- `block_num`: `uint32` – the block number of the target block. +Returns delta of the account states in the range from `from_block_num` (exclusive) to `to_block_num` (inclusive). -**Returns:** +--- -- `block`: `Block` – block data encoded using [winter_utils::Serializable](https://github.com/facebook/winterfell/blob/main/utils/core/src/serde/mod.rs#L26) implementation for [miden_objects::block::Block](https://github.com/0xPolygonMiden/miden-base/blob/main/objects/src/block/mod.rs#L43). +### GetBlockByNumber -### GetBlockInputs +Returns raw block data for the specified block number. -Returns data required to prove the next block. +--- -**Parameters** +### GetBlockHeaderByNumber -- `account_ids`: `[AccountId]` – array of account IDs. -- `nullifiers`: `[Digest]` – array of nullifier hashes (not currently in use). +Retrieves block header by given block number. Optionally, it also returns the MMR path and current chain length to +authenticate the block's inclusion. -**Returns** +--- -- `block_header`: `[BlockHeader]` – the latest block header. -- `mmr_peaks`: `[Digest]` – peaks of the above block's mmr, The `forest` value is equal to the block number. -- `account_states`: `[AccountBlockInputRecord]` – the hashes of the requested accounts and their authentication paths. -- `nullifiers`: `[NullifierBlockInputRecord]` – the requested nullifiers and their authentication paths. +### GetBlockInputs -### GetTransactionInputs +Used by the `block-producer` to query state required to prove the next block. -Returns data required to validate a new transaction. +--- -**Parameters** +### GetNoteAuthenticationInfo -- `account_id`: `AccountId` – ID of the account against which a transaction is executed. -- `nullifiers`: `[Digest]` – set of nullifiers consumed by this transaction. +Returns a list of Note inclusion proofs for the specified Note IDs. -**Returns** +This is used by the `block-producer` as part of the batch proving process. -- `account_state`: `AccountTransactionInputRecord` – account's descriptors. -- `nullifiers`: `[NullifierTransactionInputRecord]` – the block numbers at which corresponding nullifiers have been consumed, zero if not consumed. +--- ### GetNotesById Returns a list of notes matching the provided note IDs. -**Parameters** +--- -- `note_ids`: `[NoteId]` - list of IDs of the notes we want to query. +### GetTransactionInputs -**Returns** +Used by the `block-producer` to query state required to verify a submitted transaction. -- `notes`: `[Note]` - List of notes matching the list of requested NoteIds. +--- -### GetAccountDetails +### SyncNotes -Returns the latest state of an account with the specified ID. +Returns info which can be used by the client to sync up to the tip of chain for the notes they are interested in. -**Parameters** +Client specifies the `note_tags` they are interested in, and the block height from which to search for new for matching +notes for. The request will then return the next block containing any note matching the provided tags. -- `account_id`: `AccountId` – account ID. +The response includes each note's metadata and inclusion proof. -**Returns** +A basic note sync can be implemented by repeatedly requesting the previous response's block until reaching the tip of +the chain. -- `account`: `AccountInfo` – latest state of the account. For public accounts, this will include full details describing the current account state. For private accounts, only the hash of the latest state and the time of the last update is returned. +--- ### SyncState -Returns info which can be used by the client to sync up to the latest state of the chain -for the objects (accounts, notes, nullifiers) the client is interested in. - -This request returns the next block containing requested data. It also returns `chain_tip` which is the latest block number in the chain. -Client is expected to repeat these requests in a loop until `response.block_header.block_num == response.chain_tip`, at which point the client is fully synchronized with the chain. - -Each request also returns info about new notes, nullifiers etc. created. It also returns Chain MMR delta that can be used to update the state of Chain MMR. -This includes both chain MMR peaks and chain MMR nodes. - -For preserving some degree of privacy, note tags and nullifiers filters contain only high part of hashes. Thus, returned data -contains excessive notes and nullifiers, client can make additional filtering of that data on its side. +Returns info which can be used by the client to sync up to the latest state of the chain for the objects (accounts, +notes, nullifiers) the client is interested in. -**Parameters** +This request returns the next block containing requested data. It also returns `chain_tip` which is the latest block +number in the chain. Client is expected to repeat these requests in a loop until +`response.block_header.block_num == response.chain_tip`, at which point the client is fully synchronized with the chain. -- `block_num`: `uint32` – send updates to the client starting at this block. -- `account_ids`: `[AccountId]` – accounts filter. -- `note_tags`: `[uint32]` – note tags filter. Corresponds to the high 16 bits of the real values. -- `nullifiers`: `[uint32]` – nullifiers filter. Corresponds to the high 16 bits of the real values. +Each request also returns info about new notes, nullifiers etc. created. It also returns Chain MMR delta that can be +used to update the state of Chain MMR. This includes both chain MMR peaks and chain MMR nodes. -**Returns** +For preserving some degree of privacy, note tags and nullifiers filters contain only high part of hashes. Thus, returned +data contains excessive notes and nullifiers, client can make additional filtering of that data on its side. -- `chain_tip`: `uint32` – number of the latest block in the chain. -- `block_header`: `BlockHeader` – block header of the block with the first note matching the specified criteria. -- `mmr_delta`: `MmrDelta` – data needed to update the partial MMR from `request.block_num + 1` to `response.block_header.block_num`. -- `accounts`: `[AccountSummary]` – account summaries for accounts updated after `request.block_num + 1` but not after `response.block_header.block_num`. -- `transactions`: `[TransactionSummary]` – transaction summaries for transactions included after `request.block_num + 1` but not after `response.block_header.block_num`. - - Each `TransactionSummary` consists of the `transaction_id` the transaction identifier, `account_id` of the account that executed that transaction, `block_num` the block number in which the transaction was included. -- `notes`: `[NoteSyncRecord]` – a list of all notes together with the Merkle paths from `response.block_header.note_root`. -- `nullifiers`: `[NullifierUpdate]` – a list of nullifiers created between `request.block_num + 1` and `response.block_header.block_num`. - - Each `NullifierUpdate` consists of the `nullifier` and `block_num` the block number in which the note corresponding to that nullifier was consumed. +--- ## License diff --git a/proto/requests.proto b/proto/requests.proto index b8e2e0817..1230eac78 100644 --- a/proto/requests.proto +++ b/proto/requests.proto @@ -20,7 +20,7 @@ message CheckNullifiersByPrefixRequest { repeated uint32 nullifiers = 2; } -// Get a list of proofs for given nullifier hashes, each proof as a sparse Merkle Tree. +// Returns a nullifier proof for each of the requested nullifiers. message CheckNullifiersRequest { // List of nullifiers to return proofs for. repeated digest.Digest nullifiers = 1; diff --git a/proto/rpc.proto b/proto/rpc.proto index bd5d359ad..82da3e20c 100644 --- a/proto/rpc.proto +++ b/proto/rpc.proto @@ -6,10 +6,12 @@ import "requests.proto"; import "responses.proto"; service Api { - // Gets a list of proofs for given nullifier hashes, each proof as a sparse Merkle Tree. + // Returns a nullifier proof for each of the requested nullifiers. rpc CheckNullifiers(requests.CheckNullifiersRequest) returns (responses.CheckNullifiersResponse) {} // Returns a list of nullifiers that match the specified prefixes and are recorded in the node. + // + // Note that only 16-bit prefixes are supported at this time. rpc CheckNullifiersByPrefix(requests.CheckNullifiersByPrefixRequest) returns (responses.CheckNullifiersByPrefixResponse) {} // Returns the latest state of an account with the specified ID. @@ -22,7 +24,7 @@ service Api { // `to_block_num` (inclusive). rpc GetAccountStateDelta(requests.GetAccountStateDeltaRequest) returns (responses.GetAccountStateDeltaResponse) {} - // Retrieves block data by given block number. + // Returns raw block data for the specified block number. rpc GetBlockByNumber(requests.GetBlockByNumberRequest) returns (responses.GetBlockByNumberResponse) {} // Retrieves block header by given block number. Optionally, it also returns the MMR path @@ -35,10 +37,15 @@ service Api { // Submits proven transaction to the Miden network. rpc SubmitProvenTransaction(requests.SubmitProvenTransactionRequest) returns (responses.SubmitProvenTransactionResponse) {} - // Note synchronization request. + // Returns info which can be used by the client to sync up to the tip of chain for the notes they are interested in. + // + // Client specifies the `note_tags` they are interested in, and the block height from which to search for new for + // matching notes for. The request will then return the next block containing any note matching the provided tags. + // + // The response includes each note's metadata and inclusion proof. // - // Specifies note tags that client is interested in. The server will return the first block which - // contains a note matching `note_tags` or the chain tip. + // A basic note sync can be implemented by repeatedly requesting the previous response's block until reaching the + // tip of the chain. rpc SyncNotes(requests.SyncNoteRequest) returns (responses.SyncNoteResponse) {} // Returns info which can be used by the client to sync up to the latest state of the chain diff --git a/proto/store.proto b/proto/store.proto index 5ed980aba..0562b8c54 100644 --- a/proto/store.proto +++ b/proto/store.proto @@ -11,10 +11,12 @@ service Api { // Applies changes of a new block to the DB and in-memory data structures. rpc ApplyBlock(requests.ApplyBlockRequest) returns (responses.ApplyBlockResponse) {} - // Gets a list of proofs for given nullifier hashes, each proof as a sparse Merkle Tree. + // Returns a nullifier proof for each of the requested nullifiers. rpc CheckNullifiers(requests.CheckNullifiersRequest) returns (responses.CheckNullifiersResponse) {} // Returns a list of nullifiers that match the specified prefixes and are recorded in the node. + // + // Note that only 16-bit prefixes are supported at this time. rpc CheckNullifiersByPrefix(requests.CheckNullifiersByPrefixRequest) returns (responses.CheckNullifiersByPrefixResponse) {} // Returns the latest state of an account with the specified ID. @@ -27,7 +29,7 @@ service Api { // `to_block_num` (inclusive). rpc GetAccountStateDelta(requests.GetAccountStateDeltaRequest) returns (responses.GetAccountStateDeltaResponse) {} - // Retrieves block data by given block number. + // Returns raw block data for the specified block number. rpc GetBlockByNumber(requests.GetBlockByNumberRequest) returns (responses.GetBlockByNumberResponse) {} // Retrieves block header by given block number. Optionally, it also returns the MMR path @@ -46,10 +48,15 @@ service Api { // Returns data required to validate a new transaction. rpc GetTransactionInputs(requests.GetTransactionInputsRequest) returns (responses.GetTransactionInputsResponse) {} - // Note synchronization request. + // Returns info which can be used by the client to sync up to the tip of chain for the notes they are interested in. + // + // Client specifies the `note_tags` they are interested in, and the block height from which to search for new for + // matching notes for. The request will then return the next block containing any note matching the provided tags. + // + // The response includes each note's metadata and inclusion proof. // - // Specifies note tags that client is interested in. The server will return the first block which - // contains a note matching `note_tags` or the chain tip. + // A basic note sync can be implemented by repeatedly requesting the previous response's block until reaching the + // tip of the chain. rpc SyncNotes(requests.SyncNoteRequest) returns (responses.SyncNoteResponse) {} // Returns info which can be used by the client to sync up to the latest state of the chain From 1d5d5f8629f51a0f8ae7961cdd6739a998d28983 Mon Sep 17 00:00:00 2001 From: igamigo Date: Wed, 22 Jan 2025 16:38:46 -0300 Subject: [PATCH 48/50] reformat: Use latest next from miden-base; rename modules to singular (#636) * chore: update with miden-base/next latest changes * reformat: Use latest next from miden-base; rename modules to singular where applicable --------- Co-authored-by: SantiagoPittella --- CHANGELOG.md | 1 + Cargo.lock | 129 ++++++++---------- Cargo.toml | 6 +- bin/faucet/src/client.rs | 8 +- bin/faucet/src/handlers.rs | 4 +- bin/faucet/src/main.rs | 6 +- bin/faucet/src/state.rs | 2 +- bin/faucet/src/store.rs | 4 +- bin/node/src/commands/genesis/mod.rs | 8 +- crates/block-producer/Cargo.toml | 2 +- .../block-producer/src/batch_builder/batch.rs | 10 +- .../block-producer/src/batch_builder/mod.rs | 2 +- crates/block-producer/src/block.rs | 6 +- .../block-producer/src/block_builder/mod.rs | 4 +- .../prover/asm/block_kernel.masm | 6 +- .../src/block_builder/prover/block_witness.rs | 4 +- .../src/block_builder/prover/mod.rs | 7 +- .../src/block_builder/prover/tests.rs | 28 ++-- .../block-producer/src/domain/transaction.rs | 4 +- crates/block-producer/src/errors.rs | 4 +- .../src/mempool/inflight_state/mod.rs | 4 +- crates/block-producer/src/mempool/mod.rs | 2 +- crates/block-producer/src/mempool/tests.rs | 2 +- crates/block-producer/src/state_view/mod.rs | 2 +- .../src/state_view/tests/apply_block.rs | 2 +- .../src/state_view/tests/verify_tx.rs | 2 +- crates/block-producer/src/store/mod.rs | 6 +- .../block-producer/src/test_utils/account.rs | 2 +- crates/block-producer/src/test_utils/batch.rs | 2 +- crates/block-producer/src/test_utils/block.rs | 2 +- crates/block-producer/src/test_utils/mod.rs | 2 +- crates/block-producer/src/test_utils/note.rs | 4 +- .../src/test_utils/proven_tx.rs | 4 +- crates/block-producer/src/test_utils/store.rs | 4 +- .../src/domain/{accounts.rs => account.rs} | 2 +- .../proto/src/domain/{blocks.rs => block.rs} | 0 crates/proto/src/domain/digest.rs | 2 +- crates/proto/src/domain/mod.rs | 10 +- crates/proto/src/domain/{notes.rs => note.rs} | 4 +- .../domain/{nullifiers.rs => nullifier.rs} | 2 +- .../{transactions.rs => transaction.rs} | 0 crates/proto/src/generated/account.rs | 4 +- crates/proto/src/generated/note.rs | 4 +- crates/proto/src/generated/responses.rs | 2 +- crates/proto/src/lib.rs | 4 +- crates/rpc-proto/proto/account.proto | 4 +- crates/rpc-proto/proto/note.proto | 4 +- crates/rpc-proto/proto/responses.proto | 2 +- crates/rpc/src/server/api.rs | 2 +- crates/store/src/db/mod.rs | 6 +- crates/store/src/db/sql/mod.rs | 8 +- crates/store/src/db/sql/utils.rs | 6 +- crates/store/src/db/tests.rs | 10 +- crates/store/src/errors.rs | 4 +- crates/store/src/genesis.rs | 2 +- crates/store/src/nullifier_tree.rs | 2 +- crates/store/src/server/api.rs | 8 +- crates/store/src/state.rs | 10 +- proto/account.proto | 4 +- proto/note.proto | 4 +- proto/responses.proto | 2 +- 61 files changed, 191 insertions(+), 206 deletions(-) rename crates/proto/src/domain/{accounts.rs => account.rs} (99%) rename crates/proto/src/domain/{blocks.rs => block.rs} (100%) rename crates/proto/src/domain/{notes.rs => note.rs} (96%) rename crates/proto/src/domain/{nullifiers.rs => nullifier.rs} (99%) rename crates/proto/src/domain/{transactions.rs => transaction.rs} (100%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 87319b3a4..d4eec5ccb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ - [BREAKING] Remove store's `ListXXX` endpoints which were intended for test purposes (#608). - [BREAKING] Added support for storage maps on `GetAccountProofs` endpoint (#598). - [BREAKING] Removed the `testing` feature (#619). +- [BREAKING] Renamed modules to singular (#636). ## v0.6.0 (2024-11-05) diff --git a/Cargo.lock b/Cargo.lock index 2f78ddeb6..e62c02307 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -570,9 +570,9 @@ checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" [[package]] name = "crunchy" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" +checksum = "43da5946c66ffcc7745f48db692ffbb10a83bfe0afd96235c5c2a4fb23994929" [[package]] name = "crypto-common" @@ -1411,39 +1411,39 @@ checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" [[package]] name = "miden-air" -version = "0.11.0" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea00ea451d1547978817a37c1904ff34f4a24de2c23f9b77b282a0bc570ac3f1" +checksum = "671c1e48a91d4652c5bd3e6a5f6147e34b5b93484421bd7d5ded8fbe22fee7ec" dependencies = [ "miden-core", - "miden-thiserror", + "thiserror 2.0.11", "winter-air", "winter-prover", ] [[package]] name = "miden-assembly" -version = "0.11.0" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58925d5ab3d625b8e828267a64dd555b1fe37cd5a1d89d09224920d3255de5a9" +checksum = "b7f5b09d4f5e4c6b041b8a2358ebf64fdbc0a5b6a4ddc22e48aed9bb20961811" dependencies = [ "aho-corasick", "lalrpop", "lalrpop-util", "miden-core", "miden-miette", - "miden-thiserror", "rustc_version 0.4.1", "smallvec", + "thiserror 2.0.11", "tracing", "unicode-width 0.2.0", ] [[package]] name = "miden-core" -version = "0.11.0" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ba2f0ffd0362c91c0eedba391a3c17a8047389e15020ec95b0d7e840375bf03" +checksum = "e9326385003c062affde18cb336a3475033df0a871fb122ecaa9b48561109e40" dependencies = [ "lock_api", "loom", @@ -1451,19 +1451,19 @@ dependencies = [ "miden-crypto", "miden-formatting", "miden-miette", - "miden-thiserror", "num-derive", "num-traits", "parking_lot", + "thiserror 2.0.11", "winter-math", "winter-utils", ] [[package]] name = "miden-crypto" -version = "0.12.0" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f50a68deed96cde1f51eb623f75828e320f699e0d798f11592f8958ba8b512c3" +checksum = "a06bf3ad2a85f3f8f0da73b6357c77e482b1ceb36cacda8a2d85caae3bd1f702" dependencies = [ "blake3", "cc", @@ -1473,6 +1473,7 @@ dependencies = [ "rand", "rand_core", "sha3", + "thiserror 2.0.11", "winter-crypto", "winter-math", "winter-utils", @@ -1518,7 +1519,7 @@ dependencies = [ [[package]] name = "miden-lib" version = "0.7.0" -source = "git+https://github.com/0xPolygonMiden/miden-base.git?branch=next#a0521f17421c2db2968be404b20280457b79d195" +source = "git+https://github.com/0xPolygonMiden/miden-base.git?branch=next#829580636d2098978ece0a06d01477e2b3a3a131" dependencies = [ "miden-assembly", "miden-objects", @@ -1530,9 +1531,9 @@ dependencies = [ [[package]] name = "miden-miette" -version = "7.1.1" +version = "8.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c532250422d933f15b148fb81e4522a5d649c178ab420d0d596c86228da35570" +checksum = "eef536978f24a179d94fa2a41e4f92b28e7d8aab14b8d23df28ad2a3d7098b20" dependencies = [ "backtrace", "backtrace-ext", @@ -1541,7 +1542,6 @@ dependencies = [ "indenter", "lazy_static", "miden-miette-derive", - "miden-thiserror", "owo-colors", "regex", "rustc_version 0.2.3", @@ -1555,15 +1555,16 @@ dependencies = [ "syn", "terminal_size", "textwrap", + "thiserror 2.0.11", "trybuild", "unicode-width 0.1.14", ] [[package]] name = "miden-miette-derive" -version = "7.1.0" +version = "8.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cc759f0a2947acae217a2f32f722105cacc57d17d5f93bc16362142943a4edd" +checksum = "86a905f3ea65634dd4d1041a4f0fd0a3e77aa4118341d265af1a94339182222f" dependencies = [ "proc-macro2", "quote", @@ -1703,7 +1704,7 @@ dependencies = [ [[package]] name = "miden-objects" version = "0.7.0" -source = "git+https://github.com/0xPolygonMiden/miden-base.git?branch=next#a0521f17421c2db2968be404b20280457b79d195" +source = "git+https://github.com/0xPolygonMiden/miden-base.git?branch=next#829580636d2098978ece0a06d01477e2b3a3a131" dependencies = [ "getrandom", "miden-assembly", @@ -1722,21 +1723,22 @@ dependencies = [ [[package]] name = "miden-processor" -version = "0.11.0" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e1a756be679bfe2bdb209b9f4f12b4b59e8cac7c9f884ad90535b32bbd75923" +checksum = "4fbc6f1d5cad60b17da51ee00c8f16e047b3deca50cc42dd18e13cc0c73ca23c" dependencies = [ "miden-air", "miden-core", + "thiserror 2.0.11", "tracing", "winter-prover", ] [[package]] name = "miden-prover" -version = "0.11.0" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b6c2fc7ed9831493e1c18fb9d15a4b780ec624e9192f494ad3900c19bfe5b17" +checksum = "8e11ce36f3c6e16e6df6c0ff3bbf025e155b151e430017d4ec65dd617bd691e8" dependencies = [ "miden-air", "miden-processor", @@ -1751,37 +1753,17 @@ version = "0.6.0" [[package]] name = "miden-stdlib" -version = "0.11.0" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09948fda011d044273d1bbc56669da410ee3167f96bf1ad9885c946cfbed5757" +checksum = "001a8c957ea655e96c91f081cf6bf99bb35109e3a20acda1c2a87e2db53a2696" dependencies = [ "miden-assembly", ] -[[package]] -name = "miden-thiserror" -version = "1.0.59" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "183ff8de338956ecfde3a38573241eb7a6f3d44d73866c210e5629c07fa00253" -dependencies = [ - "miden-thiserror-impl", -] - -[[package]] -name = "miden-thiserror-impl" -version = "1.0.59" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ee4176a0f2e7d29d2a8ee7e60b6deb14ce67a20e94c3e2c7275cdb8804e1862" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "miden-tx" version = "0.7.0" -source = "git+https://github.com/0xPolygonMiden/miden-base.git?branch=next#a0521f17421c2db2968be404b20280457b79d195" +source = "git+https://github.com/0xPolygonMiden/miden-base.git?branch=next#829580636d2098978ece0a06d01477e2b3a3a131" dependencies = [ "async-trait", "miden-lib", @@ -1797,12 +1779,13 @@ dependencies = [ [[package]] name = "miden-verifier" -version = "0.11.0" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "636b2eab9dc0a9fdd6d30d9ece3cf276b012e9ecbccd934d4c0befa07c84cfd0" +checksum = "76105a8bc8fb948d02b114ea0540de771b34f446191a08a3393d705aafc70191" dependencies = [ "miden-air", "miden-core", + "thiserror 2.0.11", "tracing", "winter-verifier", ] @@ -2293,9 +2276,9 @@ dependencies = [ [[package]] name = "prost-reflect" -version = "0.14.4" +version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc9647f03b808b79abca8408b1609be9887ba90453c940d00332a60eeb6f5748" +checksum = "e92b959d24e05a3e2da1d0beb55b48bc8a97059b8336ea617780bd6addbbfb5a" dependencies = [ "logos", "miette", @@ -2543,9 +2526,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.43" +version = "0.38.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a78891ee6bf2340288408954ac787aa063d8e8817e9f53abb37c695c6d834ef6" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" dependencies = [ "bitflags", "errno", @@ -3808,9 +3791,9 @@ dependencies = [ [[package]] name = "winter-air" -version = "0.10.3" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c92f0d9a736cb744b0a3e267fafd50d27d6625be3681f55a2b7650ddbf3a2ce" +checksum = "3a8fdb702503625f54dcaf9222aa2c7a0b2e868b3eb84b90d1837d68034bf999" dependencies = [ "libm", "winter-crypto", @@ -3821,9 +3804,9 @@ dependencies = [ [[package]] name = "winter-crypto" -version = "0.10.2" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fcae1ada055aa10554910ecffc106cb116a19dba11ac91390ef982f94adb9c5" +checksum = "67c57748fd2da77742be601f03eda639ff6046879738fd1faae86e80018263cb" dependencies = [ "blake3", "sha3", @@ -3833,9 +3816,9 @@ dependencies = [ [[package]] name = "winter-fri" -version = "0.10.2" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff88657560100f34fb83882a0adf33fb7caee235deb83193d0d251ddb28ed9c9" +checksum = "3f9f999bc248c6254138b627035cb6d2c319580eb37dadcab6672298dbf00e41" dependencies = [ "winter-crypto", "winter-math", @@ -3844,18 +3827,18 @@ dependencies = [ [[package]] name = "winter-math" -version = "0.10.2" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82479f94efc0b5374a93e2074ba46ef404384fb1ea6e35a847febec53096509b" +checksum = "6020c17839fa107ce4a7cc178e407ebbc24adfac1980f4fa2111198e052700ab" dependencies = [ "winter-utils", ] [[package]] name = "winter-maybe-async" -version = "0.10.1" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be43529f43f70306437d2c2c9f9e2b3a4d39b42e86702d8d7577f2357ea32fa6" +checksum = "91ce144fde121b98523bb8a6c15a311773e1d534d33c1cb47f5580bba9cff8e7" dependencies = [ "quote", "syn", @@ -3863,9 +3846,9 @@ dependencies = [ [[package]] name = "winter-prover" -version = "0.10.3" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cab5e02d53d5df7903ebf3e1f4ba44b918267876bfd37eade046d9a669f4e9fb" +checksum = "b6c2f3cf80955f084fd614c86883195331116b6c96dc88532d43d6836bd7adee" dependencies = [ "tracing", "winter-air", @@ -3878,9 +3861,9 @@ dependencies = [ [[package]] name = "winter-rand-utils" -version = "0.10.1" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a7616d11fcc26552dada45c803a884ac97c253218835b83a2c63e1c2a988639" +checksum = "226e4c455f6eb72f64ac6eeb7642df25e21ff2280a4f6b09db75392ad6b390ef" dependencies = [ "rand", "winter-utils", @@ -3888,18 +3871,18 @@ dependencies = [ [[package]] name = "winter-utils" -version = "0.10.2" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f948e71ffd482aa13d0ec3349047f81ecdb89f3b3287577973dcbf092a25fb4" +checksum = "1507ef312ea5569d54c2c7446a18b82143eb2a2e21f5c3ec7cfbe8200c03bd7c" dependencies = [ "rayon", ] [[package]] name = "winter-verifier" -version = "0.10.3" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "517c31712cceaafc3c7dcc9311f7d5d306b34e208bc72664c691056fd863f7b8" +checksum = "56d949eab6c26328733477ec56ca054fdc69c0c1b8267fa03d8b618ffe2193c4" dependencies = [ "winter-air", "winter-crypto", @@ -3910,9 +3893,9 @@ dependencies = [ [[package]] name = "winterfell" -version = "0.10.3" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbf1ab01d2781f7d3f1bd5c12800905c5bbf62e06778672498be798006ac463a" +checksum = "f6bdcd01333bbf4a349d8d13f269281524bd6d1a36ae3a853187f0665bf1cfd4" dependencies = [ "winter-air", "winter-prover", diff --git a/Cargo.toml b/Cargo.toml index e3fa468fa..4ec0576ae 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,7 +26,7 @@ readme = "README.md" [workspace.dependencies] assert_matches = { version = "1.5" } -miden-air = { version = "0.11" } +miden-air = { version = "0.12" } miden-lib = { git = "https://github.com/0xPolygonMiden/miden-base.git", branch = "next" } miden-node-block-producer = { path = "crates/block-producer", version = "0.6" } miden-node-proto = { path = "crates/proto", version = "0.6" } @@ -35,8 +35,8 @@ miden-node-store = { path = "crates/store", version = "0.6" } miden-node-test-macro = { path = "crates/test-macro" } miden-node-utils = { path = "crates/utils", version = "0.6" } miden-objects = { git = "https://github.com/0xPolygonMiden/miden-base.git", branch = "next" } -miden-processor = { version = "0.11" } -miden-stdlib = { version = "0.11", default-features = false } +miden-processor = { version = "0.12" } +miden-stdlib = { version = "0.12", default-features = false } miden-tx = { git = "https://github.com/0xPolygonMiden/miden-base.git", branch = "next" } prost = { version = "0.13" } rand = { version = "0.8" } diff --git a/bin/faucet/src/client.rs b/bin/faucet/src/client.rs index 10b911e52..44a9ee19a 100644 --- a/bin/faucet/src/client.rs +++ b/bin/faucet/src/client.rs @@ -1,7 +1,7 @@ use std::{sync::Arc, time::Duration}; use anyhow::Context; -use miden_lib::{notes::create_p2id_note, transaction::TransactionKernel}; +use miden_lib::{note::create_p2id_note, transaction::TransactionKernel}; use miden_node_proto::generated::{ requests::{ GetAccountDetailsRequest, GetBlockHeaderByNumberRequest, SubmitProvenTransactionRequest, @@ -9,14 +9,14 @@ use miden_node_proto::generated::{ rpc::api_client::ApiClient, }; use miden_objects::{ - accounts::{Account, AccountData, AccountId, AuthSecretKey}, - assets::FungibleAsset, + account::{Account, AccountData, AccountId, AuthSecretKey}, + asset::FungibleAsset, block::{BlockHeader, BlockNumber}, crypto::{ merkle::{MmrPeaks, PartialMmr}, rand::RpoRandomCoin, }, - notes::{Note, NoteType}, + note::{Note, NoteType}, transaction::{ChainMmr, ExecutedTransaction, TransactionArgs, TransactionScript}, utils::Deserializable, vm::AdviceMap, diff --git a/bin/faucet/src/handlers.rs b/bin/faucet/src/handlers.rs index 9037c2a24..6ff7b75d5 100644 --- a/bin/faucet/src/handlers.rs +++ b/bin/faucet/src/handlers.rs @@ -8,8 +8,8 @@ use axum::{ use http::header; use http_body_util::Full; use miden_objects::{ - accounts::AccountId, - notes::{NoteDetails, NoteExecutionMode, NoteFile, NoteId, NoteTag}, + account::AccountId, + note::{NoteDetails, NoteExecutionMode, NoteFile, NoteId, NoteTag}, utils::serde::Serializable, }; use serde::{Deserialize, Serialize}; diff --git a/bin/faucet/src/main.rs b/bin/faucet/src/main.rs index cb4053ead..4b4feeb0f 100644 --- a/bin/faucet/src/main.rs +++ b/bin/faucet/src/main.rs @@ -15,11 +15,11 @@ use axum::{ use clap::{Parser, Subcommand}; use client::initialize_faucet_client; use http::HeaderValue; -use miden_lib::{accounts::faucets::create_basic_fungible_faucet, AuthScheme}; +use miden_lib::{account::faucets::create_basic_fungible_faucet, AuthScheme}; use miden_node_utils::{config::load_config, crypto::get_rpo_random_coin, version::LongVersion}; use miden_objects::{ - accounts::{AccountData, AccountStorageMode, AuthSecretKey}, - assets::TokenSymbol, + account::{AccountData, AccountStorageMode, AuthSecretKey}, + asset::TokenSymbol, crypto::dsa::rpo_falcon512::SecretKey, Felt, }; diff --git a/bin/faucet/src/state.rs b/bin/faucet/src/state.rs index 6f70a0207..1c839228c 100644 --- a/bin/faucet/src/state.rs +++ b/bin/faucet/src/state.rs @@ -1,6 +1,6 @@ use std::{collections::HashMap, sync::Arc}; -use miden_objects::accounts::AccountId; +use miden_objects::account::AccountId; use static_files::Resource; use tokio::sync::Mutex; use tracing::info; diff --git a/bin/faucet/src/store.rs b/bin/faucet/src/store.rs index 6e36eabfa..1c61b7ebb 100644 --- a/bin/faucet/src/store.rs +++ b/bin/faucet/src/store.rs @@ -1,9 +1,9 @@ use std::sync::Mutex; use miden_objects::{ - accounts::{Account, AccountId}, + account::{Account, AccountId}, block::{BlockHeader, BlockNumber}, - notes::NoteId, + note::NoteId, transaction::{ChainMmr, InputNotes, TransactionInputs}, Word, }; diff --git a/bin/node/src/commands/genesis/mod.rs b/bin/node/src/commands/genesis/mod.rs index 1875bb940..08a65898b 100644 --- a/bin/node/src/commands/genesis/mod.rs +++ b/bin/node/src/commands/genesis/mod.rs @@ -5,12 +5,12 @@ use std::{ use anyhow::{anyhow, bail, Context, Result}; pub use inputs::{AccountInput, AuthSchemeInput, GenesisInput}; -use miden_lib::{accounts::faucets::create_basic_fungible_faucet, AuthScheme}; +use miden_lib::{account::faucets::create_basic_fungible_faucet, AuthScheme}; use miden_node_store::genesis::GenesisState; use miden_node_utils::{config::load_config, crypto::get_rpo_random_coin}; use miden_objects::{ - accounts::{Account, AccountData, AccountIdAnchor, AuthSecretKey}, - assets::TokenSymbol, + account::{Account, AccountData, AccountIdAnchor, AuthSecretKey}, + asset::TokenSymbol, crypto::{dsa::rpo_falcon512::SecretKey, utils::Serializable}, Felt, ONE, }; @@ -182,7 +182,7 @@ mod tests { use figment::Jail; use miden_node_store::genesis::GenesisState; - use miden_objects::{accounts::AccountData, utils::serde::Deserializable}; + use miden_objects::{account::AccountData, utils::serde::Deserializable}; use crate::DEFAULT_GENESIS_FILE_PATH; diff --git a/crates/block-producer/Cargo.toml b/crates/block-producer/Cargo.toml index ee65ddd0f..27d52be8a 100644 --- a/crates/block-producer/Cargo.toml +++ b/crates/block-producer/Cargo.toml @@ -45,4 +45,4 @@ miden-tx = { workspace = true, features = ["testing"] } pretty_assertions = "1.4" rand_chacha = { version = "0.3", default-features = false } tokio = { workspace = true, features = ["test-util"] } -winterfell = { version = "0.10" } +winterfell = { version = "0.11" } diff --git a/crates/block-producer/src/batch_builder/batch.rs b/crates/block-producer/src/batch_builder/batch.rs index 7806e3557..33b27b058 100644 --- a/crates/block-producer/src/batch_builder/batch.rs +++ b/crates/block-producer/src/batch_builder/batch.rs @@ -4,13 +4,13 @@ use std::{ mem, }; -use miden_node_proto::domain::notes::NoteAuthenticationInfo; +use miden_node_proto::domain::note::NoteAuthenticationInfo; use miden_node_utils::formatting::format_blake3_digest; use miden_objects::{ - accounts::{delta::AccountUpdateDetails, AccountId}, - batches::BatchNoteTree, + account::{delta::AccountUpdateDetails, AccountId}, + batch::BatchNoteTree, crypto::hash::blake::{Blake3Digest, Blake3_256}, - notes::{NoteHeader, NoteId, Nullifier}, + note::{NoteHeader, NoteId, Nullifier}, transaction::{InputNoteCommitment, OutputNote, ProvenTransaction, TransactionId}, AccountDeltaError, Digest, }; @@ -301,7 +301,7 @@ impl OutputNoteTracker { #[cfg(test)] mod tests { - use miden_objects::notes::NoteInclusionProof; + use miden_objects::note::NoteInclusionProof; use miden_processor::crypto::MerklePath; use super::*; diff --git a/crates/block-producer/src/batch_builder/mod.rs b/crates/block-producer/src/batch_builder/mod.rs index 58283d02b..68fae021d 100644 --- a/crates/block-producer/src/batch_builder/mod.rs +++ b/crates/block-producer/src/batch_builder/mod.rs @@ -1,7 +1,7 @@ use std::{num::NonZeroUsize, ops::Range, time::Duration}; use batch::BatchId; -use miden_node_proto::domain::notes::NoteAuthenticationInfo; +use miden_node_proto::domain::note::NoteAuthenticationInfo; use rand::Rng; use tokio::{task::JoinSet, time}; use tracing::{debug, info, instrument, Span}; diff --git a/crates/block-producer/src/block.rs b/crates/block-producer/src/block.rs index dd1460664..857a5b018 100644 --- a/crates/block-producer/src/block.rs +++ b/crates/block-producer/src/block.rs @@ -1,16 +1,16 @@ use std::collections::BTreeMap; use miden_node_proto::{ - domain::notes::NoteAuthenticationInfo, + domain::note::NoteAuthenticationInfo, errors::{ConversionError, MissingFieldHelper}, generated::responses::GetBlockInputsResponse, AccountInputRecord, NullifierWitness, }; use miden_objects::{ - accounts::AccountId, + account::AccountId, block::BlockHeader, crypto::merkle::{MerklePath, MmrPeaks, SmtProof}, - notes::Nullifier, + note::Nullifier, Digest, }; diff --git a/crates/block-producer/src/block_builder/mod.rs b/crates/block-producer/src/block_builder/mod.rs index 5066d267a..32023a58f 100644 --- a/crates/block-producer/src/block_builder/mod.rs +++ b/crates/block-producer/src/block_builder/mod.rs @@ -2,9 +2,9 @@ use std::{collections::BTreeSet, ops::Range}; use miden_node_utils::formatting::format_array; use miden_objects::{ - accounts::AccountId, + account::AccountId, block::Block, - notes::{NoteHeader, Nullifier}, + note::{NoteHeader, Nullifier}, transaction::{InputNoteCommitment, OutputNote}, }; use rand::Rng; diff --git a/crates/block-producer/src/block_builder/prover/asm/block_kernel.masm b/crates/block-producer/src/block_builder/prover/asm/block_kernel.masm index 63e277047..309501d59 100644 --- a/crates/block-producer/src/block_builder/prover/asm/block_kernel.masm +++ b/crates/block-producer/src/block_builder/prover/asm/block_kernel.masm @@ -226,17 +226,17 @@ begin exec.compute_account_root mem_storew.0 dropw # => [, , ] - exec.compute_note_root mem_storew.1 dropw + exec.compute_note_root mem_storew.4 dropw # => [, ] - exec.compute_nullifier_root mem_storew.2 dropw + exec.compute_nullifier_root mem_storew.8 dropw # => [] exec.compute_chain_mmr_root # => [CHAIN_MMR_ROOT] # Load output on stack - padw mem_loadw.2 padw mem_loadw.1 padw mem_loadw.0 + padw mem_loadw.8 padw mem_loadw.4 padw mem_loadw.0 # => [ACCOUNT_ROOT, NOTE_ROOT, NULLIFIER_ROOT, CHAIN_MMR_ROOT] # truncate the stack diff --git a/crates/block-producer/src/block_builder/prover/block_witness.rs b/crates/block-producer/src/block_builder/prover/block_witness.rs index 5101a421d..9f453b598 100644 --- a/crates/block-producer/src/block_builder/prover/block_witness.rs +++ b/crates/block-producer/src/block_builder/prover/block_witness.rs @@ -1,10 +1,10 @@ use std::collections::{BTreeMap, BTreeSet}; use miden_objects::{ - accounts::{delta::AccountUpdateDetails, AccountId}, + account::{delta::AccountUpdateDetails, AccountId}, block::{BlockAccountUpdate, BlockHeader}, crypto::merkle::{EmptySubtreeRoots, MerklePath, MerkleStore, MmrPeaks, SmtProof}, - notes::Nullifier, + note::Nullifier, transaction::TransactionId, vm::{AdviceInputs, StackInputs}, Digest, Felt, BLOCK_NOTE_TREE_DEPTH, MAX_BATCHES_PER_BLOCK, ZERO, diff --git a/crates/block-producer/src/block_builder/prover/mod.rs b/crates/block-producer/src/block_builder/prover/mod.rs index 0ef259f7c..ded93a7d4 100644 --- a/crates/block-producer/src/block_builder/prover/mod.rs +++ b/crates/block-producer/src/block_builder/prover/mod.rs @@ -88,17 +88,18 @@ impl BlockProver { witness: BlockWitness, ) -> Result<(Digest, Digest, Digest, Digest), BlockProverError> { let (advice_inputs, stack_inputs) = witness.into_program_inputs()?; - let host = { + let mut host = { let advice_provider = MemAdviceProvider::from(advice_inputs); let mut host = DefaultHost::new(advice_provider); - host.load_mast_forest(StdLibrary::default().mast_forest().clone()); + host.load_mast_forest(StdLibrary::default().mast_forest().clone()) + .expect("failed to load mast forest"); host }; let execution_output = - execute(&self.kernel, stack_inputs, host, ExecutionOptions::default()) + execute(&self.kernel, stack_inputs, &mut host, ExecutionOptions::default()) .map_err(BlockProverError::ProgramExecutionFailed)?; let new_account_root = execution_output diff --git a/crates/block-producer/src/block_builder/prover/tests.rs b/crates/block-producer/src/block_builder/prover/tests.rs index b739f1132..d688cdcfd 100644 --- a/crates/block-producer/src/block_builder/prover/tests.rs +++ b/crates/block-producer/src/block_builder/prover/tests.rs @@ -1,16 +1,16 @@ use std::{collections::BTreeMap, iter}; use assert_matches::assert_matches; -use miden_node_proto::domain::notes::NoteAuthenticationInfo; +use miden_node_proto::domain::note::NoteAuthenticationInfo; use miden_objects::{ - accounts::{ + account::{ delta::AccountUpdateDetails, AccountId, AccountIdVersion, AccountStorageMode, AccountType, }, block::{BlockAccountUpdate, BlockNoteIndex, BlockNoteTree, BlockNumber}, crypto::merkle::{ EmptySubtreeRoots, LeafIndex, MerklePath, Mmr, MmrPeaks, Smt, SmtLeaf, SmtProof, SMT_DEPTH, }, - notes::{NoteExecutionHint, NoteHeader, NoteMetadata, NoteTag, NoteType, Nullifier}, + note::{NoteExecutionHint, NoteHeader, NoteMetadata, NoteTag, NoteType, Nullifier}, testing::account_id::{ ACCOUNT_ID_OFF_CHAIN_SENDER, ACCOUNT_ID_REGULAR_ACCOUNT_UPDATABLE_CODE_OFF_CHAIN, }, @@ -42,19 +42,19 @@ fn block_witness_validation_inconsistent_account_ids() { [0; 15], AccountIdVersion::Version0, AccountType::RegularAccountImmutableCode, - miden_objects::accounts::AccountStorageMode::Private, + miden_objects::account::AccountStorageMode::Private, ); let account_id_2 = AccountId::dummy( [1; 15], AccountIdVersion::Version0, AccountType::RegularAccountImmutableCode, - miden_objects::accounts::AccountStorageMode::Private, + miden_objects::account::AccountStorageMode::Private, ); let account_id_3 = AccountId::dummy( [2; 15], AccountIdVersion::Version0, AccountType::RegularAccountImmutableCode, - miden_objects::accounts::AccountStorageMode::Private, + miden_objects::account::AccountStorageMode::Private, ); let block_inputs_from_store: BlockInputs = { @@ -296,31 +296,31 @@ async fn compute_account_root_success() { [0; 15], AccountIdVersion::Version0, AccountType::RegularAccountImmutableCode, - miden_objects::accounts::AccountStorageMode::Private, + miden_objects::account::AccountStorageMode::Private, ), AccountId::dummy( [1; 15], AccountIdVersion::Version0, AccountType::RegularAccountImmutableCode, - miden_objects::accounts::AccountStorageMode::Private, + miden_objects::account::AccountStorageMode::Private, ), AccountId::dummy( [2; 15], AccountIdVersion::Version0, AccountType::RegularAccountImmutableCode, - miden_objects::accounts::AccountStorageMode::Private, + miden_objects::account::AccountStorageMode::Private, ), AccountId::dummy( [3; 15], AccountIdVersion::Version0, AccountType::RegularAccountImmutableCode, - miden_objects::accounts::AccountStorageMode::Private, + miden_objects::account::AccountStorageMode::Private, ), AccountId::dummy( [4; 15], AccountIdVersion::Version0, AccountType::RegularAccountImmutableCode, - miden_objects::accounts::AccountStorageMode::Private, + miden_objects::account::AccountStorageMode::Private, ), ]; @@ -568,19 +568,19 @@ async fn compute_note_root_success() { [0; 15], AccountIdVersion::Version0, AccountType::RegularAccountImmutableCode, - miden_objects::accounts::AccountStorageMode::Private, + miden_objects::account::AccountStorageMode::Private, ), AccountId::dummy( [1; 15], AccountIdVersion::Version0, AccountType::RegularAccountImmutableCode, - miden_objects::accounts::AccountStorageMode::Private, + miden_objects::account::AccountStorageMode::Private, ), AccountId::dummy( [2; 15], AccountIdVersion::Version0, AccountType::RegularAccountImmutableCode, - miden_objects::accounts::AccountStorageMode::Private, + miden_objects::account::AccountStorageMode::Private, ), ]; diff --git a/crates/block-producer/src/domain/transaction.rs b/crates/block-producer/src/domain/transaction.rs index c23644d54..10c2f236d 100644 --- a/crates/block-producer/src/domain/transaction.rs +++ b/crates/block-producer/src/domain/transaction.rs @@ -1,9 +1,9 @@ use std::{collections::BTreeSet, sync::Arc}; use miden_objects::{ - accounts::AccountId, + account::AccountId, block::BlockNumber, - notes::{NoteId, Nullifier}, + note::{NoteId, Nullifier}, transaction::{ProvenTransaction, TransactionId, TxAccountUpdate}, Digest, }; diff --git a/crates/block-producer/src/errors.rs b/crates/block-producer/src/errors.rs index e3a6a670c..785de16a2 100644 --- a/crates/block-producer/src/errors.rs +++ b/crates/block-producer/src/errors.rs @@ -1,10 +1,10 @@ use miden_node_proto::errors::ConversionError; use miden_node_utils::formatting::format_opt; use miden_objects::{ - accounts::AccountId, + account::AccountId, block::BlockNumber, crypto::merkle::MerkleError, - notes::{NoteId, Nullifier}, + note::{NoteId, Nullifier}, transaction::TransactionId, AccountDeltaError, Digest, }; diff --git a/crates/block-producer/src/mempool/inflight_state/mod.rs b/crates/block-producer/src/mempool/inflight_state/mod.rs index 5734eeeb7..e9945aca8 100644 --- a/crates/block-producer/src/mempool/inflight_state/mod.rs +++ b/crates/block-producer/src/mempool/inflight_state/mod.rs @@ -1,9 +1,9 @@ use std::collections::{BTreeMap, BTreeSet, VecDeque}; use miden_objects::{ - accounts::AccountId, + account::AccountId, block::BlockNumber, - notes::{NoteId, Nullifier}, + note::{NoteId, Nullifier}, transaction::TransactionId, }; diff --git a/crates/block-producer/src/mempool/mod.rs b/crates/block-producer/src/mempool/mod.rs index a772dfb50..08e332324 100644 --- a/crates/block-producer/src/mempool/mod.rs +++ b/crates/block-producer/src/mempool/mod.rs @@ -86,7 +86,7 @@ impl BatchBudget { // This type assertion reminds us to update the account check if we ever support multiple // account updates per tx. const ACCOUNT_UPDATES_PER_TX: usize = 1; - let _: miden_objects::accounts::AccountId = tx.account_update().account_id(); + let _: miden_objects::account::AccountId = tx.account_update().account_id(); let output_notes = tx.output_note_count(); let input_notes = tx.input_note_count(); diff --git a/crates/block-producer/src/mempool/tests.rs b/crates/block-producer/src/mempool/tests.rs index 1e68933d1..8e93892fb 100644 --- a/crates/block-producer/src/mempool/tests.rs +++ b/crates/block-producer/src/mempool/tests.rs @@ -1,4 +1,4 @@ -use miden_node_proto::domain::notes::NoteAuthenticationInfo; +use miden_node_proto::domain::note::NoteAuthenticationInfo; use miden_objects::block::BlockNumber; use pretty_assertions::assert_eq; diff --git a/crates/block-producer/src/state_view/mod.rs b/crates/block-producer/src/state_view/mod.rs index 15710eadf..5295eece9 100644 --- a/crates/block-producer/src/state_view/mod.rs +++ b/crates/block-producer/src/state_view/mod.rs @@ -4,7 +4,7 @@ use async_trait::async_trait; use miden_node_utils::formatting::format_array; use miden_objects::{ block::Block, - notes::{NoteId, Nullifier}, + note::{NoteId, Nullifier}, transaction::OutputNote, Digest, MIN_PROOF_SECURITY_LEVEL, }; diff --git a/crates/block-producer/src/state_view/tests/apply_block.rs b/crates/block-producer/src/state_view/tests/apply_block.rs index 5bec67698..6a6e6ec54 100644 --- a/crates/block-producer/src/state_view/tests/apply_block.rs +++ b/crates/block-producer/src/state_view/tests/apply_block.rs @@ -8,7 +8,7 @@ use std::iter; use assert_matches::assert_matches; -use miden_objects::{accounts::delta::AccountUpdateDetails, block::BlockAccountUpdate}; +use miden_objects::{account::delta::AccountUpdateDetails, block::BlockAccountUpdate}; use super::*; use crate::test_utils::{block::MockBlockBuilder, MockStoreSuccessBuilder}; diff --git a/crates/block-producer/src/state_view/tests/verify_tx.rs b/crates/block-producer/src/state_view/tests/verify_tx.rs index 6df558dbb..ed8f0f3c1 100644 --- a/crates/block-producer/src/state_view/tests/verify_tx.rs +++ b/crates/block-producer/src/state_view/tests/verify_tx.rs @@ -13,7 +13,7 @@ use std::iter; use assert_matches::assert_matches; -use miden_objects::notes::Note; +use miden_objects::note::Note; use tokio::task::JoinSet; use super::*; diff --git a/crates/block-producer/src/store/mod.rs b/crates/block-producer/src/store/mod.rs index 9d2f76bd3..2608975b5 100644 --- a/crates/block-producer/src/store/mod.rs +++ b/crates/block-producer/src/store/mod.rs @@ -6,7 +6,7 @@ use std::{ use itertools::Itertools; use miden_node_proto::{ - domain::notes::NoteAuthenticationInfo, + domain::note::NoteAuthenticationInfo, errors::{ConversionError, MissingFieldHelper}, generated::{ digest, @@ -21,9 +21,9 @@ use miden_node_proto::{ }; use miden_node_utils::formatting::format_opt; use miden_objects::{ - accounts::AccountId, + account::AccountId, block::{Block, BlockHeader, BlockNumber}, - notes::{NoteId, Nullifier}, + note::{NoteId, Nullifier}, transaction::ProvenTransaction, utils::Serializable, Digest, diff --git a/crates/block-producer/src/test_utils/account.rs b/crates/block-producer/src/test_utils/account.rs index 98d1b9f12..2247a39a3 100644 --- a/crates/block-producer/src/test_utils/account.rs +++ b/crates/block-producer/src/test_utils/account.rs @@ -1,7 +1,7 @@ use std::{collections::HashMap, ops::Not, sync::LazyLock}; use miden_objects::{ - accounts::{AccountIdAnchor, AccountIdVersion, AccountStorageMode, AccountType}, + account::{AccountIdAnchor, AccountIdVersion, AccountStorageMode, AccountType}, Hasher, }; diff --git a/crates/block-producer/src/test_utils/batch.rs b/crates/block-producer/src/test_utils/batch.rs index d27eb9e0b..45346941a 100644 --- a/crates/block-producer/src/test_utils/batch.rs +++ b/crates/block-producer/src/test_utils/batch.rs @@ -1,4 +1,4 @@ -use miden_node_proto::domain::notes::NoteAuthenticationInfo; +use miden_node_proto::domain::note::NoteAuthenticationInfo; use crate::{batch_builder::TransactionBatch, test_utils::MockProvenTxBuilder}; diff --git a/crates/block-producer/src/test_utils/block.rs b/crates/block-producer/src/test_utils/block.rs index 27276a6ee..2a940b1ae 100644 --- a/crates/block-producer/src/test_utils/block.rs +++ b/crates/block-producer/src/test_utils/block.rs @@ -3,7 +3,7 @@ use std::iter; use miden_objects::{ block::{Block, BlockAccountUpdate, BlockHeader, BlockNoteIndex, BlockNoteTree, NoteBatch}, crypto::merkle::{Mmr, SimpleSmt}, - notes::Nullifier, + note::Nullifier, transaction::OutputNote, Digest, ACCOUNT_TREE_DEPTH, }; diff --git a/crates/block-producer/src/test_utils/mod.rs b/crates/block-producer/src/test_utils/mod.rs index b246839d2..28ba4349f 100644 --- a/crates/block-producer/src/test_utils/mod.rs +++ b/crates/block-producer/src/test_utils/mod.rs @@ -1,7 +1,7 @@ use std::sync::Arc; use miden_objects::{ - accounts::AccountId, + account::AccountId, crypto::rand::{FeltRng, RpoRandomCoin}, transaction::TransactionId, Digest, diff --git a/crates/block-producer/src/test_utils/note.rs b/crates/block-producer/src/test_utils/note.rs index 0a3b0a7d2..8ed060c6e 100644 --- a/crates/block-producer/src/test_utils/note.rs +++ b/crates/block-producer/src/test_utils/note.rs @@ -1,7 +1,7 @@ use miden_lib::transaction::TransactionKernel; use miden_objects::{ - notes::Note, - testing::notes::NoteBuilder, + note::Note, + testing::note::NoteBuilder, transaction::{InputNote, InputNoteCommitment, OutputNote}, }; use rand_chacha::{rand_core::SeedableRng, ChaCha20Rng}; diff --git a/crates/block-producer/src/test_utils/proven_tx.rs b/crates/block-producer/src/test_utils/proven_tx.rs index 55a69dba7..3de0fd2c1 100644 --- a/crates/block-producer/src/test_utils/proven_tx.rs +++ b/crates/block-producer/src/test_utils/proven_tx.rs @@ -3,9 +3,9 @@ use std::ops::Range; use itertools::Itertools; use miden_air::HashFunction; use miden_objects::{ - accounts::AccountId, + account::AccountId, block::BlockNumber, - notes::{Note, NoteExecutionHint, NoteHeader, NoteMetadata, NoteType, Nullifier}, + note::{Note, NoteExecutionHint, NoteHeader, NoteMetadata, NoteType, Nullifier}, transaction::{InputNote, OutputNote, ProvenTransaction, ProvenTransactionBuilder}, vm::ExecutionProof, Digest, Felt, Hasher, ONE, diff --git a/crates/block-producer/src/test_utils/store.rs b/crates/block-producer/src/test_utils/store.rs index 63dd1f31f..ac8e694af 100644 --- a/crates/block-producer/src/test_utils/store.rs +++ b/crates/block-producer/src/test_utils/store.rs @@ -3,11 +3,11 @@ use std::{ num::NonZeroU32, }; -use miden_node_proto::domain::{blocks::BlockInclusionProof, notes::NoteAuthenticationInfo}; +use miden_node_proto::domain::{block::BlockInclusionProof, note::NoteAuthenticationInfo}; use miden_objects::{ block::{Block, BlockHeader, BlockNumber, NoteBatch}, crypto::merkle::{Mmr, SimpleSmt, Smt, ValuePath}, - notes::{NoteId, NoteInclusionProof, Nullifier}, + note::{NoteId, NoteInclusionProof, Nullifier}, transaction::ProvenTransaction, ACCOUNT_TREE_DEPTH, EMPTY_WORD, ZERO, }; diff --git a/crates/proto/src/domain/accounts.rs b/crates/proto/src/domain/account.rs similarity index 99% rename from crates/proto/src/domain/accounts.rs rename to crates/proto/src/domain/account.rs index a5e387a3f..3d6680065 100644 --- a/crates/proto/src/domain/accounts.rs +++ b/crates/proto/src/domain/account.rs @@ -2,7 +2,7 @@ use std::fmt::{Debug, Display, Formatter}; use miden_node_utils::formatting::format_opt; use miden_objects::{ - accounts::{Account, AccountHeader, AccountId}, + account::{Account, AccountHeader, AccountId}, block::BlockNumber, crypto::{hash::rpo::RpoDigest, merkle::MerklePath}, utils::{Deserializable, Serializable}, diff --git a/crates/proto/src/domain/blocks.rs b/crates/proto/src/domain/block.rs similarity index 100% rename from crates/proto/src/domain/blocks.rs rename to crates/proto/src/domain/block.rs diff --git a/crates/proto/src/domain/digest.rs b/crates/proto/src/domain/digest.rs index b0bf19fac..87c8e459d 100644 --- a/crates/proto/src/domain/digest.rs +++ b/crates/proto/src/domain/digest.rs @@ -1,7 +1,7 @@ use std::fmt::{Debug, Display, Formatter}; use hex::{FromHex, ToHex}; -use miden_objects::{notes::NoteId, Digest, Felt, StarkField}; +use miden_objects::{note::NoteId, Digest, Felt, StarkField}; use crate::{errors::ConversionError, generated::digest as proto}; diff --git a/crates/proto/src/domain/mod.rs b/crates/proto/src/domain/mod.rs index b3e682664..83959535e 100644 --- a/crates/proto/src/domain/mod.rs +++ b/crates/proto/src/domain/mod.rs @@ -1,10 +1,10 @@ -pub mod accounts; -pub mod blocks; +pub mod account; +pub mod block; pub mod digest; pub mod merkle; -pub mod notes; -pub mod nullifiers; -pub mod transactions; +pub mod note; +pub mod nullifier; +pub mod transaction; // UTILITIES // ================================================================================================ diff --git a/crates/proto/src/domain/notes.rs b/crates/proto/src/domain/note.rs similarity index 96% rename from crates/proto/src/domain/notes.rs rename to crates/proto/src/domain/note.rs index 6535c3a47..14131bc5b 100644 --- a/crates/proto/src/domain/notes.rs +++ b/crates/proto/src/domain/note.rs @@ -1,13 +1,13 @@ use std::collections::{BTreeMap, BTreeSet}; use miden_objects::{ - notes::{NoteExecutionHint, NoteId, NoteInclusionProof, NoteMetadata, NoteTag, NoteType}, + note::{NoteExecutionHint, NoteId, NoteInclusionProof, NoteMetadata, NoteTag, NoteType}, Digest, Felt, }; use crate::{ convert, - domain::blocks::BlockInclusionProof, + domain::block::BlockInclusionProof, errors::{ConversionError, MissingFieldHelper}, generated::note as proto, try_convert, diff --git a/crates/proto/src/domain/nullifiers.rs b/crates/proto/src/domain/nullifier.rs similarity index 99% rename from crates/proto/src/domain/nullifiers.rs rename to crates/proto/src/domain/nullifier.rs index 3f7bebc03..482183a0f 100644 --- a/crates/proto/src/domain/nullifiers.rs +++ b/crates/proto/src/domain/nullifier.rs @@ -1,6 +1,6 @@ use miden_objects::{ crypto::{hash::rpo::RpoDigest, merkle::SmtProof}, - notes::Nullifier, + note::Nullifier, }; use crate::{ diff --git a/crates/proto/src/domain/transactions.rs b/crates/proto/src/domain/transaction.rs similarity index 100% rename from crates/proto/src/domain/transactions.rs rename to crates/proto/src/domain/transaction.rs diff --git a/crates/proto/src/generated/account.rs b/crates/proto/src/generated/account.rs index a2683eb0d..eb4a9c022 100644 --- a/crates/proto/src/generated/account.rs +++ b/crates/proto/src/generated/account.rs @@ -7,7 +7,7 @@ #[prost(skip_debug)] pub struct AccountId { /// 15 bytes (120 bits) encoded using \[winter_utils::Serializable\] implementation for - /// \[miden_objects::accounts::account_id::AccountId\]. + /// \[miden_objects::account::account_id::AccountId\]. #[prost(bytes = "vec", tag = "1")] pub id: ::prost::alloc::vec::Vec, } @@ -31,7 +31,7 @@ pub struct AccountInfo { #[prost(message, optional, tag = "1")] pub summary: ::core::option::Option, /// Account details encoded using \[winter_utils::Serializable\] implementation for - /// \[miden_objects::accounts::Account\]. + /// \[miden_objects::account::Account\]. #[prost(bytes = "vec", optional, tag = "2")] pub details: ::core::option::Option<::prost::alloc::vec::Vec>, } diff --git a/crates/proto/src/generated/note.rs b/crates/proto/src/generated/note.rs index b64565915..77a9bae41 100644 --- a/crates/proto/src/generated/note.rs +++ b/crates/proto/src/generated/note.rs @@ -10,12 +10,12 @@ pub struct NoteMetadata { pub note_type: u32, /// A value which can be used by the recipient(s) to identify notes intended for them. /// - /// See `miden_objects::notes::note_tag` for more info. + /// See `miden_objects::note::note_tag` for more info. #[prost(fixed32, tag = "3")] pub tag: u32, /// Specifies when a note is ready to be consumed. /// - /// See `miden_objects::notes::execution_hint` for more info. + /// See `miden_objects::note::execution_hint` for more info. #[prost(fixed64, tag = "4")] pub execution_hint: u64, /// An arbitrary user-defined value. diff --git a/crates/proto/src/generated/responses.rs b/crates/proto/src/generated/responses.rs index ee222ce8e..5b526ae67 100644 --- a/crates/proto/src/generated/responses.rs +++ b/crates/proto/src/generated/responses.rs @@ -204,7 +204,7 @@ pub struct GetBlockByNumberResponse { #[derive(Clone, PartialEq, ::prost::Message)] pub struct GetAccountStateDeltaResponse { /// The calculated account delta encoded using \[winter_utils::Serializable\] implementation - /// for \[miden_objects::accounts::delta::AccountDelta\]. + /// for \[miden_objects::account::delta::AccountDelta\]. #[prost(bytes = "vec", optional, tag = "1")] pub delta: ::core::option::Option<::prost::alloc::vec::Vec>, } diff --git a/crates/proto/src/lib.rs b/crates/proto/src/lib.rs index 991bbfeb2..9290fc739 100644 --- a/crates/proto/src/lib.rs +++ b/crates/proto/src/lib.rs @@ -8,8 +8,8 @@ pub mod generated; // ================================================================================================ pub use domain::{ - accounts::{AccountInputRecord, AccountState}, + account::{AccountInputRecord, AccountState}, convert, - nullifiers::NullifierWitness, + nullifier::NullifierWitness, try_convert, }; diff --git a/crates/rpc-proto/proto/account.proto b/crates/rpc-proto/proto/account.proto index f2c1d6e50..a09482e4a 100644 --- a/crates/rpc-proto/proto/account.proto +++ b/crates/rpc-proto/proto/account.proto @@ -9,7 +9,7 @@ import "digest.proto"; // and a random user-provided seed. message AccountId { // 15 bytes (120 bits) encoded using [winter_utils::Serializable] implementation for - // [miden_objects::accounts::account_id::AccountId]. + // [miden_objects::account::account_id::AccountId]. bytes id = 1; } @@ -31,7 +31,7 @@ message AccountInfo { AccountSummary summary = 1; // Account details encoded using [winter_utils::Serializable] implementation for - // [miden_objects::accounts::Account]. + // [miden_objects::account::Account]. optional bytes details = 2; } diff --git a/crates/rpc-proto/proto/note.proto b/crates/rpc-proto/proto/note.proto index f65960d67..9acfbd847 100644 --- a/crates/rpc-proto/proto/note.proto +++ b/crates/rpc-proto/proto/note.proto @@ -16,12 +16,12 @@ message NoteMetadata { // A value which can be used by the recipient(s) to identify notes intended for them. // - // See `miden_objects::notes::note_tag` for more info. + // See `miden_objects::note::note_tag` for more info. fixed32 tag = 3; // Specifies when a note is ready to be consumed. // - // See `miden_objects::notes::execution_hint` for more info. + // See `miden_objects::note::execution_hint` for more info. fixed64 execution_hint = 4; // An arbitrary user-defined value. diff --git a/crates/rpc-proto/proto/responses.proto b/crates/rpc-proto/proto/responses.proto index 860368e67..36e175d1e 100644 --- a/crates/rpc-proto/proto/responses.proto +++ b/crates/rpc-proto/proto/responses.proto @@ -195,7 +195,7 @@ message GetBlockByNumberResponse { // Represents the result of getting account state delta. message GetAccountStateDeltaResponse { // The calculated account delta encoded using [winter_utils::Serializable] implementation - // for [miden_objects::accounts::delta::AccountDelta]. + // for [miden_objects::account::delta::AccountDelta]. optional bytes delta = 1; } diff --git a/crates/rpc/src/server/api.rs b/crates/rpc/src/server/api.rs index 6afad672f..609b87435 100644 --- a/crates/rpc/src/server/api.rs +++ b/crates/rpc/src/server/api.rs @@ -19,7 +19,7 @@ use miden_node_proto::{ try_convert, }; use miden_objects::{ - accounts::AccountId, crypto::hash::rpo::RpoDigest, transaction::ProvenTransaction, + account::AccountId, crypto::hash::rpo::RpoDigest, transaction::ProvenTransaction, utils::serde::Deserializable, Digest, MAX_NUM_FOREIGN_ACCOUNTS, MIN_PROOF_SECURITY_LEVEL, }; use miden_tx::TransactionVerifier; diff --git a/crates/store/src/db/mod.rs b/crates/store/src/db/mod.rs index ba589a1e9..ad7219dce 100644 --- a/crates/store/src/db/mod.rs +++ b/crates/store/src/db/mod.rs @@ -6,14 +6,14 @@ use std::{ use deadpool_sqlite::{Config as SqliteConfig, Hook, HookError, Pool, Runtime}; use miden_node_proto::{ - domain::accounts::{AccountInfo, AccountSummary}, + domain::account::{AccountInfo, AccountSummary}, generated::note as proto, }; use miden_objects::{ - accounts::{AccountDelta, AccountId}, + account::{AccountDelta, AccountId}, block::{Block, BlockHeader, BlockNoteIndex, BlockNumber}, crypto::{hash::rpo::RpoDigest, merkle::MerklePath, utils::Deserializable}, - notes::{NoteId, NoteInclusionProof, NoteMetadata, Nullifier}, + note::{NoteId, NoteInclusionProof, NoteMetadata, Nullifier}, transaction::TransactionId, utils::Serializable, }; diff --git a/crates/store/src/db/sql/mod.rs b/crates/store/src/db/sql/mod.rs index ca1592429..62d0d92f0 100644 --- a/crates/store/src/db/sql/mod.rs +++ b/crates/store/src/db/sql/mod.rs @@ -9,17 +9,17 @@ use std::{ rc::Rc, }; -use miden_node_proto::domain::accounts::{AccountInfo, AccountSummary}; +use miden_node_proto::domain::account::{AccountInfo, AccountSummary}; use miden_objects::{ - accounts::{ + account::{ delta::AccountUpdateDetails, AccountDelta, AccountId, AccountStorageDelta, AccountVaultDelta, FungibleAssetDelta, NonFungibleAssetDelta, NonFungibleDeltaAction, StorageMapDelta, }, - assets::NonFungibleAsset, + asset::NonFungibleAsset, block::{BlockAccountUpdate, BlockHeader, BlockNoteIndex, BlockNumber}, crypto::{hash::rpo::RpoDigest, merkle::MerklePath}, - notes::{NoteId, NoteInclusionProof, NoteMetadata, NoteType, Nullifier}, + note::{NoteId, NoteInclusionProof, NoteMetadata, NoteType, Nullifier}, transaction::TransactionId, utils::serde::{Deserializable, Serializable}, Digest, Word, diff --git a/crates/store/src/db/sql/utils.rs b/crates/store/src/db/sql/utils.rs index 58dd72afa..284c3dde8 100644 --- a/crates/store/src/db/sql/utils.rs +++ b/crates/store/src/db/sql/utils.rs @@ -1,9 +1,9 @@ -use miden_node_proto::domain::accounts::{AccountInfo, AccountSummary}; +use miden_node_proto::domain::account::{AccountInfo, AccountSummary}; use miden_objects::{ - accounts::{Account, AccountDelta, AccountId}, + account::{Account, AccountDelta, AccountId}, block::BlockNumber, crypto::hash::rpo::RpoDigest, - notes::Nullifier, + note::Nullifier, utils::Deserializable, }; use rusqlite::{ diff --git a/crates/store/src/db/tests.rs b/crates/store/src/db/tests.rs index c0ecd927e..fe8d1684c 100644 --- a/crates/store/src/db/tests.rs +++ b/crates/store/src/db/tests.rs @@ -2,17 +2,17 @@ #![allow(clippy::too_many_lines, reason = "test code can be long")] use miden_lib::transaction::TransactionKernel; -use miden_node_proto::domain::accounts::AccountSummary; +use miden_node_proto::domain::account::AccountSummary; use miden_objects::{ - accounts::{ + account::{ delta::AccountUpdateDetails, Account, AccountBuilder, AccountComponent, AccountDelta, AccountId, AccountIdVersion, AccountStorageDelta, AccountStorageMode, AccountType, AccountVaultDelta, StorageSlot, }, - assets::{Asset, FungibleAsset, NonFungibleAsset, NonFungibleAssetDetails}, + asset::{Asset, FungibleAsset, NonFungibleAsset, NonFungibleAssetDetails}, block::{BlockAccountUpdate, BlockHeader, BlockNoteIndex, BlockNoteTree, BlockNumber}, crypto::{hash::rpo::RpoDigest, merkle::MerklePath}, - notes::{NoteExecutionHint, NoteId, NoteMetadata, NoteType, Nullifier}, + note::{NoteExecutionHint, NoteId, NoteMetadata, NoteType, Nullifier}, testing::account_id::{ ACCOUNT_ID_FUNGIBLE_FAUCET_ON_CHAIN, ACCOUNT_ID_NON_FUNGIBLE_FAUCET_ON_CHAIN, ACCOUNT_ID_OFF_CHAIN_SENDER, ACCOUNT_ID_REGULAR_ACCOUNT_UPDATABLE_CODE_OFF_CHAIN, @@ -306,7 +306,7 @@ fn sql_select_accounts() { [i; 15], AccountIdVersion::Version0, AccountType::RegularAccountImmutableCode, - miden_objects::accounts::AccountStorageMode::Private, + miden_objects::account::AccountStorageMode::Private, ); let account_hash = num_to_rpo_digest(u64::from(i)); state.push(AccountInfo { diff --git a/crates/store/src/errors.rs b/crates/store/src/errors.rs index 72fa993e6..462341bf5 100644 --- a/crates/store/src/errors.rs +++ b/crates/store/src/errors.rs @@ -2,14 +2,14 @@ use std::io; use deadpool_sqlite::{InteractError, PoolError}; use miden_objects::{ - accounts::AccountId, + account::AccountId, block::{BlockHeader, BlockNumber}, crypto::{ hash::rpo::RpoDigest, merkle::{MerkleError, MmrError}, utils::DeserializationError, }, - notes::Nullifier, + note::Nullifier, transaction::OutputNote, AccountDeltaError, AccountError, BlockError, NoteError, }; diff --git a/crates/store/src/genesis.rs b/crates/store/src/genesis.rs index 402d820d7..2232cc2da 100644 --- a/crates/store/src/genesis.rs +++ b/crates/store/src/genesis.rs @@ -1,6 +1,6 @@ use miden_lib::transaction::TransactionKernel; use miden_objects::{ - accounts::{delta::AccountUpdateDetails, Account}, + account::{delta::AccountUpdateDetails, Account}, block::{Block, BlockAccountUpdate, BlockHeader, BlockNumber}, crypto::merkle::{EmptySubtreeRoots, MmrPeaks, SimpleSmt, Smt}, utils::serde::{ByteReader, ByteWriter, Deserializable, DeserializationError, Serializable}, diff --git a/crates/store/src/nullifier_tree.rs b/crates/store/src/nullifier_tree.rs index 8f275428f..d34bc01d3 100644 --- a/crates/store/src/nullifier_tree.rs +++ b/crates/store/src/nullifier_tree.rs @@ -4,7 +4,7 @@ use miden_objects::{ hash::rpo::RpoDigest, merkle::{MutationSet, Smt, SmtProof, SMT_DEPTH}, }, - notes::Nullifier, + note::Nullifier, Felt, FieldElement, Word, }; diff --git a/crates/store/src/server/api.rs b/crates/store/src/server/api.rs index fa9978e31..9cb196861 100644 --- a/crates/store/src/server/api.rs +++ b/crates/store/src/server/api.rs @@ -3,8 +3,8 @@ use std::{collections::BTreeSet, sync::Arc}; use miden_node_proto::{ convert, domain::{ - accounts::{AccountInfo, AccountProofRequest}, - notes::NoteAuthenticationInfo, + account::{AccountInfo, AccountProofRequest}, + note::NoteAuthenticationInfo, }, errors::ConversionError, generated::{ @@ -32,10 +32,10 @@ use miden_node_proto::{ try_convert, }; use miden_objects::{ - accounts::AccountId, + account::AccountId, block::{Block, BlockNumber}, crypto::hash::rpo::RpoDigest, - notes::{NoteId, Nullifier}, + note::{NoteId, Nullifier}, utils::{Deserializable, Serializable}, }; use tonic::{Request, Response, Status}; diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index 92da06f41..f3ae2dc99 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -12,9 +12,9 @@ use std::{ use miden_node_proto::{ convert, domain::{ - accounts::{AccountInfo, AccountProofRequest, StorageMapKeysProof}, - blocks::BlockInclusionProof, - notes::NoteAuthenticationInfo, + account::{AccountInfo, AccountProofRequest, StorageMapKeysProof}, + block::BlockInclusionProof, + note::NoteAuthenticationInfo, }, generated::responses::{ AccountProofsResponse, AccountStateHeader, GetBlockInputsResponse, StorageSlotMapProof, @@ -23,7 +23,7 @@ use miden_node_proto::{ }; use miden_node_utils::formatting::format_array; use miden_objects::{ - accounts::{AccountDelta, AccountHeader, AccountId, StorageSlot}, + account::{AccountDelta, AccountHeader, AccountId, StorageSlot}, block::{Block, BlockHeader, BlockNumber}, crypto::{ hash::rpo::RpoDigest, @@ -31,7 +31,7 @@ use miden_objects::{ LeafIndex, Mmr, MmrDelta, MmrError, MmrPeaks, MmrProof, SimpleSmt, SmtProof, ValuePath, }, }, - notes::{NoteId, Nullifier}, + note::{NoteId, Nullifier}, transaction::OutputNote, utils::Serializable, AccountError, ACCOUNT_TREE_DEPTH, diff --git a/proto/account.proto b/proto/account.proto index f2c1d6e50..a09482e4a 100644 --- a/proto/account.proto +++ b/proto/account.proto @@ -9,7 +9,7 @@ import "digest.proto"; // and a random user-provided seed. message AccountId { // 15 bytes (120 bits) encoded using [winter_utils::Serializable] implementation for - // [miden_objects::accounts::account_id::AccountId]. + // [miden_objects::account::account_id::AccountId]. bytes id = 1; } @@ -31,7 +31,7 @@ message AccountInfo { AccountSummary summary = 1; // Account details encoded using [winter_utils::Serializable] implementation for - // [miden_objects::accounts::Account]. + // [miden_objects::account::Account]. optional bytes details = 2; } diff --git a/proto/note.proto b/proto/note.proto index f65960d67..9acfbd847 100644 --- a/proto/note.proto +++ b/proto/note.proto @@ -16,12 +16,12 @@ message NoteMetadata { // A value which can be used by the recipient(s) to identify notes intended for them. // - // See `miden_objects::notes::note_tag` for more info. + // See `miden_objects::note::note_tag` for more info. fixed32 tag = 3; // Specifies when a note is ready to be consumed. // - // See `miden_objects::notes::execution_hint` for more info. + // See `miden_objects::note::execution_hint` for more info. fixed64 execution_hint = 4; // An arbitrary user-defined value. diff --git a/proto/responses.proto b/proto/responses.proto index 860368e67..36e175d1e 100644 --- a/proto/responses.proto +++ b/proto/responses.proto @@ -195,7 +195,7 @@ message GetBlockByNumberResponse { // Represents the result of getting account state delta. message GetAccountStateDeltaResponse { // The calculated account delta encoded using [winter_utils::Serializable] implementation - // for [miden_objects::accounts::delta::AccountDelta]. + // for [miden_objects::account::delta::AccountDelta]. optional bytes delta = 1; } From 6df28b482a58a380d6805938b56799516cb68c75 Mon Sep 17 00:00:00 2001 From: Mirko <48352201+Mirko-von-Leipzig@users.noreply.github.com> Date: Thu, 23 Jan 2025 10:25:58 +0200 Subject: [PATCH 49/50] chore: merge main into next (#637) --- .github/actions/build_package/action.yml | 120 ++++++++++++ .github/actions/ssm_execute/action.yml | 4 +- .github/workflows/amd_deb_packager.yml | 134 -------------- .github/workflows/arm_deb_packager.yml | 195 -------------------- .github/workflows/deploy.yml | 154 ++++++++++++++++ .github/workflows/deploy_package.yml | 136 -------------- .github/workflows/package.yml | 69 +++++++ CHANGELOG.md | 1 + README.md | 6 +- bin/node/Dockerfile | 2 +- packaging/{ => faucet}/miden-faucet.service | 6 +- packaging/faucet/postinst | 26 +++ packaging/{postrm.faucet => faucet/postrm} | 7 +- packaging/{ => node}/miden-node.service | 6 +- packaging/node/postinst | 26 +++ packaging/{ => node}/postrm | 8 +- packaging/postinst | 15 -- 17 files changed, 416 insertions(+), 499 deletions(-) create mode 100644 .github/actions/build_package/action.yml delete mode 100644 .github/workflows/amd_deb_packager.yml delete mode 100644 .github/workflows/arm_deb_packager.yml create mode 100644 .github/workflows/deploy.yml delete mode 100644 .github/workflows/deploy_package.yml create mode 100644 .github/workflows/package.yml rename packaging/{ => faucet}/miden-faucet.service (55%) create mode 100644 packaging/faucet/postinst rename packaging/{postrm.faucet => faucet/postrm} (51%) mode change 100755 => 100644 rename packaging/{ => node}/miden-node.service (56%) create mode 100644 packaging/node/postinst rename packaging/{ => node}/postrm (52%) mode change 100755 => 100644 delete mode 100755 packaging/postinst diff --git a/.github/actions/build_package/action.yml b/.github/actions/build_package/action.yml new file mode 100644 index 000000000..dd1d3091c --- /dev/null +++ b/.github/actions/build_package/action.yml @@ -0,0 +1,120 @@ +# Creates miden-node.deb and miden-faucet.deb DEBIAN packages. +name: build-package +description: Builds miden-node and miden-faucet debian packages for the given git reference +inputs: + gitref: + required: true + description: The git ref to build the packages from. + +runs: + using: "composite" + steps: + - name: Identify target git SHA + id: git-sha + shell: bash + run: | + if git show-ref -q --verify "refs/remotes/origin/${{ inputs.gitref }}" 2>/dev/null; then + echo "sha=$(git show-ref --hash --verify 'refs/remotes/origin/${{ inputs.gitref }}')" >> $GITHUB_OUTPUT + elif git show-ref -q --verify "refs/tags/${{ inputs.gitref }}" 2>/dev/null; then + echo "sha=$(git show-ref --hash --verify 'refs/tags/${{ inputs.gitref }}')" >> $GITHUB_OUTPUT + elif git rev-parse --verify "${{ inputs.gitref }}^{commit}" >/dev/null 2>&1; then + echo "sha=$(git rev-parse --verify '${{ inputs.gitref }}^{commit}')" >> $GITHUB_OUTPUT + else + echo "::error::Unknown git reference type" + exit 1 + fi + + - name: Create package directories + shell: bash + run: | + for pkg in miden-node miden-faucet; do + mkdir -p \ + packaging/deb/$pkg/DEBIAN \ + packaging/deb/$pkg/usr/bin \ + packaging/deb/$pkg/lib/systemd/system \ + packaging/deb/$pkg/etc/opt/$pkg \ + packaging/deb/$pkg/opt/$pkg + done + + # These have to be downloaded as the current repo source isn't necessarily the target git reference. + - name: Copy package install scripts + shell: bash + run: | + # git show ${{ steps.git-sha.outputs.sha }}:packaging/node/miden-node.service > packaging/deb/miden-node/lib/systemd/system/miden-node.service + # git show ${{ steps.git-sha.outputs.sha }}:packaging/node/postinst > packaging/deb/miden-node/DEBIAN/postinst + # git show ${{ steps.git-sha.outputs.sha }}:packaging/node/postrm > packaging/deb/miden-node/DEBIAN/postrm + # git show ${{ steps.git-sha.outputs.sha }}:packaging/faucet/miden-faucet.service > packaging/deb/miden-faucet/lib/systemd/system/miden-faucet.service + # git show ${{ steps.git-sha.outputs.sha }}:packaging/faucet/postinst > packaging/deb/miden-faucet/DEBIAN/postinst + # git show ${{ steps.git-sha.outputs.sha }}:packaging/faucet/postrm > packaging/deb/miden-faucet/DEBIAN/postrm + + # This is temporary until these files land on main. + cp packaging/node/miden-node.service packaging/deb/miden-node/lib/systemd/system/miden-node.service + cp packaging/node/postinst packaging/deb/miden-node/DEBIAN/postinst + cp packaging/node/postrm packaging/deb/miden-node/DEBIAN/postrm + cp packaging/faucet/miden-faucet.service packaging/deb/miden-faucet/lib/systemd/system/miden-faucet.service + cp packaging/faucet/postinst packaging/deb/miden-faucet/DEBIAN/postinst + cp packaging/faucet/postrm packaging/deb/miden-faucet/DEBIAN/postrm + + chmod 0775 packaging/deb/miden-node/DEBIAN/postinst + chmod 0775 packaging/deb/miden-node/DEBIAN/postrm + chmod 0775 packaging/deb/miden-faucet/DEBIAN/postinst + chmod 0775 packaging/deb/miden-faucet/DEBIAN/postrm + + - name: Create control files + shell: bash + run: | + # Map the architecture to the format required by Debian. + # i.e. arm64 and amd64 instead of aarch64 and x86_64. + arch=$(uname -m | sed "s/x86_64/amd64/" | sed "s/aarch64/arm64/") + # Control file's version field must be x.y.z format so strip the rest. + version=$(git describe --tags --abbrev=0 | sed 's/[^0-9.]//g' ) + + cat > packaging/deb/miden-node/DEBIAN/control << EOF + Package: miden-node + Version: $version + Section: base + Priority: optional + Architecture: $arch + Maintainer: Polygon Devops + Description: miden-node binary package + Homepage: https://polygon.technology/polygon-miden + Vcs-Git: git@github.com:0xPolygonMiden/miden-node.git + Vcs-Browser: https://github.com/0xPolygonMiden/miden-node + EOF + + cat > packaging/deb/miden-faucet/DEBIAN/control << EOF + Package: miden-faucet + Version: $version + Section: base + Priority: optional + Architecture: $arch + Maintainer: Polygon Devops + Description: miden-faucet binary package + Homepage: https://polygon.technology/polygon-miden + Vcs-Git: git@github.com:0xPolygonMiden/miden-node.git + Vcs-Browser: https://github.com/0xPolygonMiden/miden-node + EOF + + - name: Build binaries + shell: bash + env: + repo-url: ${{ github.server_url }}/${{ github.repository }} + run: | + cargo install miden-node --root . --locked --features testing --git ${{ env.repo-url }} --rev ${{ steps.git-sha.outputs.sha }} + cargo install miden-faucet --root . --locked --features testing --git ${{ env.repo-url }} --rev ${{ steps.git-sha.outputs.sha }} + + - name: Copy binary files + shell: bash + run: | + cp -p ./bin/miden-node packaging/deb/miden-node/usr/bin/ + cp -p ./bin/miden-faucet packaging/deb/miden-faucet/usr/bin/ + + - name: Build packages + shell: bash + run: | + dpkg-deb --build --root-owner-group packaging/deb/miden-node + dpkg-deb --build --root-owner-group packaging/deb/miden-faucet + + # Save the .deb files, delete the rest. + mv packaging/deb/*.deb . + rm -rf packaging diff --git a/.github/actions/ssm_execute/action.yml b/.github/actions/ssm_execute/action.yml index 933fee59c..f730d9c3e 100644 --- a/.github/actions/ssm_execute/action.yml +++ b/.github/actions/ssm_execute/action.yml @@ -35,7 +35,7 @@ runs: --output text \ --query "Command.CommandId") echo "command_id=$COMMAND_ID" >> $GITHUB_OUTPUT - continue-on-error: true + continue-on-error: false - name: Poll command status and retrieve live logs id: poll_output @@ -64,7 +64,7 @@ runs: break elif [ "$STATUS" == "Failed" ] || [ "$STATUS" == "Cancelled" ]; then echo "Command failed with status: $STATUS" - break + exit 1 else elapsed_time=$(( $(date +%s) - start_time )) if [ "$elapsed_time" -gt "$timeout" ]; then diff --git a/.github/workflows/amd_deb_packager.yml b/.github/workflows/amd_deb_packager.yml deleted file mode 100644 index 2b17fe9ff..000000000 --- a/.github/workflows/amd_deb_packager.yml +++ /dev/null @@ -1,134 +0,0 @@ -name: amd_deb_packager - -on: - release: - types: [released, prereleased] - -jobs: - build: - permissions: - id-token: write - contents: write - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@main - with: - fetch-depth: 0 - ##### TAG Variable ##### - - name: Adding TAG to ENV - run: echo "GIT_TAG=`echo $(git describe --tags --abbrev=0)`" >> $GITHUB_ENV - - name: adding version - run: | - NUMERIC_VERSION=$( echo ${{ env.GIT_TAG }} | sed 's/[^0-9.]//g' ) - echo "VERSION=$NUMERIC_VERSION" >> $GITHUB_ENV - - - name: cleaning repo - run: cargo clean - - - name: Building for amd64 - run: | - cargo build --release --locked --bin miden-node - cargo build --release --locked --bin miden-faucet - - - name: create package directories - run: | - mkdir -p packaging/deb/miden-node/DEBIAN - mkdir -p packaging/deb/miden-node/usr/bin - mkdir -p packaging/deb/miden-node/lib/systemd/system - mkdir -p packaging/deb/miden-node/etc/miden - mkdir -p packaging/deb/miden-node/opt/miden/miden-node - - - name: copy package files - run: | - cp -p target/release/miden-node packaging/deb/miden-node/usr/bin/ - cp packaging/miden-node.service packaging/deb/miden-node/lib/systemd/system/ - cp packaging/postinst packaging/deb/miden-node/DEBIAN/postinst - cp packaging/postrm packaging/deb/miden-node/DEBIAN/postrm - - ########### Control file creation for amd64 miden-node ########## - - name: create control file - run: | - touch packaging/deb/miden-node/DEBIAN/control - echo "Package: miden-node" >> packaging/deb/miden-node/DEBIAN/control - echo "Version: ${{ env.VERSION }}" >> packaging/deb/miden-node/DEBIAN/control - echo "Section: base" >> packaging/deb/miden-node/DEBIAN/control - echo "Priority: optional" >> packaging/deb/miden-node/DEBIAN/control - echo "Architecture: amd64" >> packaging/deb/miden-node/DEBIAN/control - echo "Maintainer: Polygon Devops " >> packaging/deb/miden-node/DEBIAN/control - echo "Description: miden-node binary package" >> packaging/deb/miden-node/DEBIAN/control - echo "Homepage: https://polygon.technology/polygon-miden" >> packaging/deb/miden-node/DEBIAN/control - echo "Vcs-Git: git@github.com:0xPolygonMiden/miden-node.git" >> packaging/deb/miden-node/DEBIAN/control - echo "Vcs-Browser: https://github.com/0xPolygonMiden/miden-node" >> packaging/deb/miden-node/DEBIAN/control - - - name: Creating package for binary for miden-node ${{ env.ARCH }} - run: cp -rp packaging/deb/miden-node packaging/deb/miden-node-${{ env.GIT_TAG }}-${{ env.ARCH }} - env: - ARCH: amd64 - - - name: Running package build - run: dpkg-deb --build --root-owner-group packaging/deb/miden-node-${{ env.GIT_TAG }}-${{ env.ARCH }} - env: - ARCH: amd64 - - ########## Miden Faucet Package ########################################## - - name: create package directories - run: | - mkdir -p packaging/deb/miden-faucet/DEBIAN - mkdir -p packaging/deb/miden-faucet/usr/bin - mkdir -p packaging/deb/miden-faucet/lib/systemd/system - mkdir -p packaging/deb/miden-faucet/etc/miden - mkdir -p packaging/deb/miden-faucet/opt/miden/miden-faucet - - - name: copy package files - run: | - cp -p target/release/miden-faucet packaging/deb/miden-faucet/usr/bin/ - cp packaging/miden-faucet.service packaging/deb/miden-faucet/lib/systemd/system/ - cp packaging/postinst packaging/deb/miden-faucet/DEBIAN/postinst - cp packaging/postrm.faucet packaging/deb/miden-faucet/DEBIAN/postrm - - ########### Control file creation for amd64 miden-faucet ########## - - name: create control file - run: | - touch packaging/deb/miden-faucet/DEBIAN/control - echo "Package: miden-faucet" >> packaging/deb/miden-faucet/DEBIAN/control - echo "Version: ${{ env.VERSION }}" >> packaging/deb/miden-faucet/DEBIAN/control - echo "Section: base" >> packaging/deb/miden-faucet/DEBIAN/control - echo "Priority: optional" >> packaging/deb/miden-faucet/DEBIAN/control - echo "Architecture: amd64" >> packaging/deb/miden-faucet/DEBIAN/control - echo "Maintainer: Polygon Devops " >> packaging/deb/miden-faucet/DEBIAN/control - echo "Description: miden-faucet binary package" >> packaging/deb/miden-faucet/DEBIAN/control - echo "Homepage: https://polygon.technology/polygon-miden" >> packaging/deb/miden-faucet/DEBIAN/control - echo "Vcs-Git: git@github.com:0xPolygonMiden/miden-node.git" >> packaging/deb/miden-faucet/DEBIAN/control - echo "Vcs-Browser: https://github.com/0xPolygonMiden/miden-node" >> packaging/deb/miden-faucet/DEBIAN/control - - - name: Creating package for binary for miden-faucet ${{ env.ARCH }} - run: cp -rp packaging/deb/miden-faucet packaging/deb/miden-faucet-${{ env.GIT_TAG }}-${{ env.ARCH }} - env: - ARCH: amd64 - - - name: Running package build - run: dpkg-deb --build --root-owner-group packaging/deb/miden-faucet-${{ env.GIT_TAG }}-${{ env.ARCH }} - env: - ARCH: amd64 - - - name: shasum the package - run: cd packaging/deb/ && sha256sum miden-node-${{ env.GIT_TAG }}-${{ env.ARCH }}.deb > miden-node-${{ env.GIT_TAG }}-${{ env.ARCH }}.deb.checksum - env: - ARCH: amd64 - - - name: shasum the package - run: cd packaging/deb/ && sha256sum miden-faucet-${{ env.GIT_TAG }}-${{ env.ARCH }}.deb > miden-faucet-${{ env.GIT_TAG }}-${{ env.ARCH }}.deb.checksum - env: - ARCH: amd64 - - - name: release miden-node Packages - uses: softprops/action-gh-release@v1 - with: - tag_name: ${{ env.GIT_TAG }} - prerelease: true - files: | - packaging/deb/miden-node**.deb - packaging/deb/miden-node**.deb.checksum - packaging/deb/miden-faucet**.deb - packaging/deb/miden-faucet**.deb.checksum diff --git a/.github/workflows/arm_deb_packager.yml b/.github/workflows/arm_deb_packager.yml deleted file mode 100644 index cb88d772b..000000000 --- a/.github/workflows/arm_deb_packager.yml +++ /dev/null @@ -1,195 +0,0 @@ -name: arm_deb_packager - -on: - workflow_dispatch: - release: - types: [released, prereleased] - push: - branches: - - next - -jobs: - build_package: - permissions: - id-token: write - contents: write - outputs: - target: ${{ steps.build_type_vars_setup.outputs.target }} - tag: ${{ steps.build_type_vars_setup.outputs.tag }} - account_id: ${{ steps.build_type_vars_setup.outputs.account_id }} - instance_id: ${{ steps.build_type_vars_setup.outputs.instance_id }} - runs-on: - labels: ubuntu22-arm-4core - steps: - - name: Checkout - uses: actions/checkout@main - with: - fetch-depth: 0 - - - name: Adding TAG to ENV - run: | - if [[ ${{ github.event_name }} == "release" || ${{ github.event_name }} == "prereleased" ]]; then - echo "GIT_TAG=`echo $(git describe --tags --abbrev=0)`" >> $GITHUB_ENV - else - echo "GIT_TAG=next" >> $GITHUB_ENV - fi - - - name: adding version - run: | - if [[ ${{ github.event_name }} == "release" || ${{ github.event_name }} == "prereleased" ]]; then - NUMERIC_VERSION=$( echo ${{ env.GIT_TAG }} | sed 's/[^0-9.]//g' ) - echo "VERSION=$NUMERIC_VERSION" >> $GITHUB_ENV - else - echo "VERSION=0.1" >> $GITHUB_ENV - fi - - - name: cleaning repo - run: cargo clean - - - name: Building for arm64 - run: | - cargo build --release --locked --bin miden-node - cargo build --release --locked --bin miden-faucet - - - name: create package directories - run: | - mkdir -p packaging/deb/miden-node/DEBIAN - mkdir -p packaging/deb/miden-node/usr/bin - mkdir -p packaging/deb/miden-node/lib/systemd/system - mkdir -p packaging/deb/miden-node/etc/miden - mkdir -p packaging/deb/miden-node/opt/miden/miden-node - - - name: copy package files - run: | - cp -p target/release/miden-node packaging/deb/miden-node/usr/bin/ - cp packaging/miden-node.service packaging/deb/miden-node/lib/systemd/system/ - cp packaging/postinst packaging/deb/miden-node/DEBIAN/postinst - cp packaging/postrm packaging/deb/miden-node/DEBIAN/postrm - - ########### Control file creation for arm64 miden-node########## - - name: create control file - run: | - touch packaging/deb/miden-node/DEBIAN/control - echo "Package: miden-node" >> packaging/deb/miden-node/DEBIAN/control - echo "Version: ${{ env.VERSION }}" >> packaging/deb/miden-node/DEBIAN/control - echo "Section: base" >> packaging/deb/miden-node/DEBIAN/control - echo "Priority: optional" >> packaging/deb/miden-node/DEBIAN/control - echo "Architecture: arm64" >> packaging/deb/miden-node/DEBIAN/control - echo "Maintainer: Polygon Devops " >> packaging/deb/miden-node/DEBIAN/control - echo "Description: miden-node binary package" >> packaging/deb/miden-node/DEBIAN/control - echo "Homepage: https://polygon.technology/polygon-miden" >> packaging/deb/miden-node/DEBIAN/control - echo "Vcs-Git: git@github.com:0xPolygonMiden/miden-node.git" >> packaging/deb/miden-node/DEBIAN/control - echo "Vcs-Browser: https://github.com/0xPolygonMiden/miden-node" >> packaging/deb/miden-node/DEBIAN/control - - - name: Creating package for binary for miden-node ${{ env.ARCH }} - run: cp -rp packaging/deb/miden-node packaging/deb/miden-node-${{ env.GIT_TAG }}-${{ env.ARCH }} - env: - ARCH: arm64 - - - name: Running package build - run: dpkg-deb --build --root-owner-group packaging/deb/miden-node-${{ env.GIT_TAG }}-${{ env.ARCH }} - env: - ARCH: arm64 - - ########## Miden Faucet Package ########################################## - - name: create package directories - run: | - mkdir -p packaging/deb/miden-faucet/DEBIAN - mkdir -p packaging/deb/miden-faucet/usr/bin - mkdir -p packaging/deb/miden-faucet/lib/systemd/system - mkdir -p packaging/deb/miden-faucet/etc/miden - mkdir -p packaging/deb/miden-faucet/opt/miden/miden-faucet - - - name: copy package files - run: | - cp -p target/release/miden-faucet packaging/deb/miden-faucet/usr/bin/ - cp packaging/miden-faucet.service packaging/deb/miden-faucet/lib/systemd/system/ - cp packaging/postinst packaging/deb/miden-faucet/DEBIAN/postinst - cp packaging/postrm.faucet packaging/deb/miden-faucet/DEBIAN/postrm - - ########### Control file creation for arm64 miden-faucet ########## - - name: create control file - run: | - touch packaging/deb/miden-faucet/DEBIAN/control - echo "Package: miden-faucet" >> packaging/deb/miden-faucet/DEBIAN/control - echo "Version: ${{ env.VERSION }}" >> packaging/deb/miden-faucet/DEBIAN/control - echo "Section: base" >> packaging/deb/miden-faucet/DEBIAN/control - echo "Priority: optional" >> packaging/deb/miden-faucet/DEBIAN/control - echo "Architecture: arm64" >> packaging/deb/miden-faucet/DEBIAN/control - echo "Maintainer: Polygon Devops " >> packaging/deb/miden-faucet/DEBIAN/control - echo "Description: miden-faucet binary package" >> packaging/deb/miden-faucet/DEBIAN/control - echo "Homepage: https://polygon.technology/polygon-miden" >> packaging/deb/miden-faucet/DEBIAN/control - echo "Vcs-Git: git@github.com:0xPolygonMiden/miden-node.git" >> packaging/deb/miden-faucet/DEBIAN/control - echo "Vcs-Browser: https://github.com/0xPolygonMiden/miden-node" >> packaging/deb/miden-faucet/DEBIAN/control - - - name: Creating package for binary for miden-faucet ${{ env.ARCH }} - run: cp -rp packaging/deb/miden-faucet packaging/deb/miden-faucet-${{ env.GIT_TAG }}-${{ env.ARCH }} - env: - ARCH: arm64 - - - name: Running package build - run: dpkg-deb --build --root-owner-group packaging/deb/miden-faucet-${{ env.GIT_TAG }}-${{ env.ARCH }} - env: - ARCH: arm64 - - - name: shasum the package - run: cd packaging/deb/ && sha256sum miden-node-${{ env.GIT_TAG }}-${{ env.ARCH }}.deb > miden-node-${{ env.GIT_TAG }}-${{ env.ARCH }}.deb.checksum - env: - ARCH: arm64 - - - name: shasum the package - run: cd packaging/deb/ && sha256sum miden-faucet-${{ env.GIT_TAG }}-${{ env.ARCH }}.deb > miden-faucet-${{ env.GIT_TAG }}-${{ env.ARCH }}.deb.checksum - env: - ARCH: arm64 - - - name: release miden-node Packages for testnet - uses: softprops/action-gh-release@v1 - with: - tag_name: ${{ env.GIT_TAG }} - prerelease: true - files: | - packaging/deb/miden-node**.deb - packaging/deb/miden-node**.deb.checksum - packaging/deb/miden-faucet**.deb - packaging/deb/miden-faucet**.deb.checksum - if: ${{ github.event_name == 'release' || github.event_name == 'prereleased' }} - - - name: release miden-node Packages for devnet - uses: actions/upload-artifact@v3 - with: - name: devnet_package - path: | - packaging/deb/miden-node-${{ env.GIT_TAG }}-${{ env.ARCH }}.deb - packaging/deb/miden-node-${{ env.GIT_TAG }}-${{ env.ARCH }}.deb.checksum - packaging/deb/miden-faucet-${{ env.GIT_TAG }}-${{ env.ARCH }}.deb - packaging/deb/miden-faucet-${{ env.GIT_TAG }}-${{ env.ARCH }}.deb.checksum - env: - ARCH: arm64 - if: ${{ github.event_name == 'push' }} - - - name: Determine build type - id: build_type_vars_setup - run: | - if [[ ${{ github.event_name }} == "release" || ${{ github.event_name }} == "prereleased" ]]; then - echo "target=testnet" >> $GITHUB_OUTPUT - echo "tag=${{ env.GIT_TAG }}" >> $GITHUB_OUTPUT - echo "account_id=MIDEN_DEV_ACCOUNT_ID" >> $GITHUB_OUTPUT - echo "instance_id=TESTNET_INSTANCE_TF" >> $GITHUB_OUTPUT - else - echo "target=devnet" >> $GITHUB_OUTPUT - echo "tag=next" >> $GITHUB_OUTPUT - echo "account_id=MIDEN_DEV_ACCOUNT_ID" >> $GITHUB_OUTPUT - echo "instance_id=DEVNET_INSTANCE_TF" >> $GITHUB_OUTPUT - fi - - - deploy: - name: Deploy to ${{ needs.build_package.outputs.target }} - needs: build_package - uses: ./.github/workflows/deploy_package.yml - with: - target: ${{ needs.build_package.outputs.target }} - tag: ${{ needs.build_package.outputs.tag }} - account_id: ${{ needs.build_package.outputs.account_id }} - instance_id: ${{ needs.build_package.outputs.instance_id }} - secrets: inherit diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 000000000..98b9a0755 --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,154 @@ +name: Deploy +run-name: Deploy ${{ inputs.network }} - ${{ inputs.gitref }} +on: + workflow_dispatch: + inputs: + network: + description: 'Deployment instance' + required: true + type: choice + options: + - devnet + - testnet + + gitref: + description: 'Version, branch or commit to deploy' + required: true + type: string + +permissions: + id-token: write + contents: write + +jobs: + deploy: + name: ${{ inputs.network }} - ${{ inputs.gitref }} + # This is our arm64 runner which matches the AWS instance. + runs-on: + labels: ubuntu22-arm-4core + + env: + # Define the instance information. + account-id: MIDEN_DEV_ACCOUNT_ID + oidcrole: midendev + instance-id: ${{ inputs.network == 'testnet' && 'TESTNET_INSTANCE_TF' || 'DEVNET_INSTANCE_TF' }} + + # Define the expected package names. + node-package: miden-node-${{ inputs.gitref }}-arm64.deb + faucet-package: miden-faucet-${{ inputs.gitref }}-arm64.deb + + + steps: + # S3 path where packages are stored; used to send packages to instance as this isn't trivially possible directly. + # This cannot be done in the global env setup as it requires another env variable. + - name: Setup S3 path + run: echo "s3-path=s3://release-artifacts-${{ secrets[env.account-id] }}" >> $GITHUB_ENV + + # Checkout repo so we have access to the required workflow actions. + - name: Checkout repo + uses: actions/checkout@main + with: + fetch-depth: 0 + + # Download from github if its a version tag referece. + - name: Download from releases + if: ${{ startsWith(inputs.gitref, 'v') }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh release download ${{ inputs.gitref }} -p ${{ env.node-package }} + gh release download ${{ inputs.gitref }} -p ${{ env.node-package }}.checksum + gh release download ${{ inputs.gitref }} -p ${{ env.faucet-package }} + gh release download ${{ inputs.gitref }} -p ${{ env.faucet-package }}.checksum + + sha256sum --check ${{ env.node-package }}.checksum + sha256sum --check ${{ env.faucet-package }}.checksum + + # Otherwise build the packages from source. + # + # Note that we cannot build from the currently checked out repo source since that source + # defines our workflow actions, and not the compilation source target. For this reason we + # prefer building the binary using `cargo install ...`. + - name: Build from source + if: ${{ !startsWith(inputs.gitref, 'v') }} + uses: ./.github/actions/build_package + with: + gitref: ${{ inputs.gitref }} + + - name: Rename built packages + if: ${{ !startsWith(inputs.gitref, 'v') }} + run: | + mv miden-node.deb ${{ env.node-package }} + mv miden-faucet.deb ${{ env.faucet-package }} + + + # Configure AWS communication via SSM. + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: eu-west-1 + role-to-assume: "arn:aws:iam::${{ secrets[env.account-id] }}:role/${{ env.oidcrole }}-GithubActionsRole" + role-session-name: GithubActionsSession + + - name: Install awscli + uses: ./.github/actions/ssm_execute + with: + instance_id: ${{ secrets[env.instance-id] }} + command: | + sudo apt-get udpate; \ + sudo apt install awscli -y + + # Move packages to instance using S3. Note that this will clobber the files. + - name: Upload packages to S3 + run: | + aws s3 cp ${{ env.node-package }} ${{ env.s3-path }}/${{ env.node-package }} + aws s3 cp ${{ env.faucet-package }} ${{ env.s3-path }}/${{ env.faucet-package }} + + - name: Download packages to instance + uses: ./.github/actions/ssm_execute + with: + instance_id: ${{ secrets[env.instance-id] }} + command: | + aws s3 cp ${{ env.s3-path }}/${{ env.node-package }} ${{ env.node-package}}; \ + aws s3 cp ${{ env.s3-path }}/${{ env.faucet-package }} ${{ env.faucet-package}} + + # Install and launch services on the instance. + - name: Stop miden services + uses: ./.github/actions/ssm_execute + with: + instance_id: ${{ secrets[env.instance-id] }} + command: | + sudo systemctl stop miden-node; \ + sudo systemctl stop miden-faucet; \ + sudo apt remove miden-node miden-faucet -y; + + - name: Install packages + uses: ./.github/actions/ssm_execute + with: + instance_id: ${{ secrets[env.instance-id] }} + command: | + dpkg -i ${{ env.node-package }}; \ + dpkg -i ${{ env.faucet-package }} + + # The faucet uses the public faucet generated in the genesis block. + - name: Configure environment + uses: ./.github/actions/ssm_execute + with: + instance_id: ${{ secrets[env.instance-id] }} + command: | + sudo /usr/bin/miden-node init -c /etc/opt/miden-node/miden-node.toml -g /etc/opt/miden-node/genesis.toml; \ + sudo /usr/bin/miden-node make-genesis -i /etc/opt/miden-node/genesis.toml -o /opt/miden-node/genesis.dat --force; \ + sudo /usr/bin/miden-faucet init -c /etc/opt/miden-faucet/miden-faucet.toml -f /opt/miden-faucet/accounts/faucet.mac; \ + sudo mkdir /opt/miden-faucet/accounts; \ + sudo cp /opt/miden-node/accounts/faucet.mac /opt/miden-faucet/accounts/faucet.mac; \ + sudo chown -R miden-node /opt/miden-node; \ + sudo chown -R miden-faucet /opt/miden-faucet; + + - name: Start miden services + uses: ./.github/actions/ssm_execute + with: + instance_id: ${{ secrets[env.instance-id] }} + command: | + sudo systemctl daemon-reload; \ + sudo systemctl start miden-node; \ + sudo systemctl start miden-faucet; diff --git a/.github/workflows/deploy_package.yml b/.github/workflows/deploy_package.yml deleted file mode 100644 index d2de0c2ba..000000000 --- a/.github/workflows/deploy_package.yml +++ /dev/null @@ -1,136 +0,0 @@ -on: - workflow_call: - inputs: - target: - type: string - default: "devnet" - tag: - type: string - default: "next" - oidcrole: - type: string - default: midendev - account_id: - type: string - instance_id: - type: string - -permissions: - id-token: write - contents: write - -jobs: - deploy: - runs-on: ubuntu-latest - steps: - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-region: eu-west-1 - role-to-assume: "arn:aws:iam::${{ secrets[inputs.account_id] }}:role/${{ inputs.oidcrole }}-GithubActionsRole" - role-session-name: GithubActionsSession - - - name: Checkout Code Repository for reusable action - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Install awscli - uses: ./.github/actions/ssm_execute - with: - instance_id: ${{ secrets[inputs.instance_id] }} - command: | - sudo apt udpate; \ - sudo apt install awscli -y - if: ${{ inputs.target == 'devnet' }} - - - name: Execute Architecture for instance - id: get_arch - uses: ./.github/actions/ssm_execute - with: - instance_id: ${{ secrets[inputs.instance_id] }} - command: | - uname -m - - - name: Determine architecture - run: | - if [[ "${{ steps.get_arch.outputs.cmd_result }}" =~ ^x86 ]]; then - echo "package_postfix=amd64.deb" >> $GITHUB_ENV - else - echo "package_postfix=arm64.deb" >> $GITHUB_ENV - fi - - - name: Stop miden services - uses: ./.github/actions/ssm_execute - with: - instance_id: ${{ secrets[inputs.instance_id] }} - command: | - sudo systemctl stop miden-node; sudo systemctl stop miden-faucet; sudo apt remove miden-node miden-faucet -y; sudo rm -f miden-* - - - name: Package download testnet - uses: ./.github/actions/ssm_execute - with: - instance_id: ${{ secrets[inputs.instance_id] }} - command: | - wget https://github.com/0xPolygonMiden/miden-node/releases/download/${{ inputs.tag }}/miden-faucet-${{ inputs.tag }}-${{ env.package_postfix }}; \ - wget https://github.com/0xPolygonMiden/miden-node/releases/download/${{ inputs.tag }}/miden-node-${{ inputs.tag }}-${{ env.package_postfix }} - if: ${{ inputs.target == 'testnet' }} - - - name: Download package to github runner - uses: actions/download-artifact@v3 - with: - name: devnet_package - if: ${{ inputs.target == 'devnet' }} - - - name: Upload packages to S3 - run: | - # Copy miden-node package to S3 - aws s3 cp ${{ runner.workspace }}/miden-node/miden-node-${{ inputs.tag }}-${{ env.package_postfix }} \ - s3://release-artifacts-${{ secrets[inputs.account_id] }}/miden-node-${{ inputs.tag }}-${{ env.package_postfix }} - # Copy miden-faucet package to S3 - aws s3 cp ${{ runner.workspace }}/miden-node/miden-faucet-${{ inputs.tag }}-${{ env.package_postfix }} \ - s3://release-artifacts-${{ secrets[inputs.account_id] }}/miden-faucet-${{ inputs.tag }}-${{ env.package_postfix }} - if: ${{ inputs.target == 'devnet' }} - - - name: Download package to devnet - uses: ./.github/actions/ssm_execute - with: - instance_id: ${{ secrets[inputs.instance_id] }} - command: | - aws s3 cp s3://release-artifacts-${{ secrets[inputs.account_id] }}/miden-node-${{ inputs.tag }}-${{ env.package_postfix }} miden-node-${{ inputs.tag }}-${{ env.package_postfix }}; \ - aws s3 cp s3://release-artifacts-${{ secrets[inputs.account_id] }}/miden-faucet-${{ inputs.tag }}-${{ env.package_postfix }} miden-faucet-${{ inputs.tag }}-${{ env.package_postfix }} - if: ${{ inputs.target == 'devnet' }} - - - name: Package install on ${{ inputs.target }} - uses: ./.github/actions/ssm_execute - with: - instance_id: ${{ secrets[inputs.instance_id] }} - command: | - dpkg -i miden-node-${{ inputs.tag }}-${{ env.package_postfix }}; \ - dpkg -i miden-faucet-${{ inputs.tag }}-${{ env.package_postfix }} - - - name: Configure environment - uses: ./.github/actions/ssm_execute - with: - instance_id: ${{ secrets[inputs.instance_id] }} - command: | - sudo chown -R miden /opt/miden; \ - sudo /usr/bin/miden-node init -c /etc/miden/miden-node.toml -g /opt/miden/miden-node/genesis.toml; \ - sudo /usr/bin/miden-node make-genesis -i /opt/miden/miden-node/genesis.toml -o /opt/miden/miden-node/genesis.dat --force; \ - sudo /usr/bin/miden-faucet init -c /opt/miden/miden-faucet/miden-faucet.toml -f /opt/miden/miden-node/accounts/faucet.mac - - - name: Start miden node service - uses: ./.github/actions/ssm_execute - with: - instance_id: ${{ secrets[inputs.instance_id] }} - command: | - sudo systemctl daemon-reload; \ - sudo systemctl start miden-node - - - name: Start miden faucet service - uses: ./.github/actions/ssm_execute - with: - instance_id: ${{ secrets[inputs.instance_id] }} - command: | - sudo systemctl daemon-reload; \ - sudo systemctl start miden-faucet diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml new file mode 100644 index 000000000..65701f275 --- /dev/null +++ b/.github/workflows/package.yml @@ -0,0 +1,69 @@ +# Release debian packages for miden node and faucet. +name: Release packages +run-name: Release packaging for ${{ inputs.version || github.ref }} + +env: + version: ${{ inputs.version || github.ref }} + +on: + release: + types: [released, prereleased] + + workflow_dispatch: + inputs: + version: + description: 'Version tag' + required: true + type: string + +permissions: + id-token: write + contents: write + +jobs: + package: + name: ${{ inputs.version }} for ${{ matrix.arch }} + strategy: + matrix: + arch: [amd64, arm64] + runs-on: + labels: ${{ matrix.arch == 'arm64' && 'ubuntu22-arm-4core' || 'ubuntu-latest' }} + steps: + # Note that this checkout is _not_ used as the source for the package. + # Instead this is required to access the workflow actions. Package source + # selection is handled by the packaging action. + - name: Checkout repo + uses: actions/checkout@main + with: + fetch-depth: 0 + + - name: Build packages + uses: ./.github/actions/build_package + with: + gitref: ${{ env.version }} + + - name: Package names + run: | + echo "node-package=miden-node-${{ env.version }}-${{ matrix.arch }}.deb" >> $GITHUB_ENV + echo "faucet-package=miden-faucet-${{ env.version }}-${{ matrix.arch }}.deb" >> $GITHUB_ENV + + - name: Rename package files + run: | + mv miden-node.deb ${{ env.node-package }} + mv miden-faucet.deb ${{ env.faucet-package }} + + - name: shasum packages + run: | + sha256sum ${{ env.node-package }} > ${{ env.node-package }}.checksum + sha256sum ${{ env.faucet-package }} > ${{ env.faucet-package }}.checksum + + - name: Publish packages + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh release upload ${{ env.version }} \ + ${{ env.node-package }} \ + ${{ env.node-package }}.checksum \ + ${{ env.faucet-package }} \ + ${{ env.faucet-package }}.checksum \ + --clobber diff --git a/CHANGELOG.md b/CHANGELOG.md index d4eec5ccb..3070e8ea4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,7 @@ - Reduce cloning in the store's `apply_block` (#532). - [BREAKING] Changed faucet storage type in the genesis to public. Using faucet from the genesis for faucet web app. Added support for faucet restarting without blockchain restarting (#517). - [BREAKING] Improved `ApplyBlockError` in the store (#535). +- [BREAKING] Updated minimum Rust version to 1.82. ## 0.5.1 (2024-09-12) diff --git a/README.md b/README.md index 76874276f..6ea1de0c4 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![LICENSE](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/0xPolygonMiden/miden-node/blob/main/LICENSE) [![test](https://github.com/0xPolygonMiden/miden-node/actions/workflows/test.yml/badge.svg)](https://github.com/0xPolygonMiden/miden-node/actions/workflows/test.yml) -[![RUST_VERSION](https://img.shields.io/badge/rustc-1.80+-lightgray.svg)](https://www.rust-lang.org/tools/install) +[![RUST_VERSION](https://img.shields.io/badge/rustc-1.82+-lightgray.svg)](https://www.rust-lang.org/tools/install) [![crates.io](https://img.shields.io/crates/v/miden-node)](https://crates.io/crates/miden-node) This repository holds the Miden node; that is, the software which processes transactions and creates blocks for the Miden rollup. @@ -58,7 +58,7 @@ sudo dpkg -i $package_name.deb ### Install using `cargo` -Install Rust version **1.80** or greater using the official Rust installation [instructions](https://www.rust-lang.org/tools/install). +Install Rust version **1.82** or greater using the official Rust installation [instructions](https://www.rust-lang.org/tools/install). Depending on the platform, you may need to install additional libraries. For example, on Ubuntu 22.04 the following command ensures that all required libraries are installed. @@ -121,7 +121,7 @@ Next, bootstrap the chain by generating the genesis data: ```sh miden-node make-genesis \ - --input-path /genesis.toml \ + --inputs-path /genesis.toml \ --output-path /genesis.dat ``` diff --git a/bin/node/Dockerfile b/bin/node/Dockerfile index 0195f0068..f29a750e9 100644 --- a/bin/node/Dockerfile +++ b/bin/node/Dockerfile @@ -1,4 +1,4 @@ -FROM rust:1.80-slim-bookworm AS builder +FROM rust:1.82-slim-bookworm AS builder RUN apt-get update && \ apt-get -y upgrade && \ diff --git a/packaging/miden-faucet.service b/packaging/faucet/miden-faucet.service similarity index 55% rename from packaging/miden-faucet.service rename to packaging/faucet/miden-faucet.service index 50c0cbb3c..d73f6a793 100644 --- a/packaging/miden-faucet.service +++ b/packaging/faucet/miden-faucet.service @@ -8,8 +8,8 @@ WantedBy=multi-user.target [Service] Type=exec Environment="RUST_LOG=info" -ExecStart=/usr/bin/miden-faucet start --config /opt/miden/miden-faucet/miden-faucet.toml -WorkingDirectory=/opt/miden/miden-faucet -User=miden +ExecStart=/usr/bin/miden-faucet start --config /etc/opt/miden-faucet/miden-faucet.toml +WorkingDirectory=/opt/miden-faucet +User=miden-faucet RestartSec=5 Restart=always diff --git a/packaging/faucet/postinst b/packaging/faucet/postinst new file mode 100644 index 000000000..3b2c4bad1 --- /dev/null +++ b/packaging/faucet/postinst @@ -0,0 +1,26 @@ +#!/bin/bash +# +# This is a postinstallation script so the service can be configured and started when requested. + +# user is expected by the systemd service file and `/opt/` is its working directory, +sudo adduser --disabled-password --disabled-login --shell /usr/sbin/nologin --quiet --system --no-create-home --home /nonexistent miden-faucet + +# Working folder. +if [ -d "/opt/miden-faucet" ] +then + echo "Directory /opt/miden-faucet exists." +else + mkdir -p /opt/miden-faucet + sudo chown -R miden-faucet /opt/miden-faucet +fi + +# Configuration folder +if [ -d "/etc/opt/miden-faucet" ] +then + echo "Directory /etc/opt/miden-faucet exists." +else + mkdir -p /etc/opt/miden-faucet + sudo chown -R miden-faucet /etc/opt/miden-faucet +fi + +sudo systemctl daemon-reload diff --git a/packaging/postrm.faucet b/packaging/faucet/postrm old mode 100755 new mode 100644 similarity index 51% rename from packaging/postrm.faucet rename to packaging/faucet/postrm index fb2e11765..16a51a388 --- a/packaging/postrm.faucet +++ b/packaging/faucet/postrm @@ -1,9 +1,10 @@ #!/bin/bash # ############### -# Remove miden installs +# Remove miden-faucet installs ############## sudo rm -rf /lib/systemd/system/miden-faucet.service -sudo rm -rf /opt/miden/miden-faucet -sudo deluser miden +sudo rm -rf /etc/opt/miden-faucet +sudo rm -rf /opt/miden-faucet +sudo deluser miden-faucet sudo systemctl daemon-reload diff --git a/packaging/miden-node.service b/packaging/node/miden-node.service similarity index 56% rename from packaging/miden-node.service rename to packaging/node/miden-node.service index 24021397e..7c520422d 100644 --- a/packaging/miden-node.service +++ b/packaging/node/miden-node.service @@ -8,8 +8,8 @@ WantedBy=multi-user.target [Service] Type=exec Environment="RUST_LOG=info" -ExecStart=/usr/bin/miden-node start --config /etc/miden/miden-node.toml node -WorkingDirectory=/opt/miden/miden-node -User=miden +ExecStart=/usr/bin/miden-node start --config /etc/opt/miden-node/miden-node.toml node +WorkingDirectory=/opt/miden-node +User=miden-node RestartSec=5 Restart=always diff --git a/packaging/node/postinst b/packaging/node/postinst new file mode 100644 index 000000000..ba2595059 --- /dev/null +++ b/packaging/node/postinst @@ -0,0 +1,26 @@ +#!/bin/bash +# +# This is a postinstallation script so the service can be configured and started when requested. + +# user is expected by the systemd service file and `/opt/` is its working directory, +sudo adduser --disabled-password --disabled-login --shell /usr/sbin/nologin --quiet --system --no-create-home --home /nonexistent miden-node + +# Working folder. +if [ -d "/opt/miden-node" ] +then + echo "Directory /opt/miden-node exists." +else + mkdir -p /opt/miden-node + sudo chown -R miden-node /opt/miden-node +fi + +# Configuration folder +if [ -d "/etc/opt/miden-node" ] +then + echo "Directory /etc/opt/miden-node exists." +else + mkdir -p /etc/opt/miden-node + sudo chown -R miden-node /etc/opt/miden-node +fi + +sudo systemctl daemon-reload diff --git a/packaging/postrm b/packaging/node/postrm old mode 100755 new mode 100644 similarity index 52% rename from packaging/postrm rename to packaging/node/postrm index 120a95228..376c91b8b --- a/packaging/postrm +++ b/packaging/node/postrm @@ -1,10 +1,10 @@ #!/bin/bash # ############### -# Remove miden installs +# Remove miden-node installs ############## sudo rm -rf /lib/systemd/system/miden-node.service -sudo rm -rf /etc/miden -sudo rm -rf /opt/miden/miden-node -sudo deluser miden +sudo rm -rf /etc/opt/miden-node +sudo rm -rf /opt/miden-node +sudo deluser miden-node sudo systemctl daemon-reload diff --git a/packaging/postinst b/packaging/postinst deleted file mode 100755 index 23a37dfa3..000000000 --- a/packaging/postinst +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -# -# This is a postinstallation script so the service can be configured and started when requested. - -# `miden` user is expected by the systemd service file and `/opt/miden` is its working directory. -sudo adduser --disabled-password --disabled-login --shell /usr/sbin/nologin --quiet --system --no-create-home --home /nonexistent miden - -if [ -d "/opt/miden" ] -then - echo "Directory /opt/miden exists." -else - mkdir -p /opt/miden - sudo chown -R miden /opt/miden -fi -sudo systemctl daemon-reload From fd85af8cdabb1442ae234b25537fbb1a075c53f8 Mon Sep 17 00:00:00 2001 From: Bobbin Threadbare <43513081+bobbinth@users.noreply.github.com> Date: Thu, 23 Jan 2025 07:22:55 -0800 Subject: [PATCH 50/50] chore: prepare v0.7.0 release (#638) --- CHANGELOG.md | 2 ++ Cargo.lock | 29 ++++++++++++++++------------- Cargo.toml | 18 +++++++++--------- LICENSE | 2 +- 4 files changed, 28 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3070e8ea4..8ed656eaf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## Unreleased +## v0.7.0 (2025-01-23) + ### Enhancements - Support Https in endpoint configuration (#556). diff --git a/Cargo.lock b/Cargo.lock index e62c02307..a1363a3c7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1481,7 +1481,7 @@ dependencies = [ [[package]] name = "miden-faucet" -version = "0.6.0" +version = "0.7.0" dependencies = [ "anyhow", "axum", @@ -1519,7 +1519,8 @@ dependencies = [ [[package]] name = "miden-lib" version = "0.7.0" -source = "git+https://github.com/0xPolygonMiden/miden-base.git?branch=next#829580636d2098978ece0a06d01477e2b3a3a131" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ee8babd17ea380c6c5b948761ca63208b633b7130379ee2a57c6d3732d2f8bc" dependencies = [ "miden-assembly", "miden-objects", @@ -1573,7 +1574,7 @@ dependencies = [ [[package]] name = "miden-node" -version = "0.6.0" +version = "0.7.0" dependencies = [ "anyhow", "clap", @@ -1594,7 +1595,7 @@ dependencies = [ [[package]] name = "miden-node-block-producer" -version = "0.6.0" +version = "0.7.0" dependencies = [ "assert_matches", "async-trait", @@ -1622,7 +1623,7 @@ dependencies = [ [[package]] name = "miden-node-proto" -version = "0.6.0" +version = "0.7.0" dependencies = [ "anyhow", "hex", @@ -1639,7 +1640,7 @@ dependencies = [ [[package]] name = "miden-node-rpc" -version = "0.6.0" +version = "0.7.0" dependencies = [ "miden-node-proto", "miden-node-utils", @@ -1655,7 +1656,7 @@ dependencies = [ [[package]] name = "miden-node-store" -version = "0.6.0" +version = "0.7.0" dependencies = [ "assert_matches", "deadpool-sqlite", @@ -1684,7 +1685,7 @@ dependencies = [ [[package]] name = "miden-node-utils" -version = "0.6.0" +version = "0.7.0" dependencies = [ "anyhow", "figment", @@ -1704,7 +1705,8 @@ dependencies = [ [[package]] name = "miden-objects" version = "0.7.0" -source = "git+https://github.com/0xPolygonMiden/miden-base.git?branch=next#829580636d2098978ece0a06d01477e2b3a3a131" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7532561ba51c1edcdc510e4e4a097357d49a2b6ea899d9982176dc5608bfd32e" dependencies = [ "getrandom", "miden-assembly", @@ -1749,7 +1751,7 @@ dependencies = [ [[package]] name = "miden-rpc-proto" -version = "0.6.0" +version = "0.7.0" [[package]] name = "miden-stdlib" @@ -1763,7 +1765,8 @@ dependencies = [ [[package]] name = "miden-tx" version = "0.7.0" -source = "git+https://github.com/0xPolygonMiden/miden-base.git?branch=next#829580636d2098978ece0a06d01477e2b3a3a131" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4371509f1e4c25dfe26b7ffcffbb34aaa152c6eaad400f2624240a941baed2d0" dependencies = [ "async-trait", "miden-lib", @@ -3292,9 +3295,9 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "trybuild" -version = "1.0.102" +version = "1.0.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f14b5c02a137632f68194ec657ecb92304138948e8957c932127eb1b58c23be" +checksum = "b812699e0c4f813b872b373a4471717d9eb550da14b311058a4d9cf4173cbca6" dependencies = [ "dissimilar", "glob", diff --git a/Cargo.toml b/Cargo.toml index 4ec0576ae..92dbb1847 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,7 +16,7 @@ resolver = "2" [workspace.package] edition = "2021" rust-version = "1.82" -version = "0.6.0" +version = "0.7.0" license = "MIT" authors = ["Miden contributors"] homepage = "https://polygon.technology/polygon-miden" @@ -27,17 +27,17 @@ readme = "README.md" [workspace.dependencies] assert_matches = { version = "1.5" } miden-air = { version = "0.12" } -miden-lib = { git = "https://github.com/0xPolygonMiden/miden-base.git", branch = "next" } -miden-node-block-producer = { path = "crates/block-producer", version = "0.6" } -miden-node-proto = { path = "crates/proto", version = "0.6" } -miden-node-rpc = { path = "crates/rpc", version = "0.6" } -miden-node-store = { path = "crates/store", version = "0.6" } +miden-lib = { version = "0.7" } +miden-node-block-producer = { path = "crates/block-producer", version = "0.7" } +miden-node-proto = { path = "crates/proto", version = "0.7" } +miden-node-rpc = { path = "crates/rpc", version = "0.7" } +miden-node-store = { path = "crates/store", version = "0.7" } miden-node-test-macro = { path = "crates/test-macro" } -miden-node-utils = { path = "crates/utils", version = "0.6" } -miden-objects = { git = "https://github.com/0xPolygonMiden/miden-base.git", branch = "next" } +miden-node-utils = { path = "crates/utils", version = "0.7" } +miden-objects = { version = "0.7" } miden-processor = { version = "0.12" } miden-stdlib = { version = "0.12", default-features = false } -miden-tx = { git = "https://github.com/0xPolygonMiden/miden-base.git", branch = "next" } +miden-tx = { version = "0.7" } prost = { version = "0.13" } rand = { version = "0.8" } thiserror = { version = "2.0", default-features = false } diff --git a/LICENSE b/LICENSE index 7e84649f0..58fc0d5f7 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2024 Polygon Miden +Copyright (c) 2025 Polygon Miden Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal