feat(storage, mdbx): transaction manager (#6126)

paradigmxyz · Jan 23, 2024 · a6f8e44 · a6f8e44
1 parent 9a5120a
commit a6f8e44
Show file tree

Hide file tree

Showing 36 changed files with 822 additions and 263 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/bin/reth/src/cli/db_type.rs b/bin/reth/src/cli/db_type.rs
@@ -4,6 +4,7 @@ use crate::dirs::{ChainPath, DataDirPath, MaybePlatformPath};
 use alloy_chains::Chain;
 use reth_db::{
     init_db,
+    mdbx::DatabaseArguments,
     test_utils::{create_test_rw_db, TempDatabase},
     DatabaseEnv,
 };
@@ -54,7 +55,10 @@ impl DatabaseBuilder {
                 let db_path = data_dir.db_path();
 
                 tracing::info!(target: "reth::cli", path = ?db_path, "Opening database");
-                let db = Arc::new(init_db(db_path.clone(), log_level)?.with_metrics());
+                let db = Arc::new(
+                    init_db(db_path.clone(), DatabaseArguments::default().log_level(log_level))?
+                        .with_metrics(),
+                );
                 Ok(DatabaseInstance::Real { db, data_dir })
             }
         }

diff --git a/bin/reth/src/commands/db/diff.rs b/bin/reth/src/commands/db/diff.rs
@@ -15,12 +15,12 @@ use crate::{
     dirs::{DataDirPath, PlatformPath},
 };
 use reth_db::{
-    cursor::DbCursorRO, database::Database, open_db_read_only, table::Table, transaction::DbTx,
-    AccountChangeSet, AccountHistory, AccountsTrie, BlockBodyIndices, BlockOmmers,
-    BlockWithdrawals, Bytecodes, CanonicalHeaders, DatabaseEnv, HashedAccount, HashedStorage,
-    HeaderNumbers, HeaderTD, Headers, PlainAccountState, PlainStorageState, PruneCheckpoints,
-    Receipts, StorageChangeSet, StorageHistory, StoragesTrie, SyncStage, SyncStageProgress, Tables,
-    TransactionBlock, Transactions, TxHashNumber, TxSenders,
+    cursor::DbCursorRO, database::Database, mdbx::DatabaseArguments, open_db_read_only,
+    table::Table, transaction::DbTx, AccountChangeSet, AccountHistory, AccountsTrie,
+    BlockBodyIndices, BlockOmmers, BlockWithdrawals, Bytecodes, CanonicalHeaders, DatabaseEnv,
+    HashedAccount, HashedStorage, HeaderNumbers, HeaderTD, Headers, PlainAccountState,
+    PlainStorageState, PruneCheckpoints, Receipts, StorageChangeSet, StorageHistory, StoragesTrie,
+    SyncStage, SyncStageProgress, Tables, TransactionBlock, Transactions, TxHashNumber, TxSenders,
 };
 use tracing::info;
 
@@ -61,7 +61,10 @@ impl Command {
     pub fn execute(self, tool: &DbTool<'_, DatabaseEnv>) -> eyre::Result<()> {
         // open second db
         let second_db_path: PathBuf = self.secondary_datadir.join("db").into();
-        let second_db = open_db_read_only(&second_db_path, self.second_db.log_level)?;
+        let second_db = open_db_read_only(
+            &second_db_path,
+            DatabaseArguments::default().log_level(self.second_db.log_level),
+        )?;
 
         let tables = match self.table {
             Some(table) => vec![table],

diff --git a/bin/reth/src/commands/db/mod.rs b/bin/reth/src/commands/db/mod.rs
@@ -14,7 +14,9 @@ use eyre::WrapErr;
 use human_bytes::human_bytes;
 use reth_db::{
     database::Database,
-    mdbx, open_db, open_db_read_only,
+    mdbx,
+    mdbx::DatabaseArguments,
+    open_db, open_db_read_only,
     version::{get_db_version, DatabaseVersionError, DB_VERSION},
     Tables,
 };
@@ -102,7 +104,10 @@ impl Command {
         match self.command {
             // TODO: We'll need to add this on the DB trait.
             Subcommands::Stats { .. } => {
-                let db = open_db_read_only(&db_path, self.db.log_level)?;
+                let db = open_db_read_only(
+                    &db_path,
+                    DatabaseArguments::default().log_level(self.db.log_level),
+                )?;
                 let tool = DbTool::new(&db, self.chain.clone())?;
                 let mut stats_table = ComfyTable::new();
                 stats_table.load_preset(comfy_table::presets::ASCII_MARKDOWN);
@@ -186,17 +191,26 @@ impl Command {
                 println!("{stats_table}");
             }
             Subcommands::List(command) => {
-                let db = open_db_read_only(&db_path, self.db.log_level)?;
+                let db = open_db_read_only(
+                    &db_path,
+                    DatabaseArguments::default().log_level(self.db.log_level),
+                )?;
                 let tool = DbTool::new(&db, self.chain.clone())?;
                 command.execute(&tool)?;
             }
             Subcommands::Diff(command) => {
-                let db = open_db_read_only(&db_path, self.db.log_level)?;
+                let db = open_db_read_only(
+                    &db_path,
+                    DatabaseArguments::default().log_level(self.db.log_level),
+                )?;
                 let tool = DbTool::new(&db, self.chain.clone())?;
                 command.execute(&tool)?;
             }
             Subcommands::Get(command) => {
-                let db = open_db_read_only(&db_path, self.db.log_level)?;
+                let db = open_db_read_only(
+                    &db_path,
+                    DatabaseArguments::default().log_level(self.db.log_level),
+                )?;
                 let tool = DbTool::new(&db, self.chain.clone())?;
                 command.execute(&tool)?;
             }
@@ -216,12 +230,14 @@ impl Command {
                     }
                 }
 
-                let db = open_db(&db_path, self.db.log_level)?;
+                let db =
+                    open_db(&db_path, DatabaseArguments::default().log_level(self.db.log_level))?;
                 let mut tool = DbTool::new(&db, self.chain.clone())?;
                 tool.drop(db_path)?;
             }
             Subcommands::Clear(command) => {
-                let db = open_db(&db_path, self.db.log_level)?;
+                let db =
+                    open_db(&db_path, DatabaseArguments::default().log_level(self.db.log_level))?;
                 command.execute(&db)?;
             }
             Subcommands::Snapshot(command) => {

diff --git a/bin/reth/src/commands/db/snapshots/headers.rs b/bin/reth/src/commands/db/snapshots/headers.rs
@@ -3,7 +3,7 @@ use super::{
     Command,
 };
 use rand::{seq::SliceRandom, Rng};
-use reth_db::{open_db_read_only, snapshot::HeaderMask};
+use reth_db::{mdbx::DatabaseArguments, open_db_read_only, snapshot::HeaderMask};
 use reth_interfaces::db::LogLevel;
 use reth_primitives::{
     snapshot::{Compression, Filters, InclusionFilter, PerfectHashingFunction},
@@ -28,7 +28,9 @@ impl Command {
         inclusion_filter: InclusionFilter,
         phf: Option<PerfectHashingFunction>,
     ) -> eyre::Result<()> {
-        let factory = ProviderFactory::new(open_db_read_only(db_path, log_level)?, chain.clone());
+        let db_args = DatabaseArguments::default().log_level(log_level);
+
+        let factory = ProviderFactory::new(open_db_read_only(db_path, db_args)?, chain.clone());
         let provider = factory.provider()?;
         let tip = provider.last_block_number()?;
         let block_range =
@@ -43,7 +45,7 @@ impl Command {
         let mut row_indexes = block_range.clone().collect::<Vec<_>>();
         let mut rng = rand::thread_rng();
 
-        let tx_range = ProviderFactory::new(open_db_read_only(db_path, log_level)?, chain.clone())
+        let tx_range = ProviderFactory::new(open_db_read_only(db_path, db_args)?, chain.clone())
             .provider()?
             .transaction_range_by_block_range(block_range.clone())?;
 
@@ -61,7 +63,7 @@ impl Command {
         for bench_kind in [BenchKind::Walk, BenchKind::RandomAll] {
             bench(
                 bench_kind,
-                (open_db_read_only(db_path, log_level)?, chain.clone()),
+                (open_db_read_only(db_path, db_args)?, chain.clone()),
                 SnapshotSegment::Headers,
                 filters,
                 compression,
@@ -92,7 +94,7 @@ impl Command {
             let num = row_indexes[rng.gen_range(0..row_indexes.len())];
             bench(
                 BenchKind::RandomOne,
-                (open_db_read_only(db_path, log_level)?, chain.clone()),
+                (open_db_read_only(db_path, db_args)?, chain.clone()),
                 SnapshotSegment::Headers,
                 filters,
                 compression,
@@ -113,14 +115,14 @@ impl Command {
         {
             let num = row_indexes[rng.gen_range(0..row_indexes.len())] as u64;
             let header_hash =
-                ProviderFactory::new(open_db_read_only(db_path, log_level)?, chain.clone())
+                ProviderFactory::new(open_db_read_only(db_path, db_args)?, chain.clone())
                     .header_by_number(num)?
                     .ok_or(ProviderError::HeaderNotFound(num.into()))?
                     .hash_slow();
 
             bench(
                 BenchKind::RandomHash,
-                (open_db_read_only(db_path, log_level)?, chain.clone()),
+                (open_db_read_only(db_path, db_args)?, chain.clone()),
                 SnapshotSegment::Headers,
                 filters,
                 compression,

diff --git a/bin/reth/src/commands/db/snapshots/mod.rs b/bin/reth/src/commands/db/snapshots/mod.rs
@@ -2,7 +2,11 @@ use clap::{builder::RangedU64ValueParser, Parser};
 use human_bytes::human_bytes;
 use itertools::Itertools;
 use rayon::iter::{IntoParallelIterator, ParallelIterator};
-use reth_db::{database::Database, open_db_read_only, DatabaseEnv};
+use reth_db::{
+    database::Database,
+    mdbx::{DatabaseArguments, MaxReadTransactionDuration},
+    open_db_read_only, DatabaseEnv,
+};
 use reth_interfaces::db::LogLevel;
 use reth_nippy_jar::{NippyJar, NippyJarCursor};
 use reth_primitives::{
@@ -89,7 +93,11 @@ impl Command {
             );
 
         {
-            let db = open_db_read_only(db_path, None)?;
+            let db = open_db_read_only(
+                db_path,
+                DatabaseArguments::default()
+                    .max_read_transaction_duration(Some(MaxReadTransactionDuration::Unbounded)),
+            )?;
             let factory = Arc::new(ProviderFactory::new(db, chain.clone()));
 
             if !self.only_bench {

diff --git a/bin/reth/src/commands/db/snapshots/receipts.rs b/bin/reth/src/commands/db/snapshots/receipts.rs
@@ -14,6 +14,7 @@ use reth_provider::{
     TransactionsProvider, TransactionsProviderExt,
 };
 
+use reth_db::mdbx::DatabaseArguments;
 use std::{
     path::{Path, PathBuf},
     sync::Arc,
@@ -29,7 +30,9 @@ impl Command {
         inclusion_filter: InclusionFilter,
         phf: Option<PerfectHashingFunction>,
     ) -> eyre::Result<()> {
-        let factory = ProviderFactory::new(open_db_read_only(db_path, log_level)?, chain.clone());
+        let db_args = DatabaseArguments::default().log_level(log_level);
+
+        let factory = ProviderFactory::new(open_db_read_only(db_path, db_args)?, chain.clone());
         let provider = factory.provider()?;
         let tip = provider.last_block_number()?;
         let block_range =
@@ -43,7 +46,7 @@ impl Command {
 
         let mut rng = rand::thread_rng();
 
-        let tx_range = ProviderFactory::new(open_db_read_only(db_path, log_level)?, chain.clone())
+        let tx_range = ProviderFactory::new(open_db_read_only(db_path, db_args)?, chain.clone())
             .provider()?
             .transaction_range_by_block_range(block_range.clone())?;
 
@@ -64,7 +67,7 @@ impl Command {
         for bench_kind in [BenchKind::Walk, BenchKind::RandomAll] {
             bench(
                 bench_kind,
-                (open_db_read_only(db_path, log_level)?, chain.clone()),
+                (open_db_read_only(db_path, db_args)?, chain.clone()),
                 SnapshotSegment::Receipts,
                 filters,
                 compression,
@@ -95,7 +98,7 @@ impl Command {
             let num = row_indexes[rng.gen_range(0..row_indexes.len())];
             bench(
                 BenchKind::RandomOne,
-                (open_db_read_only(db_path, log_level)?, chain.clone()),
+                (open_db_read_only(db_path, db_args)?, chain.clone()),
                 SnapshotSegment::Receipts,
                 filters,
                 compression,
@@ -115,15 +118,14 @@ impl Command {
         // BENCHMARK QUERYING A RANDOM RECEIPT BY HASH
         {
             let num = row_indexes[rng.gen_range(0..row_indexes.len())] as u64;
-            let tx_hash =
-                ProviderFactory::new(open_db_read_only(db_path, log_level)?, chain.clone())
-                    .transaction_by_id(num)?
-                    .ok_or(ProviderError::ReceiptNotFound(num.into()))?
-                    .hash();
+            let tx_hash = ProviderFactory::new(open_db_read_only(db_path, db_args)?, chain.clone())
+                .transaction_by_id(num)?
+                .ok_or(ProviderError::ReceiptNotFound(num.into()))?
+                .hash();
 
             bench(
                 BenchKind::RandomHash,
-                (open_db_read_only(db_path, log_level)?, chain.clone()),
+                (open_db_read_only(db_path, db_args)?, chain.clone()),
                 SnapshotSegment::Receipts,
                 filters,
                 compression,

diff --git a/bin/reth/src/commands/db/snapshots/transactions.rs b/bin/reth/src/commands/db/snapshots/transactions.rs
@@ -14,6 +14,7 @@ use reth_provider::{
     TransactionsProvider, TransactionsProviderExt,
 };
 
+use reth_db::mdbx::DatabaseArguments;
 use std::{
     path::{Path, PathBuf},
     sync::Arc,
@@ -29,7 +30,9 @@ impl Command {
         inclusion_filter: InclusionFilter,
         phf: Option<PerfectHashingFunction>,
     ) -> eyre::Result<()> {
-        let factory = ProviderFactory::new(open_db_read_only(db_path, log_level)?, chain.clone());
+        let db_args = DatabaseArguments::default().log_level(log_level);
+
+        let factory = ProviderFactory::new(open_db_read_only(db_path, db_args)?, chain.clone());
         let provider = factory.provider()?;
         let tip = provider.last_block_number()?;
         let block_range =
@@ -61,7 +64,7 @@ impl Command {
         for bench_kind in [BenchKind::Walk, BenchKind::RandomAll] {
             bench(
                 bench_kind,
-                (open_db_read_only(db_path, log_level)?, chain.clone()),
+                (open_db_read_only(db_path, db_args)?, chain.clone()),
                 SnapshotSegment::Transactions,
                 filters,
                 compression,
@@ -93,7 +96,7 @@ impl Command {
             let num = row_indexes[rng.gen_range(0..row_indexes.len())];
             bench(
                 BenchKind::RandomOne,
-                (open_db_read_only(db_path, log_level)?, chain.clone()),
+                (open_db_read_only(db_path, db_args)?, chain.clone()),
                 SnapshotSegment::Transactions,
                 filters,
                 compression,
@@ -115,14 +118,14 @@ impl Command {
         {
             let num = row_indexes[rng.gen_range(0..row_indexes.len())] as u64;
             let transaction_hash =
-                ProviderFactory::new(open_db_read_only(db_path, log_level)?, chain.clone())
+                ProviderFactory::new(open_db_read_only(db_path, db_args)?, chain.clone())
                     .transaction_by_id(num)?
                     .ok_or(ProviderError::TransactionNotFound(num.into()))?
                     .hash();
 
             bench(
                 BenchKind::RandomHash,
-                (open_db_read_only(db_path, log_level)?, chain.clone()),
+                (open_db_read_only(db_path, db_args)?, chain.clone()),
                 SnapshotSegment::Transactions,
                 filters,
                 compression,

diff --git a/bin/reth/src/commands/debug_cmd/build_block.rs b/bin/reth/src/commands/debug_cmd/build_block.rs
@@ -11,7 +11,7 @@ use reth_beacon_consensus::BeaconConsensus;
 use reth_blockchain_tree::{
     BlockchainTree, BlockchainTreeConfig, ShareableBlockchainTree, TreeExternals,
 };
-use reth_db::{init_db, DatabaseEnv};
+use reth_db::{init_db, mdbx::DatabaseArguments, DatabaseEnv};
 use reth_interfaces::{consensus::Consensus, RethResult};
 use reth_node_api::PayloadBuilderAttributes;
 use reth_payload_builder::database::CachedReads;
@@ -150,7 +150,8 @@ impl Command {
         fs::create_dir_all(&db_path)?;
 
         // initialize the database
-        let db = Arc::new(init_db(db_path, self.db.log_level)?);
+        let db =
+            Arc::new(init_db(db_path, DatabaseArguments::default().log_level(self.db.log_level))?);
         let provider_factory = ProviderFactory::new(Arc::clone(&db), Arc::clone(&self.chain));
 
         let consensus: Arc<dyn Consensus> = Arc::new(BeaconConsensus::new(Arc::clone(&self.chain)));

diff --git a/bin/reth/src/commands/debug_cmd/execution.rs b/bin/reth/src/commands/debug_cmd/execution.rs
@@ -16,7 +16,7 @@ use clap::Parser;
 use futures::{stream::select as stream_select, StreamExt};
 use reth_beacon_consensus::BeaconConsensus;
 use reth_config::Config;
-use reth_db::{database::Database, init_db, DatabaseEnv};
+use reth_db::{database::Database, init_db, mdbx::DatabaseArguments, DatabaseEnv};
 use reth_downloaders::{
     bodies::bodies::BodiesDownloaderBuilder,
     headers::reverse_headers::ReverseHeadersDownloaderBuilder,
@@ -204,7 +204,8 @@ impl Command {
         let data_dir = self.datadir.unwrap_or_chain_default(self.chain.chain);
         let db_path = data_dir.db_path();
         fs::create_dir_all(&db_path)?;
-        let db = Arc::new(init_db(db_path, self.db.log_level)?);
+        let db =
+            Arc::new(init_db(db_path, DatabaseArguments::default().log_level(self.db.log_level))?);
         let provider_factory = ProviderFactory::new(db.clone(), self.chain.clone());
 
         debug!(target: "reth::cli", chain=%self.chain.chain, genesis=?self.chain.genesis_hash(), "Initializing genesis");