Add new column policy

FuelLabs · Jan 2, 2025 · b4c20a0 · b4c20a0
1 parent 3ca4d25
commit b4c20a0
Show file tree

Hide file tree

Showing 10 changed files with 144 additions and 38 deletions.
diff --git a/benches/src/db_lookup_times_utils/utils.rs b/benches/src/db_lookup_times_utils/utils.rs
@@ -5,7 +5,10 @@ use crate::db_lookup_times_utils::full_block_table::{
 use anyhow::anyhow;
 use fuel_core::{
     database::database_description::DatabaseDescription,
-    state::rocks_db::RocksDb,
+    state::rocks_db::{
+        ColumnsPolicy,
+        RocksDb,
+    },
 };
 use fuel_core_storage::kv_store::{
     KeyValueInspect,
@@ -39,7 +42,7 @@ pub fn get_random_block_height(
 pub fn open_rocks_db<Description: DatabaseDescription>(
     path: &Path,
 ) -> Result<RocksDb<Description>> {
-    let db = RocksDb::default_open(path, None, -1)?;
+    let db = RocksDb::default_open(path, None, -1, ColumnsPolicy::OnCreation)?;
     Ok(db)
 }
 

diff --git a/bin/fuel-core/src/cli/rollback.rs b/bin/fuel-core/src/cli/rollback.rs
@@ -3,7 +3,10 @@ use anyhow::Context;
 use clap::Parser;
 use fuel_core::{
     combined_database::CombinedDatabase,
-    state::historical_rocksdb::StateRewindPolicy,
+    state::{
+        historical_rocksdb::StateRewindPolicy,
+        rocks_db::ColumnsPolicy,
+    },
 };
 use rlimit::{
     getrlimit,
@@ -54,6 +57,10 @@ pub async fn exec(command: Command) -> anyhow::Result<()> {
         64 * 1024 * 1024,
         StateRewindPolicy::RewindFullRange,
         command.rocksdb_max_fds,
+        #[cfg(feature = "production")]
+        ColumnsPolicy::OnCreation,
+        #[cfg(not(feature = "production"))]
+        ColumnsPolicy::Lazy,
     )
     .map_err(Into::<anyhow::Error>::into)
     .context(format!("failed to open combined database at path {path:?}"))?;

diff --git a/bin/fuel-core/src/cli/run.rs b/bin/fuel-core/src/cli/run.rs
@@ -83,7 +83,10 @@ use tracing::{
 };
 
 #[cfg(feature = "rocksdb")]
-use fuel_core::state::historical_rocksdb::StateRewindPolicy;
+use fuel_core::state::{
+    historical_rocksdb::StateRewindPolicy,
+    rocks_db::ColumnsPolicy,
+};
 
 #[cfg(feature = "p2p")]
 mod p2p;
@@ -451,6 +454,10 @@ impl Command {
             database_path,
             database_type,
             max_database_cache_size,
+            #[cfg(all(feature = "rocksdb", feature = "production"))]
+            columns_policy: ColumnsPolicy::OnCreation,
+            #[cfg(all(feature = "rocksdb", not(feature = "production")))]
+            columns_policy: ColumnsPolicy::Lazy,
             #[cfg(feature = "rocksdb")]
             state_rewind_policy,
             #[cfg(feature = "rocksdb")]

diff --git a/bin/fuel-core/src/cli/snapshot.rs b/bin/fuel-core/src/cli/snapshot.rs
@@ -6,7 +6,10 @@ use clap::{
 };
 use fuel_core::{
     combined_database::CombinedDatabase,
-    state::historical_rocksdb::StateRewindPolicy,
+    state::{
+        historical_rocksdb::StateRewindPolicy,
+        rocks_db::ColumnsPolicy,
+    },
     types::fuel_types::ContractId,
 };
 use fuel_core_chain_config::ChainConfig;
@@ -212,6 +215,7 @@ fn open_db(
         capacity.unwrap_or(1024 * 1024 * 1024),
         StateRewindPolicy::NoRewind,
         max_fds,
+        ColumnsPolicy::OnCreation,
     )
     .map_err(Into::<anyhow::Error>::into)
     .context(format!("failed to open combined database at path {path:?}",))

diff --git a/crates/fuel-core/src/combined_database.rs b/crates/fuel-core/src/combined_database.rs
@@ -1,5 +1,9 @@
 #[cfg(feature = "rocksdb")]
-use crate::state::historical_rocksdb::StateRewindPolicy;
+use crate::state::{
+    historical_rocksdb::StateRewindPolicy,
+    rocks_db::ColumnsPolicy,
+};
+
 use crate::{
     database::{
         database_description::{
@@ -38,6 +42,8 @@ pub struct CombinedDatabaseConfig {
     pub database_type: DbType,
     pub max_database_cache_size: Option<usize>,
     #[cfg(feature = "rocksdb")]
+    pub columns_policy: ColumnsPolicy,
+    #[cfg(feature = "rocksdb")]
     pub state_rewind_policy: StateRewindPolicy,
     #[cfg(feature = "rocksdb")]
     pub max_fds: i32,
@@ -82,6 +88,7 @@ impl CombinedDatabase {
         capacity: impl Into<Option<usize>>,
         state_rewind_policy: StateRewindPolicy,
         max_fds: i32,
+        columns_policy: ColumnsPolicy,
     ) -> crate::database::Result<Self> {
         // Split the fds in equitable manner between the databases
         let max_fds = match max_fds {
@@ -90,14 +97,34 @@ impl CombinedDatabase {
         };
         let capacity = capacity.into();
         // TODO: Use different cache sizes for different databases
-        let on_chain =
-            Database::open_rocksdb(path, capacity, state_rewind_policy, max_fds)?;
-        let off_chain =
-            Database::open_rocksdb(path, capacity, state_rewind_policy, max_fds)?;
-        let relayer =
-            Database::open_rocksdb(path, capacity, StateRewindPolicy::NoRewind, max_fds)?;
-        let gas_price =
-            Database::open_rocksdb(path, capacity, state_rewind_policy, max_fds)?;
+        let on_chain = Database::open_rocksdb(
+            path,
+            capacity,
+            state_rewind_policy,
+            max_fds,
+            columns_policy,
+        )?;
+        let off_chain = Database::open_rocksdb(
+            path,
+            capacity,
+            state_rewind_policy,
+            max_fds,
+            columns_policy,
+        )?;
+        let relayer = Database::open_rocksdb(
+            path,
+            capacity,
+            StateRewindPolicy::NoRewind,
+            max_fds,
+            columns_policy,
+        )?;
+        let gas_price = Database::open_rocksdb(
+            path,
+            capacity,
+            state_rewind_policy,
+            max_fds,
+            columns_policy,
+        )?;
         Ok(Self {
             on_chain,
             off_chain,
@@ -148,6 +175,7 @@ impl CombinedDatabase {
                         config.max_database_cache_size,
                         config.state_rewind_policy,
                         config.max_fds,
+                        config.columns_policy,
                     )?
                 }
             }

diff --git a/crates/fuel-core/src/database.rs b/crates/fuel-core/src/database.rs
@@ -204,13 +204,15 @@ where
         capacity: impl Into<Option<usize>>,
         state_rewind_policy: StateRewindPolicy,
         max_fds: i32,
+        columns_policy: crate::state::rocks_db::ColumnsPolicy,
     ) -> Result<Self> {
         use anyhow::Context;
         let db = HistoricalRocksDB::<Description>::default_open(
             path,
             capacity.into(),
             state_rewind_policy,
             max_fds,
+            columns_policy,
         )
         .map_err(Into::<anyhow::Error>::into)
         .with_context(|| {
@@ -263,6 +265,10 @@ where
         let db = RocksDb::<Historical<Description>>::default_open_temp_with_params(
             max_database_cache_size.into(),
             max_fds,
+            #[cfg(feature = "rocksdb-production")]
+            crate::state::rocks_db::ColumnsPolicy::Lazy,
+            #[cfg(not(feature = "rocksdb-production"))]
+            crate::state::rocks_db::ColumnsPolicy::OnCreation,
         )?;
         let historical_db = HistoricalRocksDB::new(db, state_rewind_policy)?;
         let data = Arc::new(historical_db);
@@ -1117,6 +1123,7 @@ mod tests {
             1024 * 1024 * 1024,
             Default::default(),
             512,
+            crate::state::rocks_db::ColumnsPolicy::Lazy,
         )
         .unwrap();
         // rocks db fails

diff --git a/crates/fuel-core/src/graphql_api/indexation.rs b/crates/fuel-core/src/graphql_api/indexation.rs
@@ -238,6 +238,7 @@ mod tests {
                 MessageBalances,
             },
         },
+        state::rocks_db::ColumnsPolicy,
     };
 
     impl PartialEq for IndexationError {
@@ -352,9 +353,14 @@ mod tests {
     fn balances_enabled_flag_is_respected() {
         use tempfile::TempDir;
         let tmp_dir = TempDir::new().unwrap();
-        let mut db: Database<OffChain> =
-            Database::open_rocksdb(tmp_dir.path(), None, Default::default(), 512)
-                .unwrap();
+        let mut db: Database<OffChain> = Database::open_rocksdb(
+            tmp_dir.path(),
+            None,
+            Default::default(),
+            512,
+            ColumnsPolicy::Lazy,
+        )
+        .unwrap();
         let mut tx = db.write_transaction();
 
         const BALANCES_ARE_DISABLED: bool = false;
@@ -409,9 +415,14 @@ mod tests {
     fn coins() {
         use tempfile::TempDir;
         let tmp_dir = TempDir::new().unwrap();
-        let mut db: Database<OffChain> =
-            Database::open_rocksdb(tmp_dir.path(), None, Default::default(), 512)
-                .unwrap();
+        let mut db: Database<OffChain> = Database::open_rocksdb(
+            tmp_dir.path(),
+            None,
+            Default::default(),
+            512,
+            ColumnsPolicy::Lazy,
+        )
+        .unwrap();
         let mut tx = db.write_transaction();
 
         const BALANCES_ARE_ENABLED: bool = true;
@@ -481,9 +492,14 @@ mod tests {
     fn messages() {
         use tempfile::TempDir;
         let tmp_dir = TempDir::new().unwrap();
-        let mut db: Database<OffChain> =
-            Database::open_rocksdb(tmp_dir.path(), None, Default::default(), 512)
-                .unwrap();
+        let mut db: Database<OffChain> = Database::open_rocksdb(
+            tmp_dir.path(),
+            None,
+            Default::default(),
+            512,
+            ColumnsPolicy::Lazy,
+        )
+        .unwrap();
         let mut tx = db.write_transaction();
 
         const BALANCES_ARE_ENABLED: bool = true;
@@ -589,9 +605,14 @@ mod tests {
     fn coin_balance_overflow_does_not_error() {
         use tempfile::TempDir;
         let tmp_dir = TempDir::new().unwrap();
-        let mut db: Database<OffChain> =
-            Database::open_rocksdb(tmp_dir.path(), None, Default::default(), 512)
-                .unwrap();
+        let mut db: Database<OffChain> = Database::open_rocksdb(
+            tmp_dir.path(),
+            None,
+            Default::default(),
+            512,
+            ColumnsPolicy::Lazy,
+        )
+        .unwrap();
         let mut tx = db.write_transaction();
 
         const BALANCES_ARE_ENABLED: bool = true;
@@ -623,9 +644,14 @@ mod tests {
     fn message_balance_overflow_does_not_error() {
         use tempfile::TempDir;
         let tmp_dir = TempDir::new().unwrap();
-        let mut db: Database<OffChain> =
-            Database::open_rocksdb(tmp_dir.path(), None, Default::default(), 512)
-                .unwrap();
+        let mut db: Database<OffChain> = Database::open_rocksdb(
+            tmp_dir.path(),
+            None,
+            Default::default(),
+            512,
+            ColumnsPolicy::Lazy,
+        )
+        .unwrap();
         let mut tx = db.write_transaction();
 
         const BALANCES_ARE_ENABLED: bool = true;
@@ -661,9 +687,14 @@ mod tests {
     fn coin_balance_underflow_causes_error() {
         use tempfile::TempDir;
         let tmp_dir = TempDir::new().unwrap();
-        let mut db: Database<OffChain> =
-            Database::open_rocksdb(tmp_dir.path(), None, Default::default(), 512)
-                .unwrap();
+        let mut db: Database<OffChain> = Database::open_rocksdb(
+            tmp_dir.path(),
+            None,
+            Default::default(),
+            512,
+            ColumnsPolicy::Lazy,
+        )
+        .unwrap();
         let mut tx = db.write_transaction();
 
         const BALANCES_ARE_ENABLED: bool = true;

diff --git a/crates/fuel-core/src/service/config.rs b/crates/fuel-core/src/service/config.rs
@@ -120,6 +120,8 @@ impl Config {
             max_database_cache_size: Some(10 * 1024 * 1024),
             database_path: Default::default(),
             #[cfg(feature = "rocksdb")]
+            columns_policy: crate::state::rocks_db::ColumnsPolicy::Lazy,
+            #[cfg(feature = "rocksdb")]
             database_type: DbType::RocksDb,
             #[cfg(not(feature = "rocksdb"))]
             database_type: DbType::InMemory,

diff --git a/crates/fuel-core/src/state/historical_rocksdb.rs b/crates/fuel-core/src/state/historical_rocksdb.rs
@@ -18,7 +18,10 @@ use crate::{
         },
         iterable_key_value_view::IterableKeyValueViewWrapper,
         key_value_view::KeyValueViewWrapper,
-        rocks_db::RocksDb,
+        rocks_db::{
+            ColumnsPolicy,
+            RocksDb,
+        },
         ColumnType,
         IterableKeyValueView,
         KeyValueView,
@@ -109,9 +112,14 @@ where
         capacity: Option<usize>,
         state_rewind_policy: StateRewindPolicy,
         max_fds: i32,
+        columns_policy: ColumnsPolicy,
     ) -> DatabaseResult<Self> {
-        let db =
-            RocksDb::<Historical<Description>>::default_open(path, capacity, max_fds)?;
+        let db = RocksDb::<Historical<Description>>::default_open(
+            path,
+            capacity,
+            max_fds,
+            columns_policy,
+        )?;
         Ok(Self {
             state_rewind_policy,
             db,

diff --git a/tests/tests/aws_kms.rs b/tests/tests/aws_kms.rs
@@ -1,4 +1,7 @@
-use fuel_core::combined_database::CombinedDatabase;
+use fuel_core::{
+    combined_database::CombinedDatabase,
+    state::rocks_db::ColumnsPolicy,
+};
 use fuel_core_storage::transactional::AtomicView;
 use fuel_core_types::blockchain::consensus::Consensus;
 use test_helpers::fuel_core_driver::FuelCoreDriver;
@@ -47,8 +50,14 @@ async fn can_get_sealed_block_from_poa_produced_block_when_signing_with_kms() {
 
     // stop the node and just grab the database
     let db_path = driver.kill().await;
-    let db = CombinedDatabase::open(db_path.path(), 1024 * 1024, Default::default(), 512)
-        .unwrap();
+    let db = CombinedDatabase::open(
+        db_path.path(),
+        1024 * 1024,
+        Default::default(),
+        512,
+        ColumnsPolicy::Lazy,
+    )
+    .unwrap();
 
     let view = db.on_chain().latest_view().unwrap();