Merge branch 'mathias-FI-1296-make-build-index-interval-configurable'…

… into 'master'

feat(icrc1-index-ng): FI-1296: Make index-ng interval for retrieving blocks from the ledger configurable

Make the interval at which the ICRC1 index-ng retrieves blocks from the ledger configurable. The default interval of 1s leads to a more responsive UI/UX, but higher costs for the index-ng, ledger and archive canisters, since the index is rebuilt every second. Longer intervals means that it takes longer for transactions to show up in the index, but the cost in terms of cycles burned by the canister suite is also lower. 

See merge request dfinity-lab/public/ic!19140

rs/ethereum/ledger-suite-orchestrator/src/scheduler/mod.rs

@@ -781,6 +781,7 @@ async fn install_ledger_suite<R: CanisterRuntime>(
             .await?;
     let index_arg = Some(IndexArg::Init(IndexInitArg {
         ledger_id: ledger_canister_id,
+        retrieve_blocks_from_ledger_interval_seconds: None,
     }));
     install_canister_once::<Index, _, _>(
         &args.contract,

rs/rosetta-api/icrc1/index-ng/BUILD.bazel

@@ -105,6 +105,7 @@ rust_test(
         deps = [
             ":index-ng",
             "//packages/icrc-ledger-types:icrc_ledger_types",
+            "//rs/registry/subnet_type",
             "//rs/rosetta-api/icrc1",
             "//rs/rosetta-api/icrc1/index",
             "//rs/rosetta-api/icrc1/ledger",
@@ -116,7 +117,6 @@ rust_test(
             "//rs/state_machine_tests",
             "//rs/test_utilities/load_wasm",
             "//rs/types/base_types",
-            "@crate_index//:assert_matches",
             "@crate_index//:candid",
             "@crate_index//:ic-agent",
             "@crate_index//:num-traits",

rs/rosetta-api/icrc1/index-ng/Cargo.toml

@@ -21,7 +21,7 @@ ic-cdk = { workspace = true }
 ic-cdk-macros = { workspace = true }
 ic-cdk-timers = { workspace = true }
 ic-crypto-sha2 = { path = "../../../crypto/sha2" }
-ic-icrc1 = { path = "../" }
+ic-icrc1 = { path = ".." }
 ic-icrc1-tokens-u256 = { path = "../tokens_u256", optional = true }
 ic-icrc1-tokens-u64 = { path = "../tokens_u64" }
 ic-ledger-core = { path = "../../ledger_core" }
@@ -41,6 +41,7 @@ ic-icrc1-index = { path = "../index" }
 ic-icrc1-ledger = { path = "../ledger" }
 ic-icrc1-test-utils = { path = "../test_utils" }
 ic-ledger-canister-core = { path = "../../ledger_canister_core" }
+ic-registry-subnet-type = { path = "../../../registry/subnet_type" }
 ic-state-machine-tests = { path = "../../../state_machine_tests" }
 ic-test-utilities-load-wasm = { path = "../../../test_utilities/load_wasm" }
 proptest = "1.0"

rs/rosetta-api/icrc1/index-ng/index-ng.did

@@ -2,10 +2,18 @@ type Tokens = nat;
 
 type InitArg = record {
     ledger_id: principal;
+    // The interval in seconds in which to retrieve blocks from the ledger. A lower value makes the index more
+    // responsive in showing new blocks, but increases the consumption of cycles of both the index and ledger canisters.
+    // A higher values means that it takes longer for new blocks to show up in the index.
+    retrieve_blocks_from_ledger_interval_seconds : opt nat64;
 };
 
 type UpgradeArg = record {
     ledger_id: opt principal;
+    // The interval in seconds in which to retrieve blocks from the ledger. A lower value makes the index more
+    // responsive in showing new blocks, but increases the consumption of cycles of both the index and ledger canisters.
+    // A higher values means that it takes longer for new blocks to show up in the index.
+    retrieve_blocks_from_ledger_interval_seconds : opt nat64;
 };
 
 type IndexArg = variant {

rs/rosetta-api/icrc1/index-ng/src/lib.rs

@@ -16,11 +16,13 @@ pub enum IndexArg {
 #[derive(CandidType, Debug, Deserialize, Clone)]
 pub struct InitArg {
     pub ledger_id: Principal,
+    pub retrieve_blocks_from_ledger_interval_seconds: Option<u64>,
 }
 
 #[derive(CandidType, Debug, Deserialize, Clone)]
 pub struct UpgradeArg {
     pub ledger_id: Option<Principal>,
+    pub retrieve_blocks_from_ledger_interval_seconds: Option<u64>,
 }
 
 #[derive(CandidType, Debug, Deserialize, Eq, PartialEq)]

rs/rosetta-api/icrc1/index-ng/src/main.rs

@@ -11,8 +11,8 @@ use ic_icrc1::endpoints::StandardRecord;
 use ic_icrc1::{Block, Operation};
 use ic_icrc1_index_ng::{
     FeeCollectorRanges, GetAccountTransactionsArgs, GetAccountTransactionsResponse,
-    GetAccountTransactionsResult, GetBlocksMethod, IndexArg, ListSubaccountsArgs, Log, LogEntry,
-    Status, TransactionWithId, DEFAULT_MAX_BLOCKS_PER_RESPONSE,
+    GetAccountTransactionsResult, GetBlocksMethod, IndexArg, InitArg, ListSubaccountsArgs, Log,
+    LogEntry, Status, TransactionWithId, UpgradeArg, DEFAULT_MAX_BLOCKS_PER_RESPONSE,
 };
 use ic_ledger_core::block::{BlockIndex as BlockIndex64, BlockType, EncodedBlock};
 use ic_ledger_core::tokens::{CheckedAdd, CheckedSub, Zero};
@@ -55,7 +55,7 @@ const BLOCK_LOG_DATA_MEMORY_ID: MemoryId = MemoryId::new(2);
 const ACCOUNT_BLOCK_IDS_MEMORY_ID: MemoryId = MemoryId::new(3);
 const ACCOUNT_DATA_MEMORY_ID: MemoryId = MemoryId::new(4);
 
-const DEFAULT_MAX_WAIT_TIME: Duration = Duration::from_secs(1);
+const DEFAULT_RETRIEVE_BLOCKS_FROM_LEDGER_INTERVAL: Duration = Duration::from_secs(1);
 
 #[cfg(not(feature = "u256-tokens"))]
 type Tokens = ic_icrc1_tokens_u64::U64;
@@ -130,6 +130,18 @@ struct State {
 
     /// This fee is used if no fee nor effetive_fee is found in Approve blocks.
     pub last_fee: Option<Tokens>,
+
+    /// The interval for retrieving blocks from the ledger and archive(s) for (re)building the
+    /// index. Lower values will result in a more responsive UI, but higher costs due to increased
+    /// cycle burn for the index, ledger and archive(s).
+    retrieve_blocks_from_ledger_interval: Option<Duration>,
+}
+
+impl State {
+    pub fn retrieve_blocks_from_ledger_interval(&self) -> Duration {
+        self.retrieve_blocks_from_ledger_interval
+            .unwrap_or(DEFAULT_RETRIEVE_BLOCKS_FROM_LEDGER_INTERVAL)
+    }
 }
 
 // NOTE: the default configuration is dysfunctional, but it's convenient to have
@@ -143,6 +155,7 @@ impl Default for State {
             last_wait_time: Duration::from_secs(0),
             fee_collectors: Default::default(),
             last_fee: None,
+            retrieve_blocks_from_ledger_interval: None,
         }
     }
 }
@@ -284,18 +297,25 @@ fn balance_key(account: Account) -> (AccountDataType, (Blob<29>, [u8; 32])) {
 #[init]
 #[candid_method(init)]
 fn init(index_arg: Option<IndexArg>) {
-    let init_arg = match index_arg {
+    let InitArg {
+        ledger_id,
+        retrieve_blocks_from_ledger_interval_seconds,
+    } = match index_arg {
         Some(IndexArg::Init(arg)) => arg,
         _ => trap("Index initialization must take in input an InitArg argument"),
     };
 
     // stable memory initialization
     mutate_state(|state| {
-        state.ledger_id = init_arg.ledger_id;
+        state.ledger_id = ledger_id;
+        state.retrieve_blocks_from_ledger_interval =
+            retrieve_blocks_from_ledger_interval_seconds.map(Duration::from_secs);
     });
 
     // set the first build_index to be called after init
-    set_build_index_timer(DEFAULT_MAX_WAIT_TIME);
+    set_build_index_timer(with_state(|state| {
+        state.retrieve_blocks_from_ledger_interval()
+    }));
 }
 
 // The part of the legacy index (//rs/rosetta-api/icrc1/index) state
@@ -328,24 +348,33 @@ fn post_upgrade(index_arg: Option<IndexArg>) {
     }
 
     match index_arg {
-        Some(IndexArg::Upgrade(arg)) => {
-            if let Some(ledger_id) = arg.ledger_id {
-                log!(
-                    P1,
-                    "Found ledger_id in the upgrade arguments. ledger-id: {}",
-                    ledger_id
-                );
-                mutate_state(|state| {
-                    state.ledger_id = ledger_id;
-                });
-            }
+        Some(IndexArg::Upgrade(upgrade)) => {
+            log!(P1, "Possible upgrade configuration changes: {:#?}", upgrade,);
+
+            let UpgradeArg {
+                ledger_id,
+                retrieve_blocks_from_ledger_interval_seconds,
+            } = upgrade;
+
+            mutate_state(|state| {
+                if let Some(new_value) = ledger_id {
+                    state.ledger_id = new_value;
+                }
+
+                if let Some(new_value) = retrieve_blocks_from_ledger_interval_seconds {
+                    state.retrieve_blocks_from_ledger_interval =
+                        Some(Duration::from_secs(new_value));
+                }
+            });
         }
         Some(IndexArg::Init(..)) => trap("Index upgrade argument cannot be of variant Init"),
         _ => (),
     };
 
     // set the first build_index to be called after init
-    set_build_index_timer(DEFAULT_MAX_WAIT_TIME);
+    set_build_index_timer(with_state(|state| {
+        state.retrieve_blocks_from_ledger_interval()
+    }));
 }
 
 async fn get_supported_standards_from_ledger() -> Vec<String> {
@@ -535,11 +564,13 @@ pub async fn build_index() -> Option<()> {
         GetBlocksMethod::GetBlocks => fetch_blocks_via_get_blocks().await?,
         GetBlocksMethod::ICRC3GetBlocks => fetch_blocks_via_icrc3().await?,
     };
+    let retrieve_blocks_from_ledger_interval =
+        with_state(|state| state.retrieve_blocks_from_ledger_interval());
     log!(
         P1,
         "Indexed: {} waiting : {:?}",
         num_indexed,
-        DEFAULT_MAX_WAIT_TIME
+        retrieve_blocks_from_ledger_interval
     );
     Some(())
 }